Skip to content

Instantly share code, notes, and snippets.

@mattn

mattn/README.md

Last active February 14, 2017 15:13
Show Gist options
  • Save mattn/603cf72f66e3ca471889b95dad2b9588 to your computer and use it in GitHub Desktop.
Save mattn/603cf72f66e3ca471889b95dad2b9588 to your computer and use it in GitHub Desktop.
Download ZIP
jsonstore vs bolt
Raw
README.md 
c:\dev\go-sandbox\jsonstore>go test -bench . -count 10
goos: windows
goarch: amd64
BenchmarkJsonstore-4        2000            897051 ns/op
BenchmarkJsonstore-4        2000            885050 ns/op
BenchmarkJsonstore-4        2000           1131564 ns/op
BenchmarkJsonstore-4        2000            839048 ns/op
BenchmarkJsonstore-4        2000            919052 ns/op
BenchmarkJsonstore-4        2000            838547 ns/op
BenchmarkJsonstore-4        2000            826547 ns/op
BenchmarkJsonstore-4        2000            826047 ns/op
BenchmarkJsonstore-4        2000           1108563 ns/op
BenchmarkJsonstore-4        2000            887050 ns/op
BenchmarkBolt-4               20          52753015 ns/op
BenchmarkBolt-4               20          63603640 ns/op
BenchmarkBolt-4               20          92955320 ns/op
BenchmarkBolt-4               20          60703470 ns/op
BenchmarkBolt-4               20          62403570 ns/op
BenchmarkBolt-4               20          65103725 ns/op
BenchmarkBolt-4               20          67653870 ns/op
BenchmarkBolt-4               20          84204815 ns/op
BenchmarkBolt-4               20          99555695 ns/op
BenchmarkBolt-4               20         100305740 ns/op
PASS
ok      _/c_/dev/go-sandbox/jsonstore   45.451s

goos: windows
goarch: amd64
BenchmarkJsonstore-4                2000            884050 ns/op
BenchmarkBolt-4                       20          60253445 ns/op
BenchmarkJsonstore1000-4              10         106406090 ns/op
BenchmarkBolt1000-4                   10         135007730 ns/op
BenchmarkJsonstore50000-4              1        3646208500 ns/op
BenchmarkBolt50000-4                   1        1452083100 ns/op
PASS
ok      _/C_/dev/go-sandbox/jsonstore   13.091s
Raw
store_test.go
package main
import (
"encoding/json"
"os"
"strings"
"testing"
"github.com/boltdb/bolt"
"github.com/schollz/jsonstore"
)
type Tweet struct {
Coordinates interface{} `json:"coordinates"`
Favorited bool `json:"favorited"`
Truncated bool `json:"truncated"`
CreatedAt string `json:"created_at"`
IDStr string `json:"id_str"`
Entities struct {
Urls []struct {
ExpandedURL string `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
DisplayURL string `json:"display_url"`
} `json:"urls"`
Hashtags []interface{} `json:"hashtags"`
UserMentions []interface{} `json:"user_mentions"`
} `json:"entities"`
InReplyToUserIDStr interface{} `json:"in_reply_to_user_id_str"`
Contributors interface{} `json:"contributors"`
Text string `json:"text"`
RetweetCount int `json:"retweet_count"`
InReplyToStatusIDStr interface{} `json:"in_reply_to_status_id_str"`
ID int64 `json:"id"`
Geo interface{} `json:"geo"`
Retweeted bool `json:"retweeted"`
PossiblySensitive bool `json:"possibly_sensitive"`
InReplyToUserID interface{} `json:"in_reply_to_user_id"`
Place interface{} `json:"place"`
User struct {
ProfileSidebarFillColor string `json:"profile_sidebar_fill_color"`
ProfileSidebarBorderColor string `json:"profile_sidebar_border_color"`
ProfileBackgroundTile bool `json:"profile_background_tile"`
Name string `json:"name"`
ProfileImageURL string `json:"profile_image_url"`
CreatedAt string `json:"created_at"`
Location string `json:"location"`
FollowRequestSent bool `json:"follow_request_sent"`
ProfileLinkColor string `json:"profile_link_color"`
IsTranslator bool `json:"is_translator"`
IDStr string `json:"id_str"`
Entities struct {
URL struct {
Urls []struct {
ExpandedURL interface{} `json:"expanded_url"`
URL string `json:"url"`
Indices []int `json:"indices"`
} `json:"urls"`
} `json:"url"`
Description struct {
Urls []interface{} `json:"urls"`
} `json:"description"`
} `json:"entities"`
DefaultProfile bool `json:"default_profile"`
ContributorsEnabled bool `json:"contributors_enabled"`
FavouritesCount int `json:"favourites_count"`
URL string `json:"url"`
ProfileImageURLHTTPS string `json:"profile_image_url_https"`
UtcOffset int `json:"utc_offset"`
ID int `json:"id"`
ProfileUseBackgroundImage bool `json:"profile_use_background_image"`
ListedCount int `json:"listed_count"`
ProfileTextColor string `json:"profile_text_color"`
Lang string `json:"lang"`
FollowersCount int `json:"followers_count"`
Protected bool `json:"protected"`
Notifications interface{} `json:"notifications"`
ProfileBackgroundImageURLHTTPS string `json:"profile_background_image_url_https"`
ProfileBackgroundColor string `json:"profile_background_color"`
Verified bool `json:"verified"`
GeoEnabled bool `json:"geo_enabled"`
TimeZone string `json:"time_zone"`
Description string `json:"description"`
DefaultProfileImage bool `json:"default_profile_image"`
ProfileBackgroundImageURL string `json:"profile_background_image_url"`
StatusesCount int `json:"statuses_count"`
FriendsCount int `json:"friends_count"`
Following interface{} `json:"following"`
ShowAllInlineMedia bool `json:"show_all_inline_media"`
ScreenName string `json:"screen_name"`
} `json:"user"`
InReplyToScreenName interface{} `json:"in_reply_to_screen_name"`
Source string `json:"source"`
InReplyToStatusID interface{} `json:"in_reply_to_status_id"`
}
const data = `
[
{
"coordinates": null,
"favorited": false,
"truncated": false,
"created_at": "Wed Aug 29 17:12:58 +0000 2012",
"id_str": "240859602684612608",
"entities": {
"urls": [
{
"expanded_url": "/blog/twitter-certified-products",
"url": "https://t.co/MjJ8xAnT",
"indices": [
52,
73
],
"display_url": "dev.twitter.com/blog/twitter-c\u2026"
}
],
"hashtags": [
],
"user_mentions": [
]
},
"in_reply_to_user_id_str": null,
"contributors": null,
"text": "Introducing the Twitter Certified Products Program: https://t.co/MjJ8xAnT",
"retweet_count": 121,
"in_reply_to_status_id_str": null,
"id": 240859602684612608,
"geo": null,
"retweeted": false,
"possibly_sensitive": false,
"in_reply_to_user_id": null,
"place": null,
"user": {
"profile_sidebar_fill_color": "DDEEF6",
"profile_sidebar_border_color": "C0DEED",
"profile_background_tile": false,
"name": "Twitter API",
"profile_image_url": "http://a0.twimg.com/profile_images/2284174872/7df3h38zabcvjylnyfe3_normal.png",
"created_at": "Wed May 23 06:01:13 +0000 2007",
"location": "San Francisco, CA",
"follow_request_sent": false,
"profile_link_color": "0084B4",
"is_translator": false,
"id_str": "6253282",
"entities": {
"url": {
"urls": [
{
"expanded_url": null,
"url": "",
"indices": [
0,
22
]
}
]
},
"description": {
"urls": [
]
}
},
"default_profile": true,
"contributors_enabled": true,
"favourites_count": 24,
"url": "",
"profile_image_url_https": "https://si0.twimg.com/profile_images/2284174872/7df3h38zabcvjylnyfe3_normal.png",
"utc_offset": -28800,
"id": 6253282,
"profile_use_background_image": true,
"listed_count": 10775,
"profile_text_color": "333333",
"lang": "en",
"followers_count": 1212864,
"protected": false,
"notifications": null,
"profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png",
"profile_background_color": "C0DEED",
"verified": true,
"geo_enabled": true,
"time_zone": "Pacific Time (US & Canada)",
"description": "The Real Twitter API. I tweet about API changes, service issues and happily answer questions about Twitter and our API. Don't get an answer? It's on my website.",
"default_profile_image": false,
"profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png",
"statuses_count": 3333,
"friends_count": 31,
"following": null,
"show_all_inline_media": false,
"screen_name": "twitterapi"
},
"in_reply_to_screen_name": null,
"source": "YoruFukurou",
"in_reply_to_status_id": null
},
{
"coordinates": null,
"favorited": false,
"truncated": false,
"created_at": "Sat Aug 25 17:26:51 +0000 2012",
"id_str": "239413543487819778",
"entities": {
"urls": [
{
"expanded_url": "/issues/485",
"url": "https://t.co/p5bOzH0k",
"indices": [
97,
118
],
"display_url": "dev.twitter.com/issues/485"
}
],
"hashtags": [
],
"user_mentions": [
]
},
"in_reply_to_user_id_str": null,
"contributors": null,
"text": "We are working to resolve issues with application management & logging in to the dev portal: https://t.co/p5bOzH0k ^TS",
"retweet_count": 105,
"in_reply_to_status_id_str": null,
"id": 239413543487819778,
"geo": null,
"retweeted": false,
"possibly_sensitive": false,
"in_reply_to_user_id": null,
"place": null,
"user": {
"profile_sidebar_fill_color": "DDEEF6",
"profile_sidebar_border_color": "C0DEED",
"profile_background_tile": false,
"name": "Twitter API",
"profile_image_url": "http://a0.twimg.com/profile_images/2284174872/7df3h38zabcvjylnyfe3_normal.png",
"created_at": "Wed May 23 06:01:13 +0000 2007",
"location": "San Francisco, CA",
"follow_request_sent": false,
"profile_link_color": "0084B4",
"is_translator": false,
"id_str": "6253282",
"entities": {
"url": {
"urls": [
{
"expanded_url": null,
"url": "",
"indices": [
0,
22
]
}
]
},
"description": {
"urls": [
]
}
},
"default_profile": true,
"contributors_enabled": true,
"favourites_count": 24,
"url": "",
"profile_image_url_https": "https://si0.twimg.com/profile_images/2284174872/7df3h38zabcvjylnyfe3_normal.png",
"utc_offset": -28800,
"id": 6253282,
"profile_use_background_image": true,
"listed_count": 10775,
"profile_text_color": "333333",
"lang": "en",
"followers_count": 1212864,
"protected": false,
"notifications": null,
"profile_background_image_url_https": "https://si0.twimg.com/images/themes/theme1/bg.png",
"profile_background_color": "C0DEED",
"verified": true,
"geo_enabled": true,
"time_zone": "Pacific Time (US & Canada)",
"description": "The Real Twitter API. I tweet about API changes, service issues and happily answer questions about Twitter and our API. Don't get an answer? It's on my website.",
"default_profile_image": false,
"profile_background_image_url": "http://a0.twimg.com/images/themes/theme1/bg.png",
"statuses_count": 3333,
"friends_count": 31,
"following": null,
"show_all_inline_media": false,
"screen_name": "twitterapi"
},
"in_reply_to_screen_name": null,
"source": "YoruFukurou",
"in_reply_to_status_id": null
}
]
`
func makeData() (tweets []Tweet) {
err := json.NewDecoder(strings.NewReader(data)).Decode(&tweets)
if err != nil {
panic(err)
}
return
}
func makeDataN(n int) (tweets []Tweet) {
err := json.NewDecoder(strings.NewReader(data)).Decode(&tweets)
if err != nil {
panic(err)
}
for len(tweets) < n {
tweets = append(tweets, tweets...)
}
return tweets[:n]
}
func BenchmarkJsonstore(b *testing.B) {
os.Remove("foo.gz")
sample := makeData()
b.ResetTimer()
js := new(jsonstore.JSONStore)
err := jsonstore.Save(js, "foo.gz")
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
err = js.Set("data", sample)
if err != nil {
b.Fatal(err)
}
err = jsonstore.Save(js, "foo.gz")
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkBolt(b *testing.B) {
os.Remove("foo.db")
sample := makeData()
b.ResetTimer()
db, err := bolt.Open("foo.db", 0600, nil)
if err != nil {
b.Fatal(err)
}
defer db.Close()
db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucket([]byte("MyBucket"))
if err != nil {
b.Fatal(err)
}
return nil
})
for i := 0; i < b.N; i++ {
err := db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("MyBucket"))
bJSON, err := json.Marshal(sample)
if err != nil {
return err
}
return b.Put([]byte("data"), bJSON)
})
if err != nil {
b.Fatal(err)
}
err = db.Sync()
if err != nil {
b.Fatal(err)
}
}
}
func benchmarkJsonstoreN(b *testing.B, n int) {
os.Remove("foo.gz")
sample := makeDataN(n)
js := new(jsonstore.JSONStore)
err := jsonstore.Save(js, "foo.gz")
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
js, err = jsonstore.Open("foo.gz")
if err != nil {
b.Fatal(err)
}
err = js.Set("data", sample)
if err != nil {
b.Fatal(err)
}
err = jsonstore.Save(js, "foo.gz")
if err != nil {
b.Fatal(err)
}
}
}
func benchmarkBoltN(b *testing.B, n int) {
os.Remove("foo.db")
sample := makeDataN(n)
db, err := bolt.Open("foo.db", 0600, nil)
if err != nil {
b.Fatal(err)
}
defer db.Close()
db.Update(func(tx *bolt.Tx) error {
_, err := tx.CreateBucket([]byte("MyBucket"))
if err != nil {
b.Fatal(err)
}
return nil
})
db.Close()
b.ResetTimer()
for i := 0; i < b.N; i++ {
db, err := bolt.Open("foo.db", 0600, nil)
if err != nil {
b.Fatal(err)
}
err = db.Update(func(tx *bolt.Tx) error {
b := tx.Bucket([]byte("MyBucket"))
bJSON, err := json.Marshal(sample)
if err != nil {
return err
}
return b.Put([]byte("data"), bJSON)
})
if err != nil {
b.Fatal(err)
}
err = db.Sync()
if err != nil {
b.Fatal(err)
}
db.Close()
}
}
func BenchmarkJsonstore1000(b *testing.B) {
benchmarkJsonstoreN(b, 1000)
}
func BenchmarkBolt1000(b *testing.B) {
benchmarkBoltN(b, 1000)
}
func BenchmarkJsonstore50000(b *testing.B) {
benchmarkJsonstoreN(b, 50000)
}
func BenchmarkBolt50000(b *testing.B) {
benchmarkBoltN(b, 50000)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment