benchmark graph

draw_benchmark.py

import seaborn as sns
import matplotlib.pyplot as plt


qps = [25.69, 18.91, 17.13, 16.18, 14.45, 7.24, 4.51]
recall = [0.9079, 0.9461, 0.9497, 0.9535, 0.9608, 0.9650, 0.9672]
p99_latency = [148.25, 260.68, 269.5, 302.29, 456.09, 1042.41, 1675.42]

fig, (ax1, ax3) = plt.subplots(1, 2, figsize=(12, 5), gridspec_kw={'width_ratios': [20, 7]})


ax2 = ax1.twinx()

bar = sns.barplot(x=recall, y=p99_latency, ax=ax1, color="lightsteelblue", label="p99 latency", legend=False, width=0.2)
line = sns.lineplot(x=ax1.get_xticks(), y=qps, ax=ax2, marker='o', color='orange', label="QPS", legend=False)

lines, labels = ax1.get_legend_handles_labels()
bars, bar_labels = ax2.get_legend_handles_labels()
plt.legend(lines + bars, labels + bar_labels, loc='upper center')

ax1.yaxis.tick_right()
ax2.yaxis.tick_left()
ax1.yaxis.set_label_position('right')
ax2.yaxis.set_label_position('left')

ax1.set_title("VectorChord / 100m Vectors / 1 Thread / Top 10")
ax1.set_xlabel("Recall")
ax2.set_ylabel("QPS")
ax1.set_ylabel("P99 Latency")

# 调整布局
fig.tight_layout()

qps = [5.41, 4.26, 3.78]
recall = [0.9231, 0.9502, 0.9563]
p99_latency = [627.86, 885.59, 1006.84]


ax4 = ax3.twinx()

bar = sns.barplot(x=recall, y=p99_latency, ax=ax3, color="lightsteelblue", label="p99 latency", legend=False, width=0.2)
line = sns.lineplot(x=ax3.get_xticks(), y=qps, ax=ax4, marker='o', color='orange', label="QPS", legend=False)

lines, labels = ax3.get_legend_handles_labels()
bars, bar_labels = ax4.get_legend_handles_labels()
plt.legend(lines + bars, labels + bar_labels, loc='upper center')

ax3.yaxis.tick_right()
ax4.yaxis.tick_left()
ax3.yaxis.set_label_position('right')
ax4.yaxis.set_label_position('left')

ax3.set_title("Top 100")
ax3.set_xlabel("Recall")
ax4.set_ylabel("QPS")
ax3.set_ylabel("P99 Latency")

plt.subplots_adjust(wspace=0.27, right=0.94)

fig.savefig("benchmark.jpg")

draw_cost.py

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

sns.set_theme(style="white", context="talk")
rs = np.random.RandomState(8)

highlight = pd.DataFrame(
    {
        "DB": [
            "VectorChord",
        ],
        "Dataset": [
            "on 100m",
        ],
        "Cost": [247],
    }
)

on_100m = pd.DataFrame(
    {
        "DB": [
            "Myscale(Cap)",
            "Myscale(Std)",
            "Qdrant",
            "Zillis Cloud(Cap)",
            "Zillis Cloud(Perf)",
        ],
        "Dataset": [
            "on 100m",
            "on 100m",
            "on 100m",
            "on 100m",
            "on 100m",
        ],
        "Cost": [1088, 3840, 4374, 2305, 7800],
    }
)

on_5m = pd.DataFrame(
    {
        "DB": [
            "Pinecone(Cap)",
            "Pinecone(Perf)",
            "PGVector / PGVecto.rs",
            "Weaviate",
            "Elastic Cloud",
        ],
        "Cost": [80, 600, 329, 365, 983],
    }
)

est_on_100m = pd.DataFrame(
    {
        "DB": [
            "Pinecone(Cap)",
            "Pinecone(Perf)",
            "PGVector / PGVecto.rs",
            "Weaviate",
            "Elastic Cloud",
        ],
        "Cost": [1600, 7200, 6580, 7296, 19656],
    }
)

fig, ax = plt.subplots(figsize=(40, 15))

width = 0.5

sns.barplot(
    x="Cost",
    y="DB",
    data=highlight,
    width=width,
    label="on 100m(best)",
    color="#FA7F6F",
)
sns.barplot(
    x="Cost",
    y="DB",
    data=on_100m,
    width=width,
    label="on 100m",
    color="#8ECFC9",
)
sns.barplot(
    x="Cost",
    y="DB",
    data=est_on_100m,
    hatch="////",
    width=width,
    label="estimated on 100m",
    linewidth=2.5,
    color="w",
    edgecolor="#FFBE7A",
)

sns.barplot(
    x="Cost",
    y="DB",
    data=on_5m,
    hatch=None,
    width=width,
    label="on 5m",
    color="#FFBE7A",
)
ax.bar_label(ax.containers[0], fontsize=30, fmt=lambda x: f"${int(x):d}")
ax.bar_label(ax.containers[1], fontsize=30, fmt=lambda x: f"${int(x):d}")
ax.bar_label(ax.containers[2], fontsize=30, fmt=lambda x: f"${int(x):d}")

plt.setp(ax.get_legend().get_texts(), fontsize=40)
plt.setp(ax.get_legend().get_title(), fontsize=40)

plt.title(
    "Cost to store 100 million vectors per month", fontsize=45, y=1.05
)
plt.xlabel("Monthly Cost", fontsize=45)
plt.ylabel("")
plt.xticks(fontsize=45)
plt.yticks(fontsize=35)

plt.xscale("log")
sns.despine(right=True)
plt.subplots_adjust(left=0.15)

fig.savefig("cost.jpg")

draw_cost_rev.py

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

sns.set_theme(style="white", context="talk")
rs = np.random.RandomState(8)

SIZE = 100000000

highlight = pd.DataFrame(
    {
        "DB": [
            "VectorChord",
        ],
        "Dataset": [
            "on 100m",
        ],
        "Cost": [SIZE / 247],
    }
)

on_100m = pd.DataFrame(
    {
        "DB": [
            "Myscale(Cap)",
            "Myscale(Std)",
            "Qdrant",
            "Zillis Cloud(Cap)",
            "Zillis Cloud(Perf)",
        ],
        "Dataset": [
            "on 100m",
            "on 100m",
            "on 100m",
            "on 100m",
            "on 100m",
        ],
        "Cost": [SIZE / 1088, SIZE / 3840, SIZE / 4374, SIZE / 2305, SIZE / 7800],
    }
)

on_5m = pd.DataFrame(
    {
        "DB": [
            "Pinecone(Cap)",
            "Pinecone(Perf)",
            "PGVector / PGVecto.rs",
            "Weaviate",
            "Elastic Cloud",
        ],
        "Cost": [SIZE / 80 / 20, SIZE / 600 / 20, SIZE / 329 / 20, SIZE / 365 / 20, SIZE / 983 / 20],
    }
)

fig, ax = plt.subplots(figsize=(40, 15))

width = 0.5

sns.barplot(
    x="Cost",
    y="DB",
    data=highlight,
    width=width,
    label="on 100m(best)",
    color="#FA7F6F",
    legend=False,
)
sns.barplot(
    x="Cost",
    y="DB",
    data=on_100m,
    width=width,
    label="on 100m",
    color="#8ECFC9",
    legend=False,
)

sns.barplot(
    x="Cost",
    y="DB",
    data=on_5m,
    hatch=None,
    width=width,
    label="on 5m",
    color="#FFBE7A",
    legend=False,
)


plt.title(
    "How many vectors can be stored per $1", fontsize=55, y=1.05
)
plt.xlabel("Vectors", fontsize=35)
plt.ylabel("")
plt.xticks(fontsize=25)
plt.yticks(fontsize=35)

sns.despine(right=True)
plt.subplots_adjust(left=0.15)

fig.savefig("cost_rev.jpg")

draw_threads.py

import seaborn as sns
import matplotlib.pyplot as plt


qps = [14.84, 30.14, 58.08, 91.44, 131.84]
threads = [1, 2, 4, 8, 16]

fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))

line = sns.barplot(x=threads, y=qps, ax=ax1, color="lightsteelblue", width=0.4)

ax1.set_title("VectorChord / 100m Vectors / Recall=0.9536 / Top 10")
ax1.set_xlabel("Threads")
ax1.set_ylabel("QPS")

ax1.bar_label(ax1.containers[0], fontsize=8)

fig.tight_layout()

qps = [4.26, 9.24, 18.47, 34.07]
threads = [1, 2, 4, 8]

line = sns.barplot(x=threads, y=qps, ax=ax2, color="lightsteelblue", width=0.3)

ax2.set_title("Recall=0.9502 / Top 100")
ax2.set_xlabel("Threads")
ax2.set_ylabel("QPS")

ax2.bar_label(ax2.containers[0], fontsize=8)

plt.subplots_adjust(wspace=0.2)

fig.savefig("threads.jpg")