kemingy

# CNPG only support Debian 12 (Bookworm)
FROM ubuntu:22.04

ARG PGRX_VERSION=0.12.8
ARG SCCACHE_VERSION=0.8.2

ENV DEBIAN_FRONTEND=noninteractive \
    LANG=en_US.UTF-8 \
    LC_ALL=en_US.UTF-8 \
    RUSTFLAGS="-Dwarnings" \

kemingy

• ref to https://pythonspeed.com/articles/uv-python-production/

runs on i7 13700k

• code: https://github.com/dundee/pybenchmarks/blob/master/bencher/programs/pystone/pystone.python3
• run for 1_000_000

Ubuntu:24.04

install from apt: 3.12.3

kemingy

vecs file format

A binary format to store the vectors.

| dim (4 bytes) | vector (4 * dim bytes) |
| dim (4 bytes) | vector (4 * dim bytes) |
...
| dim (4 bytes) | vector (4 * dim bytes) |

kemingy

Test disk speed

• https://cloud.google.com/compute/docs/disks/benchmarking-pd-performance

• write

# sequential write
sudo fio --name=write_throughput --directory=. --numjobs=4 \
      --size=2G --time_based --runtime=5m --ramp_time=2s --ioengine=libaio \

kemingy

# use devel since vllm need to compile the paged attention
ARG base=nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04

FROM ${base}

ENV DEBIAN_FRONTEND=noninteractive LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
ARG commit=main
ARG CONDA_VERSION=py310_23.3.1-0

RUN apt update && \

kemingy

import json
import base64

import msgpack
import numpy as np

dim = 1536
emb = np.random.rand(dim).astype(np.float32)
emb_list = emb.tolist()

kemingy

How to use

requires:

• https://github.com/vi/websocat
• https://github.com/websocket-client/websocket-client

# terminal 1
python mosec_websocket.py
# terminal 2

kemingy

package main

import (
	"fmt"
	"sync"

	"go.starlark.net/starlark"
	"go.starlark.net/starlarkstruct"
)

kemingy

from flask import Flask, request


app = Flask(__name__)


@app.route("/inference", methods=["POST"])
def inference():
    return request.get_data()

kemingy