|
| 1 | +import http from 'k6/http' |
| 2 | +import {check, sleep} from 'k6' |
| 3 | +import {SharedArray} from 'k6/data' |
| 4 | +import {Counter, Rate, Trend} from 'k6/metrics' |
| 5 | +import exec from 'k6/execution'; |
| 6 | + |
| 7 | +// Server chat completions prefix |
| 8 | +const server_url = __ENV.SERVER_BENCH_URL ? __ENV.SERVER_BENCH_URL : 'http://localhost:8080/v1' |
| 9 | + |
| 10 | +// Number of total prompts in the dataset - default 10m / 10 seconds/request * number of users |
| 11 | +const n_prompt = __ENV.SERVER_BENCH_N_PROMPTS ? parseInt(__ENV.SERVER_BENCH_N_PROMPTS) : 600 / 10 * 8 |
| 12 | + |
| 13 | +// Model name to request |
| 14 | +const model = __ENV.SERVER_BENCH_MODEL_ALIAS ? __ENV.SERVER_BENCH_MODEL_ALIAS : 'my-model' |
| 15 | + |
| 16 | +// Dataset path |
| 17 | +const dataset_path = __ENV.SERVER_BENCH_DATASET ? __ENV.SERVER_BENCH_DATASET : './ShareGPT_V3_unfiltered_cleaned_split.json' |
| 18 | + |
| 19 | +// Max tokens to predict |
| 20 | +const max_tokens = __ENV.SERVER_BENCH_MAX_TOKENS ? parseInt(__ENV.SERVER_BENCH_MAX_TOKENS) : 512 |
| 21 | + |
| 22 | +// Max prompt tokens |
| 23 | +const n_prompt_tokens = __ENV.SERVER_BENCH_MAX_PROMPT_TOKENS ? parseInt(__ENV.SERVER_BENCH_MAX_PROMPT_TOKENS) : 1024 |
| 24 | + |
| 25 | +// Max slot context |
| 26 | +const n_ctx_slot = __ENV.SERVER_BENCH_MAX_CONTEXT ? parseInt(__ENV.SERVER_BENCH_MAX_CONTEXT) : 2048 |
| 27 | + |
| 28 | +export function setup() { |
| 29 | + console.info(`Benchmark config: server_url=${server_url} n_prompt=${n_prompt} model=${model} dataset_path=${dataset_path} max_tokens=${max_tokens}`) |
| 30 | +} |
| 31 | + |
| 32 | +const data = new SharedArray('conversations', function () { |
| 33 | + const tokenizer = (message) => message.split(/[\s,'".?]/) |
| 34 | + |
| 35 | + return JSON.parse(open(dataset_path)) |
| 36 | + // Filter out the conversations with less than 2 turns. |
| 37 | + .filter(data => data["conversations"].length >= 2) |
| 38 | + .filter(data => data["conversations"][0]["from"] === "human") |
| 39 | + .map(data => { |
| 40 | + return { |
| 41 | + prompt: data["conversations"][0]["value"], |
| 42 | + n_prompt_tokens: tokenizer(data["conversations"][0]["value"]).length, |
| 43 | + n_completion_tokens: tokenizer(data["conversations"][1]["value"]).length, |
| 44 | + } |
| 45 | + }) |
| 46 | + // Filter out too short sequences |
| 47 | + .filter(conv => conv.n_prompt_tokens >= 4 && conv.n_completion_tokens >= 4) |
| 48 | + // Filter out too long sequences. |
| 49 | + .filter(conv => conv.n_prompt_tokens <= n_prompt_tokens && conv.n_prompt_tokens + conv.n_completion_tokens <= n_ctx_slot) |
| 50 | + // Keep only first n prompts |
| 51 | + .slice(0, n_prompt) |
| 52 | +}) |
| 53 | + |
| 54 | +const llamacpp_prompt_tokens = new Trend('llamacpp_prompt_tokens') |
| 55 | +const llamacpp_completion_tokens = new Trend('llamacpp_completion_tokens') |
| 56 | +const llamacpp_tokens_second = new Trend('llamacpp_tokens_second') |
| 57 | + |
| 58 | +const llamacpp_prompt_tokens_total_counter = new Counter('llamacpp_prompt_tokens_total_counter') |
| 59 | +const llamacpp_completion_tokens_total_counter = new Counter('llamacpp_completion_tokens_total_counter') |
| 60 | + |
| 61 | +const llamacpp_completions_truncated_rate = new Rate('llamacpp_completions_truncated_rate') |
| 62 | +const llamacpp_completions_stop_rate = new Rate('llamacpp_completions_stop_rate') |
| 63 | + |
| 64 | +export const options = { |
| 65 | + thresholds: { |
| 66 | + llamacpp_completions_truncated_rate: [ |
| 67 | + // more than 80% of truncated input will abort the test |
| 68 | + {threshold: 'rate < 0.8', abortOnFail: true, delayAbortEval: '1m'}, |
| 69 | + ], |
| 70 | + }, |
| 71 | + duration: '10m', |
| 72 | + vus: 8, |
| 73 | +} |
| 74 | + |
| 75 | +export default function () { |
| 76 | + const conversation = data[exec.scenario.iterationInInstance % data.length] |
| 77 | + const payload = { |
| 78 | + "messages": [ |
| 79 | + { |
| 80 | + "role": "system", |
| 81 | + "content": "You are ChatGPT, an AI assistant.", |
| 82 | + }, |
| 83 | + { |
| 84 | + "role": "user", |
| 85 | + "content": conversation.prompt, |
| 86 | + } |
| 87 | + ], |
| 88 | + "model": model, |
| 89 | + "stream": false, |
| 90 | + "max_tokens": max_tokens |
| 91 | + } |
| 92 | + |
| 93 | + const body = JSON.stringify(payload) |
| 94 | + |
| 95 | + let res = http.post(`${server_url}/chat/completions`, body, { |
| 96 | + headers: {'Content-Type': 'application/json'}, |
| 97 | + timeout: '300s' |
| 98 | + }) |
| 99 | + |
| 100 | + check(res, {'success completion': (r) => r.status === 200}) |
| 101 | + |
| 102 | + if (res.status === 200) { |
| 103 | + const completions = res.json() |
| 104 | + |
| 105 | + llamacpp_prompt_tokens.add(completions.usage.prompt_tokens) |
| 106 | + llamacpp_prompt_tokens_total_counter.add(completions.usage.prompt_tokens) |
| 107 | + |
| 108 | + llamacpp_completion_tokens.add(completions.usage.completion_tokens) |
| 109 | + llamacpp_completion_tokens_total_counter.add(completions.usage.completion_tokens) |
| 110 | + |
| 111 | + llamacpp_completions_truncated_rate.add(completions.choices[0].finish_reason === 'length') |
| 112 | + llamacpp_completions_stop_rate.add(completions.choices[0].finish_reason === 'stop') |
| 113 | + |
| 114 | + llamacpp_tokens_second.add(completions.usage.total_tokens / res.timings.duration * 1.e3) |
| 115 | + } else { |
| 116 | + console.error(`response: ${res.body} request=${payload}`) |
| 117 | + } |
| 118 | + |
| 119 | + sleep(0.3) |
| 120 | +} |
0 commit comments