{
  "hardware": {
    "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
    "vram_mib": 97887,
    "compute_capability": "12.0",
    "driver": "580.82.07",
    "host_ram_gb": 176,
    "disk_gb": 236
  },
  "model": {
    "repo": "nvidia/Qwen3.8-Flash-Next-NVFP4",
    "revision": "fc694b54fb0174e0913e6adf86691ef85a4ead47",
    "architecture": "Qwen4ExpForConditionalGeneration",
    "params": "125B core / 6B active + 51B n-gram PLE + 4B MTP",
    "quantization": "ModelOpt mixed (NVFP4 W4A4 experts, FP8 PLE/MTP, BF16 rest)",
    "disk_gb": 132.7
  },
  "server": {
    "vllm": "0.32.0",
    "torch": "2.13.0+cu130",
    "cuda": "13.0",
    "flags": "TP1, --language-model-only, --enforce-eager, --max-model-len 16384, --gpu-memory-utilization 0.95, --quantization modelopt, --no-enable-flashinfer-autotune, no MTP",
    "gpu_weights_gib": 73.6,
    "kv_cache_gib": 14.75,
    "kv_cache_tokens": 326041,
    "model_load_s": 396.6
  },
  "cold_start": {
    "note": "first inference after Application startup complete (run 1)",
    "prompt_tokens": 523,
    "ttft_s": 4.7935,
    "decode_tps": 22.22,
    "server_prefill_tps": 109.23
  },
  "direct_localhost": {
    "method": "streaming OpenAI API on VM localhost, unique prompts, enable_thinking=false, ignore_eos+min_tokens=256, 3 repeats after 1 warmup",
    "by_input": [
      {
        "target": 128,
        "actual_prompt_tokens_median": 139,
        "n": 3,
        "ttft_s_median": 0.11,
        "ttft_s_range": [
          0.11,
          0.11
        ],
        "decode_tps_median": 21.87,
        "decode_tps_range": [
          21.85,
          21.88
        ],
        "server_prefill_tps_median": 1276.79,
        "server_prefill_tps_range": [
          1267.91,
          1298.86
        ],
        "client_prefill_est_tps_median": 1240.13,
        "overall_tps_median": 21.75,
        "prefix_cache_hits_total": 0.0
      },
      {
        "target": 512,
        "actual_prompt_tokens_median": 525,
        "n": 3,
        "ttft_s_median": 0.11,
        "ttft_s_range": [
          0.11,
          0.11
        ],
        "decode_tps_median": 22.27,
        "decode_tps_range": [
          22.26,
          22.28
        ],
        "server_prefill_tps_median": 5061.97,
        "server_prefill_tps_range": [
          5008.4,
          5077.37
        ],
        "client_prefill_est_tps_median": 4895.8,
        "overall_tps_median": 22.15,
        "prefix_cache_hits_total": 0.0
      },
      {
        "target": 2048,
        "actual_prompt_tokens_median": 2066,
        "n": 3,
        "ttft_s_median": 0.2,
        "ttft_s_range": [
          0.2,
          0.2
        ],
        "decode_tps_median": 21.89,
        "decode_tps_range": [
          21.88,
          21.94
        ],
        "server_prefill_tps_median": 10620.83,
        "server_prefill_tps_range": [
          10513.71,
          10648.32
        ],
        "client_prefill_est_tps_median": 10350.55,
        "overall_tps_median": 21.61,
        "prefix_cache_hits_total": 0.0
      },
      {
        "target": 8192,
        "actual_prompt_tokens_median": 8242,
        "n": 3,
        "ttft_s_median": 0.58,
        "ttft_s_range": [
          0.58,
          0.58
        ],
        "decode_tps_median": 22.07,
        "decode_tps_range": [
          22.03,
          22.53
        ],
        "server_prefill_tps_median": 14536.05,
        "server_prefill_tps_range": [
          14497.58,
          14545.74
        ],
        "client_prefill_est_tps_median": 14260.88,
        "overall_tps_median": 21.1,
        "prefix_cache_hits_total": 0.0
      },
      {
        "target": 12000,
        "actual_prompt_tokens_median": 12071,
        "n": 3,
        "ttft_s_median": 0.81,
        "ttft_s_range": [
          0.81,
          0.81
        ],
        "decode_tps_median": 21.93,
        "decode_tps_range": [
          21.87,
          21.94
        ],
        "server_prefill_tps_median": 15124.95,
        "server_prefill_tps_range": [
          15101.85,
          15135.14
        ],
        "client_prefill_est_tps_median": 14862.82,
        "overall_tps_median": 20.58,
        "prefix_cache_hits_total": 0.0
      }
    ],
    "concurrency": [
      {
        "concurrency": 2,
        "wall_s": 12.314,
        "requests": 2,
        "total_completion_tokens": 512,
        "total_prompt_tokens": 4127,
        "aggregate_output_tps": 41.58,
        "aggregate_prompt_tps": 335.14,
        "per_stream_decode_tps_median": 21.0,
        "per_stream_ttft_s_median": 0.15
      },
      {
        "concurrency": 4,
        "wall_s": 12.353,
        "requests": 4,
        "total_completion_tokens": 1024,
        "total_prompt_tokens": 8204,
        "aggregate_output_tps": 82.89,
        "aggregate_prompt_tps": 664.13,
        "per_stream_decode_tps_median": 21.02,
        "per_stream_ttft_s_median": 0.19
      }
    ]
  },
  "thinking_mode": {
    "prompt_tokens": 1748,
    "completion_tokens": 61,
    "completion_tokens_details": {
      "reasoning_tokens": 55
    },
    "ttft_s": 2.754,
    "total_s": 2.936,
    "reasoning_chars": 0,
    "content_chars": 5
  },
  "term_llm": {
    "quick": {
      "runs": 6,
      "aggregates": [
        {
          "workload": "decode",
          "requested_input": 2000,
          "actual_input_median": 1979,
          "output_median": 125,
          "ttft_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 534.084501,
            "median": 535.036518,
            "max": 538.080573,
            "p95": null
          },
          "decode_tps": {
            "count": 3,
            "label": "min_median_max",
            "min": 22.032766734450195,
            "median": 22.060358435622902,
            "max": 22.23559807413784,
            "p95": null
          },
          "e2e_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 6023.636283,
            "median": 6156.936367,
            "max": 6251.361389,
            "p95": null
          },
          "cache_state": "miss",
          "measured_runs": 3
        }
      ],
      "fits": null,
      "limitations": [
        "Cold, sequential measurements only; warm-cache and concurrent workloads are intentionally not implemented in schema version 1.",
        "Timings are client-observed at the llm.Provider/llm.Stream boundary, not server-side phase telemetry.",
        "Transport chunks are not treated as token boundaries; throughput uses provider-reported token counts only."
      ]
    },
    "decode": {
      "runs": 6,
      "aggregates": [
        {
          "workload": "decode",
          "requested_input": 2000,
          "actual_input_median": 1979,
          "output_median": 216,
          "ttft_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 441.106526,
            "median": 475.216922,
            "max": 477.974012,
            "p95": null
          },
          "decode_tps": {
            "count": 3,
            "label": "min_median_max",
            "min": 22.256042244744794,
            "median": 22.261279327329973,
            "max": 22.2760423365571,
            "p95": null
          },
          "e2e_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 7714.069222,
            "median": 10179.212049,
            "max": 10723.35344,
            "p95": null
          },
          "cache_state": "miss",
          "measured_runs": 3
        }
      ],
      "fits": null,
      "limitations": [
        "Cold, sequential measurements only; warm-cache and concurrent workloads are intentionally not implemented in schema version 1.",
        "Timings are client-observed at the llm.Provider/llm.Stream boundary, not server-side phase telemetry.",
        "Transport chunks are not treated as token boundaries; throughput uses provider-reported token counts only."
      ]
    },
    "prefill": {
      "runs": 12,
      "aggregates": [
        {
          "workload": "prefill",
          "requested_input": 1000,
          "actual_input_median": 975,
          "output_median": 2,
          "ttft_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 437.296331,
            "median": 439.980033,
            "max": 441.063212,
            "p95": null
          },
          "decode_tps": {
            "count": 0,
            "label": "unavailable",
            "min": null,
            "median": null,
            "max": null,
            "p95": null
          },
          "e2e_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 482.587328,
            "median": 484.645839,
            "max": 486.380082,
            "p95": null
          },
          "cache_state": "miss",
          "measured_runs": 3
        },
        {
          "workload": "prefill",
          "requested_input": 4000,
          "actual_input_median": 3971,
          "output_median": 2,
          "ttft_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 672.097868,
            "median": 678.747304,
            "max": 694.409265,
            "p95": null
          },
          "decode_tps": {
            "count": 0,
            "label": "unavailable",
            "min": null,
            "median": null,
            "max": null,
            "p95": null
          },
          "e2e_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 716.523065,
            "median": 723.353119,
            "max": 740.263093,
            "p95": null
          },
          "cache_state": "miss",
          "measured_runs": 3
        }
      ],
      "fits": null,
      "limitations": [
        "Cold, sequential measurements only; warm-cache and concurrent workloads are intentionally not implemented in schema version 1.",
        "Timings are client-observed at the llm.Provider/llm.Stream boundary, not server-side phase telemetry.",
        "Transport chunks are not treated as token boundaries; throughput uses provider-reported token counts only."
      ]
    },
    "balanced": {
      "runs": 30,
      "aggregates": [
        {
          "workload": "prefill",
          "requested_input": 128,
          "actual_input_median": 131,
          "output_median": 2,
          "ttft_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 433.352019,
            "median": 434.906735,
            "max": 435.055467,
            "p95": null
          },
          "decode_tps": {
            "count": 0,
            "label": "unavailable",
            "min": null,
            "median": null,
            "max": null,
            "p95": null
          },
          "e2e_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 479.231751,
            "median": 480.462479,
            "max": 482.398153,
            "p95": null
          },
          "cache_state": "miss",
          "measured_runs": 3
        },
        {
          "workload": "prefill",
          "requested_input": 512,
          "actual_input_median": 504,
          "output_median": 2,
          "ttft_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 432.362336,
            "median": 433.291887,
            "max": 433.394627,
            "p95": null
          },
          "decode_tps": {
            "count": 0,
            "label": "unavailable",
            "min": null,
            "median": null,
            "max": null,
            "p95": null
          },
          "e2e_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 478.081986,
            "median": 478.947212,
            "max": 479.918474,
            "p95": null
          },
          "cache_state": "miss",
          "measured_runs": 3
        },
        {
          "workload": "prefill",
          "requested_input": 2048,
          "actual_input_median": 2101,
          "output_median": 2,
          "ttft_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 562.32301,
            "median": 565.077162,
            "max": 565.09112,
            "p95": null
          },
          "decode_tps": {
            "count": 0,
            "label": "unavailable",
            "min": null,
            "median": null,
            "max": null,
            "p95": null
          },
          "e2e_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 609.221598,
            "median": 610.910887,
            "max": 632.714696,
            "p95": null
          },
          "cache_state": "miss",
          "measured_runs": 3
        },
        {
          "workload": "prefill",
          "requested_input": 8192,
          "actual_input_median": 8159,
          "output_median": 2,
          "ttft_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 943.119577,
            "median": 945.552826,
            "max": 946.626484,
            "p95": null
          },
          "decode_tps": {
            "count": 0,
            "label": "unavailable",
            "min": null,
            "median": null,
            "max": null,
            "p95": null
          },
          "e2e_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 987.258148,
            "median": 989.272685,
            "max": 990.82581,
            "p95": null
          },
          "cache_state": "miss",
          "measured_runs": 3
        },
        {
          "workload": "prefill",
          "requested_input": 12000,
          "actual_input_median": 12002,
          "output_median": 2,
          "ttft_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 1167.500809,
            "median": 1173.246054,
            "max": 1175.94392,
            "p95": null
          },
          "decode_tps": {
            "count": 0,
            "label": "unavailable",
            "min": null,
            "median": null,
            "max": null,
            "p95": null
          },
          "e2e_ms": {
            "count": 3,
            "label": "min_median_max",
            "min": 1211.700473,
            "median": 1215.79919,
            "max": 1219.591937,
            "p95": null
          },
          "cache_state": "miss",
          "measured_runs": 3
        }
      ],
      "fits": [
        {
          "provider": "colabvllm",
          "requested_model": "qwen3.8-flash-next",
          "range": "1K-16K",
          "minimum_input_tokens": 2101,
          "maximum_input_tokens": 12002,
          "distinct_lengths": 3,
          "valid_runs": 9,
          "slope_seconds_per_token": 6.142499666700334e-05,
          "client_observed_effective_prefill_tokens_per_second": 16280.017163390197,
          "intercept_seconds": 0.43602324400262604,
          "median_absolute_error_seconds": 0,
          "method": "median_pairwise_slopes"
        }
      ],
      "limitations": [
        "Cold, sequential measurements only; warm-cache and concurrent workloads are intentionally not implemented in schema version 1.",
        "Timings are client-observed at the llm.Provider/llm.Stream boundary, not server-side phase telemetry.",
        "Transport chunks are not treated as token boundaries; throughput uses provider-reported token counts only."
      ]
    }
  }
}