mirror of
https://github.com/tdurieux/anonymous_github.git
synced 2026-11-16 01:06:48 +01:00
354 lines
10 KiB
Go
354 lines
10 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"math"
|
|
"net/http"
|
|
"os"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/redis/go-redis/v9"
|
|
)
|
|
|
|
var requestLatencyBuckets = []float64{10, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000}
|
|
|
|
type requestMetricKey struct {
|
|
minute int64
|
|
metric, route, method string
|
|
}
|
|
type requestSample struct {
|
|
count, sumMs, bytes, aborted, errors, slow int64
|
|
buckets [12]int64
|
|
}
|
|
type requestMetrics struct {
|
|
mu sync.Mutex
|
|
flushMu sync.Mutex
|
|
pending map[requestMetricKey]requestSample
|
|
dropped atomic.Int64
|
|
droppedBatches atomic.Int64
|
|
active atomic.Int64
|
|
stop, done chan struct{}
|
|
instance string
|
|
sampledAt, cpuAt, lastFlush time.Time
|
|
cpuSeconds float64
|
|
}
|
|
|
|
func (s *Server) startRequestMetrics() {
|
|
if s.Store == nil || s.Store.Redis == nil {
|
|
return
|
|
}
|
|
m := &requestMetrics{pending: map[requestMetricKey]requestSample{}, stop: make(chan struct{}), done: make(chan struct{})}
|
|
host, _ := os.Hostname()
|
|
m.instance = "api:" + host + ":" + strconv.Itoa(os.Getpid())
|
|
if seconds, ok := processCPUSeconds(); ok {
|
|
m.cpuAt, m.cpuSeconds = time.Now(), seconds
|
|
}
|
|
s.requestMetrics = m
|
|
go func() {
|
|
defer close(m.done)
|
|
ticker := time.NewTicker(15 * time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-m.stop:
|
|
return
|
|
case <-ticker.C:
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
s.flushRequestMetrics(ctx)
|
|
cancel()
|
|
}
|
|
}
|
|
}()
|
|
}
|
|
|
|
func (m *requestMetrics) observe(key requestMetricKey, elapsed time.Duration, bytes int64, aborted, failed, slow bool) {
|
|
ms := math.Max(0, float64(elapsed)/float64(time.Millisecond))
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
v, exists := m.pending[key]
|
|
if !exists && len(m.pending) >= 512 {
|
|
m.dropped.Add(1)
|
|
return
|
|
}
|
|
v.count++
|
|
v.sumMs += int64(math.Round(ms))
|
|
v.bytes += bytes
|
|
if aborted {
|
|
v.aborted++
|
|
}
|
|
if failed {
|
|
v.errors++
|
|
}
|
|
if slow {
|
|
v.slow++
|
|
}
|
|
i := sort.SearchFloat64s(requestLatencyBuckets, ms)
|
|
v.buckets[i]++
|
|
m.pending[key] = v
|
|
}
|
|
|
|
// Labels are registered route templates or fixed fallbacks, never caller paths.
|
|
func (s *Server) performanceRoute(r *http.Request) string {
|
|
for _, route := range s.routes {
|
|
if route.pattern.MatchString(r.URL.Path) {
|
|
return route.template
|
|
}
|
|
}
|
|
path := r.URL.Path
|
|
for _, prefix := range []string{"/script/", "/css/", "/fonts/", "/partials/", "/favicon/"} {
|
|
if strings.HasPrefix(path, prefix) {
|
|
return "/assets/:path"
|
|
}
|
|
}
|
|
if path == "/healthcheck" || path == "/api/healthcheck" || path == "/github/app/webhook" {
|
|
return path
|
|
}
|
|
return "other"
|
|
}
|
|
|
|
func (s *Server) monitorRequest(state *responseState, r *http.Request) func() {
|
|
m := s.requestMetrics
|
|
started := time.Now()
|
|
route := s.performanceRoute(r)
|
|
method := r.Method
|
|
switch method {
|
|
case "GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS":
|
|
default:
|
|
method = "OTHER"
|
|
}
|
|
if m != nil {
|
|
m.active.Add(1)
|
|
}
|
|
return func() {
|
|
if m != nil {
|
|
m.active.Add(-1)
|
|
}
|
|
elapsed := time.Since(started)
|
|
threshold := time.Second
|
|
if strings.Contains(route, "/file/") || strings.HasSuffix(route, "/zip") || strings.HasPrefix(route, "/w/") {
|
|
threshold = 10 * time.Second
|
|
}
|
|
bytes := state.bytes
|
|
if method == "HEAD" {
|
|
bytes = 0
|
|
}
|
|
aborted := r.Context().Err() != nil || state.writeFailed
|
|
detail := traceFrom(r.Context()).log(r, state, route, method, started, elapsed, bytes, aborted, elapsed > threshold)
|
|
if detail != nil {
|
|
message := "slow request"
|
|
if aborted {
|
|
message = "request interrupted"
|
|
}
|
|
s.enqueueError("warn", "requests", message, "_", int(number(detail["status"])), detail)
|
|
}
|
|
if m == nil {
|
|
return
|
|
}
|
|
key := requestMetricKey{time.Now().UnixMilli() / 60000 * 60000, "request", route, method}
|
|
m.observe(key, elapsed, bytes, r.Context().Err() != nil || state.writeFailed, state.status >= 500, elapsed > threshold)
|
|
if !state.firstWrite.IsZero() {
|
|
key.metric = "first_byte"
|
|
m.observe(key, state.firstWrite.Sub(started), 0, false, false, false)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s *Server) flushRequestMetrics(ctx context.Context) error {
|
|
// Error persistence shares the existing bounded periodic flush lifecycle.
|
|
if err := s.flushErrorLog(ctx); err != nil {
|
|
return err
|
|
}
|
|
m := s.requestMetrics
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
m.flushMu.Lock()
|
|
defer m.flushMu.Unlock()
|
|
m.mu.Lock()
|
|
entries := m.pending
|
|
m.pending = map[requestMetricKey]requestSample{}
|
|
m.mu.Unlock()
|
|
now := time.Now()
|
|
sample := s.processSample(now)
|
|
if len(entries) == 0 && sample == nil {
|
|
return nil
|
|
}
|
|
pipe := s.Store.Redis.TxPipeline()
|
|
if sample != nil {
|
|
writeProcessSample(ctx, pipe, sample)
|
|
}
|
|
for k, v := range entries {
|
|
key := "performance:v1:minute:api:" + strconv.FormatInt(k.minute, 10)
|
|
label, _ := json.Marshal([]string{k.metric, k.route, k.method})
|
|
for field, n := range map[string]int64{"count": v.count, "sumMs": v.sumMs, "bytes": v.bytes, "aborted": v.aborted, "errors": v.errors, "slow": v.slow} {
|
|
pipe.HIncrBy(ctx, key, string(label)+"|"+field, n)
|
|
}
|
|
for i, n := range v.buckets {
|
|
if n > 0 {
|
|
pipe.HIncrBy(ctx, key, string(label)+"|b"+strconv.Itoa(i), n)
|
|
}
|
|
}
|
|
pipe.Expire(ctx, key, 2*time.Hour)
|
|
}
|
|
_, err := pipe.Exec(ctx)
|
|
// Do not replay an uncertain write: that would duplicate counts.
|
|
if err != nil {
|
|
m.droppedBatches.Add(1)
|
|
} else {
|
|
m.lastFlush = now
|
|
}
|
|
return err
|
|
}
|
|
|
|
func latencyUpper(buckets [12]int64, quantile float64) any {
|
|
var total, cumulative int64
|
|
for _, n := range buckets {
|
|
total += n
|
|
}
|
|
if total == 0 {
|
|
return nil
|
|
}
|
|
for i, n := range buckets {
|
|
cumulative += n
|
|
if float64(cumulative) >= math.Ceil(float64(total)*quantile) {
|
|
if i == len(requestLatencyBuckets) {
|
|
return nil
|
|
}
|
|
return requestLatencyBuckets[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *Server) requestPerformance(q *request) (any, error) {
|
|
rawMinutes, err := extendedQueryString(q.R.URL.RawQuery, "minutes")
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
minutes := int64(15)
|
|
if queryWindowNumber(rawMinutes) == 60 {
|
|
minutes = 60
|
|
}
|
|
out := M{"available": false, "windowMinutes": minutes, "instances": []any{}, "routes": []M{}, "stages": []M{}, "series": []M{}, "runtimeSeries": []any{}, "latencyBucketsMs": requestLatencyBuckets, "percentileNote": "Percentiles are histogram bucket upper bounds; null means no samples or above 60s."}
|
|
if s.requestMetrics == nil {
|
|
return out, nil
|
|
}
|
|
ctx, cancel := context.WithTimeout(q.ctx(), 3*time.Second)
|
|
defer cancel()
|
|
if s.flushRequestMetrics(ctx) != nil {
|
|
return out, nil
|
|
}
|
|
now := time.Now()
|
|
minute := now.UnixMilli() / 60000 * 60000
|
|
type read struct {
|
|
service string
|
|
minute int64
|
|
cmd *redis.MapStringStringCmd
|
|
}
|
|
reads := []read{}
|
|
pipe := s.Store.Redis.Pipeline()
|
|
for _, service := range []string{"api", "streamer"} {
|
|
for i := int64(0); i < minutes; i++ {
|
|
at := minute - i*60000
|
|
reads = append(reads, read{service, at, pipe.HGetAll(ctx, "performance:v1:minute:"+service+":"+strconv.FormatInt(at, 10))})
|
|
}
|
|
}
|
|
if _, err := pipe.Exec(ctx); err != nil {
|
|
return out, nil
|
|
}
|
|
type group struct{ service, metric, route, method string }
|
|
groups := map[group]requestSample{}
|
|
series := map[int64]M{}
|
|
for _, read := range reads {
|
|
for label, raw := range read.cmd.Val() {
|
|
split := strings.LastIndexByte(label, '|')
|
|
if split < 0 {
|
|
continue
|
|
}
|
|
var fields []string
|
|
if json.Unmarshal([]byte(label[:split]), &fields) != nil || len(fields) != 3 {
|
|
continue
|
|
}
|
|
n, err := strconv.ParseInt(raw, 10, 64)
|
|
if err != nil || n < 0 {
|
|
continue
|
|
}
|
|
g := group{read.service, fields[0], fields[1], fields[2]}
|
|
v := groups[g]
|
|
field := label[split+1:]
|
|
switch field {
|
|
case "count":
|
|
v.count += n
|
|
case "sumMs":
|
|
v.sumMs += n
|
|
case "bytes":
|
|
v.bytes += n
|
|
case "aborted":
|
|
v.aborted += n
|
|
case "errors":
|
|
v.errors += n
|
|
case "slow":
|
|
v.slow += n
|
|
default:
|
|
if strings.HasPrefix(field, "b") {
|
|
i, e := strconv.Atoi(field[1:])
|
|
if e == nil && i >= 0 && i < len(v.buckets) {
|
|
v.buckets[i] += n
|
|
}
|
|
}
|
|
}
|
|
groups[g] = v
|
|
if g.service == "api" && g.metric == "request" {
|
|
point := series[read.minute]
|
|
if point == nil {
|
|
point = M{"minute": read.minute, "requests": int64(0), "aborted": int64(0), "errors": int64(0), "sumMs": int64(0)}
|
|
}
|
|
target := field
|
|
if field == "count" {
|
|
target = "requests"
|
|
}
|
|
if _, ok := point[target]; ok && target != "minute" {
|
|
point[target] = number(point[target]) + n
|
|
}
|
|
series[read.minute] = point
|
|
}
|
|
}
|
|
}
|
|
routes, stages := []M{}, []M{}
|
|
for g, v := range groups {
|
|
avg := float64(0)
|
|
if v.count > 0 {
|
|
avg = math.Round(float64(v.sumMs) / float64(v.count))
|
|
}
|
|
row := M{"service": g.service, "metric": g.metric, "route": g.route, "method": g.method, "count": v.count, "sumMs": v.sumMs, "bytes": v.bytes, "aborted": v.aborted, "errors": v.errors, "slow": v.slow, "buckets": v.buckets, "avgMs": avg, "p95UpperMs": latencyUpper(v.buckets, .95), "p99UpperMs": latencyUpper(v.buckets, .99)}
|
|
if g.metric == "request" {
|
|
first := groups[group{g.service, "first_byte", g.route, g.method}]
|
|
row["firstByteCount"] = first.count
|
|
row["firstByteP95UpperMs"] = latencyUpper(first.buckets, .95)
|
|
routes = append(routes, row)
|
|
} else {
|
|
stages = append(stages, row)
|
|
}
|
|
}
|
|
sort.Slice(routes, func(i, j int) bool { return number(routes[i]["sumMs"]) > number(routes[j]["sumMs"]) })
|
|
points := []M{}
|
|
for _, p := range series {
|
|
points = append(points, p)
|
|
}
|
|
sort.Slice(points, func(i, j int) bool { return number(points[i]["minute"]) < number(points[j]["minute"]) })
|
|
instances, runtimeSeries, err := s.readProcessSamples(ctx, now, minutes)
|
|
if err != nil {
|
|
return out, nil
|
|
}
|
|
out["instances"], out["runtimeSeries"] = instances, runtimeSeries
|
|
out["available"], out["generatedAt"], out["routes"], out["stages"], out["series"] = true, now.UnixMilli(), routes, stages, points
|
|
out["droppedMetrics"], out["droppedBatches"] = s.requestMetrics.dropped.Load(), s.requestMetrics.droppedBatches.Load()
|
|
return out, nil
|
|
}
|