GitHub - A1darbek/ayder

# Produce (raw bytes in body)
curl -X POST 'localhost:1109/broker/topics/orders/produce?partition=0' \
  -H 'Authorization: Bearer dev' \
  -d '{"item":"widget"}'

# Consume (base64 for binary-safe payloads)
curl 'localhost:1109/broker/consume/orders/mygroup/0?encoding=b64' \
  -H 'Authorization: Bearer dev'

chmod +x test_broker_v2.sh
./test_broker_v2.sh

BROKER SMOKE TEST: ALL GREEN ✅

Handler:     1.22ms
Queue wait:  0.47ms
HTTP parse:  0.41ms

Handler:     0.65ms
Queue wait:  0.29ms
HTTP parse:  0.29ms

# 3-node cluster with 8 million offsets
1. SIGKILL a follower mid-write
2. Leader continues, follower misses offsets
3. Restart follower
4. Follower replays local AOF → asks leader for missing offsets
5. Leader streams missing data → follower catches up
6. Cluster fully healthy in 40-50 seconds
7. Zero data loss

# Clone and run with Docker Compose
git clone https://github.com/A1darbek/ayder.git
cd ayder
docker compose up -d --build

# Or build and run standalone
docker build -t ayder .
docker run -p 1109:1109 --shm-size=2g ayder

# That's it. Now produce:
curl -X POST localhost:1109/broker/topics \
  -H 'Authorization: Bearer dev' \
  -H 'Content-Type: application/json' \
  -d '{"name":"events","partitions":4}'

curl -X POST 'localhost:1109/broker/topics/events/produce?partition=0' \
  -H 'Authorization: Bearer dev' \
  -d 'hello world'

# Dependencies:
# - libuv (>= 1.51 recommended)
# - libevent
# - libcurl
# - openssl (libssl + libcrypto)
# - zlib
# - liburing (Linux)
# - pthreads + standard Linux headers

# Ubuntu/Debian
sudo apt-get update
sudo apt-get install -y build-essential pkg-config \
  libuv1-dev libevent-dev libcurl4-openssl-dev libssl-dev zlib1g-dev liburing-dev

# Alpine (musl)
apk add --no-cache build-base linux-headers pkgconf \
  libuv-dev libevent-dev curl-dev openssl-dev zlib-dev liburing-dev

# Build
make clean && make
./ayder --port 1109

make clean && make CFLAGS="-O2 -g -Wall -Wextra -Wshadow -Werror"

# Works on small shm
docker run --rm -it --shm-size=64m -e RF_SHARED_ENTRIES=131072 ayder:musl

# Larger table (needs larger shm)
docker run --rm -it --shm-size=256m -e RF_SHARED_ENTRIES=262144 ayder:musl

# Intentionally too big (will fail gracefully with a helpful error)
docker run --rm -it --shm-size=64m -e RF_SHARED_ENTRIES=2097152 ayder:musl

docker build -f Dockerfile.musl -t ayder:musl .

# Default (auto-picks SharedStorage size to fit /dev/shm)
docker run --rm -it --shm-size=64m -p 1109:1109 ayder:musl

# docker-compose.yml
services:
  ayder:
    build: .
    ports:
      - "1109:1109"
    shm_size: 2g

# Create a topic
curl -X POST localhost:1109/broker/topics \
  -H 'Authorization: Bearer dev' \
  -H 'Content-Type: application/json' \
  -d '{"name":"events","partitions":8}'

# Produce a message
curl -X POST 'localhost:1109/broker/topics/events/produce?partition=0' \
  -H 'Authorization: Bearer dev' \
  -d 'hello world'

# Consume messages (binary-safe with base64)
curl 'localhost:1109/broker/consume/events/mygroup/0?offset=0' \
  -H 'Authorization: Bearer dev'

# Commit offset
curl -X POST localhost:1109/broker/commit \
  -H 'Authorization: Bearer dev' \
  -H 'Content-Type: application/json' \
  -d '{"topic":"events","group":"mygroup","partition":0,"offset":10}'

GET  /health      # → {"ok":true}
GET  /ready       # → {"ready":true}
GET  /metrics     # → Prometheus format
GET  /metrics_ha  # → HA cluster metrics

POST /broker/topics
{"name":"events","partitions":8}

{"ok":true,"topic":"events","partitions":8}

POST /broker/topics/{topic}/produce

{
  "ok": true,
  "offset": 123,
  "partition": 0,
  "batch_id": 9991,
  "sealed": true,
  "durable": true,
  "mode": "sealed",
  "synced": true
}

{"ok":true,"offset":123,"partition":0,"sealed":true,"synced":null,"duplicate":true}

POST /broker/topics/{topic}/produce-ndjson

{
  "ok": true,
  "first_offset": 1000,
  "count": 250,
  "partition": 0,
  "batch_id": 424242,
  "sealed": true,
  "durable": true,
  "mode": "sealed",
  "synced": false
}

GET /broker/consume/{topic}/{group}/{partition}

{
  "messages": [
    {"offset": 0, "partition": 0, "value_b64": "aGVsbG8=", "key_b64": "a2V5"}
  ],
  "count": 1,
  "next_offset": 1,
  "committed_offset": 0,
  "truncated": false
}

POST /broker/commit
{"topic":"events","group":"g1","partition":0,"offset":124}

POST /broker/delete-before
{"topic":"events","partition":0,"before_offset":100000}

{"ok":true,"deleted_count":12345,"freed_bytes":987654}

// TTL + size cap for specific partition
{"topic":"events","partition":0,"ttl_ms":60000,"max_bytes":104857600}

// TTL for all topics
{"topic":"*","ttl_ms":300000}

POST /kv/{namespace}/{key}?cas=<u64>&ttl_ms=<u64>

{"ok":true,"cas":2,"sealed":true,"durable":true,"mode":"sealed","synced":true,"batch_id":123}

GET /kv/{namespace}/{key}

{"value":"<base64>","cas":2}

GET /kv/{namespace}/{key}/meta

DELETE /kv/{namespace}/{key}?cas=<u64>

{"ok":true,"deleted":true,"sealed":true,"durable":true,"mode":"sealed","synced":false,"batch_id":456}

# 3-node cluster
node1 (7001) = LEADER
node2 (8001) = FOLLOWER
node3 (9001) = FOLLOWER

# Write to leader
curl -X POST 'localhost:7001/broker/topics/test/produce?partition=0' \
  -H 'Authorization: Bearer dev' -d 'msg-0'
# → offset 0

# Kill node2
kill -9 $(pgrep -f "port 8001")

# Write while node2 is down
curl -X POST 'localhost:7001/broker/topics/test/produce?partition=0' \
  -H 'Authorization: Bearer dev' -d 'msg-1'
# → offset 1

curl -X POST 'localhost:7001/broker/topics/test/produce?partition=0' \
  -H 'Authorization: Bearer dev' -d 'msg-2'
# → offset 2

# Restart node2 — automatically catches up to offset 2

# Verify all data is present on recovered node
curl 'localhost:8001/broker/consume/test/g1/0?offset=0&limit=10' \
  -H 'Authorization: Bearer dev'
# → offsets 0, 1, 2 all present

# Default port is 1109
./ayder --port 1109

# Or specify custom port
./ayder --port 7001

# Node 1 (bootstrap leader)
export RF_HA_ENABLED=1
export RF_HA_NODE_ID=node1
export RF_HA_BOOTSTRAP_LEADER=1
export RF_HA_NODES='node1:10.0.0.1:7000:100,node2:10.0.0.2:8000:50,node3:10.0.0.3:9000:25'
export RF_HA_WRITE_CONCERN=2  # sync-majority (2/3 nodes)
export RF_HA_DEDICATED_WORKER=0  # critical for low P99
export RF_BEARER_TOKENS='dev@scope:token2:token3'
export RF_HA_TLS=1
export RF_HA_TLS_CA=./certs/ca.crt
export RF_HA_TLS_CERT=./certs/node1.crt
export RF_HA_TLS_KEY=./certs/node1.key
./ayder --port 7001

# Node 2
export RF_HA_ENABLED=1
export RF_HA_NODE_ID=node2
export RF_HA_NODES='node1:10.0.0.1:7000:100,node2:10.0.0.2:8000:50,node3:10.0.0.3:9000:25'
export RF_HA_WRITE_CONCERN=2
export RF_HA_DEDICATED_WORKER=0
export RF_BEARER_TOKENS='dev@scope:token2:token3'
export RF_HA_TLS=1
export RF_HA_TLS_CA=./certs/ca.crt
export RF_HA_TLS_CERT=./certs/node2.crt
export RF_HA_TLS_KEY=./certs/node2.key
./ayder --port 8001

# Node 3 (same pattern, port 9001)

# 5-node topology
export RF_HA_NODES='node1:host1:7000:100,node2:host2:8000:80,node3:host3:9000:60,node4:host4:10000:40,node5:host5:11000:20'
export RF_HA_WRITE_CONCERN=3  # majority of 5

# 7-node topology
export RF_HA_NODES='node1:host1:7000:100,node2:host2:8000:90,node3:host3:9000:80,node4:host4:10000:70,node5:host5:11000:60,node6:host6:12000:50,node7:host7:13000:40'
export RF_HA_WRITE_CONCERN=4  # majority of 7

# Create CA
openssl req -x509 -newkey rsa:4096 -keyout ca.key -out ca.crt \
  -days 365 -nodes -subj "/CN=ayder-ca"

# Create node certificate (repeat for each node)
openssl req -newkey rsa:2048 -nodes -keyout node1.key -out node1.csr \
  -subj "/CN=node1" -addext "subjectAltName=DNS:node1,IP:10.0.0.1"

openssl x509 -req -in node1.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
  -out node1.crt -days 365 -copy_extensions copy

{
  "ok": false,
  "error": "missing_topic",
  "message": "Topic name is required",
  "docs": "https://ayder.dev/docs/api/produce"
}