GitHub - amr8t/blobsearch

docker run -d -p 8080:8080 \
  -e ENDPOINT=https://s3.amazonaws.com \
  -e ACCESS_KEY=your-key \
  -e SECRET_KEY=your-secret \
  -e BUCKET=your-bucket \
  -e REGION=us-east-1 \
  ghcr.io/amr8t/blobsearch/ingestor:latest

# Send logs (any JSON format with timestamp and level fields)
echo '{"timestamp":"2024-01-15T10:30:00Z","level":"error","message":"Database connection failed"}' | \
  curl -X POST --data-binary @- http://localhost:8080/ingest

# Flush to S3
curl -X POST http://localhost:8080/flush

INSTALL httpfs; LOAD httpfs;
SET s3_region='us-east-1';
SET s3_access_key_id='your-key';
SET s3_secret_access_key='your-secret';

SELECT * FROM read_parquet('s3://your-bucket/logs/date=*/level=*/*', hive_partitioning=true)
WHERE date = '2024-01-15' AND level = 'error'
LIMIT 10;

# Run ingestor on host
docker run -d \
  --name blobsearch-ingestor \
  --restart unless-stopped \
  -p 8080:8080 \
  -p 12201:12201 \
  -e ENDPOINT=https://s3.amazonaws.com \
  -e ACCESS_KEY=your-key \
  -e SECRET_KEY=your-secret \
  -e BUCKET=my-logs \
  -e REGION=us-east-1 \
  ghcr.io/amr8t/blobsearch/ingestor:latest

# Configure containers to use GELF
docker run -d \
  --log-driver=gelf \
  --log-opt gelf-address=tcp://172.17.0.1:12201 \
  --log-opt tag=my-app \
  your-app:latest

version: '3.8'

services:
  ingestor:
    image: ghcr.io/amr8t/blobsearch/ingestor:latest
    ports:
      - "8080:8080"
      - "12201:12201"
    environment:
      ENDPOINT: https://s3.amazonaws.com
      ACCESS_KEY: ${AWS_ACCESS_KEY}
      SECRET_KEY: ${AWS_SECRET_KEY}
      BUCKET: my-logs
      REGION: us-east-1

  app:
    image: your-app:latest
    logging:
      driver: gelf
      options:
        gelf-address: "tcp://ingestor:12201"
        tag: "my-app"
    depends_on:
      - ingestor

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: blobsearch-ingestor
spec:
  selector:
    matchLabels:
      app: blobsearch-ingestor
  template:
    metadata:
      labels:
        app: blobsearch-ingestor
    spec:
      hostNetwork: true
      containers:
      - name: ingestor
        image: ghcr.io/amr8t/blobsearch/ingestor:latest
        ports:
        - containerPort: 12201
          hostPort: 12201
        - containerPort: 8080
          hostPort: 8080
        env:
        - name: ENDPOINT
          value: "https://s3.amazonaws.com"
        - name: ACCESS_KEY
          valueFrom:
            secretKeyRef:
              name: s3-credentials
              key: access-key
        - name: SECRET_KEY
          valueFrom:
            secretKeyRef:
              name: s3-credentials
              key: secret-key
        - name: BUCKET
          value: "my-logs"
        - name: REGION
          value: "us-east-1"

apiVersion: v1
kind: Pod
metadata:
  name: my-app
spec:
  containers:
  - name: app
    image: your-app:latest
    # Docker GELF driver configuration
    # Add to docker daemon.json or use logging sidecar

TIMESTAMP_FIELDS="timestamp,time,@timestamp"  # Default

{"timestamp": "2024-01-15T10:30:00Z", ...}        # ✓ Works
{"time": "2024-01-15T10:30:00.123Z", ...}         # ✓ Works
{"@timestamp": "2024-01-15 10:30:00", ...}        # ✓ Works

LEVEL_FIELDS="level,severity,severityText"  # Default

{"level": "error", ...}                           # ✓ Works
{"severity": "ERROR", ...}                        # ✓ Works (normalized to lowercase)
{"severityText": "WARN", ...}                     # ✓ Works (OpenTelemetry)
{"severityNumber": 17, ...}                       # ✓ Works (OTLP numeric)

cat app.log | curl -X POST --data-binary @- http://localhost:8080/ingest

# GELF messages can be sent via HTTP POST
curl -X POST -H "Content-Type: application/json" \
  --data '{"version":"1.1","host":"myhost","short_message":"Log message","timestamp":1234567890.123,"level":6}' \
  http://localhost:8080/gelf

logging:
  driver: gelf
  options:
    gelf-address: "tcp://ingestor:12201"
    tag: "my-app"

curl -X POST http://localhost:8080/flush

curl http://localhost:8080/stats

-- Count logs by level
SELECT level, COUNT(*) as count
FROM read_parquet('s3://your-bucket/logs/date=*/level=*/*', hive_partitioning=true)
GROUP BY level;

-- Recent errors
SELECT timestamp, message
FROM read_parquet('s3://your-bucket/logs/date=*/level=*/*', hive_partitioning=true)
WHERE level = 'error'
ORDER BY timestamp DESC
LIMIT 10;

-- Time series
SELECT date, COUNT(*) as log_count
FROM read_parquet('s3://your-bucket/logs/date=*/level=*/*', hive_partitioning=true)
GROUP BY date
ORDER BY date DESC;

-- Extract fields from JSON messages
SELECT 
    timestamp,
    level,
    json_extract_string(message, '$.service') as service,
    json_extract_string(message, '$.error_code') as error_code,
    message
FROM read_parquet('s3://your-bucket/logs/date=*/level=*/*', hive_partitioning=true)
WHERE level = 'error'
LIMIT 10;

-- Count by service
SELECT 
    json_extract_string(message, '$.service') as service,
    COUNT(*) as count
FROM read_parquet('s3://your-bucket/logs/date=*/level=*/*', hive_partitioning=true)
WHERE message LIKE '{%'
GROUP BY service
ORDER BY count DESC;

services:
  app:
    image: your-app:latest
    logging:
      driver: gelf
      options:
        gelf-address: "tcp://ingestor:12201"
        tag: "my-app"