GitHub - basecamp/upright: Synthetic monitoring engine with Playwright and Prometheus metrics

Note

Upright is designed to be run in its own Rails app and deployed with Kamal.

rails new my-upright --database=sqlite3 --skip-test
cd my-upright
bundle add upright
bin/rails generate upright:install
bin/rails db:migrate

# config/initializers/upright.rb
Upright.configure do |config|
  config.hostname = "upright.com"
end

shared:
  sites:
    - code: nyc
      city: New York City
      country: US
      geohash: dr5reg
      provider: digitalocean

    - code: ams
      city: Amsterdam
      country: NL
      geohash: u17982
      provider: digitalocean

    - code: sfo
      city: San Francisco
      country: US
      geohash: 9q8yy
      provider: hetzner

Warning

Change the default password before deploying to production by setting the ADMIN_PASSWORD environment variable.

# config/initializers/upright.rb
Upright.configure do |config|
  config.auth_provider = :openid_connect
  config.auth_options = {
    issuer: "https://your-tenant.logto.app/oidc",
    client_id: ENV["OIDC_CLIENT_ID"],
    client_secret: ENV["OIDC_CLIENT_SECRET"]
  }
end

Upright.configure do |config|
  config.stale_success_threshold = 24.hours     # Delete successful results older than this (default: 24 hours)
  config.stale_failure_threshold = 30.days       # Delete failed results older than this (default: 30 days)
  config.failure_retention_limit = 20_000        # Keep at most this many failed results (default: 20,000)
end

- name: Main Website
  url: https://example.com
  expected_status: 200

- name: API Health
  url: https://api.example.com/health
  expected_status: 200
  alert_severity: critical

- name: Admin Panel
  url: https://admin.example.com
  basic_auth_credentials: admin_auth  # Key in Rails credentials

- name: Primary Mail Server
  host: mail.example.com

- name: Backup Mail Server
  host: mail2.example.com

bin/rails generate upright:playwright_probe MyServiceAuth

# probes/my_service_auth_probe.rb
class Probes::Playwright::MyServiceAuthProbe < Upright::Probes::Playwright::Base
  # Optionally authenticate before running
  # authenticate_with_form :my_service

  def check
    page.goto("https://app.example.com")
    page.fill('[name="email"]', "test@example.com")
    page.click('button[type="submit"]')
    page.wait_for_selector(".dashboard")
  end
end

# probes/authenticators/my_service.rb
class Playwright::Authenticator::MyService < Upright::Playwright::Authenticator::Base
  def signin_redirect_url = "https://app.example.com/dashboard"
  def signin_path = "/login"
  def service_name = :my_service

  def authenticate
    page.goto("https://app.example.com/login")
    page.get_by_label("Email").fill(credentials.my_service.email)
    page.get_by_label("Password").fill(credentials.my_service.password)
    page.get_by_role("button", name: "Sign in").click
  end
end

production:
  http_probes:
    command: "Upright::Probes::HTTPProbe.check_and_record_all_later"
    schedule: every 30 seconds

  smtp_probes:
    command: "Upright::Probes::SMTPProbe.check_and_record_all_later"
    schedule: every 30 seconds

  my_service_auth:
    command: "Probes::Playwright::MyServiceAuthProbe.check_and_record_later"
    schedule: every 15 minutes

; Primary dashboard
app.upright.example.com      A    203.0.113.10

; Monitoring nodes
ams.upright.example.com      A    203.0.113.10    ; Amsterdam
nyc.upright.example.com      A    198.51.100.20   ; New York
sfo.upright.example.com      A    192.0.2.30      ; San Francisco

service: upright
image: your-org/upright

servers:
  web:
    hosts:
      - ams.upright.example.com: [amsterdam]
      - nyc.upright.example.com: [new_york]
      - sfo.upright.example.com: [san_francisco]
  jobs:
    hosts:
      - ams.upright.example.com: [amsterdam]
      - nyc.upright.example.com: [new_york]
      - sfo.upright.example.com: [san_francisco]
    cmd: bin/jobs

proxy:
  app_port: 3000
  ssl: true
  hosts:
    - "*.upright.example.com"

env:
  secret:
    - RAILS_MASTER_KEY
  tags:
    amsterdam:
      SITE_SUBDOMAIN: ams
    new_york:
      SITE_SUBDOMAIN: nyc
    san_francisco:
      SITE_SUBDOMAIN: sfo

accessories:
  playwright:
    image: jacoblincool/playwright:chromium-server-1.55.0
    port: "127.0.0.1:53333:53333"
    roles:
      - jobs

  prometheus:
    image: prom/prometheus:v3.2.1
    hosts:
      - ams.upright.example.com
    cmd: >-
      --config.file=/etc/prometheus/prometheus.yml
      --storage.tsdb.path=/prometheus
      --storage.tsdb.retention.time=30d
      --web.enable-otlp-receiver
    files:
      - config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - config/prometheus/rules/upright.rules.yml:/etc/prometheus/rules/upright.rules.yml

  alertmanager:
    image: prom/alertmanager:v0.28.1
    hosts:
      - ams.upright.example.com
    cmd: --config.file=/etc/alertmanager/alertmanager.yml
    files:
      - config/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml

scrape_configs:
  - job_name: upright
    static_configs:
      - targets: ['localhost:9394']

groups:
  - name: upright
    rules:
      - alert: ProbeDown
        expr: upright_probe_up == 0
        for: 5m
        labels:
          severity: "{{ $labels.alert_severity }}"
        annotations:
          summary: "Probe {{ $labels.name }} is down"

Upright.configure do |config|
  config.otel_endpoint = "https://otel.example.com:4318"
end

LOCAL_PLAYWRIGHT=1 bin/rails console

Probes::Playwright::MyServiceAuthProbe.check