GitHub - autospec-ai/playwright: AutoSpec AI is a GitHub Action that analyzes your code changes (via diff), understands what user-facing behavior changed, and generates production-quality Playwright E2E tests that match your existing test style.

Commit / PR  →  Diff Analysis  →  LLM Test Planning  →  Playwright Code Gen  →  PR with Tests

# .github/workflows/autospec.yml
name: AutoSpec AI

on:
  pull_request:
    types: [opened, synchronize]

permissions:
  contents: write
  pull-requests: write

jobs:
  generate-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0  # Required for diff analysis

      - uses: autospec-ai/action@v1
        with:
          llm_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
          base_url: 'http://localhost:3000'
          framework: 'react'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- uses: autospec-ai/action@v1
  with:
    llm_provider: anthropic
    llm_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
    llm_model: claude-sonnet-4-20250514  # optional, this is the default

- uses: autospec-ai/action@v1
  with:
    llm_provider: openai
    llm_api_key: ${{ secrets.OPENAI_API_KEY }}
    llm_model: gpt-4o  # optional, this is the default

- uses: autospec-ai/action@v1
  with:
    llm_provider: custom
    llm_api_key: ${{ secrets.TOGETHER_API_KEY }}
    llm_model: meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo
    llm_base_url: https://api.together.xyz/v1

trace_on_failure: 'true'
trace_mode: 'retain-on-failure'  # on | off | retain-on-failure | on-first-retry

npx playwright show-trace test-results/<test-name>/trace.zip

generate_api_mocks: 'true'
mock_error_states: 'true'            # Also generate 4xx/5xx error test cases
fixture_extraction_threshold: '3'    # Extract to fixtures when route count exceeds this

visual_regression: 'true'
visual_threshold: '0.2'          # Pixel comparison threshold (0-1)
visual_max_diff_ratio: '0.05'    # Max diff pixel ratio before failure
visual_full_page: 'false'        # Viewport-only or full-page capture

npx playwright test --update-snapshots

- uses: autospec-ai/action@v1
  with:
    llm_api_key: ${{ secrets.OPENAI_API_KEY }}
    test_directory: 'e2e/tests'                        # where to write generated tests
    pom_patterns: '**/*.po.ts,**/pageobjects/**/*.ts'  # match your POM convention
    utility_patterns: '**/helpers/**/*.ts'              # match your utility convention
    project_context_budget: '10000'                     # increase if you have many POMs

accessibility_assertions: 'true'
axe_scan: 'true'
axe_standard: 'wcag2aa'  # wcag2a | wcag2aa | wcag21a | wcag21aa | best-practice

- uses: autospec-ai/action@v1
  with:
    llm_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
    base_url: 'http://localhost:3000'
    framework: 'react'

    # Trace
    trace_on_failure: 'true'
    trace_mode: 'retain-on-failure'

    # API Mocks
    generate_api_mocks: 'true'
    mock_error_states: 'true'
    fixture_extraction_threshold: '3'

    # Visual Regression
    visual_regression: 'true'
    visual_threshold: '0.2'
    visual_max_diff_ratio: '0.05'
    visual_full_page: 'false'

    # Accessibility
    accessibility_assertions: 'true'
    axe_scan: 'true'
    axe_standard: 'wcag2aa'
  env:
    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- uses: autospec-ai/action@v1
  with:
    llm_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
    include_paths: 'src/components/,src/pages/'
    exclude_paths: 'src/components/__tests__/'

- uses: autospec-ai/action@v1
  id: autospec
  with:
    llm_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
    dry_run: 'true'

- name: Comment preview
  if: github.event_name == 'pull_request' && steps.autospec.outputs.tests_generated != '0'
  uses: actions/github-script@v7
  env:
    TESTS_GENERATED: ${{ steps.autospec.outputs.tests_generated }}
    SUMMARY: ${{ steps.autospec.outputs.summary }}
  with:
    script: |
      const testsGenerated = process.env.TESTS_GENERATED;
      const summary = process.env.SUMMARY;
      github.rest.issues.createComment({
        owner: context.repo.owner,
        repo: context.repo.repo,
        issue_number: context.issue.number,
        body: `### 🤖 AutoSpec Preview\nGenerated **${testsGenerated}** test(s).\n\n${summary}`
      });

- uses: autospec-ai/action@v1
  with:
    llm_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
    auto_pr: 'false'
    auto_commit: 'false'

- name: Install Playwright
  run: npx playwright install --with-deps

- name: Run generated tests
  run: npx playwright test e2e/generated/

# Run only critical tests for hotfix branches
- name: Run critical tests
  if: startsWith(github.head_ref, 'hotfix/')
  run: npx playwright test --grep "@sev1"

# Run sev1 + sev2 for staging
- name: Run high-priority tests
  run: npx playwright test --grep "@sev1|@sev2"

- uses: autospec-ai/action@v1
  with:
    llm_api_key: ${{ secrets.ANTHROPIC_API_KEY }}
    custom_instructions: |
      - Our app uses Clerk for auth. Mock auth with: await clerk.signIn(page)
      - All API calls go through /api/v2/ prefix
      - Use data-cy attributes for selectors (our convention)
      - We use MSW for API mocking in tests

src/
├── index.ts                    # Action entry point, config parsing
├── types.ts                    # Shared TypeScript types
├── providers/
│   ├── index.ts                # Provider factory
│   ├── anthropic.ts            # Anthropic Claude client
│   └── openai.ts               # OpenAI / compatible client
├── diff/
│   └── analyzer.ts             # Git diff extraction & filtering
├── discovery/
│   └── project-scanner.ts      # Scans for existing POMs, utilities, and test coverage
├── generator/
│   ├── prompts.ts              # Two-phase prompt construction with feature-conditional sections
│   └── test-generator.ts       # Orchestrates planning + generation + post-processing
└── utils/
    ├── git-ops.ts              # Commit & PR creation
    ├── test-post-processor.ts  # Trace injection, axe imports, screenshot normalization
    ├── fixture-extractor.ts    # Extracts page.route() mocks into shared fixture files
    └── trace-uploader.ts       # Uploads traces as GitHub Actions artifacts

npm install
npm run build       # Compile with ncc
npm run lint
npm test