Documentation

$ npm install -g deflaky-cli

# Verify installation

$ deflaky-cli --version

$ npx deflaky-cli --help

# Run Playwright tests 5 times (default)

$ deflaky-cli run -- npx playwright test

# Run 10 times with a custom threshold

$ deflaky-cli run --runs 10 --threshold 95 -- npx playwright test

# Specify number of runs

$ deflaky-cli run --runs 5 -- npx playwright test

# Set flakiness threshold (fail if FlakeScore is below)

$ deflaky-cli run --threshold 90 -- npx playwright test

# Output as JSON

$ deflaky-cli run --format json -- npx playwright test

# Output as JUnit XML

$ deflaky-cli run --format junit -- pytest

# Save report to a file

$ deflaky-cli run --format json --output report.json -- npx jest

# Basic detection

$ deflaky-cli run -- npx playwright test

# Run specific test file 10 times

$ deflaky-cli run --runs 10 -- npx playwright test tests/login.spec.ts

# With a specific project (e.g. chromium only)

$ deflaky-cli run -- npx playwright test --project=chromium

# Push results to dashboard

$ deflaky-cli run --push -- npx playwright test

# Detect flaky Cypress tests

$ deflaky-cli run -- npx cypress run

# Specific spec file

$ deflaky-cli run --runs 5 -- npx cypress run --spec cypress/e2e/checkout.cy.ts

# With a specific browser

$ deflaky-cli run -- npx cypress run --browser chrome

# Run Maven tests

$ deflaky-cli run -- mvn test

# Specific test class

$ deflaky-cli run -- mvn test -Dtest="LoginTest"

# With Gradle

$ deflaky-cli run -- gradle test

# Run all Jest tests

$ deflaky-cli run -- npx jest

# Specific test file

$ deflaky-cli run --runs 10 -- npx jest src/__tests__/api.test.ts

# With coverage disabled for speed

$ deflaky-cli run -- npx jest --no-coverage

# Run all pytest tests

$ deflaky-cli run -- pytest

# Specific test module

$ deflaky-cli run --runs 5 -- pytest tests/test_auth.py

# With JUnit XML output for richer reports

$ deflaky-cli run -- pytest --junitxml=report.xml

# Run in verbose mode

$ deflaky-cli run -- pytest -v

# Run all Mocha tests

$ deflaky-cli run -- npx mocha

# With specific test directory

$ deflaky-cli run -- npx mocha "test/**/*.spec.js" --exit

# With TypeScript

$ deflaky-cli run -- npx mocha --require ts-node/register 'test/**/*.spec.ts'

# Run TestNG suite

$ deflaky-cli run -- mvn test -Dsurefire.suiteXmlFiles=testng.xml

# Specific test group

$ deflaky-cli run -- mvn test -Dgroups="smoke"

name: Flaky Test Check

on: [pull_request]

jobs:

flaky-check:

runs-on: ubuntu-latest

steps:

- uses: actions/checkout@v4

- name: Setup Node.js

uses: actions/setup-node@v4

with:

node-version: 20

- name: Install dependencies

run: npm ci

- name: Install Playwright browsers

run: npx playwright install --with-deps

- name: Run DeFlaky

run: npx deflaky-cli run --runs 3 --threshold 90 --push -- npx playwright test

env:

DEFLAKY_API_TOKEN: ${{ secrets.DEFLAKY_API_TOKEN }}

flaky_check:

stage: test

image: mcr.microsoft.com/playwright:v1.44.0-jammy

script:

- npm ci

- npx deflaky-cli run --runs 3 --threshold 90 --push -- npx playwright test

variables:

DEFLAKY_API_TOKEN: $DEFLAKY_API_TOKEN

only:

- merge_requests

pipeline {

agent { docker { image 'node:20' } }

environment {

DEFLAKY_API_TOKEN = credentials('deflaky-token')

}

stages {

stage('Install') {

steps {

sh 'npm ci'

}

}

stage('Flaky Check') {

steps {

sh 'npx deflaky-cli run --runs 3 --threshold 90 --push -- npx playwright test'

}

}

}

}

# 1. Install your project dependencies

$ npm ci

# 2. Run DeFlaky with your test command

$ npx deflaky-cli run --runs 3 --threshold 90 --push -- <your-test-command>

# Make sure DEFLAKY_API_TOKEN is set in your CI environment

# DeFlaky exits with code 1 if FlakeScore is below --threshold

# Option 1: Pass token as a flag

$ deflaky-cli run --push --token df_abc123 -- npx playwright test

# Option 2: Set as environment variable (recommended for CI)

$ export DEFLAKY_API_TOKEN=df_abc123

$ deflaky-cli run --push -- npx playwright test

# Option 3: Save in config file

$ deflaky-cli config set token df_abc123

$ deflaky-cli run --push -- npx playwright test

{

"runs": 5,

"threshold": 90,

"format": "json",

"push": true,

"token": "df_abc123",

"project": "my-app",

"apiUrl": "https://api.deflaky.com",

"parallel": 1,

"verbose": false,

"failOnFlaky": false

}

$ deflaky-cli config init