perf(ci): pytest-xdist with per-worker DBs — 22m → ~4m backend suite #151

Closed
chihlasm wants to merge 1 commits from fix/ci-pytest-xdist into main
3 changed files with 60 additions and 2 deletions

View File

@@ -66,11 +66,15 @@ jobs:
run: cd backend && python scripts/check_tenant_filters.py
- name: Run tests with coverage
# `-n auto` parallelizes across all runner cores via pytest-xdist.
# conftest.py creates a per-worker DB (resolutionflow_test_gw0,
# resolutionflow_test_gw1, …) so the per-test DROP SCHEMA doesn't
# race across workers. Master/serial runs keep the base DB.
# term-missing dropped — the custom "Display coverage summary" step
# below parses coverage.json and prints the same info more concisely.
# --maxfail=10 short-circuits on structural breakage so we don't burn
# 25 minutes when a fixture explodes.
run: cd backend && python -m pytest --override-ini="addopts=" --maxfail=10 --cov=app --cov-report=json:coverage.json --cov-fail-under=50
run: cd backend && python -m pytest --override-ini="addopts=" -n auto --maxfail=10 --cov=app --cov-report=json:coverage.json --cov-fail-under=50
- name: Display coverage summary
if: always()

View File

@@ -4,6 +4,7 @@
# Testing — pytest-asyncio 0.24+ requires pytest>=8.2
pytest==8.4.2
pytest-asyncio==0.24.0
pytest-xdist==3.6.1
httpx>=0.27.0
pytest-cov==5.0.0

View File

@@ -35,11 +35,64 @@ settings.REQUIRE_INVITE_CODE = False
# would silently nuke the dev database. Only DATABASE_TEST_URL is honored,
# and the safety assertion below refuses to run against a DB whose name
# doesn't contain "test".
TEST_DATABASE_URL = os.environ.get(
_BASE_TEST_DATABASE_URL = os.environ.get(
"DATABASE_TEST_URL",
"postgresql+asyncpg://postgres:postgres@localhost:5432/resolutionflow_test",
)
def _worker_db_url(base_url: str) -> str:
"""Per-worker DB URL for pytest-xdist parallelization.
pytest-xdist sets PYTEST_XDIST_WORKER to 'gw0', 'gw1', ... per worker
process. Each worker needs its own database so the per-test
`DROP SCHEMA public CASCADE` doesn't race across workers. Master/serial
runs (no xdist) keep the base DB. The base DB is created by the postgres
service container; per-worker DBs are CREATE DATABASE-d on first import
by `_ensure_worker_db_exists` below.
"""
worker = os.environ.get("PYTEST_XDIST_WORKER")
if not worker or worker == "master":
return base_url
head, tail = base_url.rsplit("/", 1)
db_name, _, query = tail.partition("?")
suffix = f"?{query}" if query else ""
return f"{head}/{db_name}_{worker}{suffix}"
def _ensure_worker_db_exists(worker_url: str, base_url: str) -> None:
"""Create the per-worker DB if it doesn't exist. Runs synchronously at
conftest import time (before any async test machinery), using psycopg2
against the postgres maintenance DB. No-op when not running under xdist.
"""
if worker_url == base_url:
return
head, tail = worker_url.rsplit("/", 1)
worker_db = tail.partition("?")[0]
# Strip the +asyncpg dialect for sync psycopg2 + connect to 'postgres'.
sync_head = head.replace("+asyncpg", "")
admin_url = f"{sync_head}/postgres"
# Lazy import — psycopg2 is a transitive backend dep; not imported at
# module top to keep the conftest light when xdist isn't in use.
from sqlalchemy import create_engine
engine = create_engine(admin_url, isolation_level="AUTOCOMMIT")
try:
with engine.begin() as conn:
exists = conn.execute(
sa.text("SELECT 1 FROM pg_database WHERE datname = :n"),
{"n": worker_db},
).scalar()
if not exists:
# Identifier interpolation is safe — worker_db is built from
# the trusted base URL + 'gw\d+' worker suffix.
conn.execute(sa.text(f'CREATE DATABASE "{worker_db}"'))
finally:
engine.dispose()
TEST_DATABASE_URL = _worker_db_url(_BASE_TEST_DATABASE_URL)
_ensure_worker_db_exists(TEST_DATABASE_URL, _BASE_TEST_DATABASE_URL)
# Belt-and-suspenders: refuse to run tests against a DB whose name doesn't
# contain "test". Parses the last path segment of the URL (everything after
# the final '/', with query string stripped) so credentials / hosts that