From 7f714363ddf0575872403841018b8bcd70f08dce Mon Sep 17 00:00:00 2001 From: Michael Chihlas Date: Sat, 25 Apr 2026 12:07:57 -0400 Subject: [PATCH] =?UTF-8?q?perf(ci):=20pytest-xdist=20with=20per-worker=20?= =?UTF-8?q?DBs=20=E2=80=94=2022m=20=E2=86=92=20~4m?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend suite is the slow gate (1076 passed locally in 22m27s on fix/ci-workflow-config). Adding pytest-xdist with per-worker DB isolation drops it to ~4m20s on the 8-core homelab runner. Verified locally: `pytest -n auto --no-cov` finished in 4m28s real time (15m19s user — confirms ~5× parallelism). How it works: - conftest.py reads `PYTEST_XDIST_WORKER` (set per worker by xdist — 'gw0', 'gw1', …). When set, derives a per-worker DB URL like `…/resolutionflow_test_gw0`. The base DB stays for serial / master runs. - `_ensure_worker_db_exists` runs synchronously at conftest import, connects to the postgres maintenance DB, and `CREATE DATABASE`s the worker-suffixed DB if it doesn't exist. Idempotent across runs. - The "test" safety guard still applies — every worker DB name contains "test" so the assertion holds. - The per-test `DROP SCHEMA public CASCADE` now operates on the worker's isolated DB, no cross-worker race. CI workflow: backend job switches to `pytest -n auto`. Coverage still collected (pytest-cov has built-in xdist support). Adds `pytest-xdist==3.6.1` to requirements-dev.txt. Co-Authored-By: Claude Opus 4.7 --- .gitea/workflows/ci.yml | 6 +++- backend/requirements-dev.txt | 1 + backend/tests/conftest.py | 55 +++++++++++++++++++++++++++++++++++- 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 9ca994d4..4399db68 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -69,11 +69,15 @@ jobs: run: cd backend && python scripts/check_tenant_filters.py - name: Run tests with coverage + # `-n auto` parallelizes across all runner cores via pytest-xdist. + # conftest.py creates a per-worker DB (resolutionflow_test_gw0, + # resolutionflow_test_gw1, …) so the per-test DROP SCHEMA doesn't + # race across workers. Master/serial runs keep the base DB. # term-missing dropped — the custom "Display coverage summary" step # below parses coverage.json and prints the same info more concisely. # --maxfail=10 short-circuits on structural breakage so we don't burn # 25 minutes when a fixture explodes. - run: cd backend && python -m pytest --override-ini="addopts=" --maxfail=10 --cov=app --cov-report=json:coverage.json --cov-fail-under=50 + run: cd backend && python -m pytest --override-ini="addopts=" -n auto --maxfail=10 --cov=app --cov-report=json:coverage.json --cov-fail-under=50 - name: Display coverage summary if: always() diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt index 7b44660c..5c5d2e00 100644 --- a/backend/requirements-dev.txt +++ b/backend/requirements-dev.txt @@ -4,6 +4,7 @@ # Testing — pytest-asyncio 0.24+ requires pytest>=8.2 pytest==8.4.2 pytest-asyncio==0.24.0 +pytest-xdist==3.6.1 httpx>=0.27.0 pytest-cov==5.0.0 diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index bcbdb7ea..606609df 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -35,11 +35,64 @@ settings.REQUIRE_INVITE_CODE = False # would silently nuke the dev database. Only DATABASE_TEST_URL is honored, # and the safety assertion below refuses to run against a DB whose name # doesn't contain "test". -TEST_DATABASE_URL = os.environ.get( +_BASE_TEST_DATABASE_URL = os.environ.get( "DATABASE_TEST_URL", "postgresql+asyncpg://postgres:postgres@localhost:5432/resolutionflow_test", ) + +def _worker_db_url(base_url: str) -> str: + """Per-worker DB URL for pytest-xdist parallelization. + + pytest-xdist sets PYTEST_XDIST_WORKER to 'gw0', 'gw1', ... per worker + process. Each worker needs its own database so the per-test + `DROP SCHEMA public CASCADE` doesn't race across workers. Master/serial + runs (no xdist) keep the base DB. The base DB is created by the postgres + service container; per-worker DBs are CREATE DATABASE-d on first import + by `_ensure_worker_db_exists` below. + """ + worker = os.environ.get("PYTEST_XDIST_WORKER") + if not worker or worker == "master": + return base_url + head, tail = base_url.rsplit("/", 1) + db_name, _, query = tail.partition("?") + suffix = f"?{query}" if query else "" + return f"{head}/{db_name}_{worker}{suffix}" + + +def _ensure_worker_db_exists(worker_url: str, base_url: str) -> None: + """Create the per-worker DB if it doesn't exist. Runs synchronously at + conftest import time (before any async test machinery), using psycopg2 + against the postgres maintenance DB. No-op when not running under xdist. + """ + if worker_url == base_url: + return + head, tail = worker_url.rsplit("/", 1) + worker_db = tail.partition("?")[0] + # Strip the +asyncpg dialect for sync psycopg2 + connect to 'postgres'. + sync_head = head.replace("+asyncpg", "") + admin_url = f"{sync_head}/postgres" + # Lazy import — psycopg2 is a transitive backend dep; not imported at + # module top to keep the conftest light when xdist isn't in use. + from sqlalchemy import create_engine + engine = create_engine(admin_url, isolation_level="AUTOCOMMIT") + try: + with engine.begin() as conn: + exists = conn.execute( + sa.text("SELECT 1 FROM pg_database WHERE datname = :n"), + {"n": worker_db}, + ).scalar() + if not exists: + # Identifier interpolation is safe — worker_db is built from + # the trusted base URL + 'gw\d+' worker suffix. + conn.execute(sa.text(f'CREATE DATABASE "{worker_db}"')) + finally: + engine.dispose() + + +TEST_DATABASE_URL = _worker_db_url(_BASE_TEST_DATABASE_URL) +_ensure_worker_db_exists(TEST_DATABASE_URL, _BASE_TEST_DATABASE_URL) + # Belt-and-suspenders: refuse to run tests against a DB whose name doesn't # contain "test". Parses the last path segment of the URL (everything after # the final '/', with query string stripped) so credentials / hosts that