MAINT - Ensure Playwright tests use test sites and are run in CI (#2133)

trallard · peytondmurray · drammock · web-flow · commit 83e7e51b0b52 · 2025-03-20T10:06:45.000-05:00
Ensures playwright is always available for tests following conversations in #2119 (comment) Note, however, that the `test_version_switcher_highlighting` test is currently failing due to changes to the version switcher component (as part of recent a11y improvements) and due to RTD version switcher no longer being flushed into the sidebar (#2034). I could try and add an alternative test that checks perhaps that we have at least a latest and a dev version in the version switcher? --------- Co-authored-by: Peyton Murray <peynmurray@gmail.com> Co-authored-by: Daniel McCloy <dan@mccloy.info>
diff --git a/pyproject.toml b/pyproject.toml
@@ -81,7 +81,6 @@ doc = [
   "ipywidgets",
   "graphviz",
 ]
-test = ["pytest", "pytest-cov", "pytest-regressions", "sphinx[test]"]
 dev = [
   "pyyaml",
   "pre-commit",
@@ -90,7 +89,14 @@ dev = [
   "pandoc",
   "sphinx-theme-builder[cli]",
 ]
-a11y = ["pytest-playwright"]
+test = [
+  "pytest",
+  "pytest-cov",
+  "pytest-regressions",
+  "sphinx[test]",
+  "pytest-playwright"
+]
+a11y = ["pydata-sphinx-theme[test]"]
 i18n = ["Babel", "jinja2"]
 
 [project.entry-points]
diff --git a/tests/sites/version_switcher/Makefile b/tests/sites/version_switcher/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/tests/sites/version_switcher/_static/switcher.json b/tests/sites/version_switcher/_static/switcher.json
@@ -0,0 +1,11 @@
+[
+  {
+    "version": "dev",
+    "url": "https://pydata-sphinx-theme.readthedocs.io/en/latest/"
+  },
+  {
+    "name": "0.16.1 (stable)",
+    "version": "v0.16.1",
+    "url": "https://pydata-sphinx-theme.readthedocs.io/en/stable/"
+  }
+]
diff --git a/tests/sites/version_switcher/conf.py b/tests/sites/version_switcher/conf.py
@@ -0,0 +1,22 @@
+"""Test conf file - basic site with version switcher."""
+
+# -- Project information -----------------------------------------------------
+project = "PyData Tests"
+copyright = "2020, Pydata community"
+author = "Pydata community"
+
+root_doc = "index"
+
+# -- General configuration ---------------------------------------------------
+html_theme = "pydata_sphinx_theme"
+
+html_static_path = ["_static"]
+
+# Add version switcher
+html_theme_options = {
+    "switcher": {
+        "json_url": "_static/switcher.json",
+        "version_match": "dev",
+    },
+    "navbar_start": ["navbar-logo", "version-switcher"],
+}
diff --git a/tests/sites/version_switcher/index.rst b/tests/sites/version_switcher/index.rst
@@ -0,0 +1,15 @@
+Simple site with version switcher
+=================================
+
+.. toctree::
+   :caption: Caption 1
+   :numbered:
+
+   page1
+   page2
+   section1/index
+
+Other content
+-------------
+
+Nothing to see here
diff --git a/tests/sites/version_switcher/page1.rst b/tests/sites/version_switcher/page1.rst
@@ -0,0 +1,33 @@
+Page 1
+======
+
+**normal link**
+
+- https://pydata-sphinx-theme.readthedocs.io/en/latest/
+
+**GitHub**
+
+.. container:: github-container
+
+    https://github.com
+    https://github.com/pydata
+    https://github.com/pydata/pydata-sphinx-theme
+    https://github.com/pydata/pydata-sphinx-theme/pull/1012
+    https://github.com/orgs/pydata/projects/2
+
+**GitLab**
+
+.. container:: gitlab-container
+
+    https://gitlab.com
+    https://gitlab.com/gitlab-org
+    https://gitlab.com/gitlab-org/gitlab
+    https://gitlab.com/gitlab-org/gitlab/-/issues/375583
+    https://gitlab.com/gitlab-org/gitlab/issues/375583
+    https://gitlab.com/gitlab-org/gitlab/-/issues/
+    https://gitlab.com/gitlab-org/gitlab/issues/
+    https://gitlab.com/gitlab-org/gitlab/-/issues
+    https://gitlab.com/gitlab-org/gitlab/issues
+    https://gitlab.com/gitlab-org/gitlab/-/merge_requests/84669
+    https://gitlab.com/gitlab-org/gitlab/-/pipelines/511894707
+    https://gitlab.com/gitlab-com/gl-infra/production/-/issues/6788
diff --git a/tests/sites/version_switcher/page2.rst b/tests/sites/version_switcher/page2.rst
@@ -0,0 +1,9 @@
+:html_theme.sidebar_secondary.remove: true
+
+Page :math:`\beta`
+==================
+
+.. code-block:: python
+
+    import pydata_sphinx_theme as pst
+    raise RuntimeError('Test of pygments highlighting')
diff --git a/tests/sites/version_switcher/section1/index.rst b/tests/sites/version_switcher/section1/index.rst
@@ -0,0 +1,6 @@
+Section 1 index
+===============
+.. toctree::
+
+   page1
+   https://google.com
diff --git a/tests/sites/version_switcher/section1/page1.rst b/tests/sites/version_switcher/section1/page1.rst
@@ -0,0 +1,2 @@
+Section 1 page1
+===============
diff --git a/tests/test_a11y.py b/tests/test_a11y.py
@@ -1,4 +1,8 @@
-"""Using Axe-core, scan the Kitchen Sink pages for accessibility violations."""
+"""
+Using Axe-core, scan the Kitchen Sink pages for accessibility violations.
+Note that in contrast with the rest of our tests, the accessibility tests in this file
+are run against a build of our PST documentation, not purposedly-built test sites.
+"""
 
 from urllib.parse import urljoin
 
@@ -288,17 +292,6 @@ def test_notebook_ipywidget_output_tab_stop(page: Page, url_base: str) -> None:
     expect(ipywidget).to_have_attribute("tabindex", "0")
 
 
-def test_breadcrumb_expansion(page: Page, url_base: str) -> None:
-    """Foo."""
-    # page.goto(urljoin(url_base, "community/practices/merge.html"))
-    # expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text("Merge and review policy") # noqa: E501
-    page.set_viewport_size({"width": 1440, "height": 720})
-    page.goto(urljoin(url_base, "community/topics/config.html"))
-    expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text(
-        "Update Sphinx configuration during the build"
-    )
-
-
 @pytest.mark.a11y
 def test_search_as_you_type(page: Page, url_base: str) -> None:
     """Search-as-you-type feature should support keyboard navigation.
diff --git a/tests/test_playwright.py b/tests/test_playwright.py
@@ -1,4 +1,8 @@
-"""Build minimal test sites with sphinx_build_factory and test them with Playwright."""
+"""
+Build minimal test sites with sphinx_build_factory and test them with Playwright.
+When adding new tests to this file, remember to also add the corresponding test site
+to `tests/sites/` or use an existing one.
+"""
 
 from pathlib import Path
 from typing import Callable
@@ -21,6 +25,7 @@
 test_sites_dir = repo_path / "docs" / "_build" / "html" / "playwright_tests"
 
 
+# ------------------------- Helper functions -------------------------
 def _is_overflowing(element):
     """Check if an element is being shortened via CSS due to text-overflow property.
 
@@ -42,7 +47,8 @@ def _build_test_site(site_name: str, sphinx_build_factory: Callable) -> None:
 
 def _check_test_site(site_name: str, site_path: Path, test_func: Callable):
     """Make the built test site available to Playwright, then run `test_func` on it."""
-    test_sites_dir.mkdir(exist_ok=True)
+    # Need to ensure parent directories exist in CI
+    test_sites_dir.mkdir(exist_ok=True, parents=True)
     symlink_path = test_sites_dir / site_name
     try:
         symlink_path.symlink_to(site_path, True)
@@ -55,65 +61,37 @@ def _check_test_site(site_name: str, site_path: Path, test_func: Callable):
         test_sites_dir.rmdir()
 
 
-def test_version_switcher_highlighting(page: Page, url_base: str) -> None:
+# ------------------------- Test functions: style -------------------------
+def test_version_switcher_highlighting(
+    sphinx_build_factory: Callable, page: Page, url_base: str
+) -> None:
     """
     In sidebar and topbar - version switcher should apply highlight color to currently
     selected version.
     """
-    page.goto(url=url_base)
-    # no need to include_hidden here ↓↓↓, we just need to get the active version name
-    button = page.get_by_role("button").filter(has_text="dev")
-    active_version_name = button.get_attribute("data-active-version-name")
-    # here we do include_hidden, so sidebar & topbar menus should each have a
-    # matching entry:
-    entries = page.get_by_role("option", include_hidden=True).filter(
-        has_text=active_version_name
-    )
-    assert entries.count() == 2
-    # make sure they're highlighted
-    for entry in entries.all():
-        light_mode = "rgb(10, 125, 145)"  # pst-color-primary
-        # dark_mode = "rgb(63, 177, 197)"
-        expect(entry).to_have_css("color", light_mode)
-
-
-def test_breadcrumb_expansion(page: Page, url_base: str) -> None:
-    """Test breadcrumb text-overflow."""
-    # wide viewport width → no truncation
-    page.set_viewport_size({"width": 1440, "height": 720})
-    page.goto(urljoin(url_base, "community/topics/config.html"))
-    expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text(
-        "Update Sphinx configuration during the build"
-    )
-    el = page.get_by_text("Update Sphinx configuration during the build").nth(1)
-    expect(el).to_have_css("overflow-x", "hidden")
-    expect(el).to_have_css("text-overflow", "ellipsis")
-    assert not _is_overflowing(el)
-    # narrow viewport width → truncation
-    page.set_viewport_size({"width": 150, "height": 720})
-    assert _is_overflowing(el)
-
-
-def test_breadcrumbs_everywhere(
-    sphinx_build_factory: Callable, page: Page, url_base: str
-) -> None:
-    """Test breadcrumbs truncate properly when placed in various parts of the layout."""
-    site_name = "breadcrumbs"
+    site_name = "version_switcher"
     site_path = _build_test_site(site_name, sphinx_build_factory=sphinx_build_factory)
 
-    def check_breadcrumb_truncation():
-        page.goto(
-            urljoin(url_base, f"playwright_tests/{site_name}/hansel/gretel/house.html")
+    def check_version_switcher_highlighting():
+        page.goto(urljoin(url_base, f"playwright_tests/{site_name}/index.html"))
+        # no need to include_hidden here ↓↓↓, we just need to get the active
+        # version name
+        button = page.get_by_role("button").filter(has_text="dev")
+        active_version_name = button.get_attribute("data-active-version-name")
+
+        # here we do include_hidden, since we are not adding this in the sidebar
+        # we should only get one entry
+        entries = page.get_by_role("option", include_hidden=True).filter(
+            has_text=active_version_name
         )
-        # sidebar should overflow
-        text = "In the oven with my sister, so hot right now. Soooo. Hotttt."
-        el = page.locator("#main-content").get_by_text(text).last
-        assert _is_overflowing(el)
-        # footer containers never trigger ellipsis overflow because min-width is content
-        el = page.locator(".footer-items__center > .footer-item")
-        assert not _is_overflowing(el)
+        assert entries.count() == 1
+        # make sure they're highlighted
+        for entry in entries.all():
+            light_mode = "rgb(10, 125, 145)"  # pst-color-primary
+            # dark_mode = "rgb(63, 177, 197)"
+            expect(entry).to_have_css("color", light_mode)
 
-    _check_test_site(site_name, site_path, check_breadcrumb_truncation)
+    _check_test_site(site_name, site_path, check_version_switcher_highlighting)
 
 
 def test_colors(sphinx_build_factory: Callable, page: Page, url_base: str) -> None:
@@ -143,3 +121,53 @@ def check_colors():
                 expect(el).to_have_css("color", hover_color)
 
     _check_test_site(site_name, site_path, check_colors)
+
+
+# ------------------------- Test functions: layout & components -----------------------
+
+
+def test_breadcrumb_expansion(
+    sphinx_build_factory: Callable, page: Page, url_base: str
+) -> None:
+    """Test breadcrumb text-overflow."""
+    site_name = "breadcrumbs"
+    site_path = _build_test_site(site_name, sphinx_build_factory=sphinx_build_factory)
+
+    def check_breadcrumb_expansion():
+        # wide viewport width → no truncation
+        page.set_viewport_size({"width": 1440, "height": 720})
+        page.goto(urljoin(url_base, "community/topics/config.html"))
+        expect(page.get_by_label("Breadcrumb").get_by_role("list")).to_contain_text(
+            "Update Sphinx configuration during the build"
+        )
+        el = page.get_by_text("Update Sphinx configuration during the build").nth(1)
+        expect(el).to_have_css("overflow-x", "hidden")
+        expect(el).to_have_css("text-overflow", "ellipsis")
+        assert not _is_overflowing(el)
+        # narrow viewport width → truncation
+        page.set_viewport_size({"width": 150, "height": 720})
+        assert _is_overflowing(el)
+
+        _check_test_site(site_name, site_path, check_breadcrumb_expansion)
+
+
+def test_breadcrumbs_everywhere(
+    sphinx_build_factory: Callable, page: Page, url_base: str
+) -> None:
+    """Test breadcrumbs truncate properly when placed in various parts of the layout."""
+    site_name = "breadcrumbs"
+    site_path = _build_test_site(site_name, sphinx_build_factory=sphinx_build_factory)
+
+    def check_breadcrumb_truncation():
+        page.goto(
+            urljoin(url_base, f"playwright_tests/{site_name}/hansel/gretel/house.html")
+        )
+        # sidebar should overflow
+        text = "In the oven with my sister, so hot right now. Soooo. Hotttt."
+        el = page.locator("#main-content").get_by_text(text).last
+        assert _is_overflowing(el)
+        # footer containers never trigger ellipsis overflow because min-width is content
+        el = page.locator(".footer-items__center > .footer-item")
+        assert not _is_overflowing(el)
+
+    _check_test_site(site_name, site_path, check_breadcrumb_truncation)
diff --git a/tox.ini b/tox.ini
@@ -15,7 +15,7 @@ env_list =
 # helping contributors run common tasks without needing to call all the steps
 # For example to run the tests: tox run -m tests
 labels =
-    tests = compile-assets, i18n-compile,  py312-tests
+    tests = compile-assets, i18n-compile, py312-tests
     a11y = compile-assets, i18n-compile, py312-docs, a11y-tests
     i18n = i18n-extract, i18n-compile
     live-server = compile-assets, i18n-compile, docs-live
@@ -61,15 +61,20 @@ description = "Run tests Python and Sphinx versions. If a Sphinx version is spec
 # need to ensure the package is installed in editable mode
 package = editable
 extras =
-    test # install dependencies - defined in pyproject.toml
+    test # install dependencies, includes pytest-playwright - defined in pyproject.toml
+pass_env = GITHUB_ACTIONS # so we can check if this is run on GitHub Actions
 deps =
     coverage[toml]
     py39-sphinx61-tests: sphinx~=6.1.0
     py312-sphinxdev: sphinx[test] @ git+https://github.com/sphinx-doc/sphinx.git@master
 depends =
     compile-assets,
     i18n-compile
+allowlist_externals=
+    playwright
+    bash
 commands =
+    bash -c 'if [[ "{env:GITHUB_ACTIONS:}" == "true" ]]; then playwright install --with-deps; else playwright install; fi'
     py3{9,10,11,12}{,-sphinx61,-sphinxdev,}-tests: coverage run -m pytest -m "not a11y" {posargs}
     py3{9,10,11,12}{,-sphinx61,-sphinxdev,}-tests-no-cov: pytest -m "not a11y" {posargs}
 
@@ -84,8 +89,7 @@ description = "run accessibility tests with Playwright and axe-core"
 base_python = py312 # keep in sync with tests.yml
 pass_env = GITHUB_ACTIONS # so we can check if this is run on GitHub Actions
 extras =
-    test
-    a11y
+    test # install dependencies, includes pytest-playwright - defined in pyproject.toml
 depends =
     compile-assets,
     i18n-compile
