Run subprocess-starting tests on separate runners by jkseppan · Pull Request #30278 · matplotlib/matplotlib · GitHub
Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 17 additions & 43 deletions .github/workflows/tests.yml
42 changes: 34 additions & 8 deletions azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,30 @@ stages:
- job: Pytest
strategy:
matrix:
Windows_py311:
Windows_py311_parallel:
vmImage: 'windows-2022' # Keep one job pinned to the oldest image
python.version: '3.11'
Windows_py312:
test.type: 'parallel'
Windows_py311_serial:
vmImage: 'windows-2022' # Keep one job pinned to the oldest image
python.version: '3.11'
test.type: 'serial'
Windows_py312_parallel:
vmImage: 'windows-latest'
python.version: '3.12'
Windows_py313:
test.type: 'parallel'
Windows_py312_serial:
vmImage: 'windows-latest'
python.version: '3.12'
test.type: 'serial'
Windows_py313_parallel:
vmImage: 'windows-latest'
python.version: '3.13'
test.type: 'parallel'
Windows_py313_serial:
vmImage: 'windows-latest'
python.version: '3.13'
test.type: 'serial'
maxParallel: 4
pool:
vmImage: '$(vmImage)'
Expand Down Expand Up @@ -120,10 +135,20 @@ stages:

echo "##vso[task.setvariable variable=VS_COVERAGE_TOOL]$TOOL"

PYTHONFAULTHANDLER=1 pytest -rfEsXR -n 2 \
--maxfail=50 --timeout=300 --durations=25 \
--junitxml=junit/test-results.xml --cov-report=xml --cov=lib

FLAGS=(
-rfEsXR
--maxfail=50
--timeout=300
--durations=25
--cov-report=xml
--cov=lib
--junitxml=junit/test-results.xml
)
if [[ "$TEST_TYPE" == serial ]]; then
PYTHONFAULTHANDLER=1 pytest "${FLAGS[@]}" -m subprocess -n 0
else
PYTHONFAULTHANDLER=1 pytest "${FLAGS[@]}" -m 'not subprocess' -n 2
fi
if [[ $VS_VER == 2022 ]]; then
"$TOOL" shutdown $SESSION_ID
echo "Coverage collection log"
Expand Down Expand Up @@ -153,7 +178,8 @@ stages:

- task: PublishTestResults@2
inputs:
testResultsFiles: '**/test-results.xml'
mergeTestResults: true
testResultsFiles: '**/test-results*.xml'
testRunTitle: 'Python $(python.version)'
condition: succeededOrFailed()

Expand Down
1 change: 1 addition & 0 deletions lib/matplotlib/testing/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ def pytest_configure(config):
("markers", "backend: Set alternate Matplotlib backend temporarily."),
("markers", "baseline_images: Compare output against references."),
("markers", "pytz: Tests that require pytz to be installed."),
("markers", "subprocess: Tests that start a subprocess."),
("filterwarnings", "error"),
("filterwarnings",
"ignore:.*The py23 module has been deprecated:DeprecationWarning"),
Expand Down
1 change: 1 addition & 0 deletions lib/matplotlib/tests/test_backend_inline.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
pytest.importorskip('nbconvert')
pytest.importorskip('ipykernel')
pytest.importorskip('matplotlib_inline')
pytestmark = pytest.mark.subprocess


def test_ipynb():
Expand Down
1 change: 1 addition & 0 deletions lib/matplotlib/tests/test_backend_nbagg.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
nbformat = pytest.importorskip('nbformat')
pytest.importorskip('nbconvert')
pytest.importorskip('ipykernel')
pytestmark = pytest.mark.subprocess

# From https://blog.thedataincubator.com/2016/06/testing-jupyter-notebooks/

Expand Down
1 change: 1 addition & 0 deletions lib/matplotlib/tests/test_backend_webagg.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from matplotlib.testing import subprocess_run_for_testing


@pytest.mark.subprocess
@pytest.mark.parametrize("backend", ["webagg", "nbagg"])
def test_webagg_fallback(backend):
pytest.importorskip("tornado")
Expand Down
23 changes: 16 additions & 7 deletions lib/matplotlib/tests/test_backends_interactive.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
from matplotlib.testing import subprocess_run_helper as _run_helper, is_ci_environment


pytestmark = pytest.mark.subprocess


class _WaitForStringPopen(subprocess.Popen):
"""
A Popen that passes flags that allow triggering KeyboardInterrupt.
Expand All @@ -33,17 +36,23 @@ def __init__(self, *args, **kwargs):
super().__init__(
*args, **kwargs,
# Force Agg so that each test can switch to its desired backend.
env={**os.environ, "MPLBACKEND": "Agg", "SOURCE_DATE_EPOCH": "0"},
stdout=subprocess.PIPE, universal_newlines=True)
env={**os.environ, "MPLBACKEND": "Agg", "SOURCE_DATE_EPOCH": "0",
"PYTHONUNBUFFERED": "1"},
bufsize=0, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, universal_newlines=True)

def wait_for(self, terminator):
"""Read until the terminator is reached."""
buf = ''
while True:
c = self.stdout.read(1)
if not c:
raise RuntimeError(
f'Subprocess died before emitting expected {terminator!r}')
if self.poll() is None:
os.sched_yield()
else:
raise RuntimeError(
f'Subprocess died before emitting expected {terminator!r}\n'
f'\nSubprocess output:\n{buf}')
buf += c
if buf.endswith(terminator):
return
Expand Down Expand Up @@ -237,7 +246,7 @@ def check_alt_backend(alt_backend):

@pytest.mark.parametrize("env", _get_testable_interactive_backends())
@pytest.mark.parametrize("toolbar", ["toolbar2", "toolmanager"])
@pytest.mark.flaky(reruns=3)
#@pytest.mark.flaky(reruns=3)
def test_interactive_backend(env, toolbar):
if env["MPLBACKEND"] == "macosx":
if toolbar == "toolmanager":
Expand Down Expand Up @@ -329,7 +338,7 @@ def _test_thread_impl():


@pytest.mark.parametrize("env", _thread_safe_backends)
@pytest.mark.flaky(reruns=3)
#@pytest.mark.flaky(reruns=3)
def test_interactive_thread_safety(env):
proc = _run_helper(_test_thread_impl, timeout=_test_timeout, extra_env=env)
assert proc.stdout.count("CloseEvent") == 1
Expand Down Expand Up @@ -617,7 +626,7 @@ def _test_number_of_draws_script():

@pytest.mark.parametrize("env", _blit_backends)
# subprocesses can struggle to get the display, so rerun a few times
@pytest.mark.flaky(reruns=4)
#@pytest.mark.flaky(reruns=4)
def test_blitting_events(env):
proc = _run_helper(
_test_number_of_draws_script, timeout=_test_timeout, extra_env=env)
Expand Down
3 changes: 3 additions & 0 deletions lib/matplotlib/tests/test_basic.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@

from matplotlib.testing import subprocess_run_for_testing

import pytest


def test_simple():
assert 1 + 1 == 2
Expand All @@ -28,6 +30,7 @@ def test_override_builtins():
assert overridden <= ok_to_override


@pytest.mark.subprocess
def test_lazy_imports():
source = textwrap.dedent("""
import sys
Expand Down
2 changes: 2 additions & 0 deletions lib/matplotlib/tests/test_determinism.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
from matplotlib.text import TextPath
from matplotlib.transforms import IdentityTransform

pytestmark = pytest.mark.subprocess


def _save_figure(objects='mhip', fmt="pdf", usetex=False):
mpl.use(fmt)
Expand Down
1 change: 1 addition & 0 deletions lib/matplotlib/tests/test_font_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,7 @@ def test_fontcache_thread_safe():
subprocess_run_helper(_test_threading, timeout=10)


@pytest.mark.subprocess
def test_lockfilefailure(tmp_path):
# The logic here:
# 1. get a temp directory from pytest
Expand Down
2 changes: 2 additions & 0 deletions lib/matplotlib/tests/test_matplotlib.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ def test_parse_to_version_info(version_str, version_tuple):
assert matplotlib._parse_to_version_info(version_str) == version_tuple


@pytest.mark.subprocess
@pytest.mark.skipif(sys.platform == "win32",
reason="chmod() doesn't work as is on Windows")
@pytest.mark.skipif(sys.platform != "win32" and os.geteuid() == 0,
Expand All @@ -37,6 +38,7 @@ def test_tmpconfigdir_warning(tmp_path):
os.chmod(tmp_path, mode)


@pytest.mark.subprocess
def test_importable_with_no_home(tmp_path):
subprocess_run_for_testing(
[sys.executable, "-c",
Expand Down
1 change: 1 addition & 0 deletions lib/matplotlib/tests/test_preprocess_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -245,6 +245,7 @@ def funcy(ax, x, y, z, t=None):
funcy.__doc__)


@pytest.mark.subprocess
def test_data_parameter_replacement():
"""
Test that the docstring contains the correct *data* parameter stub
Expand Down
1 change: 1 addition & 0 deletions lib/matplotlib/tests/test_pyplot.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from matplotlib import pyplot as plt


@pytest.mark.subprocess
def test_pyplot_up_to_date(tmp_path):
pytest.importorskip("black")

Expand Down
3 changes: 3 additions & 0 deletions lib/matplotlib/tests/test_rcparams.py
Original file line number Diff line number Diff line change
Expand Up @@ -528,6 +528,7 @@ def test_rcparams_reset_after_fail():
assert mpl.rcParams['text.usetex'] is False


@pytest.mark.subprocess
@pytest.mark.skipif(sys.platform != "linux", reason="Linux only")
def test_backend_fallback_headless_invalid_backend(tmp_path):
env = {**os.environ,
Expand All @@ -545,6 +546,7 @@ def test_backend_fallback_headless_invalid_backend(tmp_path):
env=env, check=True, stderr=subprocess.DEVNULL)


@pytest.mark.subprocess
@pytest.mark.skipif(sys.platform != "linux", reason="Linux only")
def test_backend_fallback_headless_auto_backend(tmp_path):
# specify a headless mpl environment, but request a graphical (tk) backend
Expand All @@ -567,6 +569,7 @@ def test_backend_fallback_headless_auto_backend(tmp_path):
assert backend.strip().lower() == "agg"


@pytest.mark.subprocess
@pytest.mark.skipif(
sys.platform == "linux" and not _c_internal_utils.xdisplay_is_valid(),
reason="headless")
Expand Down
1 change: 1 addition & 0 deletions lib/matplotlib/tests/test_sphinxext.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
tinypages = Path(__file__).parent / 'data/tinypages'


@pytest.mark.subprocess
def build_sphinx_html(source_dir, doctree_dir, html_dir, extra_args=None):
# Build the pages with warnings turned into errors
extra_args = [] if extra_args is None else extra_args
Expand Down
1 change: 1 addition & 0 deletions lib/matplotlib/tests/test_texmanager.py