sagemathgh-36989: OS-dependent doctest tags `# known bug: macos`, `# known bug: linux`
    
<!-- ^^^^^
Please provide a concise, informative and self-explanatory title.
Don't put issue numbers in there, do this in the PR body below.
For example, instead of "Fixes sagemath#1234" use "Introduce new method to
calculate 1+1"
-->
<!-- Describe your changes here in detail -->

<!-- Why is this change required? What problem does it solve? -->
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes sagemath#12345". -->
<!-- If your change requires a documentation PR, please link it
appropriately. -->

Cherry-picked from
- sagemath#36960

Author: @tobiasdiez

Reviewer: @mkoeppe

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->
<!-- If your change requires a documentation PR, please link it
appropriately -->
<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
<!-- Feel free to remove irrelevant items. -->

- [x] The title is concise, informative, and self-explanatory.
- [ ] The description explains in detail what this PR is about.
- [ ] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- sagemath#12345: short description why this is a dependency
- sagemath#34567: ...
-->

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
    
URL: sagemath#36989
Reported by: Matthias Köppe
Reviewer(s):

Author: Release Manager

src/doc/en/developer/coding_basics.rst

@@ -1181,7 +1181,7 @@ framework. Here is a comprehensive list:
   Use it for very long doctests that are only meant as documentation. It can
   also be used for todo notes of what will eventually be implemented::
 
-      sage: factor(x*y - x*z)    # todo: not implemented
+      sage: factor(x*y - x*z)    # not implemented
 
   It is also immediately clear to the user that the indicated example
   does not currently work.

src/sage/doctest/control.py

@@ -978,7 +978,7 @@ def expand_files_into_sources(self):
             sage: DC = DocTestController(DD, [dirname])
             sage: DC.expand_files_into_sources()
             sage: len(DC.sources)
-            11
+            12
             sage: DC.sources[0].options.optional
             True
 
@@ -1080,6 +1080,7 @@ def sort_sources(self):
             sage.doctest.test
             sage.doctest.sources
             sage.doctest.reporting
+            sage.doctest.parsing_test
             sage.doctest.parsing
             sage.doctest.forker
             sage.doctest.fixtures

src/sage/doctest/external.py

@@ -360,7 +360,7 @@ def external_features():
     yield CPLEX()
     yield Gurobi()
 
-def external_software():
+def external_software() -> list[str]:
     """
     Return the alphabetical list of external software supported by this module.
 

src/sage/doctest/parsing.py

@@ -35,10 +35,11 @@
 
 import collections.abc
 import doctest
+import platform
 import re
-
 from collections import defaultdict
 from functools import reduce
+from typing import Literal, Union, overload
 
 from sage.misc.cachefunc import cached_function
 from sage.repl.preparse import preparse, strip_string_literals
@@ -91,30 +92,49 @@ def fake_RIFtol(*args):
 
 
 # This is the correct pattern to match ISO/IEC 6429 ANSI escape sequences:
-ansi_escape_sequence = re.compile(r'(\x1b[@-Z\\-~]|\x1b\[.*?[@-~]|\x9b.*?[@-~])')
-
-special_optional_regex = 'arb216|arb218|py2|long time|not implemented|not tested|known bug'
-tag_with_explanation_regex = r'((?:\w|[.])+)\s*(?:\((.*?)\))?'
-optional_regex = re.compile(fr'(?P<cmd>{special_optional_regex})\s*(?:\((?P<cmd_explanation>.*?)\))?|'
-                            fr'[^ a-z]\s*(optional|needs)(?:\s|[:-])*(?P<tags>(?:(?:{tag_with_explanation_regex})\s*)*)',
-                            re.IGNORECASE)
+ansi_escape_sequence = re.compile(r"(\x1b[@-Z\\-~]|\x1b\[.*?[@-~]|\x9b.*?[@-~])")
+
+special_optional_regex = (
+    "arb216|arb218|py2|long time|not implemented|not tested|optional|needs|known bug"
+)
+tag_with_explanation_regex = r"((?:\w|[.])*)\s*(?:\((?P<cmd_explanation>.*?)\))?"
+optional_regex = re.compile(
+    rf"[^ a-z]\s*(?P<cmd>{special_optional_regex})(?:\s|[:-])*(?P<tags>(?:(?:{tag_with_explanation_regex})\s*)*)",
+    re.IGNORECASE,
+)
 special_optional_regex = re.compile(special_optional_regex, re.IGNORECASE)
 tag_with_explanation_regex = re.compile(tag_with_explanation_regex, re.IGNORECASE)
 
 nodoctest_regex = re.compile(r'\s*(#+|%+|r"+|"+|\.\.)\s*nodoctest')
-optionaltag_regex = re.compile(r'^(\w|[.])+$')
-optionalfiledirective_regex = re.compile(r'\s*(#+|%+|r"+|"+|\.\.)\s*sage\.doctest: (.*)')
+optionaltag_regex = re.compile(r"^(\w|[.])+$")
+optionalfiledirective_regex = re.compile(
+    r'\s*(#+|%+|r"+|"+|\.\.)\s*sage\.doctest: (.*)'
+)
+
+
+@overload
+def parse_optional_tags(string: str) -> dict[str, Union[str, None]]:
+    pass
+
 
+@overload
+def parse_optional_tags(
+    string: str, *, return_string_sans_tags: Literal[True]
+) -> tuple[dict[str, Union[str, None]], str, bool]:
+    pass
 
-def parse_optional_tags(string, *, return_string_sans_tags=False):
+
+def parse_optional_tags(
+    string: str, *, return_string_sans_tags: bool = False
+) -> Union[tuple[dict[str, Union[str, None]], str, bool], dict[str, Union[str, None]]]:
     r"""
     Return a dictionary whose keys are optional tags from the following
     set that occur in a comment on the first line of the input string.
 
     - ``'long time'``
     - ``'not implemented'``
     - ``'not tested'``
-    - ``'known bug'``
+    - ``'known bug'`` (possible values are ``None``, ``linux`` and ``macos``)
     - ``'py2'``
     - ``'arb216'``
     - ``'arb218'``
@@ -219,17 +239,31 @@ def parse_optional_tags(string, *, return_string_sans_tags=False):
             # no tag comment
             return {}, string, False
 
-    tags = {}
+    tags: dict[str, Union[str, None]] = {}
     for m in optional_regex.finditer(comment):
-        cmd = m.group('cmd')
-        if cmd and cmd.lower() == 'known bug':
-            tags['bug'] = None  # so that such tests will be run by sage -t ... -only-optional=bug
-        elif cmd:
-            tags[cmd.lower()] = m.group('cmd_explanation') or None
+        cmd = m.group("cmd").lower().strip()
+        if cmd == "":
+            # skip empty tags
+            continue
+        if cmd == "known bug":
+            value = None
+            if m.groups("tags") and m.group("tags").strip().lower().startswith("linux"):
+                value = "linux"
+            if m.groups("tags") and m.group("tags").strip().lower().startswith("macos"):
+                value = "macos"
+
+            # rename 'known bug' to 'bug' so that such tests will be run by sage -t ... -only-optional=bug
+            tags["bug"] = value
+        elif cmd not in ["optional", "needs"]:
+            tags[cmd] = m.group("cmd_explanation") or None
         else:
-            # optional/needs
-            tags.update({m.group(1).lower(): m.group(2) or None
-                         for m in tag_with_explanation_regex.finditer(m.group('tags'))})
+            # other tags with additional values
+            tags_with_value = {
+                m.group(1).lower().strip(): m.group(2) or None
+                for m in tag_with_explanation_regex.finditer(m.group("tags"))
+            }
+            tags_with_value.pop("", None)
+            tags.update(tags_with_value)
 
     if return_string_sans_tags:
         is_persistent = tags and first_line_sans_comments.strip() == 'sage:' and not rest  # persistent (block-scoped) tag
@@ -837,6 +871,14 @@ class SageDocTestParser(doctest.DocTestParser):
     A version of the standard doctest parser which handles Sage's
     custom options and tolerances in floating point arithmetic.
     """
+
+    long: bool
+    file_optional_tags: set[str]
+    optional_tags: Union[bool, set[str]]
+    optional_only: bool
+    optionals: dict[str, int]
+    probed_tags: set[str]
+
     def __init__(self, optional_tags=(), long=False, *, probed_tags=(), file_optional_tags=()):
         r"""
         INPUT:
@@ -874,7 +916,7 @@ def __init__(self, optional_tags=(), long=False, *, probed_tags=(), file_optiona
                 self.optional_tags.remove('sage')
             else:
                 self.optional_only = True
-        self.probed_tags = probed_tags
+        self.probed_tags = set(probed_tags)
         self.file_optional_tags = set(file_optional_tags)
 
     def __eq__(self, other):
@@ -1178,8 +1220,8 @@ def check_and_clear_tag_counts():
 
         for item in res:
             if isinstance(item, doctest.Example):
-                optional_tags, source_sans_tags, is_persistent = parse_optional_tags(item.source, return_string_sans_tags=True)
-                optional_tags = set(optional_tags)
+                optional_tags_with_values, _, is_persistent = parse_optional_tags(item.source, return_string_sans_tags=True)
+                optional_tags = set(optional_tags_with_values)
                 if is_persistent:
                     check_and_clear_tag_counts()
                     persistent_optional_tags = optional_tags
@@ -1210,11 +1252,24 @@ def check_and_clear_tag_counts():
                             continue
 
                     if self.optional_tags is not True:
-                        extra = {tag
-                                 for tag in optional_tags
-                                 if (tag not in self.optional_tags
-                                     and tag not in available_software)}
-                        if extra:
+                        extra = {
+                            tag
+                            for tag in optional_tags
+                            if (
+                                tag not in self.optional_tags
+                                and tag not in available_software
+                            )
+                        }
+                        if extra and any(tag in ["bug"] for tag in extra):
+                            # Bug only occurs on a specific platform?
+                            bug_platform = optional_tags_with_values.get("bug")
+                            # System platform as either linux or macos
+                            system_platform = (
+                                platform.system().lower().replace("darwin", "macos")
+                            )
+                            if not bug_platform or bug_platform == system_platform:
+                                continue
+                        elif extra:
                             if any(tag in external_software for tag in extra):
                                 # never probe "external" software
                                 continue

src/sage/doctest/parsing_test.py

@@ -0,0 +1,78 @@
+import sys
+
+import pytest
+from sage.doctest.parsing import SageDocTestParser, parse_optional_tags
+
+onlyLinux = pytest.mark.skipif(
+    sys.platform != "linux",
+    reason="No Linux system",
+)
+"""A decorator to specify that this function should only execute on Linux systems.
+"""
+
+
+def test_parse_optional_tags_known_bug_returns_bug():
+    tags = parse_optional_tags("sage: # known bug")
+    assert tags == {"bug": None}
+
+
+def test_parse_optional_tags_known_bug_with_value_returns_bug_and_value():
+    tags = parse_optional_tags("sage: # known bug: linux")
+    assert tags == {"bug": "linux"}
+
+
+def test_parse_optional_tags_known_bug_with_description_returns_bug():
+    tags = parse_optional_tags("sage: # known bug, #34506")
+    assert tags == {"bug": None}
+
+
+def test_parse_optional_tags_known_bug_with_description_in_parentheses_returns_bug():
+    tags = parse_optional_tags("sage: # known bug (#34506)")
+    assert tags == {"bug": None}
+
+
+def test_parse_optional_tags_known_bug_with_value_and_description_returns_bug_and_value():
+    tags = parse_optional_tags("sage: # known bug: linux (#34506)")
+    assert tags == {"bug": "linux"}
+
+
+def test_parse_known_bug_returns_empty():
+    parser = SageDocTestParser(("sage",))
+    parsed = parser.parse("sage: x = int('1'*4301) # known bug")
+    assert parsed == ["", ""]
+
+
+def test_parse_known_bug_returns_code_if_requested():
+    parser = SageDocTestParser(("sage", "bug"))
+    parsed = parser.parse("sage: x = int('1'*4301) # known bug")
+    assert len(parsed) == 3
+    assert parsed[1].sage_source == "x = int('1'*4301) # known bug\n"
+
+
+@onlyLinux
+def test_parse_known_bug_returns_code_if_requested_even_on_affected_os():
+    parser = SageDocTestParser(("sage", "bug"))
+    parsed = parser.parse("sage: x = int('1'*4301) # known bug: macos")
+    assert len(parsed) == 3
+    assert parsed[1].sage_source == "x = int('1'*4301) # known bug: macos\n"
+
+
+@onlyLinux
+def test_parse_known_bug_returns_code_on_not_affected_os():
+    parser = SageDocTestParser(("sage",))
+    parsed = parser.parse("sage: x = int('1'*4301) # known bug: macos")
+    assert len(parsed) == 3
+    assert parsed[1].sage_source == "x = int('1'*4301) # known bug: macos\n"
+
+
+@onlyLinux
+def test_parse_known_bug_returns_empty_on_affected_os():
+    parser = SageDocTestParser(("sage",))
+    parsed = parser.parse("sage: x = int('1'*4301) # known bug: linux")
+    assert parsed == ["", ""]
+
+
+def test_parse_known_bug_with_description_returns_empty():
+    parser = SageDocTestParser(("sage",))
+    parsed = parser.parse("sage: x = int('1'*4301) # known bug, #34506")
+    assert parsed == ["", ""]

src/sage/interfaces/maxima_abstract.py

@@ -392,7 +392,7 @@ def console(self):
 
         ::
 
-            sage: maxima.interact()     # this is not tested either
+            sage: maxima.interact()     # not tested
               --> Switching to Maxima <--
             maxima: 2+2
             4

src/sage/structure/coerce_maps.pyx

@@ -376,7 +376,7 @@ cdef class CallableConvertMap(Map):
             sage: def foo(P, x): return x^2
             sage: f = CallableConvertMap(ZZ, ZZ, foo)
             sage: g = copy(f)     # indirect doctest
-            sage: f == g          # todo: comparison not implemented
+            sage: f == g          # not implemented (todo: implement comparison)
             True
             sage: f(3) == g(3)
             True
@@ -396,7 +396,7 @@ cdef class CallableConvertMap(Map):
             sage: def foo(P, x): return x^2
             sage: f = CallableConvertMap(ZZ, ZZ, foo)
             sage: g = copy(f)     # indirect doctest
-            sage: f == g          # todo: comparison not implemented
+            sage: f == g          # not implemented (todo: implement comparison)
             True
             sage: f(3) == g(3)
             True
@@ -642,7 +642,7 @@ cdef class TryMap(Map):
             sage: map2 = QQ.coerce_map_from(ZZ)
             sage: map = sage.structure.coerce_maps.TryMap(map1, map2, error_types=(ZeroDivisionError,))
             sage: cmap = copy(map)     # indirect doctest
-            sage: cmap == map          # todo: comparison not implemented
+            sage: cmap == map          # not implemented (todo: implement comparison)
             True
             sage: map(3) == cmap(3)
             True
@@ -665,7 +665,7 @@ cdef class TryMap(Map):
             sage: map2 = QQ.coerce_map_from(ZZ)
             sage: map = sage.structure.coerce_maps.TryMap(map1, map2, error_types=(ZeroDivisionError,))
             sage: cmap = copy(map)     # indirect doctest
-            sage: cmap == map          # todo: comparison not implemented
+            sage: cmap == map          # not implemented (todo: implement comparison)
             True
             sage: map(3) == cmap(3)
             True