From cef64ff5121af5156627e4057ec3b6c249481399 Mon Sep 17 00:00:00 2001 From: Navaneet Villodi <11260095+nauaneed@users.noreply.github.com> Date: Wed, 2 Apr 2025 11:01:54 +0530 Subject: [PATCH 1/3] build: automatically exclude segfaulting classes - With this, tvtk installation won't be broken due to segfaulting vtk classes with vtk updates or on untested distros or python versions. - This makes the setup slower by a few seconds. --- setup.py | 2 + tvtk/_setup.py | 2 +- tvtk/code_gen.py | 56 ++++++------ tvtk/filter_nosegfault.py | 180 ++++++++++++++++++++++++++++++++++++++ tvtk/vtk_parser.py | 3 +- 5 files changed, 215 insertions(+), 28 deletions(-) create mode 100644 tvtk/filter_nosegfault.py diff --git a/setup.py b/setup.py index 6f978b4c4..e06a5aa44 100644 --- a/setup.py +++ b/setup.py @@ -35,6 +35,8 @@ DEFAULT_HTML_TARGET_DIR = join('docs', 'build') DEFAULT_INPUT_DIR = join('docs', 'source',) +from tvtk.filter_nosegfault import exclude_segfault_classes +exclude_segfault_classes() class GenDocs(Command): diff --git a/tvtk/_setup.py b/tvtk/_setup.py index 767ab638d..b989418c7 100755 --- a/tvtk/_setup.py +++ b/tvtk/_setup.py @@ -8,7 +8,6 @@ from setuptools import Distribution - def can_compile_extensions(): try: import numpy # noqa @@ -40,6 +39,7 @@ def gen_tvtk_classes_zip(): MY_DIR = os.path.dirname(__file__) sys.path.append(MY_DIR) from tvtk.code_gen import TVTKGenerator + target = os.path.join(MY_DIR, 'tvtk_classes.zip') output_dir = os.path.dirname(target) try: diff --git a/tvtk/code_gen.py b/tvtk/code_gen.py index 54aa72487..41f73e5dc 100644 --- a/tvtk/code_gen.py +++ b/tvtk/code_gen.py @@ -111,32 +111,7 @@ def generate_code(self): # Write the wrapper files. tree = wrap_gen.get_tree().tree - - classes = ['vtkObjectBase'] - # This is another class we should not wrap and exists - # in version 8.1.0. - ignore = ['vtkOpenGLGL2PSHelperImpl'] + [ - 'vtkSOADataArrayTemplate_I%sE' % l - for l in 'acdfhijlmstxy'] - include = ['VTKPythonAlgorithmBase'] - for node in wrap_gen.get_tree(): - name = node.name - if name in ignore: - continue - if (name not in include and not name.startswith('vtk')) or \ - name.startswith('vtkQt'): - continue - if not hasattr(vtk, name) or \ - not hasattr(getattr(vtk, name), 'AddObserver'): # noqa - # We need to wrap VTK classes that are derived - # from vtkObjectBase, the others are - # straightforward VTK classes that can be used as - # such. All of these have an 'AddObserver' method so we - # check for that. Only the vtkObjectBase - # subclasses support observers etc. and hence only - # those make sense to wrap into TVTK. - continue - classes.append(name) + classes = self.get_classes() for ti, nodes in enumerate(tree, 1): for ni, node in enumerate(nodes, 1): @@ -154,6 +129,35 @@ def generate_code(self): raise helper_gen.add_class(tvtk_name, helper_file) + def get_classes(self): + wrap_gen = self.wrap_gen + classes = ['vtkObjectBase'] + # This is another class we should not wrap and exists + # in version 8.1.0. + ignore = ['vtkOpenGLGL2PSHelperImpl'] + [ + 'vtkSOADataArrayTemplate_I%sE' % l + for l in 'acdfhijlmstxy'] + include = ['VTKPythonAlgorithmBase'] + for node in wrap_gen.get_tree(): + name = node.name + if name in ignore: + continue + if (name not in include and not name.startswith('vtk')) or \ + name.startswith('vtkQt'): + continue + if not hasattr(vtk, name) or \ + not hasattr(getattr(vtk, name), 'AddObserver'): # noqa + # We need to wrap VTK classes that are derived + # from vtkObjectBase, the others are + # straightforward VTK classes that can be used as + # such. All of these have an 'AddObserver' method so we + # check for that. Only the vtkObjectBase + # subclasses support observers etc. and hence only + # those make sense to wrap into TVTK. + continue + classes.append(name) + return classes + def write_wrapper_classes(self, names): """Given VTK class names in the list `names`, write out the wrapper classes to a suitable file. This is a convenience diff --git a/tvtk/filter_nosegfault.py b/tvtk/filter_nosegfault.py new file mode 100644 index 000000000..2db0504ad --- /dev/null +++ b/tvtk/filter_nosegfault.py @@ -0,0 +1,180 @@ +""" +Some classes in VTK segfault when accessed. Random classes segfault on different distros, +vtk versions, and python versions. This script tries to identify and exclude those +classes from the tvtk module. +""" + +import argparse +import shutil +import subprocess +import tempfile +import traceback +from pathlib import Path + +from .code_gen import TVTKGenerator +from .common import get_tvtk_name +from .wrapper_gen import WrapperGenerator + +def exclude_segfault_classes(max_exclude=50): + """ + This function calls `catch` in a separate process to identify classes that cause + segmentation faults. A separate process is used as the segfault kills the its + own process. + """ + + orig_module_py = Path(__file__).parent.joinpath("vtk_module.py") + with tempfile.TemporaryDirectory() as tmp_dir: + tmp_dir = Path(tmp_dir) + bak_module_py = tmp_dir.joinpath("vtk_module.py") + + shutil.copy(orig_module_py, bak_module_py) + + tifile = tmp_dir.joinpath("ti") + nifile = tmp_dir.joinpath("ni") + + ti = 0 + ni = 0 + + with tifile.open("w") as tif, nifile.open("w") as nif: + tif.write(f"{ti}\n") + nif.write(f"{ni}\n") + + wrap_gen = WrapperGenerator() + tree = wrap_gen.get_tree().tree + nt = len(tree) + nn = len(tree[nt - 1]) + + to_exclude = [] + + while True: + cmd = f"python -m tvtk.filter_nosegfault catch 0 0 {tmp_dir}" + with subprocess.Popen(cmd.split()) as process: + process.wait(timeout=20) + + with tifile.open("r") as tif, nifile.open("r") as nif: + ti = int(tif.read().strip()) + ni = int(nif.read().strip()) + + if (ti == nt - 1 and ni == nn - 1) or len(to_exclude) > max_exclude: + break + + nodes = tree[ti] + node = nodes[ni] + if node.name not in to_exclude: + print(f'Excluding {node.name} {ti} {ni}') + to_exclude.append(node.name) + with open(orig_module_py, "a") as f: + f.write( + f"\n" + f"SKIP.append('{node.name}')\n" + f"try:\n" + f" del {node.name}\n" + f"except NameError:\n" + f" pass\n" + ) + + if len(nodes) == ni + 1: + ti += 1 + ni = 0 + else: + ni += 1 + + if not to_exclude: + print("No extra classes to exclude") + + try: + import tvtk.vtk_module + except Exception as e: + print(f"Exception: {e}") + print("Failed to import tvtk.vtk_module") + shutil.copy(bak_module_py, orig_module_py) + finally: + bak_module_py.unlink() + + + +def catch(tstart=0, nstart=0, loc="/tmp"): + """ + This function traverses the VTK class tree and attempts to access each class. + It saves the last successfully accessed class indices to temporary files. + """ + + tifile = Path(loc).joinpath("ti") + nifile = Path(loc).joinpath("ni") + + tifile.touch(exist_ok=True) + nifile.touch(exist_ok=True) + + wrap_gen = WrapperGenerator() + wrap_gen.parser._verbose = False + with tempfile.TemporaryDirectory() as tmp_dir: + tvgen = TVTKGenerator(tmp_dir) + classes = tvgen.get_classes() + + tree = wrap_gen.get_tree().tree + nt = len(tree) + + for ti in range(tstart, nt): + nodes = tree[ti] + nn = len(nodes) + for ni in range(nstart, nn): + node = nodes[ni] + if node.name in classes: + tvtk_name = get_tvtk_name(node.name) + try: + with tifile.open("w") as tif, nifile.open("w") as nif: + tif.write(f"{ti}\n") + nif.write(f"{ni}\n") + + klass = wrap_gen.get_tree().get_class(node.name) + + methods = wrap_gen.parser.get_methods(klass) + wrap_gen.parser._organize_methods(klass, methods) + except Exception: + print( + f"Failed on {tvtk_name}\n(# {ti + 1} of {nt} nodes, " + f"#{ni + 1} of {nn} subnodes):\n{traceback.format_exc()}\n" + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run try_tree with specified starting indices and file location." + ) + + parser.add_argument( + "job", + type=str, + nargs="?", + default="catch", + choices=["catch", "exclude"], + help="Starting index for the tree traversal (default: 0).", + ) + + parser.add_argument( + "tstart", + type=int, + nargs="?", + default=0, + help="Starting index for the tree traversal (default: 0).", + ) + parser.add_argument( + "nstart", + type=int, + nargs="?", + default=0, + help="Starting index for the subnodes traversal (default: 0).", + ) + parser.add_argument( + "loc", + type=str, + nargs="?", + default="/tmp", + help="Directory path to store temporary files (default: /tmp).", + ) + args = parser.parse_args() + + if args.job == "exclude": + exclude_segfault_classes() + elif args.job == "catch": + catch(tstart=args.tstart, nstart=args.nstart, loc=args.loc) diff --git a/tvtk/vtk_parser.py b/tvtk/vtk_parser.py index d3e108b0d..4e3e478f2 100644 --- a/tvtk/vtk_parser.py +++ b/tvtk/vtk_parser.py @@ -771,6 +771,7 @@ def _get_instance(self, klass, *, do_print=True): for c in n.children: obj = self._get_instance(t.get_class(c.name), do_print=False) if obj: - print(f" Using super {t.get_class(c.name)} instead of {klass}") + if self._verbose: + print(f" Using super {t.get_class(c.name)} instead of {klass}") break return obj From 9e0f20ca8dd99cc81b7023266662fad29c77035c Mon Sep 17 00:00:00 2001 From: Navaneet Villodi <11260095+nauaneed@users.noreply.github.com> Date: Fri, 25 Jul 2025 15:41:10 +0530 Subject: [PATCH 2/3] filter segfault: formatting and minor changes --- tvtk/filter_nosegfault.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/tvtk/filter_nosegfault.py b/tvtk/filter_nosegfault.py index 2db0504ad..41bf8da30 100644 --- a/tvtk/filter_nosegfault.py +++ b/tvtk/filter_nosegfault.py @@ -1,7 +1,7 @@ """ -Some classes in VTK segfault when accessed. Random classes segfault on different distros, -vtk versions, and python versions. This script tries to identify and exclude those -classes from the tvtk module. +Some classes in VTK segfault when accessed. Random classes segfault on +different distros, vtk versions, and python versions. This script tries to +identify and exclude those classes from the tvtk module. """ import argparse @@ -15,11 +15,12 @@ from .common import get_tvtk_name from .wrapper_gen import WrapperGenerator + def exclude_segfault_classes(max_exclude=50): """ - This function calls `catch` in a separate process to identify classes that cause - segmentation faults. A separate process is used as the segfault kills the its - own process. + This function calls `catch` in a separate process to identify classes that + cause segmentation faults. A separate process is used as the segfault kills + the its own process. """ orig_module_py = Path(__file__).parent.joinpath("vtk_module.py") @@ -55,13 +56,15 @@ def exclude_segfault_classes(max_exclude=50): ti = int(tif.read().strip()) ni = int(nif.read().strip()) - if (ti == nt - 1 and ni == nn - 1) or len(to_exclude) > max_exclude: + if (ti == nt - 1 and ni == nn - 1) or len( + to_exclude + ) > max_exclude: break nodes = tree[ti] node = nodes[ni] if node.name not in to_exclude: - print(f'Excluding {node.name} {ti} {ni}') + print(f"Excluding {node.name} {ti} {ni}") to_exclude.append(node.name) with open(orig_module_py, "a") as f: f.write( @@ -79,7 +82,7 @@ def exclude_segfault_classes(max_exclude=50): else: ni += 1 - if not to_exclude: + if len(to_exclude) == 0: print("No extra classes to exclude") try: @@ -92,11 +95,11 @@ def exclude_segfault_classes(max_exclude=50): bak_module_py.unlink() - def catch(tstart=0, nstart=0, loc="/tmp"): """ - This function traverses the VTK class tree and attempts to access each class. - It saves the last successfully accessed class indices to temporary files. + This function traverses the VTK class tree and attempts to access each + class. It saves the last successfully accessed class indices to temporary + files. """ tifile = Path(loc).joinpath("ti") @@ -122,7 +125,10 @@ def catch(tstart=0, nstart=0, loc="/tmp"): if node.name in classes: tvtk_name = get_tvtk_name(node.name) try: - with tifile.open("w") as tif, nifile.open("w") as nif: + with ( + tifile.open("w", encoding="utf-8") as tif, + nifile.open("w", encoding="utf-8") as nif, + ): tif.write(f"{ti}\n") nif.write(f"{ni}\n") @@ -133,13 +139,15 @@ def catch(tstart=0, nstart=0, loc="/tmp"): except Exception: print( f"Failed on {tvtk_name}\n(# {ti + 1} of {nt} nodes, " - f"#{ni + 1} of {nn} subnodes):\n{traceback.format_exc()}\n" + f"#{ni + 1} of {nn} subnodes):\n" + f"{traceback.format_exc()}\n" ) if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Run try_tree with specified starting indices and file location." + description="Run try_tree with specified starting indices and file " + "location." ) parser.add_argument( From 5c5199e4a9a1b6ae1c0dc2d9e416856b3b63010b Mon Sep 17 00:00:00 2001 From: Navaneet Villodi <11260095+nauaneed@users.noreply.github.com> Date: Fri, 25 Jul 2025 15:57:02 +0530 Subject: [PATCH 3/3] build: FORCE_INSTALL env-var for filter segfault --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index e06a5aa44..863278d84 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,9 @@ DEFAULT_HTML_TARGET_DIR = join('docs', 'build') DEFAULT_INPUT_DIR = join('docs', 'source',) -from tvtk.filter_nosegfault import exclude_segfault_classes -exclude_segfault_classes() +if os.environ.get("FORCE_INSTALL", 'False').lower() in ['true', '1']: + from tvtk.filter_nosegfault import exclude_segfault_classes + exclude_segfault_classes() class GenDocs(Command):