Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update SuiteSparse easyblock to only install SuiteSparse libraries with make install #3004

Merged
merged 5 commits into from
Nov 6, 2023
Merged

update SuiteSparse easyblock to only install SuiteSparse libraries with make install #3004

Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
1195cd7
Install only suitespare c libraries with
Sep 18, 2023
bfdefd9
Clean up the codes
Sep 18, 2023
1821a28
Merge branch 'develop' of https://github.com/easybuilders/easybuild-e…
akesandgren Oct 31, 2023
5f60a3d
Simplify `if` statement
appolloford Oct 31, 2023
4d4bc91
Remove unused imports
appolloford Nov 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
230 changes: 99 additions & 131 deletions easybuild/easyblocks/s/suitesparse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,24 +61,22 @@ def configure_step(self):

if LooseVersion(self.version) < LooseVersion('4.0'):
self.config_name = 'UFconfig'
else:
elif LooseVersion(self.version) < LooseVersion('6.0.0'):
self.config_name = 'SuiteSparse_config'
else:
# config file is removed after v6.0.0
self.config_name = ''

cfgvars = {
'CC': os.getenv('MPICC'),
'CC': os.getenv('CC'),
'CFLAGS': os.getenv('CFLAGS'),
'CXX': os.getenv('MPICXX'),
'F77': os.getenv('MPIF77'),
'CXX': os.getenv('CXX'),
'F77': os.getenv('F77'),
'F77FLAGS': os.getenv('F77FLAGS'),
'BLAS': os.getenv('LIBBLAS_MT'),
'LAPACK': os.getenv('LIBLAPACK_MT'),
}

# avoid that (system) Intel compilers are always considered
self.cfg.update('buildopts', 'AUTOCC=no')

# Set BLAS and LAPACK libraries as specified in SuiteSparse README.txt
self.cfg.update('buildopts', 'BLAS="%s"' % os.getenv('LIBBLAS_MT'))
self.cfg.update('buildopts', 'LAPACK="%s"' % os.getenv('LIBLAPACK_MT'))

# Get CUDA and set it up appropriately
cuda = get_software_root('CUDA')
if cuda:
Expand All @@ -91,125 +89,102 @@ def configure_step(self):
# Get METIS or ParMETIS settings
metis = get_software_root('METIS')
parmetis = get_software_root('ParMETIS')
if parmetis:
metis_path = parmetis
metis_include = os.path.join(parmetis, 'include')
metis_libs = os.path.join(parmetis, get_software_libdir('ParMETIS'), 'libmetis.a')

elif metis:
metis_path = metis
metis_include = os.path.join(metis, 'include')
metis_libs = os.path.join(metis, get_software_libdir('METIS'), 'libmetis.a')
if parmetis or metis:
if parmetis:
metis_name = 'ParMETIS'
elif metis:
appolloford marked this conversation as resolved.
Show resolved Hide resolved
metis_name = 'METIS'
metis_path = get_software_root(metis_name)
metis_include = os.path.join(metis_path, 'include')
metis_libs = os.path.join(metis_path, get_software_libdir(metis_name), 'libmetis.a')

else:
raise EasyBuildError("Neither METIS or ParMETIS module loaded.")
self.log.info("Use METIS built in SuiteSparse")
# raise EasyBuildError("Neither METIS or ParMETIS module loaded.")

if LooseVersion(self.version) >= LooseVersion('4.5.1'):
# config file can catch environment variables after v4.5.0
if LooseVersion(self.version) < LooseVersion('4.5.0'):
cfgvars.update({
'MY_METIS_LIB': metis_libs,
'MY_METIS_INC': metis_include,
})
else:
cfgvars.update({
'METIS_PATH': metis_path,
'METIS': metis_libs,
'INSTALL_LIB': os.path.join(self.installdir, 'lib'),
'INSTALL_INCLUDE': os.path.join(self.installdir, 'include'),
})
if parmetis or metis:
cfgvars.update({
'METIS_PATH': metis_path,
'METIS': metis_libs,
})

# patch file
fp = os.path.join(self.cfg['start_dir'], self.config_name, '%s.mk' % self.config_name)

try:
for line in fileinput.input(fp, inplace=1, backup='.orig'):
for (var, val) in list(cfgvars.items()):
# Let's overwrite NVCCFLAGS at the end, since the line breaks and the fact that it appears multiple
# times makes it tricky to handle it properly
if var != 'NVCCFLAGS':
orig_line = line
# for variables in cfgvars, substiture lines assignment
# in the file, whatever they are, by assignments to the
# values in cfgvars
line = re.sub(r"^\s*(%s\s*=\s*).*\n$" % var,
r"\1 %s # patched by EasyBuild\n" % val,
line)
if line != orig_line:
cfgvars.pop(var)
sys.stdout.write(line)
except IOError as err:
raise EasyBuildError("Failed to patch %s in: %s", fp, err)

# add remaining entries at the end
if cfgvars:
cfgtxt = '# lines below added automatically by EasyBuild\n'
cfgtxt += '\n'.join(["%s = %s" % (var, val) for (var, val) in cfgvars.items()])
write_file(fp, cfgtxt, append=True)
# patch file
fp = os.path.join(self.cfg['start_dir'], self.config_name, '%s.mk' % self.config_name)

def install_step(self):
"""Install by copying the contents of the builddir to the installdir (preserving permissions)"""
for x in os.listdir(self.cfg['start_dir']):
src = os.path.join(self.cfg['start_dir'], x)
dst = os.path.join(self.installdir, x)
try:
if os.path.isdir(src):
# symlink points to CUDA folder that is
# not created for non GPU nodes. shutil
# throws an error in this case.
copy_dir(src, dst, symlinks=True)
# symlink
# - dst/Lib to dst/lib
# - dst/Include to dst/include
for c in ['Lib', 'Include']:
nsrc = os.path.join(dst, c)
ndst = os.path.join(dst, c.lower())
if os.path.exists(nsrc):
os.symlink(nsrc, ndst)
# enable r-x permissions for group/others
perms = stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
adjust_permissions(dst, perms, add=True, recursive=True, onlydirs=True)
for line in fileinput.input(fp, inplace=1, backup='.orig'):
for (var, val) in list(cfgvars.items()):
# Let's overwrite NVCCFLAGS at the end, since the line breaks and
# the fact that it appears multiple times makes it tricky to handle it properly
# path variables are also moved to the end
if var not in ['NVCCFLAGS', 'INSTALL_LIB', 'INSTALL_INCLUDE', 'METIS_PATH']:
orig_line = line
# for variables in cfgvars, substiture lines assignment
# in the file, whatever they are, by assignments to the
# values in cfgvars
line = re.sub(r"^\s*(%s\s*=\s*).*\n$" % var,
r"\1 %s # patched by EasyBuild\n" % val,
line)
if line != orig_line:
cfgvars.pop(var)
sys.stdout.write(line)
except IOError as err:
raise EasyBuildError("Failed to patch %s in: %s", fp, err)

# add remaining entries at the end
if cfgvars:
cfgtxt = '# lines below added automatically by EasyBuild\n'
cfgtxt += '\n'.join(["%s = %s" % (var, val) for (var, val) in cfgvars.items()])
write_file(fp, cfgtxt, append=True)

elif LooseVersion(self.version) < LooseVersion('6.0.0'):
# avoid that (system) Intel compilers are always considered
self.cfg.update('prebuildopts', 'AUTOCC=no')

# Set BLAS and LAPACK libraries as specified in SuiteSparse README.txt
self.cfg.update('buildopts', 'BLAS="%s"' % cfgvars.get('BLAS'))
self.cfg.update('buildopts', 'LAPACK="%s"' % cfgvars.get('LAPACK'))

self.cfg.update('installopts', 'INSTALL="%s"' % self.installdir)
self.cfg.update('installopts', 'BLAS="%s"' % cfgvars.get('BLAS'))
self.cfg.update('installopts', 'LAPACK="%s"' % cfgvars.get('LAPACK'))

if LooseVersion(self.version) >= LooseVersion('5.1.2'):
# graphblas exists, needs cmake
# v5.0.0 until v5.1.2 has no CMAKE_OPTIONS to set
# probably need patch
self.cfg.update('preinstallopts', 'CMAKE_OPTIONS="-DCMAKE_INSTALL_PREFIX=%s"' % self.installdir)

# set METIS library
if parmetis or metis:
if LooseVersion(self.version) == LooseVersion('4.5.0'):
self.cfg.update('buildopts', 'METIS_PATH="%s"' % metis_path)
self.cfg.update('installopts', 'METIS_PATH="%s"' % metis_path)
else:
shutil.copy2(src, dst)
except OSError as err:
raise EasyBuildError("Copying src %s to dst %s failed: %s", src, dst, err)

# some extra symlinks are necessary for UMFPACK to work.
paths = [
os.path.join('AMD', 'include', 'amd.h'),
os.path.join('AMD', 'include', 'amd_internal.h'),
os.path.join(self.config_name, '%s.h' % self.config_name),
os.path.join('AMD', 'lib', 'libamd.a')
]
for path in paths:
src = os.path.join(self.installdir, path)
dn = path.split(os.path.sep)[-2]
fn = path.split(os.path.sep)[-1]
dstdir = os.path.join(self.installdir, 'UMFPACK', dn)
mkdir(dstdir)
if os.path.exists(src):
try:
os.symlink(src, os.path.join(dstdir, fn))
except OSError as err:
raise EasyBuildError("Failed to make symbolic link from %s to %s: %s", src, dst, err)

def make_module_req_guess(self):
"""
Extra path to consider for module file:
* add config dir and include to $CPATH so include files are found
* add UMFPACK and AMD library, and lib dirs to $LD_LIBRARY_PATH
"""

guesses = super(EB_SuiteSparse, self).make_module_req_guess()

# Previous versions of SuiteSparse used specific directories for includes and libraries
if LooseVersion(self.version) < LooseVersion('4.5'):
include_dirs = [self.config_name]
ld_library_path = ['AMD/lib', 'BTF/lib', 'CAMD/lib', 'CCOLAMD/lib', 'CHOLAMD/lib', 'CHOLMOD/lib',
'COLAMD/lib/', 'CSparse/lib', 'CXSparse/lib', 'KLU/lib', 'LDL/lib', 'RBio/lib',
'UMFPACK/lib', self.config_name]
self.cfg.update('buildopts', 'MY_METIS_LIB="%s"' % metis_libs)
self.cfg.update('buildopts', 'MY_METIS_INC="%s"' % metis_include)
self.cfg.update('installopts', 'MY_METIS_LIB="%s"' % metis_libs)
self.cfg.update('installopts', 'MY_METIS_INC="%s"' % metis_include)

guesses['CPATH'].extend(include_dirs)
guesses['LD_LIBRARY_PATH'].extend(ld_library_path)
guesses['LIBRARY_PATH'].extend(ld_library_path)
else:
# after v6.0.0, no option for metis, its own metis is used anyway
# nothing to do here, set the CMAKE_OPTIONS in easyconfigs
pass

def install_step(self):
"""Install by copying the contents of the builddir to the installdir (preserving permissions)"""

if LooseVersion(self.version) < LooseVersion('4.5.0'):
mkdir(os.path.join(self.installdir, 'lib'))
mkdir(os.path.join(self.installdir, 'include'))

return guesses
super(EB_SuiteSparse, self).install_step()

def sanity_check_step(self):
"""Custom sanity check for SuiteSparse."""
Expand All @@ -219,24 +194,17 @@ def sanity_check_step(self):
raise EasyBuildError("SuiteSparse has compiled its own Metis. This will conflict with the Metis build."
" The SuiteSparse EasyBlock need to be updated!")

shlib_ext = get_shared_lib_ext()
libnames = ['AMD', 'BTF', 'CAMD', 'CCOLAMD', 'CHOLMOD', 'COLAMD', 'CXSparse', 'KLU',
'LDL', 'RBio', 'SPQR', 'UMFPACK']
libs = [os.path.join(x, 'lib', 'lib%s.a' % x.lower()) for x in libnames]

if LooseVersion(self.version) < LooseVersion('4.0'):
csparse_dir = 'CSparse3'
if LooseVersion(self.version) < LooseVersion('4.5'):
libs = [os.path.join('lib', 'lib%s.a' % x.lower()) for x in libnames]
else:
csparse_dir = 'CSparse'
libs.append(os.path.join(csparse_dir, 'lib', 'libcsparse.a'))

# Latest version of SuiteSparse also compiles shared library and put them in 'lib'
shlib_ext = get_shared_lib_ext()
if LooseVersion(self.version) >= LooseVersion('4.5.1'):
libs += [os.path.join('lib', 'lib%s.%s' % (x.lower(), shlib_ext)) for x in libnames]
libs = [os.path.join('lib', 'lib%s.%s' % (x.lower(), shlib_ext)) for x in libnames]

custom_paths = {
'files': libs,
'dirs': ['MATLAB_Tools'],
'dirs': [],
}

super(EB_SuiteSparse, self).sanity_check_step(custom_paths=custom_paths)