-
Notifications
You must be signed in to change notification settings - Fork 9
/
setup.py
184 lines (157 loc) · 7.65 KB
/
setup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
#!/usr/bin/env python
#
# Copyright 2009-2021 Ghent University
#
# This file is part of VSC-tools,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
# the Hercules foundation (http://www.herculesstichting.be/in_English)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# http://github.com/hpcugent/VSC-tools
#
# VSC-tools is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation v2.
#
# VSC-tools is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with VSC-tools. If not, see <http://www.gnu.org/licenses/>.
#
"""
Setup for mympirun
"""
import os
import sys
import vsc.install.shared_setup as shared_setup
from vsc.install.shared_setup import vsc_setup, log, sdw, kh
FAKE_SUBDIRECTORY_NAME = 'fake'
# hardcoded list, to avoid ugly hacks in order to be able to import from vsc.mympirun in setup.py...
# this list is checked to be synced via a dedicated unit test
MYMPIRUN_ALIASES = ['ihmpirun', 'impirun', 'm2hmpirun', 'm2mpirun', 'mhmpirun', 'mmpirun', 'myscoop', 'ompirun']
PACKAGE = {
'install_requires': [
'vsc-base >= 3.5.3',
'vsc-install >= 0.15.1',
'IPy',
],
'tests_require': [
'mock',
],
'version': '5.4.0',
'author': [sdw, kh],
'maintainer': [sdw, kh],
'zip_safe': False,
}
class mympirun_vsc_install_scripts(vsc_setup.vsc_install_scripts):
def run(self):
"""
make a fake mpirun, that replaces the symlinks of all the mpirun aliases.
This way any mpirun call will be passed to the fake mpirun.
Next, symlink the fake mpirun to the mympirun executable.
This way any mpirun call will be passed to the mympirun executable, but we can see that it passed through
the fake mpirun
"""
# old-style class
vsc_setup.vsc_install_scripts.run(self)
for script in self.original_outfiles:
if script.endswith(".py") or script.endswith(".sh"):
script = script[:-3]
if script.endswith('/mympirun'):
# store current working dir so we can get back to it
previous_pwd = os.getcwd()
# get script basename and dirname
rel_script = os.path.basename(script)
rel_script_dir = os.path.dirname(script)
os.chdir(rel_script_dir)
# create symlinks that point to mympirun for all mpirun aliases
for sym_name in MYMPIRUN_ALIASES:
if os.path.exists(sym_name):
os.remove(sym_name)
os.symlink(rel_script, sym_name)
newoutfile = os.path.join(rel_script_dir, sym_name)
self.outfiles.append(newoutfile)
log.info("symlink %s to %s newoutfile %s", rel_script, sym_name, newoutfile)
# create a directory for faking mpirun
os.chdir(previous_pwd)
abs_fakepath = os.path.join(self.install_dir, FAKE_SUBDIRECTORY_NAME)
if not os.path.isdir(abs_fakepath):
log.info("creating abs_fakepath %s", abs_fakepath)
os.mkdir(abs_fakepath)
else:
log.info("not creating abs_fakepath %s (already exists)", abs_fakepath)
# create a fake mpirin and symlink the real mpirun to it
os.chdir(abs_fakepath)
fake_mpirun = os.path.join(abs_fakepath, 'mpirun')
if os.path.exists(fake_mpirun):
os.remove(fake_mpirun)
# create another symlink that links mpirun to mympirun
mympirun_src = f'../{rel_script}'
os.symlink(mympirun_src, 'mpirun')
self.outfiles.append(fake_mpirun)
log.info("symlink %s to %s newoutfile %s", mympirun_src, 'mpirun', fake_mpirun)
os.chdir(previous_pwd)
class mympirun_vsc_setup(vsc_setup):
vsc_install_scripts = mympirun_vsc_install_scripts
# Monkeypatch setuptools.easy_install.install_egg_scripts.metadata_listdir
# because easy_install ignores the easy_install cmdclass
#
# The metadata_listdir assumes no subdirectories in scripts dir.
# We replace it with a function that calls the original metadata_listdir, searches through it results for the fake
# mympirun directory, appends '/mpirun' to it and returns the final result.
# The function is used through a whole bunch of Egg classes, no way we can easily intercept this
try:
from setuptools.command.easy_install import easy_install # NOQA
_orig_install_egg_scripts = sys.modules['setuptools.command.easy_install'].easy_install.install_egg_scripts
def _new_install_egg_scripts(self, dist):
orig_func = dist.metadata_listdir
def new_func(txt):
res = orig_func(txt)
if txt == 'scripts':
log.debug('mympirun easy_install.install_egg_scripts scripts res %s', res)
if FAKE_SUBDIRECTORY_NAME in res:
idx = res.index(FAKE_SUBDIRECTORY_NAME)
if idx >= 0:
res[idx] = f'{FAKE_SUBDIRECTORY_NAME}/mpirun'
return res
dist.metadata_listdir = new_func
_orig_install_egg_scripts(self, dist)
sys.modules['setuptools.command.easy_install'].easy_install.install_egg_scripts = _new_install_egg_scripts
except Exception as e:
raise Exception(f"mympirun requires setuptools: {e}")
# next monstrocity: inject header in script to filter out PYTHONPATH in mixed EB py3/py2 envs
# mympirun modules rely on the system python and dependencies, so this is fine
# however, system python might pick up loaded Python-2.X modules, and then nothing should be done
# this can be removed as soon as mympirun is py3 safe
EB_SAFE_HEADER = """
import os
import sys
ebp = os.environ.get('EBROOTPYTHON', None)
if ebp and not os.__file__.startswith(ebp):
# ebroots excpet mympirun
ignore = [v for k,v in os.environ.items() if k.startswith('EBROOT') and not k.endswith('VSCMINMYMPIRUN')]
# add realpaths to (sys.path normally ojnly has realpaths)
ignore += [os.path.realpath(x) for x in ignore if os.path.exists(x)]
# remove sys.path entries that start with eb path
sys.path = [x for x in sys.path if not [y for y in ignore if x.startswith(y)]]
"""
# this is a bit tricky, since setuptools some version has moved the function as classmethod to the ScriptWriter class
# there is also a legacy/deprecated get_script_header classmethod
# this will also trigger the magic for all dependencies pulled in
try:
orig_header = sys.modules['setuptools.command.easy_install'].ScriptWriter.get_header
def new_header(cls, *args, **kwargs): #pylint: disable=unused-argument
return orig_header(*args, **kwargs) + EB_SAFE_HEADER
sys.modules['setuptools.command.easy_install'].ScriptWriter.get_header = classmethod(new_header)
except Exception as e:
orig_header = sys.modules['setuptools.command.easy_install'].get_script_header
def new_header(*args, **kwargs):
return orig_header(*args, **kwargs) + EB_SAFE_HEADER
sys.modules['setuptools.command.easy_install'].get_script_header = new_header
if __name__ == '__main__':
shared_setup.action_target(PACKAGE)