|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +# |
| 3 | +# This file is part of Requirements-Builder |
| 4 | +# Copyright (C) 2015, 2016, 2017, 2018 CERN. |
| 5 | +# |
| 6 | +# Requirements-Builder is free software; you can redistribute it and/or |
| 7 | +# modify it under the terms of the Revised BSD License; see LICENSE |
| 8 | +# file for more details. |
| 9 | +# |
| 10 | +"""Generate requirements from `setup.py` and `requirements-devel.txt`.""" |
| 11 | + |
| 12 | +from __future__ import absolute_import, print_function |
| 13 | + |
| 14 | +import os |
| 15 | +import re |
| 16 | +import sys |
| 17 | + |
| 18 | +try: |
| 19 | + import configparser |
| 20 | +except ImportError: # pragma: no cover |
| 21 | + import ConfigParser as configparser |
| 22 | + |
| 23 | +import mock |
| 24 | +import pkg_resources |
| 25 | +import setuptools |
| 26 | + |
| 27 | + |
| 28 | +def parse_set(string): |
| 29 | + """Parse set from comma separated string.""" |
| 30 | + string = string.strip() |
| 31 | + if string: |
| 32 | + return set(string.split(",")) |
| 33 | + else: |
| 34 | + return set() |
| 35 | + |
| 36 | + |
| 37 | +def minver_error(pkg_name): |
| 38 | + """Report error about missing minimum version constraint and exit.""" |
| 39 | + print( |
| 40 | + 'ERROR: specify minimal version of "{0}" using ' |
| 41 | + '">=" or "=="'.format(pkg_name), |
| 42 | + file=sys.stderr |
| 43 | + ) |
| 44 | + sys.exit(1) |
| 45 | + |
| 46 | + |
| 47 | +def build_pkg_name(pkg): |
| 48 | + """Build package name, including extras if present.""" |
| 49 | + if pkg.extras: |
| 50 | + return '{0}[{1}]'.format( |
| 51 | + pkg.project_name, ','.join(sorted(pkg.extras))) |
| 52 | + return pkg.project_name |
| 53 | + |
| 54 | + |
| 55 | +def parse_pip_file(path): |
| 56 | + """Parse pip requirements file.""" |
| 57 | + # requirement lines sorted by importance |
| 58 | + # also collect other pip commands |
| 59 | + rdev = {} |
| 60 | + rnormal = [] |
| 61 | + stuff = [] |
| 62 | + |
| 63 | + try: |
| 64 | + with open(path) as f: |
| 65 | + for line in f: |
| 66 | + line = line.strip() |
| 67 | + |
| 68 | + # see https://pip.readthedocs.io/en/1.1/requirements.html |
| 69 | + if line.startswith('-e'): |
| 70 | + # devel requirement |
| 71 | + splitted = line.split('#egg=') |
| 72 | + rdev[splitted[1].lower()] = line |
| 73 | + |
| 74 | + elif line.startswith('-r'): |
| 75 | + # recursive file command |
| 76 | + splitted = re.split('-r\\s+', line) |
| 77 | + subrdev, subrnormal, substuff = parse_pip_file( |
| 78 | + os.path.join(os.path.dirname(path), splitted[1]) |
| 79 | + ) |
| 80 | + for k, v in subrdev.items(): |
| 81 | + if k not in rdev: |
| 82 | + rdev[k] = v |
| 83 | + rnormal.extend(subrnormal) |
| 84 | + elif line.startswith('-'): |
| 85 | + # another special command we don't recognize |
| 86 | + stuff.append(line) |
| 87 | + else: |
| 88 | + # ordinary requirement, similarly to them used in setup.py |
| 89 | + rnormal.append(line) |
| 90 | + except IOError: |
| 91 | + print( |
| 92 | + 'Warning: could not parse requirements file "{0}"!'.format(path), |
| 93 | + file=sys.stderr |
| 94 | + ) |
| 95 | + |
| 96 | + return rdev, rnormal, stuff |
| 97 | + |
| 98 | + |
| 99 | +def iter_requirements(level, extras, pip_file, setup_fp, setup_cfg_fp=None): |
| 100 | + """Iterate over requirements.""" |
| 101 | + result = dict() |
| 102 | + requires = [] |
| 103 | + stuff = [] |
| 104 | + if level == 'dev' or setup_fp is None: |
| 105 | + result, requires, stuff = parse_pip_file(pip_file) |
| 106 | + |
| 107 | + install_requires = [] |
| 108 | + requires_extras = {} |
| 109 | + if setup_fp is not None: |
| 110 | + with mock.patch.object(setuptools, 'setup') as mock_setup: |
| 111 | + sys.path.append(os.path.dirname(setup_fp.name)) |
| 112 | + g = {'__file__': setup_fp.name, '__name__': '__main__'} |
| 113 | + exec(setup_fp.read(), g) |
| 114 | + sys.path.pop() |
| 115 | + assert g['setup'] # silence warning about unused imports |
| 116 | + |
| 117 | + # called arguments are in `mock_setup.call_args` |
| 118 | + mock_args, mock_kwargs = mock_setup.call_args |
| 119 | + install_requires = mock_kwargs.get( |
| 120 | + 'install_requires', install_requires |
| 121 | + ) |
| 122 | + requires_extras = mock_kwargs.get('extras_require', requires_extras) |
| 123 | + |
| 124 | + if setup_cfg_fp is not None: |
| 125 | + parser = configparser.ConfigParser() |
| 126 | + parser.read_file(setup_cfg_fp) |
| 127 | + |
| 128 | + if parser.has_section("options"): |
| 129 | + value = parser.get("options", "install_requires", |
| 130 | + fallback="").strip() |
| 131 | + |
| 132 | + if value: |
| 133 | + install_requires = [s.strip() for s in value.splitlines()] |
| 134 | + |
| 135 | + if parser.has_section("options.extras_require"): |
| 136 | + for name, value in parser.items("options.extras_require"): |
| 137 | + requires_extras[name] = [s.strip() |
| 138 | + for s in value.strip().splitlines()] |
| 139 | + |
| 140 | + install_requires.extend(requires) |
| 141 | + |
| 142 | + for e, reqs in requires_extras.items(): |
| 143 | + # Handle conditions on extras. See pkginfo_to_metadata function |
| 144 | + # in Wheel for details. |
| 145 | + condition = '' |
| 146 | + if ':' in e: |
| 147 | + e, condition = e.split(':', 1) |
| 148 | + if not e or e in extras: |
| 149 | + if condition: |
| 150 | + reqs = ['{0}; {1}'.format(r, condition) for r in reqs] |
| 151 | + install_requires.extend(reqs) |
| 152 | + |
| 153 | + for pkg in pkg_resources.parse_requirements(install_requires): |
| 154 | + # skip things we already know |
| 155 | + # FIXME be smarter about merging things |
| 156 | + |
| 157 | + # Evaluate environment markers skip if not applicable |
| 158 | + if hasattr(pkg, 'marker') and pkg.marker is not None: |
| 159 | + if not pkg.marker.evaluate(): |
| 160 | + continue |
| 161 | + else: |
| 162 | + # Remove markers from the output |
| 163 | + pkg.marker = None |
| 164 | + |
| 165 | + if pkg.key in result: |
| 166 | + continue |
| 167 | + |
| 168 | + specs = dict(pkg.specs) |
| 169 | + if (('>=' in specs) and ('>' in specs)) \ |
| 170 | + or (('<=' in specs) and ('<' in specs)): |
| 171 | + print( |
| 172 | + 'ERROR: Do not specify such weird constraints! ' |
| 173 | + '("{0}")'.format(pkg), |
| 174 | + file=sys.stderr |
| 175 | + ) |
| 176 | + sys.exit(1) |
| 177 | + |
| 178 | + if '==' in specs: |
| 179 | + result[pkg.key] = '{0}=={1}'.format( |
| 180 | + build_pkg_name(pkg), specs['==']) |
| 181 | + |
| 182 | + elif '>=' in specs: |
| 183 | + if level == 'min': |
| 184 | + result[pkg.key] = '{0}=={1}'.format( |
| 185 | + build_pkg_name(pkg), specs['>='] |
| 186 | + ) |
| 187 | + else: |
| 188 | + result[pkg.key] = pkg |
| 189 | + |
| 190 | + elif '>' in specs: |
| 191 | + if level == 'min': |
| 192 | + minver_error(build_pkg_name(pkg)) |
| 193 | + else: |
| 194 | + result[pkg.key] = pkg |
| 195 | + |
| 196 | + elif '~=' in specs: |
| 197 | + if level == 'min': |
| 198 | + result[pkg.key] = '{0}=={1}'.format( |
| 199 | + build_pkg_name(pkg), specs['~=']) |
| 200 | + else: |
| 201 | + ver, _ = os.path.splitext(specs['~=']) |
| 202 | + result[pkg.key] = '{0}>={1},=={2}.*'.format( |
| 203 | + build_pkg_name(pkg), specs['~='], ver) |
| 204 | + |
| 205 | + else: |
| 206 | + if level == 'min': |
| 207 | + minver_error(build_pkg_name(pkg)) |
| 208 | + else: |
| 209 | + result[pkg.key] = build_pkg_name(pkg) |
| 210 | + |
| 211 | + for s in stuff: |
| 212 | + yield s |
| 213 | + |
| 214 | + for k in sorted(result.keys()): |
| 215 | + yield str(result[k]) |
0 commit comments