-
Notifications
You must be signed in to change notification settings - Fork 4.9k
/
setup.py
175 lines (142 loc) · 5.86 KB
/
setup.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
import os
import glob
from setuptools import find_packages, setup
# A special env var that allows us to disable the installation of the default extras for advanced users / containers / etc
MINDSDB_PIP_INSTALL_DEFAULT_EXTRAS = (
True
if os.getenv("MINDSDB_PIP_INSTALL_DEFAULT_EXTRAS", "true").lower() == "true"
else False
)
DEFAULT_PIP_EXTRAS = [
line.split("#")[0].rstrip()
for line in open("default_handlers.txt").read().splitlines()
if not line.strip().startswith("#")
]
class Deps:
pkgs = []
pkgs_exclude = ["tests", "tests.*"]
new_links = []
extras = {}
about = {}
with open("mindsdb/__about__.py") as fp:
exec(fp.read(), about)
with open("README.md", "r", encoding="utf8") as fh:
long_description = fh.read()
def expand_requirements_links(requirements: list) -> list:
"""Expand requirements that contain links to other requirement files"""
to_add = []
to_remove = []
for requirement in requirements:
if requirement.startswith("-r "):
if os.path.exists(requirement.split()[1]):
with open(requirement.split()[1]) as fh:
to_add += expand_requirements_links(
[req.strip() for req in fh.read().splitlines()]
)
to_remove.append(requirement)
for req in to_remove:
requirements.remove(req)
for req in to_add:
requirements.append(req)
return list(set(requirements)) # Remove duplicates
def define_deps():
"""Reads requirements.txt requirements-extra.txt files and preprocess it
to be feed into setuptools.
This is the only possible way (we found)
how requirements.txt can be reused in setup.py
using dependencies from private github repositories.
Links must be appendend by `-{StringWithAtLeastOneNumber}`
or something like that, so e.g. `-9231` works as well as
`1.1.0`. This is ignored by the setuptools, but has to be there.
Warnings:
to make pip respect the links, you have to use
`--process-dependency-links` switch. So e.g.:
`pip install --process-dependency-links {git-url}`
Returns:
list of packages, extras and dependency links.
"""
with open(os.path.normpath('requirements/requirements.txt')) as req_file:
defaults = [req.strip() for req in req_file.read().splitlines()]
links = []
requirements = []
for r in defaults:
if 'git+https' in r:
pkg = r.split('#')[-1]
links.append(r + '-9876543210')
requirements.append(pkg.replace('egg=', ''))
else:
requirements.append(r.strip())
extra_requirements = {}
full_requirements = []
for fn in os.listdir(os.path.normpath('./requirements')):
extra = []
if fn.startswith('requirements-') and fn.endswith('.txt'):
extra_name = fn.replace('requirements-', '').replace('.txt', '')
with open(os.path.normpath(f"./requirements/{fn}")) as fp:
extra = [req.strip() for req in fp.read().splitlines()]
extra_requirements[extra_name] = extra
full_requirements += extra
extra_requirements['all_extras'] = list(set(full_requirements))
full_handlers_requirements = []
handlers_dir_path = os.path.normpath('./mindsdb/integrations/handlers')
for fn in os.listdir(handlers_dir_path):
if os.path.isdir(os.path.join(handlers_dir_path, fn)) and fn.endswith(
"_handler"
):
extra = []
for req_file_path in glob.glob(
os.path.join(handlers_dir_path, fn, "requirements*.txt")
):
extra_name = fn.replace("_handler", "")
file_name = os.path.basename(req_file_path)
if file_name != "requirements.txt":
extra_name += "-" + file_name.replace("requirements_", "").replace(
".txt", ""
)
# If requirements.txt in our handler folder, import them as our extra's requirements
if os.path.exists(req_file_path):
with open(req_file_path) as fp:
extra = expand_requirements_links(
[req.strip() for req in fp.read().splitlines()]
)
extra_requirements[extra_name] = extra
full_handlers_requirements += extra
# Even with no requirements in our handler, list the handler as an extra (with no reqs)
else:
extra_requirements[extra_name] = []
# If this is a default extra and if we want to install defaults (enabled by default)
# then add it to the default requirements needing to install
if (
MINDSDB_PIP_INSTALL_DEFAULT_EXTRAS
and extra_name in DEFAULT_PIP_EXTRAS
and extra
):
requirements += extra
extra_requirements['all_handlers_extras'] = list(set(full_handlers_requirements))
Deps.pkgs = requirements
Deps.extras = extra_requirements
Deps.new_links = links
return Deps
deps = define_deps()
setup(
name=about['__title__'],
version=about['__version__'],
url=about['__github__'],
download_url=about['__pypi__'],
license=about['__license__'],
author=about['__author__'],
author_email=about['__email__'],
description=about['__description__'],
long_description=long_description,
long_description_content_type="text/markdown",
packages=find_packages(exclude=deps.pkgs_exclude),
install_requires=deps.pkgs,
dependency_links=deps.new_links,
extras_require=deps.extras,
include_package_data=True,
classifiers=[
"Programming Language :: Python :: 3",
"Operating System :: OS Independent",
],
python_requires=">=3.9,<3.12",
)