Merge pull request #33 from DavHau/dev

Dev

debug/debug.py

@@ -16,9 +16,9 @@
 
 with open(pwd + "/../mach_nix/provider_defaults.toml") as f:
     provider_settings = toml.load(f)
-provider_settings.update(dict(
-    _default="wheel,nixpkgs,sdist",
-))
+if os.path.isfile("./providers.toml"):
+    with open(pwd + "./providers.toml") as f:
+        provider_settings.update(toml.load(f))
 os.environ['providers'] = json.dumps(provider_settings)
 
 nixpkgs_json = tempfile.mktemp()

debug/pypi-deps-db.nix

@@ -1,5 +1,5 @@
 let
-  pkgs =  import (import ../mach_nix/nix/nixpkgs-src.nix).stable {};
+  pkgs =  import (import ../mach_nix/nix/nixpkgs-src.nix) {};
   commit = builtins.readFile ../mach_nix/nix/PYPI_DEPS_DB_COMMIT;
   sha256 = builtins.readFile ../mach_nix/nix/PYPI_DEPS_DB_SHA256;
   src = builtins.fetchTarball {

mach_nix/data/providers.py

@@ -233,12 +233,6 @@ def _available_versions(self, pkg_name: str) -> Iterable[Version]:
             return [p.ver for p in self.nixpkgs.get_all_candidates(name)]
         return []
 
-    # def unify_key(self, key: str) -> str:
-    #     key = super(NixpkgsDependencyProvider, self).unify_key(key)
-    #     if key in self._aliases:
-    #         return self._aliases[key]
-    #     return key
-
 
 class WheelDependencyProvider(DependencyProviderBase):
     name = 'wheel'
@@ -408,8 +402,11 @@ def get_pkg_reqs(self, pkg_name, pkg_version: Version, extras=None) -> Tuple[Lis
                 reqs_raw = pkg[t]
                 reqs = parse_reqs(reqs_raw)
                 requirements[t] = list(filter_reqs_by_eval_marker(reqs, self.context))
-        if extras:
-            requirements['install_requires'] += self.get_reqs_for_extras(pkg_name, pkg_version, extras)
+        if not extras:
+            extras = []
+        # even if no extras are selected we need to collect reqs for extras,
+        # because some extras consist of only a marker which needs to be evaluated
+        requirements['install_requires'] += self.get_reqs_for_extras(pkg_name, pkg_version, extras)
         return requirements['install_requires'], requirements['setup_requires']
 
     def _available_versions(self, pkg_name: str) -> Iterable[Version]:

mach_nix/generators/overlay_generator.py

@@ -23,25 +23,16 @@ def __init__(self, py_ver, nixpkgs: NixpkgsDirectory, pypi_fetcher_commit,
         self.pypi_fetcher_commit = pypi_fetcher_commit
         self.pypi_fetcher_sha256 = pypi_fetcher_sha256
         self.py_ver_nix = py_ver.nix()
-        nixpkgs_name = NixpkgsDependencyProvider.name
-        sdist_name = SdistDependencyProvider.name
-        providers = (nixpkgs_name, sdist_name)
-        if all(p in providers for p in providers) and providers.index(nixpkgs_name) < providers.index(sdist_name):
-            self.prefer_nixpkgs = True
-        else:
-            self.prefer_nixpkgs = False
         super(OverlaysGenerator, self).__init__(*args, **kwargs)
 
     def generate(self, reqs) -> str:
-        pkgs = self.resolver.resolve(
-            reqs,
-            prefer_nixpkgs=self.prefer_nixpkgs
-        )
+        pkgs = self.resolver.resolve(reqs)
         pkgs = dict(sorted(((p.name, p) for p in pkgs), key=lambda x: x[1].name))
         return self._gen_python_env(pkgs)
 
     def _gen_imports(self):
         out = f"""
+            with builtins;
             let
               pypi_fetcher_src = builtins.fetchTarball {{
                 name = "nix-pypi-fetcher";
@@ -52,9 +43,13 @@ def _gen_imports(self):
               fetchPypi = (import pypi_fetcher_src).fetchPypi;
               fetchPypiWheel = (import pypi_fetcher_src).fetchPypiWheel;
               try_get = obj: name:
-                if builtins.hasAttr name obj
+                if hasAttr name obj
                 then obj."${{name}}"
                 else [];
+              is_py_module = pkg:
+                isAttrs pkg && hasAttr "pythonModule" pkg;
+              filter_deps = oldAttrs: inputs_type:
+                filter (pkg: ! is_py_module pkg) (try_get oldAttrs inputs_type);
             """
         return unindent(out, 12)
 
@@ -71,14 +66,15 @@ def _gen_prop_build_inputs(self, prop_build_inputs_local, prop_build_inputs_nixp
     def _gen_overrideAttrs(self, name, ver, nix_name, build_inputs_str, prop_build_inputs_str):
         out = f"""
             {nix_name} = python-super.{nix_name}.overridePythonAttrs ( oldAttrs: {{
-              name = "{name}-{ver}";
+              pname = "{name}";
+              version = "{ver}";
               src = fetchPypi "{name}" "{ver}";"""
         if build_inputs_str:
             out += f"""
-              buildInputs = with python-self; (try_get oldAttrs "buildInputs") ++ [ {build_inputs_str} ];"""
+              buildInputs = with python-self; (filter_deps oldAttrs "buildInputs") ++ [ {build_inputs_str} ];"""
         if prop_build_inputs_str:
             out += f"""
-              propagatedBuildInputs = with python-self; (try_get oldAttrs "propagatedBuildInputs") ++ [ {prop_build_inputs_str} ];"""
+              propagatedBuildInputs = with python-self; (filter_deps oldAttrs "propagatedBuildInputs") ++ [ {prop_build_inputs_str} ];"""
         if self.disable_checks:
             out += """
               doCheck = false;
@@ -90,7 +86,8 @@ def _gen_overrideAttrs(self, name, ver, nix_name, build_inputs_str, prop_build_i
     def _gen_builPythonPackage(self, name, ver, build_inputs_str, prop_build_inputs_str):
         out = f"""
             {self._get_ref_name(name, ver)} = python-self.buildPythonPackage {{
-              name = "{name}-{ver}";
+              pname = "{name}";
+              version = "{ver}";
               src = fetchPypi "{name}" "{ver}";"""
         if build_inputs_str.strip():
             out += f"""
@@ -110,7 +107,8 @@ def _gen_wheel_buildPythonPackage(self, name, ver, prop_build_inputs_str, fname)
         manylinux = "manylinux1 ++ " if 'manylinux' in fname else ''
         out = f"""
             {self._get_ref_name(name, ver)} = python-self.buildPythonPackage {{
-              name = "{name}-{ver}";
+              pname = "{name}";
+              version = "{ver}";
               src = fetchPypiWheel "{name}" "{ver}" "{fname}";
               format = "wheel";
               doCheck = false;
@@ -132,7 +130,7 @@ def _gen_unify_nixpkgs_keys(self, master_key: str, nixpkgs_keys: List[str]):
             out += f"""    {key} = python-self.{master_key};\n"""
         return out
 
-    def _unif_nixpkgs_keys(self, name, ver):
+    def _unify_nixpkgs_keys(self, name, ver):
         master_key = self._get_ref_name(name, ver)
         other_names = (p.nix_key for p in self.nixpkgs.get_all_candidates(name) if p.nix_key != master_key)
         return self._gen_unify_nixpkgs_keys(master_key, sorted(other_names))
@@ -169,20 +167,18 @@ def _gen_overrides(self, pkgs: Dict[str, ResolvedPkg], overlay_keys, pkgs_names:
                 if self.nixpkgs.exists(pkg.name):
                     nix_name = self._get_ref_name(pkg.name, pkg.ver)
                     out += self._gen_overrideAttrs(pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str)
-                    out += self._unif_nixpkgs_keys(pkg.name, pkg.ver)
+                    out += self._unify_nixpkgs_keys(pkg.name, pkg.ver)
                 else:
                     out += self._gen_builPythonPackage(pkg.name, pkg.ver, build_inputs_str, prop_build_inputs_str)
             elif pkg.provider_info.provider == WheelDependencyProvider.name:
                 out += self._gen_wheel_buildPythonPackage(pkg.name, pkg.ver, prop_build_inputs_str,
                                                           pkg.provider_info.wheel_fname)
                 if self.nixpkgs.exists(pkg.name):
-                    out += self._unif_nixpkgs_keys(pkg.name, pkg.ver)
+                    out += self._unify_nixpkgs_keys(pkg.name, pkg.ver)
             elif pkg.provider_info.provider == NixpkgsDependencyProvider.name:
-                # we only need to touch the nixpkgs definition if extras have been selected for the package
-                if pkg.extras_selected:
-                    nix_name = self.nixpkgs.find_best_nixpkgs_candidate(pkg.name, pkg.ver)
-                    out += self._gen_overrideAttrs(pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str)
-                out += self._unif_nixpkgs_keys(pkg.name, pkg.ver)
+                nix_name = self.nixpkgs.find_best_nixpkgs_candidate(pkg.name, pkg.ver)
+                out += self._gen_overrideAttrs(pkg.name, pkg.ver, nix_name, build_inputs_str, prop_build_inputs_str)
+                out += self._unify_nixpkgs_keys(pkg.name, pkg.ver)
         end_overlay_section = f"""
                 }};
           """

mach_nix/resolver/resolvelib_resolver.py

@@ -57,9 +57,7 @@ def __init__(self, nixpkgs: NixpkgsDirectory, deps_provider: DependencyProviderB
         self.nixpkgs = nixpkgs
         self.deps_provider = deps_provider
 
-    def resolve(self,
-                reqs: Iterable[Requirement],
-                prefer_nixpkgs=True) -> List[ResolvedPkg]:
+    def resolve(self, reqs: Iterable[Requirement]) -> List[ResolvedPkg]:
         reporter = resolvelib.BaseReporter()
         result = resolvelib.Resolver(Provider(self.nixpkgs, self.deps_provider), reporter).resolve(reqs, max_rounds=1000)
         nix_py_pkgs = []

mach_nix/run.py

@@ -21,8 +21,7 @@ def gen(args, return_expr=False):
     py_ver = PyVer(args.python)
     cmd = f'nix-build {pwd}/nix/call_mach.nix -o {o_file}' \
           f' --argstr requirements "{requirements}"' \
-          f' --argstr python_attr python{py_ver.digits()}' \
-          f' --arg prefer_nixpkgs {json.dumps((not args.prefer_new))}'
+          f' --argstr python_attr python{py_ver.digits()}'
     proc = sp.run(cmd, shell=True, stdout=sys.stderr)
     if proc.returncode:
         exit(1)
@@ -95,11 +94,6 @@ def main():
             help='path to requirements.txt file',
             metavar='requirements.txt',
             required=True)),
-
-        (('--prefer-new',), dict(
-            action='store_true',
-            help='Prefer newer python package versions instead of the ones from nixpkgs. '
-                 'This might increase build times significantly since no cache can be used'))
     )
     parser = ArgumentParser()
     parser.add_argument('--version', '-V', help='show program version', action='store_true')