Fixing tests

VERSION.txt

@@ -1 +1 @@
-0.0.5
+0.6.0rc2

build_all.bat

@@ -0,0 +1,4 @@
+python feetmaker.py python -a amd64
+python feetmaker.py build -a amd64
+python feetmaker.py python -a win32
+python feetmaker.py build -a win32

feet/feet.py

@@ -1,28 +1,44 @@
 import argparse
 import fnmatch
 import glob
+import itertools
 import os
+import pkg_resources
 import shutil
 import subprocess
 import sys
 import zipfile
 
 root = os.path.abspath(os.path.dirname(__file__))
-feet_bin = root.split('_data')[0] + '.exe'
+
+def _set_root_relative():
+    global feet_bin
+    global site_packages
+    global zip_excludes
+
+    feet_bin = root.split('_data')[0] + '.exe'
+    site_packages = os.path.join(root, 'cpython', 'lib', 'site-packages')
+
+    zip_excludes = [
+        "*.pyc",
+        "*__pycache__*",
+        "dist*",
+        ".git*",
+        os.path.basename(root) + '*',
+    ]
+
+_set_root_relative()
 
 # Add path for included third-party packages with the Feet runtime
 sys.path.insert(0, os.path.join(sys.executable, 'Lib', 'site-packages'))
 
 # Add path for project required Python dependencies
-site_packages = os.path.join(root, 'cpython', 'lib', 'site-packages')
 sys.path.insert(0, site_packages)
 
 # Add path for the actual project, main script and all
 # TODO: Decide if this is necessary...?
 sys.path.insert(0, '.')
 
-import requirements
-
 
 HELP = """FEET, it makes Python run!
 
@@ -62,56 +78,68 @@
 exe_parser = subparsers.add_parser('exe')
 exe_parser.add_argument('name', type=str, action='store')
 exe_parser.add_argument('files', type=str, nargs='*')
+exe_parser.add_argument('--confirm', action='store_true')
 
 zip_parser = subparsers.add_parser('zip')
 zip_parser.add_argument('name', type=str, action='store')
 zip_parser.add_argument('files', type=str, nargs='*')
 
-zip_excludes = [
-    "*.pyc",
-    "*__pycache__*",
-]
 
 def add_to_zip(path, dest, compression, prefix=None):
-    if prefix is None:
-        prefix = os.path.join("feet", "app")
     zipf = zipfile.ZipFile(dest, 'a', compression)
 
     for root, _, files in os.walk(path):
         for file in files:
             src = os.path.join(root, file)
             name = os.path.relpath(src, ".")
             base = name.split(os.path.sep, 1)[0]
-            if base.endswith('_data'):
-                name = name.replace(base, 'feet', 1)
 
+            # print(name, '...', end='')
             excluded = False
             for pattern in zip_excludes:
-                if fnmatch.fnmatch(os.path.abspath(name), pattern):
+                if fnmatch.fnmatch(name, pattern):
                     excluded = True
+                    # print('skip')
                     break
             if not excluded:
-                name = os.path.relpath(os.path.join(prefix, name))
+                # print('ok')
+                if prefix:
+                    name = os.path.relpath(os.path.join(prefix, name))
                 name = name.replace('\\', '/')
                 try:
                     zipf.getinfo(name)
                 except KeyError:
-                    print("...", name)
                     zipf.write(src, name)
 
     zipf.close()
 
 
+def get_app_files(files, exclude=()):
+    if files:
+        yield from files
+    else:
+        for fn in os.listdir('.'):
+            include = True
+            for exc in itertools.chain(zip_excludes, exclude):
+                if fnmatch.fnmatch(fn, exc):
+                    include = False
+                    break
+            if include:
+                yield fn
+
+
+
 def main(argv):
     feet_exec = argv.pop(0)
     args = parser.parse_args(argv)
 
     assert os.path.exists(feet_bin)
     py_bin = os.path.join(root, "cpython", "python.exe")
 
-    main =  os.path.join(root, "app", "main.py")
+    main = os.path.join(root, "app", "main.py")
     if not os.path.exists(main):
-        main =  os.path.join(root, "..", "main.py")
+        main = os.path.join(root, "..", "main.py")
+    main = os.path.abspath(main)
     if not os.path.exists(main):
         print(HELP)
         exit(1)
@@ -125,6 +153,7 @@ def main(argv):
         })
         proc = subprocess.Popen(
             [py_bin, main],
+            cwd=os.path.dirname(main),
             env=env,
             stdout=sys.stdout,
             stderr=sys.stderr,
@@ -159,20 +188,23 @@ def main(argv):
             cur_libraries = {}
 
             if os.path.exists('requirements.txt'):
-                for line in requirements.parse(open('requirements.txt')):
-                    cur_libraries[line.name] = line.line
-            new_req = list(requirements.parse(args.spec))[0]
-            cur_libraries[new_req.name] = new_req.line
+                for req in pkg_resources.parse_requirements(open('requirements.txt')):
+                    cur_libraries[req.name] = req.specifier
+            new_req = list(pkg_resources.parse_requirements(args.spec))[0]
+            cur_libraries[new_req.name] = new_req.specifier
 
-            args = ['install', '--trusted-host=pypi.org', new_req.line]
+            args = ['install', '--trusted-host=pypi.org', args.spec]
             subprocess.check_call([py_bin, '-m', 'pip', *args])
 
             print("Updating project requirements.txt file...")
             with open('requirements.txt', 'w') as f:
-                for _, line in cur_libraries.items():
-                    f.write(f'{line}\n')
+                for name, spec in cur_libraries.items():
+                    f.write(f'{name}{spec}\n')
 
     elif args.command == 'exe':
+        if not args.confirm:
+            print("The exe packing command is experimental. Use --confirm to confirm opting into using it.")
+            exit(1)
         name = args.name
         if not name.endswith('.exe'):
             name += ".exe"
@@ -181,21 +213,17 @@ def main(argv):
             if not os.path.exists('dist'):
                 os.mkdir('dist')
 
-        include = args.files or [main]
-        zip_excludes.append(os.path.join(os.path.abspath(root), '*'))
-        zip_excludes.append(os.path.join(os.path.abspath("dist"), '*'))
 
         shutil.copy(feet_bin, name)
+        include = get_app_files(args.files, exclude=[feet_bin])
+        prefix = '.'
 
         zf = zipfile.ZipFile(name, 'a', zipfile.ZIP_BZIP2)
-        for pattern in include:
-            for f in glob.glob(pattern):
-                print(f, "->", os.path.join("feet", "app", f))
-                zf.write(f, os.path.join("feet", "app", f))
+        for f in include:
+            zf.write(f, os.path.join(prefix, f))
         zf.close()
 
-        if os.path.exists(site_packages):
-            add_to_zip(site_packages, name, zipfile.ZIP_BZIP2, prefix='.')
+        add_to_zip(".", name, zipfile.ZIP_BZIP2, prefix=prefix)
 
     elif args.command == 'zip':
         name = args.name
@@ -206,15 +234,11 @@ def main(argv):
         if not os.path.exists("dist"):
             os.makedirs("dist")
 
-        include = args.files or [main]
-        zip_excludes.append(os.path.join(os.path.abspath(root), '*'))
-        zip_excludes.append(os.path.join(os.path.abspath("dist"), '*'))
+        include = get_app_files(args.files)
 
-        zf = zipfile.ZipFile(name, 'a', zipfile.ZIP_BZIP2)
-        for pattern in include:
-            for f in glob.glob(pattern):
-                print(f, "->", os.path.join("feet", "app", f))
-                zf.write(f, os.path.join("feet", "app", f))
+        zf = zipfile.ZipFile(name, 'a', zipfile.ZIP_DEFLATED)
+        for f in include:
+            zf.write(f, f)
         zf.close()
 
         add_to_zip(".", name, zipfile.ZIP_DEFLATED, prefix=".")

feetmaker.py

@@ -2,6 +2,7 @@
 
 import argparse
 import fnmatch
+import json
 import logging
 import os
 import shutil
@@ -20,24 +21,19 @@
 build_parser = subparsers.add_parser('build')
 build_parser.add_argument('--debug', action='store_true')
 build_parser.add_argument('-o', action='store', default=None, dest='output')
-build_parser.add_argument('-a', dest='arch', action='store', default='amd64', help='"win32" or "amd64" architecture')
+build_parser.add_argument('-a', '--arch', dest='arch', action='store', default='win32', choices=['win32', 'amd64'])
 
 clean_parser = subparsers.add_parser('clean')
 
 setup_parser = subparsers.add_parser('setup')
 
 python_parser = subparsers.add_parser('python')
-python_parser.add_argument('-p', dest='pyversion', action='store', default='3.7')
-python_parser.add_argument('-a', dest='arch', action='store', default='amd64', help='"win32" or "amd64" architecture')
-
-argv = sys.argv[1:]
-args = parser.parse_args(argv)
+python_parser.add_argument('-p', dest='pyversion', action='store', default='v3.8.2')
+python_parser.add_argument('-a', '--arch', dest='arch', action='store', default='win32', choices=['win32', 'amd64'])
 
 version = open("VERSION.txt").read()
-arch = getattr(args, 'arch', 'amd64')
 python_loc_default = "cpython"
 python_loc = os.getenv("FEET_PYTHON_DIR", python_loc_default)
-py_bin = os.path.join(python_loc, 'PCBuild', arch, 'python.exe')
 
 # These patterns will be excluded from the generated Zip archives
 zip_excludes = [
@@ -56,7 +52,7 @@
 # These third-party packages will be included in the build
 py_deps = (
     'pip',
-    'requirements-parser',
+    # 'requirements-parser',
     'setuptools',
     # 'pkg_resources',
 )
@@ -135,7 +131,7 @@ def zipdir(path, relto, dest, compression):
     zipf.close()
 
 
-def main():
+def main(args):
 
     if not os.path.exists("feetmaker.py"):
         print("You are not in the feet repo. Cannot build.")
@@ -147,14 +143,15 @@ def main():
             subprocess.check_call(f"git clone https://github.com/python/cpython.git {python_loc}")
         os.chdir(python_loc)
         subprocess.check_call("git fetch")
-        subprocess.check_call(f"git reset --hard origin/{args.pyversion}")
+        subprocess.check_call("git reset --hard HEAD")
+        subprocess.check_call(f"git checkout {args.pyversion}")
         os.chdir('..')
 
         assert os.path.exists(f"./{python_loc}/PCBuild/build.bat")
-        if arch == "amd64":
+        if args.arch == "amd64":
             p = "x64"
         else:
-            p = arch
+            p = args.arch
         print(f"./{python_loc}/PCBuild/build.bat -c Release -p {p} -t Build")
         subprocess.check_call(f"{python_loc}\\PCBuild\\build.bat -c Release -p {p} -t Build")
 
@@ -163,11 +160,13 @@ def main():
             shutil.rmtree("feet/cpython")
 
         print("Compiling bootloader...")
-        subprocess.check_call("cargo build")
+        subprocess.check_call("cargo build --release")
 
         print("Creating runtime archive...")
+        py_exe = os.path.join(python_loc, "PCbuild", args.arch, 'python.exe')
+        subprocess.run([py_exe, '-m', 'lib2to3'], stdout=subprocess.PIPE)
         shutil.copytree(
-            os.path.join(python_loc, "PCbuild", arch),
+            os.path.join(python_loc, "PCbuild", args.arch),
             "feet/cpython",
             ignore=ignore_excludes,
         )
@@ -192,17 +191,19 @@ def main():
                             src,
                             "feet/cpython/",
                         )
+
+        feet_py = "feet/cpython/python.exe"
 
         # Create the stdlib zip to make unpacking faster
         zipdir(
             os.path.join(python_loc, "lib"),
             None,
-            os.path.join("feet", "cpython", "python37.zip"),
+            os.path.join("feet", "cpython", "python38.zip"),
             zipfile.ZIP_DEFLATED,
         )
 
         # Create the archive to attach to feet.exe to self-extract
-        feet_py = "feet/cpython/python.exe"
+
         subprocess.check_call([feet_py, '-m', 'ensurepip'])
         for name in py_deps:
             try:
@@ -226,7 +227,7 @@ def main():
         if not os.path.exists('build'):
             os.mkdir('build')
 
-        base = open('target/debug/feet.exe', 'rb')
+        base = open('target/release/feet.exe', 'rb')
         archive = open('feetruntime.zip', 'rb')
         output = getattr(args, 'output', None) or f'build/feet-{arch}-{version}'
         if not output.endswith('.exe'):
@@ -240,6 +241,15 @@ def main():
         base.close()
         archive.close()
 
+        # add metadata
+        final = zipfile.ZipFile(output, 'a')
+        final.comment = json.dumps({
+            'feet_format': '1',
+            'feet_arch': args.arch,
+            'feet_runner_size': os.stat('target/release/feet.exe').st_size,
+            'feet_archive_size': os.stat('feetruntime.zip').st_size,
+        }).encode('utf8')
+
         print("Done.")
 
     elif args.command == "clean":
@@ -250,4 +260,6 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    argv = sys.argv[1:]
+    args = parser.parse_args(argv)
+    main(args)