From 379ba631873a15dd4278080c3f28bedd8023dd38 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Thu, 6 Jul 2023 22:27:06 +0100 Subject: [PATCH 01/34] First changes to move to setup.cfg and pyproject.toml for DC install. --- donkeycar/__init__.py | 2 +- pyproject.toml | 3 +++ setup.cfg | 45 ++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 pyproject.toml diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index 6338ce4dc..dff3fe583 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -4,7 +4,7 @@ import logging from pkg_resources import get_distribution -__version__ = get_distribution('donkeycar').version +__version__ = '5.1.dev0' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..07de284aa --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index b88034e41..5f561a974 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,45 @@ [metadata] -description-file = README.md +name = donkeycar +version = attr: donkeycar.__version__ +author = Will Roscoe, Adam Conway, Tawn Kramer +url = https://github.com/autorope/donkeycar +description = Self driving library for python. +long_description = file: README.md +long_description_content_type = text/markdown +keywords = selfdriving cars donkeycar diyrobocars +license = MIT +classifiers = + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + +[options] +packages = find: +zip_safe = True +include_package_data = True +python_requires = >=3.9, <4 +install_requires = + numpy + pillow + docopt + tornado + requests + h5py + PrettyTable + paho-mqtt + simple_pid + progress + typing_extensions + pyfiglet + psutil + pynmea2 + pyserial + utm + pandas + pyyaml + +[options.package_data] +donkeycar = *.html, *.ini, *.tar.gz, *.txt + +[options.entry_points] +console_scripts = + donkey = donkeycar.management.base:execute_from_command_line \ No newline at end of file From 7682a48a69c2f7d88581e76412ca52703845cc9c Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Thu, 6 Jul 2023 22:51:12 +0100 Subject: [PATCH 02/34] First changes to move to setup.cfg and pyproject.toml for DC install. --- setup.cfg | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/setup.cfg b/setup.cfg index 5f561a974..a1a8f6fab 100644 --- a/setup.cfg +++ b/setup.cfg @@ -37,6 +37,54 @@ install_requires = pandas pyyaml +[options.extras_require] +pi = + picamera2 + Adafruit_PCA9685 + adafruit-circuitpython-ssd1306 + adafruit-circuitpython-rplidar + RPi.GPIO + tensorflow @ https://github.com/PINTO0309/Tensorflow-bin/releases/download/v2.9.0/tensorflow-2.9.0-cp39-none-linux_aarch64.whl + +nano45 = + Adafruit_PCA9685 + adafruit-circuitpython-ssd1306 + adafruit-circuitpython-rplidar + Jetson.GPIO + matplotlib + +nano = + Adafruit_PCA9685 + adafruit-circuitpython-ssd1306 + adafruit-circuitpython-rplidar + Jetson.GPIO + matplotlib + kivy-jetson + plotly + +pc = + matplotlib + kivy + pandas + plotly + albumentations + +dev = + pytest + pytest-cov + responses + mypy + +tf = + tensorflow == 2.9 + +torch = + pytorch + torchvision==0.12 + torchaudio + fastai + + [options.package_data] donkeycar = *.html, *.ini, *.tar.gz, *.txt From 86a50ad835540b6d28040f2a828f965b13577c5a Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Fri, 7 Jul 2023 22:06:04 +0100 Subject: [PATCH 03/34] Remove obsolete conda yml files, add tf to pc install and peg kivy and rename old setup.py to not infer with new install. This file will be delete ultimately. --- install/envs/jetson.yml | 41 --------------------------------- install/envs/jetson46.yml | 34 --------------------------- install/envs/mac.yml | 47 -------------------------------------- install/envs/sagemaker.yml | 16 ------------- install/envs/ubuntu.yml | 46 ------------------------------------- install/envs/windows.yml | 45 ------------------------------------ setup.cfg | 6 ++--- setup.py => setup_orig.py | 0 8 files changed, 2 insertions(+), 233 deletions(-) delete mode 100644 install/envs/jetson.yml delete mode 100644 install/envs/jetson46.yml delete mode 100644 install/envs/mac.yml delete mode 100644 install/envs/sagemaker.yml delete mode 100644 install/envs/ubuntu.yml delete mode 100644 install/envs/windows.yml rename setup.py => setup_orig.py (100%) diff --git a/install/envs/jetson.yml b/install/envs/jetson.yml deleted file mode 100644 index c27b4db2d..000000000 --- a/install/envs/jetson.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: donkey - -channels: - - defaults - - conda-forge - - fastai - -dependencies: - - python=3.8 - - h5py - - pillow - - opencv - - matplotlib - - tornado - - docopt - - pandas - - pylint - - pytest - - pytest-cov - - codecov - - pip - - progress - - moviepy - - paho-mqtt - - PrettyTable - - pyfiglet - - mypy - - numpy - - psutil - - plotly - - pyyaml - - fastai - - pynmea2 - - pyserial - - utm - - pip: - - --extra-index-url https://developer.download.nvidia.com/compute/redist/jp/v502 - - tensorflow==2.9.1+nv22.09 - - git+https://github.com/autorope/keras-vis.git - - simple-pid - - kivy diff --git a/install/envs/jetson46.yml b/install/envs/jetson46.yml deleted file mode 100644 index 1a1740ea9..000000000 --- a/install/envs/jetson46.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: donkey - -channels: - - defaults - - conda-forge - - fastai - -dependencies: - - python=3.9 - - h5py - - pillow - - matplotlib - - tornado - - docopt - - pandas - - pylint - - pytest - - pytest-cov - - codecov - - progress - - moviepy - - paho-mqtt - - PrettyTable - - pyfiglet - - mypy - - numpy - - psutil - - plotly - - pyyaml - - fastai - - pynmea2 - - pyserial - - utm - diff --git a/install/envs/mac.yml b/install/envs/mac.yml deleted file mode 100644 index be4f9844b..000000000 --- a/install/envs/mac.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: donkey - -channels: - - defaults - - conda-forge - - pytorch - - fastai - -dependencies: - - python=3.9 - - numpy - - h5py - - pillow - - opencv - - matplotlib - - tornado - - docopt - - pandas - - pylint - - pytest - - pytest-cov - - codecov - - pip - - progress - - paho-mqtt - - PrettyTable - - pyfiglet - - mypy - - pytorch - - torchvision=0.12 - - torchaudio - - pytorch-lightning>=1.9,<2.0 - - psutil - - kivy=2.1 - - plotly - - pyyaml - - fastai - - pynmea2 - - pyserial - - utm - - albumentations - - pip: - - tensorflow==2.9 - - git+https://github.com/autorope/keras-vis.git - - simple-pid - - moviepy - diff --git a/install/envs/sagemaker.yml b/install/envs/sagemaker.yml deleted file mode 100644 index f1b3d82c0..000000000 --- a/install/envs/sagemaker.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: donkey -dependencies: - -- h5py -- pillow -- tensorflow-gpu -- opencv -- matplotlib -- tornado -- docopt -- pandas - -- pip: - - moviepy - - paho-mqtt - - PrettyTable diff --git a/install/envs/ubuntu.yml b/install/envs/ubuntu.yml deleted file mode 100644 index a43798092..000000000 --- a/install/envs/ubuntu.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: donkey - -channels: - - defaults - - conda-forge - - pytorch - - fastai - -dependencies: - - python=3.9 - - numpy - - h5py - - pillow - - opencv - - matplotlib - - tornado - - docopt - - pandas - - pylint - - pytest - - pytest-cov - - codecov - - pip - - progress - - paho-mqtt - - PrettyTable - - pyfiglet - - mypy - - pytorch - - torchvision=0.12 - - torchaudio - - pytorch-lightning>=1.9,<2.0 - - psutil - - plotly - - pyyaml - - tensorflow=2.9 - - fastai - - pynmea2 - - pyserial - - utm - - albumentations - - pip: - - git+https://github.com/autorope/keras-vis.git - - simple-pid - - kivy==2.1 - - moviepy diff --git a/install/envs/windows.yml b/install/envs/windows.yml deleted file mode 100644 index 4a0d89d2a..000000000 --- a/install/envs/windows.yml +++ /dev/null @@ -1,45 +0,0 @@ -name: donkey - -channels: - - defaults - - conda-forge - - pytorch - - fastai - -dependencies: - - python=3.9 - - h5py - - pillow - - opencv - - matplotlib - - tornado - - docopt - - pandas - - pylint - - pytest - - pytest-cov - - codecov - - pip - - progress - - moviepy - - paho-mqtt - - PrettyTable - - pyfiglet - - mypy - - pytorch - - torchvision=0.12 - - torchaudio - - pytorch-lightning - - numpy - - kivy=2.1 - - plotly - - pyyaml - - psutil - - fastai - - pynmea2 - - pyserial - - utm - - albumentations - - pip: - - git+https://github.com/autorope/keras-vis.git - - simple-pid diff --git a/setup.cfg b/setup.cfg index a1a8f6fab..0c6c1cb11 100644 --- a/setup.cfg +++ b/setup.cfg @@ -63,8 +63,9 @@ nano = plotly pc = + tensorflow==2.9 matplotlib - kivy + kivy==2.1 pandas plotly albumentations @@ -75,9 +76,6 @@ dev = responses mypy -tf = - tensorflow == 2.9 - torch = pytorch torchvision==0.12 diff --git a/setup.py b/setup_orig.py similarity index 100% rename from setup.py rename to setup_orig.py From 6b9a7c607d3785a6d9bad1afa77e42c08fc9ffd6 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sat, 8 Jul 2023 18:46:02 +0100 Subject: [PATCH 04/34] fix rpi install (need to install tf manually afterwards) --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0c6c1cb11..984842185 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ pi = adafruit-circuitpython-ssd1306 adafruit-circuitpython-rplidar RPi.GPIO - tensorflow @ https://github.com/PINTO0309/Tensorflow-bin/releases/download/v2.9.0/tensorflow-2.9.0-cp39-none-linux_aarch64.whl + #tensorflow @ https://github.com/PINTO0309/Tensorflow-bin/releases/download/v2.9.0/tensorflow-2.9.0-cp39-none-linux_aarch64.whl nano45 = Adafruit_PCA9685 From 90c442d8bad01220053a6edb0718828f4502c8d2 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Mon, 10 Jul 2023 20:30:38 +0100 Subject: [PATCH 05/34] Fix setup to include web_controller module with all the sudirectories and files --- donkeycar/__init__.py | 2 +- setup.cfg | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index dff3fe583..f703d636c 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -4,7 +4,7 @@ import logging from pkg_resources import get_distribution -__version__ = '5.1.dev0' +__version__ = '5.1.dev1' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/setup.cfg b/setup.cfg index 984842185..f1416a60d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -name = donkeycar +name = donkeycar_pypi version = attr: donkeycar.__version__ author = Will Roscoe, Adam Conway, Tawn Kramer url = https://github.com/autorope/donkeycar @@ -13,10 +13,10 @@ classifiers = Programming Language :: Python :: 3.9 [options] -packages = find: +packages = find_namespace: zip_safe = True include_package_data = True -python_requires = >=3.9, <4 +python_requires = >=3.8, <4 install_requires = numpy pillow From 46dad20608c9eb97bc64c07b1fae9dcb7a56573a Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Mon, 10 Jul 2023 20:45:41 +0100 Subject: [PATCH 06/34] Remove h5py as it seems unused and restrict to py 3.8 / 3.9 --- donkeycar/__init__.py | 2 +- setup.cfg | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index f703d636c..7c58eb8cb 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -4,7 +4,7 @@ import logging from pkg_resources import get_distribution -__version__ = '5.1.dev1' +__version__ = '5.1.dev2' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/setup.cfg b/setup.cfg index f1416a60d..2ad9748b6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,14 +16,13 @@ classifiers = packages = find_namespace: zip_safe = True include_package_data = True -python_requires = >=3.8, <4 +python_requires = >=3.8, <=3.9 install_requires = numpy pillow docopt tornado requests - h5py PrettyTable paho-mqtt simple_pid From ac445f0f406df09bd9728f3345dfb4f05633182a Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Tue, 11 Jul 2023 22:17:59 +0100 Subject: [PATCH 07/34] Avoid moviepy import error and fix ui behavior of merging deleted indexes which was missing its implementation in the datastore. Added tf to pi install. --- donkeycar/__init__.py | 2 +- donkeycar/management/makemovie.py | 7 ++- donkeycar/parts/datastore_v2.py | 85 ++++++++++++++++++++++--------- setup.cfg | 5 +- 4 files changed, 72 insertions(+), 27 deletions(-) diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index 7c58eb8cb..65afa7844 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -4,7 +4,7 @@ import logging from pkg_resources import get_distribution -__version__ = '5.1.dev2' +__version__ = '5.1.dev8' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/donkeycar/management/makemovie.py b/donkeycar/management/makemovie.py index 40b1047ed..dea90616e 100755 --- a/donkeycar/management/makemovie.py +++ b/donkeycar/management/makemovie.py @@ -1,4 +1,4 @@ -import moviepy.editor as mpy + from tensorflow.python.keras import activations from tensorflow.python.keras import backend as K import tensorflow as tf @@ -24,6 +24,11 @@ def run(self, args, parser): Load the images from a tub and create a movie from them. Movie ''' + try: + import moviepy.editor as mpy + except ImportError as e: + logger.error(f'Please install moviepy first: {e}') + return if args.tub is None: print("ERR>> --tub argument missing.") diff --git a/donkeycar/parts/datastore_v2.py b/donkeycar/parts/datastore_v2.py index ac48bf503..9bd3d9810 100644 --- a/donkeycar/parts/datastore_v2.py +++ b/donkeycar/parts/datastore_v2.py @@ -1,3 +1,4 @@ +import atexit import json import mmap import os @@ -7,7 +8,6 @@ logger = logging.getLogger(__name__) - NEWLINE = '\n' NEWLINE_STRIP = '\r\n' @@ -244,6 +244,7 @@ def __init__(self, base_path, inputs=[], types=[], metadata=[], self.catalog_metadata = dict() self.deleted_indexes = set() self._updated_session = False + self._is_closed = False has_catalogs = False if self.manifest_path.exists(): @@ -258,10 +259,12 @@ def __init__(self, base_path, inputs=[], types=[], metadata=[], self.manifest_metadata['created_at'] = created_at if not self.base_path.exists(): self.base_path.mkdir(parents=True, exist_ok=True) - logger.info(f'Created a new datastore at' + logger.info(f'Creating a new datastore at' f' {self.base_path.as_posix()}') self.seekeable = Seekable(self.manifest_path, read_only=self.read_only) + logger.info(f'Creating a new manifest at ' + f'{self.manifest_path.as_posix()}') if not has_catalogs: self._write_contents() @@ -275,13 +278,21 @@ def __init__(self, base_path, inputs=[], types=[], metadata=[], start_index=self.current_index) # Create a new session_id, which will be added to each record in the # tub, when Tub.write_record() is called. - self.session_id = self.create_new_session() - - def write_record(self, record): - new_catalog = self.current_index > 0 \ - and (self.current_index % self.max_len) == 0 - if new_catalog: - self._add_catalog() + self.session_id = self.create_new_session_id() + + def exit_hook(): + if not self._is_closed: + logger.error(f"Unexpected closing manifest {self.base_path}") + self.close() + # Automatically save config when program ends + atexit.register(exit_hook) + + def write_record(self, record, index=None): + if index is None: + new_catalog = self.current_index > 0 \ + and (self.current_index % self.max_len) == 0 + if new_catalog: + self._add_catalog() self.current_catalog.write_record(record) self.current_index += 1 @@ -300,8 +311,8 @@ def delete_records(self, record_indexes): self.deleted_indexes.update(record_indexes) self._update_catalog_metadata(update=True) if record_indexes: - logger.info(f'Deleted records {min(record_indexes)} - ' - f'{max(record_indexes)}') + logger.info(f'Deleting {len(record_indexes)} records: ' + f'{min(record_indexes)} - {max(record_indexes)}') def restore_records(self, record_indexes): # Does not actually delete the record, but marks it as deleted. @@ -339,8 +350,19 @@ def _read_metadata(self, metadata=[]): def _read_contents(self): self.seekeable.seek_line_start(1) - self.inputs = json.loads(self.seekeable.readline()) - self.types = json.loads(self.seekeable.readline()) + manifest_inputs = json.loads(self.seekeable.readline()) + manifest_types = json.loads(self.seekeable.readline()) + if not self.inputs and not self.types: + self.inputs = manifest_inputs + self.types = manifest_types + else: + assert self.inputs == manifest_inputs \ + and self.types == manifest_types, \ + f'Trying to create a tub with different inputs/types than ' \ + f'the stored tub. This is only allowed when new tub ' \ + f'specifies no inputs. New inputs: {self.inputs} vs ' \ + f'stored inputs: {manifest_inputs}, new types {self.types}'\ + f' vs stored types: {manifest_types}' self.metadata = json.loads(self.seekeable.readline()) self.manifest_metadata = json.loads(self.seekeable.readline()) # Catalog metadata @@ -370,22 +392,31 @@ def _update_catalog_metadata(self, update=True): self.catalog_metadata = catalog_metadata self.seekeable.writeline(json.dumps(catalog_metadata)) - def create_new_session(self): + def _update_session_info(self): """ Creates a new session id and appends it to the metadata.""" sessions = self.manifest_metadata.get('sessions', {}) - last_id = -1 - if sessions: - last_id = sessions['last_id'] - else: + if not sessions: sessions['all_full_ids'] = [] - this_id = last_id + 1 - date = time.strftime('%y-%m-%d') - this_full_id = date + '_' + str(this_id) + this_id, this_full_id = self.session_id sessions['last_id'] = this_id sessions['last_full_id'] = this_full_id sessions['all_full_ids'].append(this_full_id) self.manifest_metadata['sessions'] = sessions - return this_full_id + + def create_new_session_id(self): + """ Creates a new session id and appends it to the metadata.""" + sessions = self.manifest_metadata.get('sessions', {}) + new_id = sessions['last_id'] + 1 if sessions else 0 + new_full_id = f"{time.strftime('%y-%m-%d')}_{new_id}" + return new_id, new_full_id + + def add_deleted_indexes(self, indexes): + if isinstance(indexes, int): + indexes = {indexes} + self.deleted_indexes.update(indexes) + self._update_catalog_metadata(update=True) + + def close(self): """ Closing tub closes open files for catalog, catalog manifest and @@ -393,9 +424,17 @@ def close(self): # If records were received, write updated session_id dictionary into # the metadata, otherwise keep the session_id information unchanged if self._updated_session: - self.seekeable.update_line(4, json.dumps(self.manifest_metadata)) + logger.info(f'Saving new session {self.session_id[1]}') + self._update_session_info() + self.write_metadata() self.current_catalog.close() self.seekeable.close() + self._is_closed = True + logger.info(f'Closing manifest {self.base_path}') + + def write_metadata(self): + self.seekeable.update_line(3, json.dumps(self.metadata)) + self.seekeable.update_line(4, json.dumps(self.manifest_metadata)) def __iter__(self): return ManifestIterator(self) diff --git a/setup.cfg b/setup.cfg index 2ad9748b6..de36dfedc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ classifiers = packages = find_namespace: zip_safe = True include_package_data = True -python_requires = >=3.8, <=3.9 +python_requires = >=3.8,<4 install_requires = numpy pillow @@ -43,6 +43,7 @@ pi = adafruit-circuitpython-ssd1306 adafruit-circuitpython-rplidar RPi.GPIO + tensorflow-aarch64==2.9.3 #tensorflow @ https://github.com/PINTO0309/Tensorflow-bin/releases/download/v2.9.0/tensorflow-2.9.0-cp39-none-linux_aarch64.whl nano45 = @@ -83,7 +84,7 @@ torch = [options.package_data] -donkeycar = *.html, *.ini, *.tar.gz, *.txt +* = *.html, *.ini, *.txt, *.kv [options.entry_points] console_scripts = From c6a1ddf41bb9d1bc1d9500a9184803d42f2f8bd6 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Wed, 12 Jul 2023 21:47:23 +0100 Subject: [PATCH 08/34] Adding Pi user and hostname, as this is required by the ui. --- donkeycar/templates/cfg_complete.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/donkeycar/templates/cfg_complete.py b/donkeycar/templates/cfg_complete.py index ac790f202..fb31d4a72 100644 --- a/donkeycar/templates/cfg_complete.py +++ b/donkeycar/templates/cfg_complete.py @@ -760,3 +760,7 @@ # FPS counter SHOW_FPS = False FPS_DEBUG_INTERVAL = 10 # the interval in seconds for printing the frequency info into the shell + +# PI connection +PI_USERNAME = "pi" +PI_HOSTNAME = "donkeypi.local" \ No newline at end of file From 582a61b74158d2d26cab82cceac915e96b8820f1 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Wed, 12 Jul 2023 22:06:25 +0100 Subject: [PATCH 09/34] Add missing opencv for RPi --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index de36dfedc..f3ace43ea 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ pi = adafruit-circuitpython-rplidar RPi.GPIO tensorflow-aarch64==2.9.3 - #tensorflow @ https://github.com/PINTO0309/Tensorflow-bin/releases/download/v2.9.0/tensorflow-2.9.0-cp39-none-linux_aarch64.whl + opencv-contrib-python nano45 = Adafruit_PCA9685 From ecc707cb1b30dad6803379ce7d7d578f938923c9 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Fri, 14 Jul 2023 21:48:37 +0100 Subject: [PATCH 10/34] Back to setup.cfg as pyproject.toml not yet fully functional and only works from withing the donkeycar project directory. --- donkeycar/__init__.py | 3 +-- pyproject.toml | 5 ++++- setup.cfg | 8 ++++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index 65afa7844..d006ab0a7 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -2,9 +2,8 @@ import sys from pyfiglet import Figlet import logging -from pkg_resources import get_distribution -__version__ = '5.1.dev8' +__version__ = '5.1.dev10' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/pyproject.toml b/pyproject.toml index 07de284aa..145c91223 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,6 @@ [build-system] requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" \ No newline at end of file +build-backend = "setuptools.build_meta" + + + diff --git a/setup.cfg b/setup.cfg index f3ace43ea..10b27d1d3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,6 +9,14 @@ long_description_content_type = text/markdown keywords = selfdriving cars donkeycar diyrobocars license = MIT classifiers = + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + Development Status :: 4 - Beta + # Indicate who your project is intended for + Intended Audience :: Developers + Topic :: Scientific/Engineering :: Artificial Intelligence Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 From 2624303c86b29052dc051f3890cc5cbafe84886c Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sat, 15 Jul 2023 21:50:22 +0100 Subject: [PATCH 11/34] include scripts folder in pypi package --- MANIFEST.in | 2 +- donkeycar/__init__.py | 2 +- install/README.md | 10 ------ install/nano/install-jp44.sh | 60 ----------------------------------- install/nano/requirements.txt | 28 ---------------- install/pi/install.sh | 46 --------------------------- install/pi/opencv.sh | 19 ----------- install/pi/prepare.sh | 40 ----------------------- scripts/profile.py | 2 +- 9 files changed, 3 insertions(+), 206 deletions(-) delete mode 100644 install/README.md delete mode 100644 install/nano/install-jp44.sh delete mode 100644 install/nano/requirements.txt delete mode 100644 install/pi/install.sh delete mode 100644 install/pi/opencv.sh delete mode 100644 install/pi/prepare.sh diff --git a/MANIFEST.in b/MANIFEST.in index 2d41b7d62..b9013ea14 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ include donkeycar/templates/* -include VERSION +include scripts recursive-include donkeycar/parts/web_controller/templates/ * diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index d006ab0a7..49197c499 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -3,7 +3,7 @@ from pyfiglet import Figlet import logging -__version__ = '5.1.dev10' +__version__ = '5.1.dev12' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/install/README.md b/install/README.md deleted file mode 100644 index 6952850c3..000000000 --- a/install/README.md +++ /dev/null @@ -1,10 +0,0 @@ -### Install on raspberry pi -The easiest way to get donkey running on a pi is with -a prebuilt disk image. To create your own disk -image you can use the scripts in /pi. - - -### Install on other systems -Create a conda environment using the env files - - diff --git a/install/nano/install-jp44.sh b/install/nano/install-jp44.sh deleted file mode 100644 index 1cb8f3cec..000000000 --- a/install/nano/install-jp44.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -e - -password='jetson' -# Get the full dir name of this script -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -# Keep updating the existing sudo time stamp -sudo -v -while true; do sudo -n true; sleep 120; kill -0 "$$" || exit; done 2>/dev/null & - -######################################## -# Install DonkeyCar -######################################## -sudo apt-get update -y -sudo apt-get upgrade -y -sudo apt-get install -y libhdf5-serial-dev hdf5-tools libhdf5-dev zlib1g-dev zip libjpeg8-dev liblapack-dev libblas-dev gfortran -sudo apt-get install -y python3-dev python3-pip -sudo apt-get install -y libxslt1-dev libxml2-dev libffi-dev libcurl4-openssl-dev libssl-dev libpng-dev libopenblas-dev -sudo apt-get install -y git -sudo apt-get install -y openmpi-doc openmpi-bin libopenmpi-dev libopenblas-dev - -# Install Tensorflow as system package -sudo -H pip3 install -r requirements.txt -#sudo -H pip3 install --pre --extra-index-url https://developer.download.nvidia.com/compute/redist/jp/v44 'tensorflow>2' -sudo -H pip3 install --pre --extra-index-url https://developer.download.nvidia.com/compute/redist/jp/v44 tensorflow==2.2.0+nv20.6 - -######################################## -# Install PyTorch v1.7 - torchvision v0.8.1 -# pytorch 1.6.0-rc2 -# https://forums.developer.nvidia.com/t/pytorch-for-jetson-nano-version-1-5-0-now-available/72048/392 -wget https://nvidia.box.com/shared/static/wa34qwrwtk9njtyarwt5nvo6imenfy26.whl -O torch-1.7.0-cp36-cp36m-linux_aarch64.whl -sudo -H pip3 install ./torch-1.7.0-cp36-cp36m-linux_aarch64.whl - -# Install PyTorch Vision -sudo apt-get install libjpeg-dev zlib1g-dev libpython3-dev libavcodec-dev libavformat-dev libswscale-dev -mkdir -p ~/projects; cd ~/projects -git clone --branch v0.8.1 https://github.com/pytorch/vision torchvision -cd torchvision -export BUILD_VERSION=0.8.1 -sudo python3 setup.py install - - -# Create virtual enviroment -pip3 install virtualenv -python3 -m virtualenv -p python3 ~/.virtualenv/donkeycar --system-site-packages -echo "source ~/.virtualenv/donkeycar/bin/activate" >> ~/.bashrc -# "source ~/.virtualenv/donkeycar/bin/activate" in the shell script -. ~/.virtualenv/donkeycar/bin/activate - - -# Install DonkeyCar as user package -cd ~/projects -git clone https://github.com/autorope/donkeycar -cd donkeycar -git checkout dev -pip install -e .[nano] - -# https://github.com/keras-team/keras-tuner/issues/317 -echo "export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libgomp.so.1" >> ~/.bashrc -#export LD_PRELOAD=/usr/lib/aarch64-linux-gnu/libgomp.so.1 \ No newline at end of file diff --git a/install/nano/requirements.txt b/install/nano/requirements.txt deleted file mode 100644 index daa2dc778..000000000 --- a/install/nano/requirements.txt +++ /dev/null @@ -1,28 +0,0 @@ -pip -testresources -setuptools -futures -protobuf -pybind11 -cython -numpy -future -mock -h5py==2.10.0 -keras_preprocessing -keras_applications -gast -grpcio -absl-py -py-cpuinfo -psutil -portpicker -six -requests -astor -termcolor -wrapt -google-pasta -scipy -pandas -gdown \ No newline at end of file diff --git a/install/pi/install.sh b/install/pi/install.sh deleted file mode 100644 index bdf69f237..000000000 --- a/install/pi/install.sh +++ /dev/null @@ -1,46 +0,0 @@ -# Script to install everything needed for donkeycar except the donkeycar library - -#standard updates (5 min) -sudo apt update -y -sudo apt upgrade -y -sudo rpi-update -y - -#helpful libraries (2 min) -sudo apt install build-essential python3-dev python3-distlib python3-setuptools python3-pip python3-wheel -y - -sudo apt-get install git cmake pkg-config -y -sudo apt-get install libjpeg-dev libtiff5-dev libjasper-dev libpng12-dev -y -sudo apt-get install libavcodec-dev libavformat-dev libswscale-dev libv4l-dev -y -sudo apt-get install libxvidcore-dev libx264-dev -y -sudo apt-get install libatlas-base-dev gfortran -y - -sudo apt install libzmq-dev -y -sudo apt install xsel xclip -y -sudo apt install python3-h5py -y - -#install numpy and pandas (3 min) -sudo apt install libxml2-dev python3-lxml -y -sudo apt install libxslt-dev -y - -#remove python2 (1 min) -sudo apt-get remove python2.7 -y -sudo apt-get autoremove -y - -#install redis-server (1 min) -sudo apt install redis-server -y - -#create a python virtualenv (2 min) -sudo apt install virtualenv -y -virtualenv ~/env --system-site-packages --python python3 -echo '#start env' >> ~/.bashrc -echo 'source ~/env/bin/activate' >> ~/.bashrc -source ~/env/bin/activate - - -#make sure the virtual environment is active -source ~/env/bin/activate - -# install pandas and numpy and Adafruit CircuitPython -pip install pandas #also installs numpy - -pip install tensorflow==1.9 \ No newline at end of file diff --git a/install/pi/opencv.sh b/install/pi/opencv.sh deleted file mode 100644 index e6008a72c..000000000 --- a/install/pi/opencv.sh +++ /dev/null @@ -1,19 +0,0 @@ -#install opencv (1 hour) -#instructions from:https://raspberrypi.stackexchange.com/questions/69169/how-to-install-opencv-on-raspberry-pi-3-in-raspbian-jessie - -# NOTE: this gets the dev version. Use tags to get specific version -git clone --branch 3.4.1 --depth 1 https://github.com/opencv/opencv.git -git clone --branch 3.4.1 --depth 1 https://github.com/opencv/opencv_contrib.git - -cd ~/opencv -mkdir build -cd build -cmake -D CMAKE_BUILD_TYPE=RELEASE \ - -D CMAKE_INSTALL_PREFIX=/usr/local \ - -D INSTALL_C_EXAMPLES=OFF \ - -D INSTALL_PYTHON_EXAMPLES=OFF \ - -D OPENCV_EXTRA_MODULES_PATH=~/opencv_contrib/modules \ - -D BUILD_EXAMPLES=OFF .. -make -j4 -sudo make install -sudo ldconfig \ No newline at end of file diff --git a/install/pi/prepare.sh b/install/pi/prepare.sh deleted file mode 100644 index b1698f14f..000000000 --- a/install/pi/prepare.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash - -#Script to setup pi disk after base rpi image. - -MEDIA=/media/$USER -BOOT=$MEDIA/boot -ROOTFS=$MEDIA/rootfs - -sudo touch $BOOT/ssh - -#enable camera -echo "start_x=1" | sudo tee -a $BOOT/config.txt -echo "gpu_mem=128" | sudo tee -a $BOOT/config.txt - -#enable i2c -echo "i2c-dev" | sudo tee -a $ROOTFS/etc/modules - -#Change hostname -hostn=$(cat $ROOTFS/etc/hostname) -echo "Existing hostname is $hostn" -echo "Enter new hostname: " -read newhost - -#change hostname in /etc/hosts & /etc/hostname -sudo sed -i "s/$hostn/$newhost/g" $ROOTFS/etc/hosts -sudo sed -i "s/$hostn/$newhost/g" $ROOTFS/etc/hostname -echo "Your new hostname is $newhost" - -# setup default wifi config -sudo truncate -s 0 $ROOTFS/etc/wpa_supplicant/wpa_supplicant.conf -cat <<'EOF' | sudo tee $ROOTFS/etc/wpa_supplicant/wpa_supplicant.conf -country=US -ctrl_interface=DIR=/var/run/wpa_supplicant GROUP=netdev -update_config=1 - -network={ - ssid="network_name" - psk="password" -} -EOF \ No newline at end of file diff --git a/scripts/profile.py b/scripts/profile.py index c6d48f3b7..cee6ca696 100755 --- a/scripts/profile.py +++ b/scripts/profile.py @@ -42,4 +42,4 @@ def profile(model_path, model_type): if __name__ == '__main__': args = docopt(__doc__) - profile(model_path = args['--model'], model_type = args['--type']) + profile(model_path=args['--model'], model_type=args['--type']) From f166b8f7f65ca91e54b489ca68fbd8aba42b64b0 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sun, 16 Jul 2023 12:54:36 +0100 Subject: [PATCH 12/34] Try fixing CI --- .github/workflows/python-package-conda.yml | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index a50c08a8f..f689de48a 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -10,11 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - include: - - os: macos-latest - ENV_FILE: install/envs/mac.yml - - os: ubuntu-latest - ENV_FILE: install/envs/ubuntu.yml + os: ["macos-latest", "ubuntu-latest"] fail-fast: false defaults: run: @@ -26,15 +22,10 @@ jobs: uses: conda-incubator/setup-miniconda@v2 with: python-version: 3.9 - mamba-version: 1.3 # "*" activate-environment: donkey - environment-file: ${{matrix.ENV_FILE}} auto-activate-base: false - channels: default, conda-forge, pytorch - channel-priority: true - - name: Conda info and list - run: | - echo Environment file ${{matrix.ENV_FILE}} + - name: Conda install and list + run: Conda list conda info conda list - name: Install donkey From c485045a22c8ce388a1c25db0d3ecb51e67e7025 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sun, 16 Jul 2023 12:59:54 +0100 Subject: [PATCH 13/34] Try fixing CI --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index f689de48a..4380f8212 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -25,7 +25,7 @@ jobs: activate-environment: donkey auto-activate-base: false - name: Conda install and list - run: Conda list + run: | conda info conda list - name: Install donkey From f8934734952a9441b13251265693cb3ec191974a Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sun, 16 Jul 2023 13:02:58 +0100 Subject: [PATCH 14/34] Try fixing CI --- .github/workflows/python-package-conda.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 4380f8212..eeda61229 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -29,6 +29,8 @@ jobs: conda info conda list - name: Install donkey - run: pip install -e .[pc] + run: | + pip install -e .[pc,dev] + pip list - name: Run tests run: pytest From dda3c49c42ab7cdeb8e6dbf8e442063dedfb1ebe Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sun, 16 Jul 2023 13:09:26 +0100 Subject: [PATCH 15/34] Make torch import local in test --- donkeycar/tests/test_torch.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/donkeycar/tests/test_torch.py b/donkeycar/tests/test_torch.py index 5f9cb38ad..75d8e3f2d 100644 --- a/donkeycar/tests/test_torch.py +++ b/donkeycar/tests/test_torch.py @@ -2,10 +2,9 @@ import tarfile import os import platform -from collections import defaultdict, namedtuple +from collections import namedtuple + -import torch -import pytorch_lightning as pl from donkeycar.parts.pytorch.torch_train import train from donkeycar.parts.pytorch.torch_data import TorchTubDataModule from donkeycar.parts.pytorch.torch_utils import get_model_by_type @@ -68,6 +67,7 @@ def test_train(config: Config, car_dir: str, data: Data) -> None: :param data: test case data :return: None """ + def pilot_path(name): pilot_name = f'pilot_{name}.ckpt' return os.path.join(car_dir, 'models', pilot_name) @@ -95,6 +95,9 @@ def test_training_pipeline(config: Config, model_type: str, car_dir: str) \ :param tub_dir: tub directory (car_dir/tub) :return: None """ + import torch + import pytorch_lightning as pl + model = get_model_by_type( model_type, config, checkpoint_path=None) From afa68b74eedbc6e1ff04bcb8b845d74e431775c4 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sun, 16 Jul 2023 16:22:16 +0100 Subject: [PATCH 16/34] Make torch import local in test --- donkeycar/tests/test_torch.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/donkeycar/tests/test_torch.py b/donkeycar/tests/test_torch.py index 75d8e3f2d..3137e50dd 100644 --- a/donkeycar/tests/test_torch.py +++ b/donkeycar/tests/test_torch.py @@ -3,12 +3,6 @@ import os import platform from collections import namedtuple - - -from donkeycar.parts.pytorch.torch_train import train -from donkeycar.parts.pytorch.torch_data import TorchTubDataModule -from donkeycar.parts.pytorch.torch_utils import get_model_by_type - from donkeycar.config import Config Data = namedtuple('Data', ['type', 'name', 'convergence', 'pretrained']) @@ -67,6 +61,7 @@ def test_train(config: Config, car_dir: str, data: Data) -> None: :param data: test case data :return: None """ + from donkeycar.parts.pytorch.torch_train import train def pilot_path(name): pilot_name = f'pilot_{name}.ckpt' @@ -97,6 +92,8 @@ def test_training_pipeline(config: Config, model_type: str, car_dir: str) \ """ import torch import pytorch_lightning as pl + from donkeycar.parts.pytorch.torch_data import TorchTubDataModule + from donkeycar.parts.pytorch.torch_utils import get_model_by_type model = get_model_by_type( model_type, config, checkpoint_path=None) @@ -123,11 +120,11 @@ def test_training_pipeline(config: Config, model_type: str, car_dir: str) \ x, y = batch # In order to use a model pre-trained on ImageNet, the image # will be re-sized to 3x224x224 regardless of what the user chooses. - assert(x.shape == (config.BATCH_SIZE, 3, 224, 224)) - assert(y.shape == (config.BATCH_SIZE, 2)) + assert x.shape == (config.BATCH_SIZE, 3, 224, 224), "shape mismatch" + assert y.shape == (config.BATCH_SIZE, 2), "shape mismatch" break # Check inference val_x, val_y = next(iter(data_module.val_dataloader())) output = model(val_x) - assert(output.shape == (config.BATCH_SIZE, 2)) + assert output.shape == (config.BATCH_SIZE, 2), "shape mismatch" From a04b78c2bc39754d5ae898641fb3cc7eac36643b Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sun, 16 Jul 2023 16:31:56 +0100 Subject: [PATCH 17/34] Fixed tubwriter test after updating of session_id to be a tuple --- donkeycar/tests/test_tubwriter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/donkeycar/tests/test_tubwriter.py b/donkeycar/tests/test_tubwriter.py index 8749b65a9..3b43be4d5 100644 --- a/donkeycar/tests/test_tubwriter.py +++ b/donkeycar/tests/test_tubwriter.py @@ -26,7 +26,7 @@ def test_tubwriter_sessions(self): total = 0 for record in tub_writer.tub: print(f'Record: {record}') - session_number = int(record['_session_id'].split('_')[1]) + session_number = int(record['_session_id'][0]) self.assertEqual(session_number, id, 'Session id not correctly generated') total += 1 From 24f746927ba0a8cbb53a39719ca46f6770ccee60 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sun, 16 Jul 2023 16:39:29 +0100 Subject: [PATCH 18/34] Fixed tub and test_tubwriter.py --- donkeycar/parts/tub_v2.py | 3 +-- donkeycar/tests/test_tubwriter.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/donkeycar/parts/tub_v2.py b/donkeycar/parts/tub_v2.py index 1a804379b..d33b353ed 100644 --- a/donkeycar/parts/tub_v2.py +++ b/donkeycar/parts/tub_v2.py @@ -78,8 +78,7 @@ def write_record(self, record=None): # Private properties contents['_timestamp_ms'] = int(round(time.time() * 1000)) contents['_index'] = self.manifest.current_index - contents['_session_id'] = self.manifest.session_id - + contents['_session_id'] = self.manifest.session_id[1] self.manifest.write_record(contents) def delete_records(self, record_indexes): diff --git a/donkeycar/tests/test_tubwriter.py b/donkeycar/tests/test_tubwriter.py index 3b43be4d5..8749b65a9 100644 --- a/donkeycar/tests/test_tubwriter.py +++ b/donkeycar/tests/test_tubwriter.py @@ -26,7 +26,7 @@ def test_tubwriter_sessions(self): total = 0 for record in tub_writer.tub: print(f'Record: {record}') - session_number = int(record['_session_id'][0]) + session_number = int(record['_session_id'].split('_')[1]) self.assertEqual(session_number, id, 'Session id not correctly generated') total += 1 From 568c0e68a1a9335ce44a9e6677c6aec0bf744cb7 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sun, 16 Jul 2023 16:55:27 +0100 Subject: [PATCH 19/34] Add windows to CI platform --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index eeda61229..43b26f860 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: ["macos-latest", "ubuntu-latest"] + os: ["macos-latest", "ubuntu-latest", "windows-latest"] fail-fast: false defaults: run: From 1ac9b2de830d124594f4a37d332608ad41dfe8fb Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sun, 16 Jul 2023 17:24:46 +0100 Subject: [PATCH 20/34] Revert "Add windows to CI platform" This reverts commit 568c0e68a1a9335ce44a9e6677c6aec0bf744cb7. --- .github/workflows/python-package-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 43b26f860..eeda61229 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -10,7 +10,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: ["macos-latest", "ubuntu-latest", "windows-latest"] + os: ["macos-latest", "ubuntu-latest"] fail-fast: false defaults: run: From 52d5df6dc2317c1f2bc39192e2d1bb7dabed6770 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Mon, 21 Aug 2023 22:23:11 +0100 Subject: [PATCH 21/34] Try to manage keras-vis but not working yet. --- donkeycar/__init__.py | 2 +- donkeycar/management/makemovie.py | 18 ++++++++++++++---- setup.cfg | 1 + 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index 49197c499..f86ba1739 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -3,7 +3,7 @@ from pyfiglet import Figlet import logging -__version__ = '5.1.dev12' +__version__ = '5.1.dev13' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/donkeycar/management/makemovie.py b/donkeycar/management/makemovie.py index dea90616e..338f5e995 100755 --- a/donkeycar/management/makemovie.py +++ b/donkeycar/management/makemovie.py @@ -4,10 +4,7 @@ import tensorflow as tf import cv2 from matplotlib import cm -try: - from vis.utils import utils -except: - raise Exception("Please install keras-vis: pip install git+https://github.com/autorope/keras-vis.git") + import donkeycar as dk from donkeycar.parts.tub_v2 import Tub @@ -167,6 +164,12 @@ def draw_steering_distribution(self, img, img_drawon): def init_salient(self, model): # Utility to search for layer index by name. # Alternatively we can specify this as -1 since it corresponds to the last layer. + try: + from vis.utils import utils + except ImportError: + raise Exception( + "Please install keras-vis: pip install " + "git+https://github.com/autorope/keras-vis.git") output_name = [] layer_idx = [] for i, layer in enumerate(model.layers): @@ -191,6 +194,13 @@ def init_salient(self, model): return True def compute_visualisation_mask(self, img): + try: + from vis.utils import utils + except ImportError: + raise Exception( + "Please install keras-vis: pip install " + "git+https://github.com/autorope/keras-vis.git") + img = img.reshape((1,) + img.shape) images = tf.Variable(img, dtype=float) diff --git a/setup.cfg b/setup.cfg index 10b27d1d3..0929c5052 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,6 +77,7 @@ pc = pandas plotly albumentations + keras-vis @ git+https://github.com/autorope/keras-vis.git dev = pytest From 28cc5fdf7100e070dc60fa47e022fcdcc9049683 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Tue, 22 Aug 2023 21:58:42 +0100 Subject: [PATCH 22/34] Try to manage keras-vis but not working yet. --- donkeycar/management/makemovie.py | 74 +++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 18 deletions(-) diff --git a/donkeycar/management/makemovie.py b/donkeycar/management/makemovie.py index 338f5e995..f02f34c4a 100755 --- a/donkeycar/management/makemovie.py +++ b/donkeycar/management/makemovie.py @@ -1,6 +1,8 @@ +import tempfile from tensorflow.python.keras import activations from tensorflow.python.keras import backend as K +from tensorflow.python.keras.models import load_model import tensorflow as tf import cv2 from matplotlib import cm @@ -14,6 +16,52 @@ DEG_TO_RAD = math.pi / 180.0 +def apply_modifications(model, custom_objects=None): + """Applies modifications to the model layers to create a new Graph. For + example, simply changing `model.layers[idx].activation = new activation` + does not change the graph. The entire graph needs to be updated with + modified inbound and outbound tensors because of change in layer building + function. + + Args: + model: The `keras.models.Model` instance. + + Returns: + The modified model with changes applied. Does not mutate the original + `model`. + """ + # The strategy is to save the modified model and load it back. This is + # done because setting the activation in a Keras layer doesnt actually + # change the graph. We have to iterate the entire graph and change the + # layer inbound and outbound nodes with modified tensors. This is doubly + # complicated in Keras 2.x since multiple inbound and outbound nodes are + # allowed with the Graph API. + model_path = os.path.join(tempfile.gettempdir(), + next(tempfile._get_candidate_names()) + '.h5') + try: + model.save(model_path) + return load_model(model_path, custom_objects=custom_objects) + finally: + os.remove(model_path) + + +def normalize(array, min_value=0., max_value=1.): + """Normalizes the numpy array to (min_value, max_value) + + Args: + array: The numpy array + min_value: The min value in normalized array (Default value = 0) + max_value: The max value in normalized array (Default value = 1) + + Returns: + The array normalized to range between (min_value, max_value) + """ + arr_min = np.min(array) + arr_max = np.max(array) + normalized = (array - arr_min) / (arr_max - arr_min + K.epsilon()) + return (max_value - min_value) * normalized + min_value + + class MakeMovie(object): def run(self, args, parser): @@ -42,15 +90,18 @@ def run(self, args, parser): if args.type is None and args.model is not None: args.type = self.cfg.DEFAULT_MODEL_TYPE - print("Model type not provided. Using default model type from config file") + print("Model type not provided. Using default model type from " + "config file") if args.salient: if args.model is None: - print("ERR>> salient visualization requires a model. Pass with the --model arg.") + print("ERR>> salient visualization requires a model. Pass " + "with the --model arg.") parser.print_help() if args.type not in ['linear', 'categorical']: - print("Model type {} is not supported. Only linear or categorical is supported for salient visualization".format(args.type)) + print(f"Model type {args.type} is not supported. Only linear " + f"or categorical is supported for salient visualization") parser.print_help() return @@ -164,12 +215,6 @@ def draw_steering_distribution(self, img, img_drawon): def init_salient(self, model): # Utility to search for layer index by name. # Alternatively we can specify this as -1 since it corresponds to the last layer. - try: - from vis.utils import utils - except ImportError: - raise Exception( - "Please install keras-vis: pip install " - "git+https://github.com/autorope/keras-vis.git") output_name = [] layer_idx = [] for i, layer in enumerate(model.layers): @@ -189,18 +234,11 @@ def init_salient(self, model): for li in layer_idx: model.layers[li].activation = activations.linear # build salient model and optimizer - sal_model = utils.apply_modifications(model) + sal_model = apply_modifications(model) self.sal_model = sal_model return True def compute_visualisation_mask(self, img): - try: - from vis.utils import utils - except ImportError: - raise Exception( - "Please install keras-vis: pip install " - "git+https://github.com/autorope/keras-vis.git") - img = img.reshape((1,) + img.shape) images = tf.Variable(img, dtype=float) @@ -225,7 +263,7 @@ def compute_visualisation_mask(self, img): channel_idx = 1 if K.image_data_format() == 'channels_first' else -1 grads = np.sum(grads, axis=channel_idx) - res = utils.normalize(grads)[0] + res = normalize(grads)[0] return res def draw_salient(self, img): From e057609e5567c968214b1b4bc7fb93b42b9deddb Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Tue, 22 Aug 2023 21:59:03 +0100 Subject: [PATCH 23/34] Try to manage keras-vis but not working yet. --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0929c5052..10b27d1d3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -77,7 +77,6 @@ pc = pandas plotly albumentations - keras-vis @ git+https://github.com/autorope/keras-vis.git dev = pytest From 09b3ed9aaf7dad7c707b05fe279f8fc6d1a233c8 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sat, 25 Nov 2023 19:11:00 +0000 Subject: [PATCH 24/34] Froze nano versions after Tom's finding that the current packages of pandas / matplotlib don't work. Updated version and pushed that to pypi test. --- donkeycar/__init__.py | 2 +- setup.cfg | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index f86ba1739..2b897d4aa 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -3,7 +3,7 @@ from pyfiglet import Figlet import logging -__version__ = '5.1.dev13' +__version__ = '5.1.dev14' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/setup.cfg b/setup.cfg index 10b27d1d3..fd4479816 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,21 +54,15 @@ pi = tensorflow-aarch64==2.9.3 opencv-contrib-python -nano45 = - Adafruit_PCA9685 - adafruit-circuitpython-ssd1306 - adafruit-circuitpython-rplidar - Jetson.GPIO - matplotlib - nano = Adafruit_PCA9685 adafruit-circuitpython-ssd1306 adafruit-circuitpython-rplidar Jetson.GPIO - matplotlib + matplotlib==3.7 kivy-jetson plotly + pandas==2.0 pc = tensorflow==2.9 From 35aae7223291189db4d6f1013706d6deffdb165a Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Sun, 26 Nov 2023 22:13:13 +0000 Subject: [PATCH 25/34] Freezing numpy for nano and pushing to test pypi --- donkeycar/__init__.py | 2 +- setup.cfg | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index 2b897d4aa..74fc1bd31 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -3,7 +3,7 @@ from pyfiglet import Figlet import logging -__version__ = '5.1.dev14' +__version__ = '5.1.dev15' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/setup.cfg b/setup.cfg index fd4479816..652177c95 100644 --- a/setup.cfg +++ b/setup.cfg @@ -59,6 +59,7 @@ nano = adafruit-circuitpython-ssd1306 adafruit-circuitpython-rplidar Jetson.GPIO + numpy==1.23 matplotlib==3.7 kivy-jetson plotly From 0844cd853e263196bdfda4bf31eea3178f6b0688 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Wed, 29 Nov 2023 22:07:16 +0000 Subject: [PATCH 26/34] Added target macos for install on apple arm --- donkeycar/__init__.py | 2 +- setup.cfg | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index 74fc1bd31..f8f02ef13 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -3,7 +3,7 @@ from pyfiglet import Figlet import logging -__version__ = '5.1.dev15' +__version__ = '5.1.dev16' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) diff --git a/setup.cfg b/setup.cfg index 652177c95..87cc4abf4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -73,6 +73,14 @@ pc = plotly albumentations +macos = + tensorflow-macos==2.9 + matplotlib + kivy==2.1 + pandas + plotly + albumentations + dev = pytest pytest-cov From 1d9d6b02e8dd27d3fb36228d6efe129096599fcd Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Tue, 5 Dec 2023 21:00:37 +0000 Subject: [PATCH 27/34] Fix kivy on jetso --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 87cc4abf4..65fd8cea0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -61,7 +61,7 @@ nano = Jetson.GPIO numpy==1.23 matplotlib==3.7 - kivy-jetson + kivy==2.1 plotly pandas==2.0 From 26e96c595a8d208224b044caf2bf0ee568cd7ec1 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Tue, 12 Dec 2023 21:49:19 +0000 Subject: [PATCH 28/34] Set version to right version for main branch commit. --- donkeycar/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index f8f02ef13..69d83cf52 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -3,7 +3,7 @@ from pyfiglet import Figlet import logging -__version__ = '5.1.dev16' +__version__ = '5.1.dev0' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) From eeec2cc79a218bfe08993a51b966e7a3dca63b31 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Tue, 12 Dec 2023 22:07:09 +0000 Subject: [PATCH 29/34] Fix merge, as we missed the latest updates on gh version changes. --- .github/workflows/python-package-conda.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index eeda61229..7c7807fc0 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -17,14 +17,15 @@ jobs: shell: bash -l {0} steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create python 3.9 conda env - uses: conda-incubator/setup-miniconda@v2 + uses: conda-incubator/setup-miniconda@v3 with: python-version: 3.9 + mamba-version: "*" activate-environment: donkey auto-activate-base: false - - name: Conda install and list + - name: Conda info and list run: | conda info conda list From e2b3981d8499db82c9691e8de445f9a904707903 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Tue, 12 Dec 2023 22:28:53 +0000 Subject: [PATCH 30/34] Remove obsolete argument in tub --- donkeycar/parts/datastore_v2.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/donkeycar/parts/datastore_v2.py b/donkeycar/parts/datastore_v2.py index 9bd3d9810..c1ae56120 100644 --- a/donkeycar/parts/datastore_v2.py +++ b/donkeycar/parts/datastore_v2.py @@ -287,12 +287,11 @@ def exit_hook(): # Automatically save config when program ends atexit.register(exit_hook) - def write_record(self, record, index=None): - if index is None: - new_catalog = self.current_index > 0 \ - and (self.current_index % self.max_len) == 0 - if new_catalog: - self._add_catalog() + def write_record(self, record): + new_catalog = self.current_index > 0 \ + and (self.current_index % self.max_len) == 0 + if new_catalog: + self._add_catalog() self.current_catalog.write_record(record) self.current_index += 1 @@ -416,8 +415,6 @@ def add_deleted_indexes(self, indexes): self.deleted_indexes.update(indexes) self._update_catalog_metadata(update=True) - - def close(self): """ Closing tub closes open files for catalog, catalog manifest and manifest.json""" From d75c7e52282e47e48559b5f7ca0024b2b292f224 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Tue, 12 Dec 2023 22:34:48 +0000 Subject: [PATCH 31/34] Add missing linebreak --- donkeycar/templates/cfg_complete.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/donkeycar/templates/cfg_complete.py b/donkeycar/templates/cfg_complete.py index fb31d4a72..464e8837f 100644 --- a/donkeycar/templates/cfg_complete.py +++ b/donkeycar/templates/cfg_complete.py @@ -763,4 +763,4 @@ # PI connection PI_USERNAME = "pi" -PI_HOSTNAME = "donkeypi.local" \ No newline at end of file +PI_HOSTNAME = "donkeypi.local" From e5fb5171930623341501149b04b4503f4a80a747 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Tue, 12 Dec 2023 22:36:52 +0000 Subject: [PATCH 32/34] Add missing linebreak --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 65fd8cea0..294f98074 100644 --- a/setup.cfg +++ b/setup.cfg @@ -99,4 +99,4 @@ torch = [options.entry_points] console_scripts = - donkey = donkeycar.management.base:execute_from_command_line \ No newline at end of file + donkey = donkeycar.management.base:execute_from_command_line From ba2de7f69a65592317983a34ccac4e1675bb519e Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Tue, 12 Dec 2023 22:37:46 +0000 Subject: [PATCH 33/34] Add missing linebreak and remove obsolete copy of old setup file --- setup_orig.py | 131 -------------------------------------------------- 1 file changed, 131 deletions(-) delete mode 100644 setup_orig.py diff --git a/setup_orig.py b/setup_orig.py deleted file mode 100644 index ca9a2b639..000000000 --- a/setup_orig.py +++ /dev/null @@ -1,131 +0,0 @@ -import os - -from setuptools import find_packages, setup - - -# include the non python files -def package_files(directory, strip_leading): - paths = [] - for (path, directories, filenames) in os.walk(directory): - for filename in filenames: - package_file = os.path.join(path, filename) - paths.append(package_file[len(strip_leading):]) - return paths - - -car_templates = ['templates/*'] -web_controller_html = package_files('donkeycar/parts/controllers/templates', - 'donkeycar/') - -extra_files = car_templates + web_controller_html -print('extra_files', extra_files) - -with open("README.md", "r") as fh: - long_description = fh.read() - -setup(name='donkeycar', - version="5.0.dev3", - long_description=long_description, - description='Self driving library for python.', - url='https://github.com/autorope/donkeycar', - author='Will Roscoe, Adam Conway, Tawn Kramer', - author_email='wroscoe@gmail.com, adam@casaconway.com, tawnkramer@gmail.com', - license='MIT', - entry_points={ - 'console_scripts': [ - 'donkey=donkeycar.management.base:execute_from_command_line', - ], - }, - install_requires=[ - 'numpy', - 'pillow', - 'docopt', - 'tornado', - 'requests', - 'h5py', - 'PrettyTable', - 'paho-mqtt', - "simple_pid", - 'progress', - 'typing_extensions', - 'pyfiglet', - 'psutil', - "pynmea2", - 'pyserial', - "utm", - 'pandas', - 'pyyaml', - ], - extras_require={ - # if installing into a conda (i.e. miniforge) env on Pi we have to - # run 'sudo apt-get install libcap-dev' first. - 'pi': [ - 'picamera2', - 'Adafruit_PCA9685', - 'adafruit-circuitpython-ssd1306', - 'adafruit-circuitpython-rplidar', - 'RPi.GPIO', - 'tensorflow @ https://github.com/PINTO0309/Tensorflow-bin/releases/download/v2.9.0/tensorflow-2.9.0-cp39-none-linux_aarch64.whl' - ], - 'nano45': [ - 'Adafruit_PCA9685', - 'adafruit-circuitpython-ssd1306', - 'adafruit-circuitpython-rplidar', - 'Jetson.GPIO', - 'matplotlib', - ], - 'nano': [ - 'Adafruit_PCA9685', - 'adafruit-circuitpython-ssd1306', - 'adafruit-circuitpython-rplidar', - 'Jetson.GPIO', - 'matplotlib', - 'kivy-jetson', - 'plotly' - ], - 'pc': [ - 'matplotlib', - 'kivy', - 'pandas', - 'plotly', - 'albumentations' - ], - 'dev': [ - 'pytest', - 'pytest-cov', - 'responses', - 'mypy' - ], - 'ci': ['codecov'], - 'tf': ['tensorflow==2.9'], - 'torch': [ - 'pytorch', - 'torchvision==0.12', - 'torchaudio', - 'fastai' - ], - 'mm1': ['pyserial'] - }, - package_data={ - 'donkeycar': extra_files, - }, - include_package_data=True, - classifiers=[ - # How mature is this project? Common values are - # 3 - Alpha - # 4 - Beta - # 5 - Production/Stable - 'Development Status :: 4 - Beta', - # Indicate who your project is intended for - 'Intended Audience :: Developers', - 'Topic :: Scientific/Engineering :: Artificial Intelligence', - # Pick your license as you wish (should match "license" above) - 'License :: OSI Approved :: MIT License', - # Specify the Python versions you support here. In particular, ensure - # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - ], - keywords='selfdriving cars donkeycar diyrobocars', - packages=find_packages(exclude=(['tests', 'docs', 'site', 'env'])), - ) From fea492783b9fe982e952e811359eca564d6da003 Mon Sep 17 00:00:00 2001 From: DocGarbanzo Date: Wed, 13 Dec 2023 20:41:21 +0000 Subject: [PATCH 34/34] Add missing linebreak and remove obsolete copy of old setup file --- .github/workflows/python-package-conda.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml index 7c7807fc0..6b0311720 100644 --- a/.github/workflows/python-package-conda.yml +++ b/.github/workflows/python-package-conda.yml @@ -25,6 +25,8 @@ jobs: mamba-version: "*" activate-environment: donkey auto-activate-base: false + channels: default, conda-forge, pytorch + channel-priority: true - name: Conda info and list run: | conda info