diff --git a/.gitignore b/.gitignore index e51bc55f..569f64af 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ wheels/ .installed.cfg *.egg + # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. @@ -68,6 +69,8 @@ instance/ # Sphinx documentation docs/_build/ +docs/source/API Reference +docs/source/Examples # Pickles *.pkl @@ -81,7 +84,6 @@ target/ # PDFs and Images *.pdf *.jpg -*.png # pyenv .python-version diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..050e0210 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,19 @@ +cff-version: 1.2.0 +message: "If you use this software, please cite it as below." +authors: +- family-names: "Stephenson" + given-names: "Mark" + # orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Mantovani" + given-names: "Lorenzzo" + # orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Herrmann" + given-names: "Adam" + # orcid: "https://orcid.org/0000-0000-0000-0000" +- family-names: "Schaub" + given-names: "Hanspeter" + # orcid: "https://orcid.org/0000-0000-0000-0000" +title: "BSK-RL" +version: 0.0.0 +date-released: 2023 +url: "https://github.com/AVSLab/bsk_rl/" diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..8ac15275 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,28 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +clean: + rm -rf "source/Examples" "source/API Reference" + + +view: + # works on macOS + open build/html/index.html \ No newline at end of file diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 00000000..747ffb7b --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/_images/static/Basilisk-Logo.png b/docs/source/_images/static/Basilisk-Logo.png new file mode 100644 index 00000000..d0b4cbd5 Binary files /dev/null and b/docs/source/_images/static/Basilisk-Logo.png differ diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 00000000..27004ba8 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,209 @@ +div.textblock a, div.textblock a:visited, p a, p a:visited, +a, a:hover, a:visited, +a.reference, a.internal, a.current { + color: #cfb87c; + font-weight: bold; +} + +a.icon-home, a.icon-home:visited { + color: #666; +} + +div.wy-menu > p.caption > span.caption-text { + color: white; + font-weight: bold; +} +a.el, a.el:visited { + color: #565A5C; +} + +.wy-side-nav-search>div.version { + color: #565A5C; # CU dark gray +} + +.wy-nav-content { + max-width: none; +} + +code span.pre, code { + color: #cb7ccf; +} + +th.head, .wy-nav-top { + color: white; + background-color: #565A5C; +} + +th.head p { + margin: 0px; +} + +tr td p, th.head p { + font-size: small; +} + +img.logo { + filter: drop-shadow(0px 0px 8px #fff); +} +/* override table no-wrap */ +.wy-table-responsive table td, .wy-table-responsive table th { + white-space: normal; +} + +.math { + text-align: left; +} +.eqno { + float: right; +} + +body, h1, h2, h3, h4, h5, .rst-content, .sidebar, .sidebar-title, p.caption { + font-family: system-ui, -apple-system, BlinkMacSystemFont, "Helvetica Neue", + "Lucida Grande", "Segoe UI" !important; +} + +:root { + color-scheme: light dark; +} + +.sidebar { + max-width: 500px; +} + +code, .rst-content tt, .rst-content code { + color: #E74C3C; +} + +ul.simple li, aside.sidebar ul li { + all: revert; +} + +ul.simple li p, aside.sidebar ul li p { + all: revert; + line-height: 24px; + font-size: 16px; + margin: 0px; +} + +ul.simple ul li { + list-style-type: circle; + margin-left: 1.5em; +} + +ul.simple, aside.sidebar ul { + all: revert; + padding-left: 1.5em; +} + +figure { + text-align: center; +} + +@media (prefers-color-scheme: dark) { + .wy-nav-content, .wy-body-for-nav, .wy-nav-content-wrap, math, span[id*='MathJax-Span'] { + background-color: black; + color: #ccc; + } + + .highlight .go { + color: #ccc; + } + + img.logo { + filter: drop-shadow(0px 0px 8px #000); + } + + .rst-content code { + background: #0004; + } + html.writer-html4 .rst-content dl:not(.docutils)>dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.glossary):not(.simple)>dt, .rst-content .note { + background: #2D4151; + } + + .rst-content .warning, .rst-content .caution, .rst-content .attention { + background: #51402F; + } + + .rst-content .important, .rst-content .hint, .rst-content .tip { + background: #275145; + } + + .rst-content .danger, .rst-content .error { + background: #523A37; + } + + /* sidebar formatting */ + .sidebar { + border-color: #666; + } + .rst-content .sidebar { + background: #222; + } + .rst-content .sidebar .sidebar-title { + background: #666; + } + + + .btn-neutral, .btn-neutral:hover, .btn:visited { + background: #333 !important; + color: #ccc !important; + } + + .highlight { + background: #4448; + } + + .rst-content table.docutils:not(.field-list) tr:nth-child(2n-1) td { + background-color: #222; + } + + .rst-content pre.literal-block, .rst-content div[class^='highlight'], .rst-content code, .rst-content table.docutils, .wy-table thead th, .rst-content table.docutils thead th, .rst-content table.field-list thead th, .wy-table-bordered-all td, .rst-content table.docutils td { + border-color: gray; + } + + img[src$="svg"] { + background-color: white; + filter: invert(100%) hue-rotate(180deg) saturate(200%);; + } + + img[src$="jpg"], img[src$="png"] { + border-radius: 5px; + } + + .rst-content dl:not(.docutils) dt { + background-color: #2D4151; + } + + .rst-content dl:not(.docutils) { + padding-top: 1em; + } + + .rst-content dl:not(.docutils) code.descclassname, .rst-content dl:not(.docutils) code.descname { + color: #ccc; + } + + html.writer-html4 .rst-content dl:not(.docutils) dl:not(.field-list)>dt, html.writer-html5 .rst-content dl[class]:not(.option-list):not(.field-list):not(.footnote):not(.glossary):not(.simple) dl:not(.field-list)>dt, .rst-content dl:not(.docutils) dl dt { + background-color: #333; + color: #ccc; + } + + .wy-table caption, .rst-content table.docutils caption, .rst-content table.field-list caption { + color: inherit; + } + + span.vm, span.nf, span.nn { + color:#66f !important; + } + + span.normal { + color: #333 !important; + } + + td.linenos pre { + background: #ccc !important; + } + + .rst-content .highlighted { + background-color: #333; + } +} \ No newline at end of file diff --git a/docs/source/citation.rst b/docs/source/citation.rst new file mode 100644 index 00000000..96cad7a5 --- /dev/null +++ b/docs/source/citation.rst @@ -0,0 +1,21 @@ +Citation +======== +If you use this code in your research, please cite the repository as follows: + +APA +--- +.. code-block:: + + Stephenson, M., Mantovani, L., Herrmann, A., & Schaub, H. BSK-RL (Version 0.0.0) [Computer software]. https://github.com/AVSLab/bsk_rl/ + +BibTeX +------ +.. code-block:: + + @software{ + Stephenson_BSK-RL, + author = {Stephenson, Mark and Mantovani, Lorenzzo and Herrmann, Adam and Schaub, Hanspeter}, + title = {{BSK-RL}}, + url = {https://github.com/AVSLab/bsk_rl/}, + version = {0.0.0} + } \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..388ede78 --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,233 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + + +import datetime + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information +import os +import re +import sys +from pathlib import Path + +# sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "src"))) +now = datetime.datetime.now() + +project = "BSK-RL" +copyright = str(now.year) + ", Autonomous Vehicle Systems (AVS) Laboratory" +author = "Mark Stephenson" +release = "0.0.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "sphinx.ext.duration", + "sphinx.ext.doctest", + "sphinx.ext.todo", + "sphinx.ext.autodoc", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", + "sphinx_rtd_theme", +] + +templates_path = ["_templates"] +exclude_patterns = [] +source_suffix = ".rst" +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = "en" + + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "sphinx_rtd_theme" +html_theme_options = { + "style_nav_header_background": "#CFB87C", + "navigation_depth": -1, +} +html_static_path = ["_static"] +html_css_files = ["custom.css"] +html_logo = "./_images/static/Basilisk-Logo.png" + +add_module_names = False + + +def skip(app, what, name, obj, would_skip, options): + if name == "__init__": + return False + return would_skip + + +def setup(app): + app.connect("autodoc-skip-member", skip) + + +class FileCrawler: + def __init__(self, base_source_dir, base_doc_dir): + self.base_source_dir = base_source_dir + self.base_doc_dir = base_doc_dir + + def grab_files(self, dir_path): + dirs_in_dir = [x for x in dir_path.iterdir() if x.is_dir()] + files_in_dir = dir_path.glob("*.py") + + # Remove any directories that shouldn't be added directly to the website + dir_filters = [ + r".*__pycache__.*", + r".*\.ruff_cache.*", + r".*\.egg-info", + r".*\/simplemaps_worldcities", + ] + dirs_in_dir = list( + filter( + lambda dir: not any( + re.match(filter, str(dir)) for filter in dir_filters + ), + dirs_in_dir, + ) + ) + + file_filters = [ + r".*__init__\.py", + r"(.*\/|)_[a-zA-Z0-9_]*\.py", + ] + files_in_dir = list( + filter( + lambda file: not any( + re.match(filter, str(file)) for filter in file_filters + ), + files_in_dir, + ) + ) + + return sorted(list(files_in_dir)), sorted(list(dirs_in_dir)) + + def populate_doc_index(self, index_path, file_paths, dir_paths, source_dir): + name = index_path.stem + lines = "" + + # if a _default.rst file exists in a folder, then use it to generate the index.rst file + try: + docFileName = source_dir / "_default.rst" + with open(docFileName, "r") as docFile: + docContents = docFile.read() + lines += docContents + "\n\n" + except FileNotFoundError: # Auto-generate the index.rst file + # add page tag + qual_name = str( + source_dir.relative_to(self.base_source_dir.parent) + ).replace("/", ".") + lines += ".. _" + qual_name.replace(" ", "_") + ":\n\n" + + # Title the page + lines += name + "\n" + "=" * len(name) + "\n\n" + lines += f"``{qual_name}``\n\n" + + # pull in folder _doc.rst file if it exists + try: + docFileName = source_dir / "_doc.rst" + if os.path.isfile(docFileName): + with open(docFileName, "r") as docFile: + docContents = docFile.read() + lines += docContents + "\n\n" + except FileNotFoundError: + pass + + # Also check for docs in the __init__.py file + lines += ( + """.. automodule:: """ + + qual_name + + """\n :members:\n :show-inheritance:\n\n""" + ) + + # Add a linking point to all local files + lines += ( + """\n\n.. toctree::\n :maxdepth: 1\n :caption: """ + "Files:\n\n" + ) + added_names = [] + for file_path in sorted(file_paths): + file_name = os.path.basename(os.path.normpath(file_path)) + file_name = file_name[: file_name.rfind(".")] + + if file_name not in added_names: + lines += " " + file_name + "\n" + added_names.append(file_name) + lines += "\n" + + # Add a linking point to all local directories + lines += ( + """.. toctree::\n :maxdepth: 1\n :caption: """ + "Directories:\n\n" + ) + + for dir_path in sorted(dir_paths): + dirName = os.path.basename(os.path.normpath(dir_path)) + lines += " " + dirName + "/index\n" + + with open(os.path.join(index_path, "index.rst"), "w") as f: + f.write(lines) + + def generate_autodoc(self, doc_path, source_file): + short_name = source_file.name.replace(".py", "") + qual_name = ( + str(source_file.relative_to(self.base_source_dir.parent)) + .replace("/", ".") + .replace(".py", "") + ) + + # Generate the autodoc file + lines = ".. _" + qual_name + ":\n\n" + lines += short_name + "\n" + "=" * len(short_name) + "\n\n" + lines += f"``{qual_name}``\n\n" + lines += """.. toctree::\n :maxdepth: 1\n :caption: """ + "Files" + ":\n\n" + lines += ( + """.. automodule:: """ + + qual_name + + """\n :members:\n :show-inheritance:\n\n""" + ) + + # Write to file + with open(doc_path / f"{short_name}.rst", "w") as f: + f.write(lines) + + def run(self, source_dir=None): + if source_dir is None: + source_dir = self.base_source_dir + + file_paths, dir_paths = self.grab_files(source_dir) + index_path = source_dir.relative_to(self.base_source_dir) + + # Populate the index.rst file of the local directory + os.makedirs(self.base_doc_dir / index_path, exist_ok=True) + self.populate_doc_index( + self.base_doc_dir / index_path, file_paths, dir_paths, source_dir + ) + + # Generate the correct auto-doc function for python modules + for file in file_paths: + self.generate_autodoc( + self.base_doc_dir / index_path, + file, + ) + + # Recursively go through all directories in source, documenting what is available. + for dir_path in sorted(dir_paths): + self.run( + source_dir=dir_path, + ) + + return + + +sys.path.append(os.path.abspath("../..")) +FileCrawler(Path("../../src/bsk_rl/"), Path("./API Reference/")).run() +FileCrawler(Path("../../examples"), Path("./Examples/")).run() diff --git a/docs/source/docsRequired.txt b/docs/source/docsRequired.txt new file mode 100644 index 00000000..438b963c --- /dev/null +++ b/docs/source/docsRequired.txt @@ -0,0 +1 @@ +sphinx_rtd_theme==2.0.0 \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..b258f077 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,90 @@ +BSK-RL: Environments and Algorithms for Spacecraft Planning and Scheduling +========================================================================== + +.. toctree:: + :hidden: + + install + Examples/index + API Reference/index + publications + citation + GitHub + + +.. note:: + + BSK-RL and its documentation are under active development. Please continue to check back for updates. + + New environments should be built using the :ref:`general satellite tasking framework `; legacy environments are in the process of being ported to this framework. + + +**BSK-RL** (`Basilisk `_ + `Reinforcement Learning `_) is a Python package for constructing `Gymnasium `_ environments for spacecraft tasking problems. It is built on top of `Basilisk `_, a modular and fast spacecraft simulation framework, making the simulation environments high-fidelity and computationally efficient. BSK-RL also includes a collection of agents, training scripts, and examples for working with these environments. + +Quickstart +---------- +Installation +^^^^^^^^^^^^ +Complete installation instructions and common troubleshooting tips can be found :doc:`here `. To install BSK-RL: + +#. Install the `Basilisk `_ spacecraft simulation framework. +#. Clone BSK-RL. + + .. code-block:: console + + $ git clone git@github.com:AVSLab/bsk_rl.git && cd bsk_rl + +#. Install BSK-RL in the same virtual environment as Basilisk. + + .. code-block:: console + + (.venv) $ python -m pip install -e . && finish_install + +#. Test the installation. + + .. code-block:: console + + (.venv) $ pytest ./tests/examples + +Construct an Environment +^^^^^^^^^^^^^^^^^^^^^^^^ +TODO: Add more detail to this example + +.. code-block:: python + + import gymnasium as gym + + from bsk_rl.envs.general_satellite_tasking.scenario import data + from bsk_rl.envs.general_satellite_tasking.scenario import satellites as sats + from bsk_rl.envs.general_satellite_tasking.scenario.environment_features import StaticTargets + from bsk_rl.envs.general_satellite_tasking.simulation import environment + from bsk_rl.envs.general_satellite_tasking.utils.orbital import random_orbit + + env = gym.make( + "SingleSatelliteTasking-v1", + satellites=sats.FullFeaturedSatellite( + "EO1", + sats.FullFeaturedSatellite.default_sat_args(oe=random_orbit), n_ahead_observe=30, + n_ahead_act=15 + ), + env_type=environment.GroundStationEnvModel, + env_args=environment.GroundStationEnvModel.default_env_args(), + env_features=StaticTargets(n_targets=1000), + data_manager=data.UniqueImagingManager, + max_step_duration=600.0, + time_limit=5700.0, + terminate_on_time_limit=True, + ) + +Train an Agent +^^^^^^^^^^^^^^ +Show RLLib or SB3 configs here. + + +Acknowledgements +---------------- +BSK-RL is developed by the `Autonomous Vehicle Systems (AVS) Lab `_ at the University of Colorado Boulder. The AVS Lab is part of the `Colorado Center for Astrodynamics Research (CCAR) `_ and the `Department of Aerospace Engineering Sciences `_. + +Development has been supported by NASA Space Technology Graduate Research Opportunity (NSTGRO) grants, 80NSSC20K1162 and 80NSSC23K1182. This work has also been supported by Air Force Research Lab grant FA9453-22-2-0050. + +Development of this software has utilized the Alpine high performance computing resource at the University of Colorado Boulder. Alpine is jointly funded by the University of Colorado Boulder, the University of Colorado Anschutz, and Colorado State University. \ No newline at end of file diff --git a/docs/source/install.rst b/docs/source/install.rst new file mode 100644 index 00000000..2d85ca5f --- /dev/null +++ b/docs/source/install.rst @@ -0,0 +1,50 @@ +Installation +============ + +.. toctree:: + :maxdepth: 1 + + +Instructions +------------ +#. Install the `Basilisk `_ spacecraft simulation framework, following instructions for the appropriate operating system. Installation on MacOS and Linux is preferable to Windows. Use a Python virtual environment as suggested in the Basilisk installation instructions. +#. Clone the BSK-RL repository. + + .. code-block:: console + + $ git clone git@github.com:AVSLab/bsk_rl.git + +#. Move to the base directory of the repository. + + .. code-block:: console + + $ cd bsk_rl + +#. Ensure that the virtual environment Basilisk is installed in is active. Install BSK-RL with the following command. + + .. code-block:: console + + (.venv) $ python -m pip install -e . && finish_install + + The first half of this command will install ``pip`` dependencies and an editable copy of the BSK-RL package. ``finish_install`` downloads data dependencies and other packages not available through ``pip``. The installation of Basilisk is also verified at this step. + +#. Test the installation by running the example scripts from the base directory. + + .. code-block:: console + + (.venv) $ pytest tests/examples + + For additional verification, the unit tests and integration tests can also be executed. + + .. code-block:: console + + (.venv) $ pytest tests/unittest + (.venv) $ pytest tests/integration + + +Common Issues +------------- + +.. note:: + + See `#51 `_ for issues with `chebpy` installation on Silicon Macs during or after ``finish_install``. diff --git a/docs/source/publications.rst b/docs/source/publications.rst new file mode 100644 index 00000000..8c842451 --- /dev/null +++ b/docs/source/publications.rst @@ -0,0 +1,30 @@ +Publications +============ +The following publications have been produced using BSK-RL and its predecessors. If you have used BSK-RL in your research, please let us know and we will add your publication to the list. + + +Journal Papers +-------------- +#. \A. P. Herrmann and H. Schaub, `"Monte Carlo Tree Search Methods for the Earth-Observing Satellite Scheduling Problem," `_ Journal of Aerospace Information Systems, Vol. 19, No. 1, January 2022, pp. 70–82. doi:10.2514/1.I010992 +#. \A. Harris, T. Valade, T. Teil and H. Schaub, `“Generation of Spacecraft Operations Procedures Using Deep Reinforcement Learning,” `_ Journal of Spacecraft and Rockets, Vol. 59, No. 2, March–April 2022, pp. 611–626. doi:10.2514/1.A35169. +#. \A. P. Herrmann and H. Schaub, `“Reinforcement Learning for the Agile Earth-Observing Satellite Scheduling Problem,” `_ IEEE Transactions of Aerospace and Electronic Systems, Vol. 59, No. 5, Oct. 2023, pp. 5235–5247. doi:10.1109/TAES.2023.3251307. +#. \A. Herrmann and H. Schaub, `“A Comparative Analysis of Reinforcement Learning Algorithms for Earth-Observing Satellite Scheduling,” `_ Frontiers in Space Technologies, November 29, 2023. doi:10.3389/frspt.2023.1263489. +#. \A. Herrmann, M. Stephenson and H. Schaub, “Single-Agent Reinforcement Learning for Scalable Earth-Observing Satellite Constellation Operations,” Journal of Spacecraft and Rockets. doi:doi.org/10.2514/1.A35736. +#. \A. Herrmann and H. Schaub, “Autonomous Small Body Science Operations Using Reinforcement Learning,” Journal of Aerospace Information Systems. + +Conference Papers +----------------- +#. \A. Harris. T. Teil and H. Schaub, `“Spacecraft Decision-Making Autonomy using Deep Reinforcement Learning,” `_ AAS Spaceflight Mechanics Meeting, Maui, Hawaii January 13–17, 2019. +#. \A. Harris and H. Schaub, `“Spacecraft Command and Control with Safety Guarantees using Shielded Deep Reinforcement Learning” `_ AIAA SciTech Forum, Orlando, Florida, Jan. 6–10, 2020. +#. \A. Herrmann and H. Schaub, `“Monte Carlo Tree Search With Value Networks For Autonomous Spacecraft Operations,” `_ AAS/AIAA Astrodynamics Specialist Conference, Lake Tahoe, CA, Aug. 9–13, 2020. +#. \A. Herrmann and H. Schaub, `“Autonomous Spacecraft Tasking using Monte Carlo Tree Search Methods,” `_ AAS/AIAA Space Flight Mechanics Meeting, Charlotte, NC, January 31–February 4, 2021. +#. \A. Herrmann and H. Schaub, `“Autonomous On-board Planning for Earth-orbiting Spacecraft,” `_ IEEE Aerospace Conference, Big Sky, MT, March 5–12, 2022. +#. \I. Nazmy, A. Harris, M. Lahijanian and H. Schaub, `“Shielded Deep Reinforcement Learning for Multi-Sensor Spacecraft Imaging,” `_ American Control Conference, Atlanta, GA, June 8–10, 2022. +#. \A. Herrmann and H. Schaub, `“A Comparison Of Deep Reinforcement Learning Algorithms For Earth-Observing Satellite Scheduling,” `_ AAS/AIAA Spaceflight Mechanics Meeting, Austin, TX, Jan. 15–19, 2023. +#. \V. Bajenaru, A. Herrmann, H. Schaub and S. Philipps, `“Command and Control of Satellite Constellations using Explainable Deep Reinforcement Learning,” `_ AAS Rocky Mountain GN&C Conference, Breckenridge, CO, Feb. 2–8, 2023. +#. \A. Herrmann, M. Stephenson and H. Schaub, `“Reinforcement Learning for Multi-Satellite Agile Earth Observing Scheduling Under Various Communication Assumptions,” `_ AAS Rocky Mountain GN&C Conference, Breckenridge, CO, Feb. 2–8, 2023. +#. \A. Herrmann and H. Schaub, `“Reinforcement Learning For Small Body Science Operations,” `_ AAS Astrodynamics Specialist Conference, Charlotte, North Carolina, August 7–10 2022, Paper No. AAS 22-563. +#. \M. Stephenson and H. Schaub, `“Optimal Target Sequencing In The Agile Earth-Observing Satellite Scheduling Problem Using Learned Dynamics,” `_ AAS/AIAA Astrodynamics Specialist Conference, Big Sky, Montana, August 13–17 2023. +#. \M. Stephenson, L. Quevedo Manotovani, S. Phillips and H. Schaub, `“Using Enhanced Simulation Environments to Accelerate Reinforcement Learning for Long-Duration Satellite Autonomy,” `_ AIAA Science and Technology Forum and Exposition (SciTech), Orlando, Florida, Jan. 8–12, 2024. doi:10.2514/6.2024-0990. +#. \M. Stephenson and H. Schaub, “Autonomous Earth-Observing Satellite Scheduling Using Reinforcement Learning With Event-Based Decision Intervals,” AAS Rocky Mountain GN&C Conference, Breckenridge, CO, Feb. 2–7, 2024. +#. \M. Stephenson, L. Quevedo Mantovani and H. Schaub, “Intent Sharing and Auctions for Emergent Collaboration in Autonomous Earth Observing Constellations,” AAS Rocky Mountain GN&C Conference, Breckenridge, CO, Feb. 2–7, 2024. \ No newline at end of file diff --git a/examples/general_satellite_tasking/multisat_aeos.py b/examples/general_satellite_tasking/multisat_aeos.py index 15b81f2f..04082d71 100644 --- a/examples/general_satellite_tasking/multisat_aeos.py +++ b/examples/general_satellite_tasking/multisat_aeos.py @@ -1,3 +1,10 @@ +""" +Multisat AEOS +============= + +some text here +""" + import gymnasium as gym import numpy as np @@ -9,104 +16,108 @@ from bsk_rl.envs.general_satellite_tasking.simulation import environment from bsk_rl.utilities.initial_conditions import leo_orbit -# This script demonstrates the configuration of an environment with multiple imaging -# satellites. - -# Data environment contains 5000 targets located near random cities, which are -# randomized on reset() -env_features = CityTargets(n_targets=500, location_offset=10e3) -# Data manager records and rewards uniquely imaged targets -data_manager = data.UniqueImagingManager(env_features) - -# Generate orbital parameters for each satellite in the constellation -oes = leo_orbit.walker_delta( - n_spacecraft=3, # Number of satellites - n_planes=1, - rel_phasing=0, - altitude=500 * 1e3, - inc=45, - clustersize=3, # Cluster all 3 satellites together - clusterspacing=30, # Space satellites by a true anomaly of 30 degrees -) -# Construct satellites of the FullFeaturedSatellite type -satellites = [] -sat_type = sats.FullFeaturedSatellite -for i, oe in enumerate(oes): - # Satellite configuration arguments are inferred from the satellite type. The - # function default_sat_args collects all of the parameters that must be set for FSW - # and dynamics in the Basilisk simulation. Any parameters that are to be overridden - # can be set as arguments to default_sat_args, and an error will be raised if the - # parameter is not valid for the satellite type. - - sat_args = sat_type.default_sat_args( - oe=oe, - imageAttErrorRequirement=0.01, # Change a default parameter - imageRateErrorRequirement=0.01, - # Parameters can also be set as a function that is called each time the - # environment is reset - panelEfficiency=lambda: 0.2 + np.random.uniform(-0.01, 0.01), +def run(): + """Demonstrate the configuration of an environment with multiple imaging satellites.""" + # Data environment contains 5000 targets located near random cities, which are + # randomized on reset() + env_features = CityTargets(n_targets=500, location_offset=10e3) + # Data manager records and rewards uniquely imaged targets + data_manager = data.UniqueImagingManager(env_features) + + # Generate orbital parameters for each satellite in the constellation + oes = leo_orbit.walker_delta( + n_spacecraft=3, # Number of satellites + n_planes=1, + rel_phasing=0, + altitude=500 * 1e3, + inc=45, + clustersize=3, # Cluster all 3 satellites together + clusterspacing=30, # Space satellites by a true anomaly of 30 degrees ) - # As an example, look at the arguments for one of the satellites - if i == 0: - print(sat_args) + # Construct satellites of the FullFeaturedSatellite type + satellites = [] + sat_type = sats.FullFeaturedSatellite + for i, oe in enumerate(oes): + # Satellite configuration arguments are inferred from the satellite type. The + # function default_sat_args collects all of the parameters that must be set for FSW + # and dynamics in the Basilisk simulation. Any parameters that are to be overridden + # can be set as arguments to default_sat_args, and an error will be raised if the + # parameter is not valid for the satellite type. - # Instantiate the satellite object. Arguments to the satellite class are set here. - satellite = sat_type( - "EO" + str(i + 1), sat_args, n_ahead_observe=15, n_ahead_act=15 - ) - satellites.append(satellite) - -# Instantiate the communication method -communicator = communication.LOSMultiCommunication(satellites) - -# Make the environment with Gymnasium -env = gym.make( - "GeneralSatelliteTasking-v1", - satellites=satellites, - # Pick the type for the Basilisk environment model. Note that it is not instantiated - # here. - env_type=environment.GroundStationEnvModel, - # Like default_sat_args, default_env_args infers model parameters from the type and - # specific parameters can be overridden or randomized. - env_args=environment.GroundStationEnvModel.default_env_args(), - # Pass configuration objects - env_features=env_features, - data_manager=data_manager, - communicator=communicator, - # Integration frequency in seconds - sim_rate=0.5, - # Environment will be propagated by at most max_step_duration before needing new - # actions selected; however, some satellites will instead end the step when the - # current task is finished - max_step_duration=600.0, - # Set 3-orbit long episodes - time_limit=95 * 60, - log_level="INFO", -) + sat_args = sat_type.default_sat_args( + oe=oe, + imageAttErrorRequirement=0.01, # Change a default parameter + imageRateErrorRequirement=0.01, + # Parameters can also be set as a function that is called each time the + # environment is reset + panelEfficiency=lambda: 0.2 + np.random.uniform(-0.01, 0.01), + ) + + # As an example, look at the arguments for one of the satellites + if i == 0: + print(sat_args) -# Run the simulation until timeout or agent failure -total_reward = 0.0 -observation, info = env.reset() - -while True: - """ - Task random actions. Look at the set_action function for the chosen satellite type - to see what actions do. In this case, the action mapping is as follows: - - 0: charge - - 1: desaturate - - 2: downlink - - 3+: image the (n-3)th upcoming target - - """ - observation, reward, terminated, truncated, info = env.step( - env.action_space.sample() + # Instantiate the satellite object. Arguments to the satellite class are set here. + satellite = sat_type( + "EO" + str(i + 1), sat_args, n_ahead_observe=15, n_ahead_act=15 + ) + satellites.append(satellite) + + # Instantiate the communication method + communicator = communication.LOSMultiCommunication(satellites) + + # Make the environment with Gymnasium + env = gym.make( + "GeneralSatelliteTasking-v1", + satellites=satellites, + # Pick the type for the Basilisk environment model. Note that it is not instantiated + # here. + env_type=environment.GroundStationEnvModel, + # Like default_sat_args, default_env_args infers model parameters from the type and + # specific parameters can be overridden or randomized. + env_args=environment.GroundStationEnvModel.default_env_args(), + # Pass configuration objects + env_features=env_features, + data_manager=data_manager, + communicator=communicator, + # Integration frequency in seconds + sim_rate=0.5, + # Environment will be propagated by at most max_step_duration before needing new + # actions selected; however, some satellites will instead end the step when the + # current task is finished + max_step_duration=600.0, + # Set 3-orbit long episodes + time_limit=95 * 60, + log_level="INFO", ) - total_reward += reward - print(f"\tReward: {reward:.3f} ({total_reward:.3f} cumulative)") + # Run the simulation until timeout or agent failure + total_reward = 0.0 + observation, info = env.reset() + + while True: + """ + Task random actions. Look at the set_action function for the chosen satellite type + to see what actions do. In this case, the action mapping is as follows: + - 0: charge + - 1: desaturate + - 2: downlink + - 3+: image the (n-3)th upcoming target + + """ + observation, reward, terminated, truncated, info = env.step( + env.action_space.sample() + ) + + total_reward += reward + print(f"\tReward: {reward:.3f} ({total_reward:.3f} cumulative)") + + if terminated or truncated: + print("Episode complete.") + break + - if terminated or truncated: - print("Episode complete.") - break +if __name__ == "__main__": + run() diff --git a/examples/general_satellite_tasking/satellite_customization.py b/examples/general_satellite_tasking/satellite_customization.py index b0f42e4b..1f02f340 100644 --- a/examples/general_satellite_tasking/satellite_customization.py +++ b/examples/general_satellite_tasking/satellite_customization.py @@ -24,195 +24,193 @@ # option (2) is to manually override methods for observations and actions in a satellite # subclass. +if __name__ == "__main__": + # OPTION 1: Define a new satellite class by composing existing types. + class CustomSatComposed( + # Action classes. Discrete actions are added in reverse order + # Thus produces an action space of the form: + # {'0': 'action_charge', '1': 'action_desat', '2-4': 'image'} + sa.ImagingActions.configure(n_ahead_act=3), + sa.DesatAction.configure(action_duration=120.0), + sa.ChargingAction.configure(action_duration=60.0), + # Observation classes. In the vectorized observation, these will be composed in + # reverse order. Default arguments for __init__ can be overriden with configure() to + # bake them into the class definition prior to instantiation. + # This produces an observaiton in the form: + # omega_BP_P_normd: [ 0.01489828 0.0004725 -0.08323254] + # c_hat_P: [ 0.66675533 -0.69281445 0.27467338] + # r_BN_P_normd: [ 0.09177786 -0.80203809 -0.7120501 ] + # v_BN_P_normd: [ 0.91321553 -0.11810811 0.25020653] + # battery_charge_fraction: 0.740410440543005 + # target_obs: {'tgt_value_0': 0.1878322060213219, + # 'tgt_loc_0_normd': array([ 0.21883092, -0.72328348, -0.6549610]), + # 'tgt_value_1': 0.8484751150377395, + # 'tgt_loc_1_normd': array([ 0.23371944, -0.73369242, -0.6380208]), + # 'tgt_value_2': 0.14482123441765427, + # 'tgt_loc_2_normd': array([ 0.23645694, -0.73721533, -0.63293101]) + # } + # normalized_time: 0.22505847953216376 + so.TimeState, + so.TargetState.configure(n_ahead_observe=3), + so.NormdPropertyState.configure( + obs_properties=[ + dict(prop="omega_BP_P", norm=0.03), + dict(prop="c_hat_P"), + dict(prop="r_BN_P", norm=orbitalMotion.REQ_EARTH * 1e3), + dict(prop="v_BN_P", norm=7616.5), + dict(prop="battery_charge_fraction"), + ] + ), + # Base class for this satellite + sats.ImagingSatellite, + ): + # Change the attitude controller by redefining fsw_type. In this case, we are using + # a MRP Feedback based controller instead of a the default PD feedback-based + # controller. + fsw_type = fsw.SteeringImagerFSWModel + + # In some cases, the specific model you want may not exactly exists. Models are + # designed to be easily composed, so a new model based on existing models can be + # quickly defined. + + class CustomDynModel(dynamics.ImagingDynModel, dynamics.LOSCommDynModel): + pass + + dyn_type = CustomDynModel + # Model compatibility between FSW, dynamics, and the environment should be + # automatically checked in most cases. + + # OPTION 2: Define a new satellite class manually, selecting a similar class as a + # starting point + # class CustomSatManual(sats.ImagingSatellite): + # # Select FSW and dynamics as in option 1 + # fsw_type = fsw.SteeringImagerFSWModel + + # class CustomDynModel(dynamics.ImagingDynModel, dynamics.LOSCommDynModel): + # pass + + # dyn_type = CustomDynModel + + # # A more common customization requirement is designing the observation and action + # # spaces. Three functions are most commonly overridden to achieve this: get_obs, + # # set_action, and n_actions + + # # Define a custom observation. Various properties from the Basilisk simulation are + # # exposed through the satellite class to make this process easier, including + # # r_BN_B, omega_BN_B, and many more. Typically, this function should return a + # # 1-dimensional numpy array. In this example, the satellite's dynamic state and + # # information about upcoming targets are normalized. + # def get_obs(self): + # dynamic_state = np.concatenate( + # [ + # self.dynamics.omega_BP_P / 0.03, + # self.fsw.c_hat_P, + # self.dynamics.r_BN_P / (orbitalMotion.REQ_EARTH * 1e3), + # self.dynamics.v_BN_P / 7616.5, + # ] + # ) + # images_state = np.array( + # [ + # np.concatenate( + # [ + # [target.priority], + # target.location / (orbitalMotion.REQ_EARTH * 1e3), + # ] + # ) + # for target in self.upcoming_targets(self.n_ahead_observe) + # ] + # ) + # images_state = images_state.flatten() + + # return np.concatenate((dynamic_state, images_state)) + + # # Define a custom action function. In most discrete RL contexts, this function + # # should accept a single integer; however, any parameterization is possible with + # # this package. An important note: it is generally undesirable to retask the same + # # action twice in a row as controller states will get reset. Good set_action + # # defintions should include protections against this. In this example: + # # - 0: charge + # # - 1: desaturate + # # - 2+: image the (n-3)th upcoming target + # def set_action(self, action): + # if action == 0 and self.current_action != 0: + # # Use functions defined in FSW with the @action decorator to interact with + # # the Basilisk sim. + # self.fsw.action_charge() + # # Save data to the info dictonary for debugging help + # self.log_info("charging tasked") + # if action == 1 and self.current_action != 1: + # self.fsw.action_desat() + # self.log_info("desat tasked") + # else: + # target_action = action + # if isinstance(target_action, int): + # target_action -= 2 + # # Use the standard ImagingSatellite tasking function + # super().set_action(target_action) + + # if action < 2: + # self.current_action = action + + # # The action space cannot be inferred; explicitly tell gymnasium how many actions + # # the satellite can take + # @property + # def action_space(self): + # return gym.spaces.Discrete(self.n_ahead_act + 2) + + # Configure the environent + env_features = CityTargets(n_targets=1000) + data_manager = data.UniqueImagingManager(env_features) + # Use the CustomSat type + sat_type = CustomSatComposed + sat_args = sat_type.default_sat_args( + imageAttErrorRequirement=0.01, + imageRateErrorRequirement=0.01, + oe=random_orbit, + ) + satellite = sat_type( + "EO1", + sat_args, + variable_interval=True, + ) + # The composed satellite action space returns a human-readable action map + print("Actions:", satellite.action_map) + + # Make the environment with Gymnasium + env = gym.make( + "SingleSatelliteTasking-v1", + satellites=satellite, + # Select an EnvironmentModel compatible with the models in the satellite + env_type=environment.BasicEnvironmentModel, + env_args=environment.BasicEnvironmentModel.default_env_args(), + env_features=env_features, + data_manager=data_manager, + sim_rate=0.5, + max_step_duration=600.0, + time_limit=95 * 60, + log_level="INFO", + ) -# OPTION 1: Define a new satellite class by composing existing types. -class CustomSatComposed( - # Action classes. Discrete actions are added in reverse order - # Thus produces an action space of the form: - # {'0': 'action_charge', '1': 'action_desat', '2-4': 'image'} - sa.ImagingActions.configure(n_ahead_act=3), - sa.DesatAction.configure(action_duration=120.0), - sa.ChargingAction.configure(action_duration=60.0), - # Observation classes. In the vectorized observation, these will be composed in - # reverse order. Default arguments for __init__ can be overriden with configure() to - # bake them into the class definition prior to instantiation. - # This produces an observaiton in the form: - # omega_BP_P_normd: [ 0.01489828 0.0004725 -0.08323254] - # c_hat_P: [ 0.66675533 -0.69281445 0.27467338] - # r_BN_P_normd: [ 0.09177786 -0.80203809 -0.7120501 ] - # v_BN_P_normd: [ 0.91321553 -0.11810811 0.25020653] - # battery_charge_fraction: 0.740410440543005 - # target_obs: {'tgt_value_0': 0.1878322060213219, - # 'tgt_loc_0_normd': array([ 0.21883092, -0.72328348, -0.6549610]), - # 'tgt_value_1': 0.8484751150377395, - # 'tgt_loc_1_normd': array([ 0.23371944, -0.73369242, -0.6380208]), - # 'tgt_value_2': 0.14482123441765427, - # 'tgt_loc_2_normd': array([ 0.23645694, -0.73721533, -0.63293101]) - # } - # normalized_time: 0.22505847953216376 - so.TimeState, - so.TargetState.configure(n_ahead_observe=3), - so.NormdPropertyState.configure( - obs_properties=[ - dict(prop="omega_BP_P", norm=0.03), - dict(prop="c_hat_P"), - dict(prop="r_BN_P", norm=orbitalMotion.REQ_EARTH * 1e3), - dict(prop="v_BN_P", norm=7616.5), - dict(prop="battery_charge_fraction"), - ] - ), - # Base class for this satellite - sats.ImagingSatellite, -): - # Change the attitude controller by redefining fsw_type. In this case, we are using - # a MRP Feedback based controller instead of a the default PD feedback-based - # controller. - fsw_type = fsw.SteeringImagerFSWModel - - # In some cases, the specific model you want may not exactly exists. Models are - # designed to be easily composed, so a new model based on existing models can be - # quickly defined. - - class CustomDynModel(dynamics.ImagingDynModel, dynamics.LOSCommDynModel): - pass - - dyn_type = CustomDynModel - # Model compatibility between FSW, dynamics, and the environment should be - # automatically checked in most cases. - - -# OPTION 2: Define a new satellite class manually, selecting a similar class as a -# starting point -# class CustomSatManual(sats.ImagingSatellite): -# # Select FSW and dynamics as in option 1 -# fsw_type = fsw.SteeringImagerFSWModel - -# class CustomDynModel(dynamics.ImagingDynModel, dynamics.LOSCommDynModel): -# pass - -# dyn_type = CustomDynModel - -# # A more common customization requirement is designing the observation and action -# # spaces. Three functions are most commonly overridden to achieve this: get_obs, -# # set_action, and n_actions - -# # Define a custom observation. Various properties from the Basilisk simulation are -# # exposed through the satellite class to make this process easier, including -# # r_BN_B, omega_BN_B, and many more. Typically, this function should return a -# # 1-dimensional numpy array. In this example, the satellite's dynamic state and -# # information about upcoming targets are normalized. -# def get_obs(self): -# dynamic_state = np.concatenate( -# [ -# self.dynamics.omega_BP_P / 0.03, -# self.fsw.c_hat_P, -# self.dynamics.r_BN_P / (orbitalMotion.REQ_EARTH * 1e3), -# self.dynamics.v_BN_P / 7616.5, -# ] -# ) -# images_state = np.array( -# [ -# np.concatenate( -# [ -# [target.priority], -# target.location / (orbitalMotion.REQ_EARTH * 1e3), -# ] -# ) -# for target in self.upcoming_targets(self.n_ahead_observe) -# ] -# ) -# images_state = images_state.flatten() - -# return np.concatenate((dynamic_state, images_state)) - -# # Define a custom action function. In most discrete RL contexts, this function -# # should accept a single integer; however, any parameterization is possible with -# # this package. An important note: it is generally undesirable to retask the same -# # action twice in a row as controller states will get reset. Good set_action -# # defintions should include protections against this. In this example: -# # - 0: charge -# # - 1: desaturate -# # - 2+: image the (n-3)th upcoming target -# def set_action(self, action): -# if action == 0 and self.current_action != 0: -# # Use functions defined in FSW with the @action decorator to interact with -# # the Basilisk sim. -# self.fsw.action_charge() -# # Save data to the info dictonary for debugging help -# self.log_info("charging tasked") -# if action == 1 and self.current_action != 1: -# self.fsw.action_desat() -# self.log_info("desat tasked") -# else: -# target_action = action -# if isinstance(target_action, int): -# target_action -= 2 -# # Use the standard ImagingSatellite tasking function -# super().set_action(target_action) - -# if action < 2: -# self.current_action = action - -# # The action space cannot be inferred; explicitly tell gymnasium how many actions -# # the satellite can take -# @property -# def action_space(self): -# return gym.spaces.Discrete(self.n_ahead_act + 2) - - -# Configure the environent -env_features = CityTargets(n_targets=1000) -data_manager = data.UniqueImagingManager(env_features) -# Use the CustomSat type -sat_type = CustomSatComposed -sat_args = sat_type.default_sat_args( - imageAttErrorRequirement=0.01, - imageRateErrorRequirement=0.01, - oe=random_orbit, -) -satellite = sat_type( - "EO1", - sat_args, - variable_interval=True, -) -# The composed satellite action space returns a human-readable action map -print("Actions:", satellite.action_map) - -# Make the environment with Gymnasium -env = gym.make( - "SingleSatelliteTasking-v1", - satellites=satellite, - # Select an EnvironmentModel compatible with the models in the satellite - env_type=environment.BasicEnvironmentModel, - env_args=environment.BasicEnvironmentModel.default_env_args(), - env_features=env_features, - data_manager=data_manager, - sim_rate=0.5, - max_step_duration=600.0, - time_limit=95 * 60, - log_level="INFO", -) - -# Run the simulation until timeout or agent failure -total_reward = 0.0 -observation, info = env.reset() + # Run the simulation until timeout or agent failure + total_reward = 0.0 + observation, info = env.reset() -while True: - print(f"") + while True: + print(f"") - observation, reward, terminated, truncated, info = env.step( - env.action_space.sample() # Task random actions - ) + observation, reward, terminated, truncated, info = env.step( + env.action_space.sample() # Task random actions + ) - # Show the custom normalized observation vector - print("\tObservation:", observation) + # Show the custom normalized observation vector + print("\tObservation:", observation) - # Using the composed satellite features also provides a human-readable state: - for k, v in env.satellite.obs_dict.items(): - print(f"\t\t{k}: {v}") + # Using the composed satellite features also provides a human-readable state: + for k, v in env.satellite.obs_dict.items(): + print(f"\t\t{k}: {v}") - total_reward += reward - print(f"\tReward: {reward:.3f} ({total_reward:.3f} cumulative)") - if terminated or truncated: - print("Episode complete.") - break + total_reward += reward + print(f"\tReward: {reward:.3f} ({total_reward:.3f} cumulative)") + if terminated or truncated: + print("Episode complete.") + break diff --git a/examples/general_satellite_tasking/single_sat.py b/examples/general_satellite_tasking/single_sat.py index 926e414f..18bbb49d 100644 --- a/examples/general_satellite_tasking/single_sat.py +++ b/examples/general_satellite_tasking/single_sat.py @@ -11,84 +11,85 @@ # This script demonstrates the configuration of an environment with a single imaging # satellite. -# Data environment contains 5000 targets randomly distributed -env_features = StaticTargets(n_targets=1000) -# Data manager records and rewards uniquely imaged targets -data_manager = data.UniqueImagingManager(env_features) +if __name__ == "__main__": + # Data environment contains 5000 targets randomly distributed + env_features = StaticTargets(n_targets=1000) + # Data manager records and rewards uniquely imaged targets + data_manager = data.UniqueImagingManager(env_features) -# Construct satellites of the FullFeaturedSatellite type -sat_type = sats.FullFeaturedSatellite -# Satellite configuration arguments are inferred from the satellite type. The function -# default_sat_args collects all of the parameters that must be set for FSW and dynamics -# in the Basilisk simulation. Any parameters that are to be overridden can be set as -# arguments to default_sat_args, and an error will be raised if the parameter is not -# valid for the satellite type. + # Construct satellites of the FullFeaturedSatellite type + sat_type = sats.FullFeaturedSatellite + # Satellite configuration arguments are inferred from the satellite type. The function + # default_sat_args collects all of the parameters that must be set for FSW and dynamics + # in the Basilisk simulation. Any parameters that are to be overridden can be set as + # arguments to default_sat_args, and an error will be raised if the parameter is not + # valid for the satellite type. -sat_args = sat_type.default_sat_args( - imageAttErrorRequirement=0.01, # Change a default parameter - imageRateErrorRequirement=0.01, - # Parameters can also be set as a function that is called each time the environment - # is reset - oe=random_orbit, -) -print(sat_args) + sat_args = sat_type.default_sat_args( + imageAttErrorRequirement=0.01, # Change a default parameter + imageRateErrorRequirement=0.01, + # Parameters can also be set as a function that is called each time the environment + # is reset + oe=random_orbit, + ) + print(sat_args) -# Instantiate the satellite object. Arguments to the satellite class are set here. -satellite = sat_type("EO1", sat_args, n_ahead_observe=30, n_ahead_act=15) + # Instantiate the satellite object. Arguments to the satellite class are set here. + satellite = sat_type("EO1", sat_args, n_ahead_observe=30, n_ahead_act=15) -# Make the environment with Gymnasium -env = gym.make( - # The SingleSatelliteTasking environment takes actions and observations directly - # from the satellite, instead of wrapping them in a tuple - "SingleSatelliteTasking-v1", - satellites=satellite, - # Pick the type for the Basilisk environment model. Note that it is not instantiated - # here. - env_type=environment.GroundStationEnvModel, - # Like default_sat_args, default_env_args infers model parameters from the type and - # specific parameters can be - # overridden or randomized. - env_args=environment.GroundStationEnvModel.default_env_args(), - # Pass configuration objects - env_features=env_features, - data_manager=data_manager, - # Integration frequency in seconds - sim_rate=0.5, - # Environment will be propagated by at most max_step_duration before needing new - # actions selected; however, some satellites will instead end the step when the - # current task is finished - max_step_duration=600.0, - # Set 3-orbit long episodes - time_limit=95 * 60, - # Send the terminated signal in addition to the truncated signal at the end of the - # episode. Needed for some RL algorithms to work correctly. - terminate_on_time_limit=True, - log_level="INFO", -) + # Make the environment with Gymnasium + env = gym.make( + # The SingleSatelliteTasking environment takes actions and observations directly + # from the satellite, instead of wrapping them in a tuple + "SingleSatelliteTasking-v1", + satellites=satellite, + # Pick the type for the Basilisk environment model. Note that it is not instantiated + # here. + env_type=environment.GroundStationEnvModel, + # Like default_sat_args, default_env_args infers model parameters from the type and + # specific parameters can be + # overridden or randomized. + env_args=environment.GroundStationEnvModel.default_env_args(), + # Pass configuration objects + env_features=env_features, + data_manager=data_manager, + # Integration frequency in seconds + sim_rate=0.5, + # Environment will be propagated by at most max_step_duration before needing new + # actions selected; however, some satellites will instead end the step when the + # current task is finished + max_step_duration=600.0, + # Set 3-orbit long episodes + time_limit=95 * 60, + # Send the terminated signal in addition to the truncated signal at the end of the + # episode. Needed for some RL algorithms to work correctly. + terminate_on_time_limit=True, + log_level="INFO", + ) -# Run the simulation until timeout or agent failure -total_reward = 0.0 -observation, info = env.reset() + # Run the simulation until timeout or agent failure + total_reward = 0.0 + observation, info = env.reset() -while True: - print(f"") + while True: + print(f"") - """ - Task random actions. Look at the set_action function for the chosen satellite type - to see what actions do. In this case, the action mapping is as follows: - - 0: charge - - 1: desaturate - - 2: downlink - - 3+: image the (n-3)th upcoming target + """ + Task random actions. Look at the set_action function for the chosen satellite type + to see what actions do. In this case, the action mapping is as follows: + - 0: charge + - 1: desaturate + - 2: downlink + - 3+: image the (n-3)th upcoming target - """ - observation, reward, terminated, truncated, info = env.step( - env.action_space.sample() - ) + """ + observation, reward, terminated, truncated, info = env.step( + env.action_space.sample() + ) - total_reward += reward - print(f"\tReward: {reward:.3f} ({total_reward:.3f} cumulative)") + total_reward += reward + print(f"\tReward: {reward:.3f} ({total_reward:.3f} cumulative)") - if terminated or truncated: - print("Episode complete.") - break + if terminated or truncated: + print("Episode complete.") + break diff --git a/examples/genetic_algorithm/ga_hp_solver.py b/examples/genetic_algorithm/ga_hp_solver.py index fefc2345..48c25f34 100644 --- a/examples/genetic_algorithm/ga_hp_solver.py +++ b/examples/genetic_algorithm/ga_hp_solver.py @@ -4,13 +4,6 @@ from bsk_rl.utilities.genetic_algorithm import experiments -# Check if any args were passed to the script -if len(sys.argv) > 1: - # Get the index of the job array - experiment_index = int(sys.argv[1]) -else: - experiment_index = None - if __name__ == "__main__": """ This script runs a hyperparameter search for the genetic algorithm on the @@ -31,6 +24,12 @@ - AgileEOS-v0 - MultiSatAgileEOS-v0 """ + # Check if any args were passed to the script + if len(sys.argv) > 1: + # Get the index of the job array + experiment_index = int(sys.argv[1]) + else: + experiment_index = None # Set the start method for multiprocessing (required for some Linux systems) multiprocessing.set_start_method("spawn") diff --git a/examples/plotting_tools/plot_a2c_hyperparams.py b/examples/plotting_tools/plot_a2c_hyperparams.py index 24f30873..47c793f1 100644 --- a/examples/plotting_tools/plot_a2c_hyperparams.py +++ b/examples/plotting_tools/plot_a2c_hyperparams.py @@ -4,7 +4,8 @@ import matplotlib as mpl import numpy as np from matplotlib import pyplot as plt -from plot_dqn_hyperparams import concatenate_results + +from .plot_dqn_hyperparams import concatenate_results SEP = os.path.sep diff --git a/examples/plotting_tools/plot_ppo_hyperparams.py b/examples/plotting_tools/plot_ppo_hyperparams.py index 0905bfe1..a15731cb 100644 --- a/examples/plotting_tools/plot_ppo_hyperparams.py +++ b/examples/plotting_tools/plot_ppo_hyperparams.py @@ -4,7 +4,8 @@ import matplotlib as mpl import numpy as np from matplotlib import pyplot as plt -from plot_dqn_hyperparams import concatenate_results + +from .plot_dqn_hyperparams import concatenate_results SEP = os.path.sep diff --git a/examples/sb3/a2c_hyperparam_search.py b/examples/sb3/a2c_hyperparam_search.py index 59a86650..f2928776 100644 --- a/examples/sb3/a2c_hyperparam_search.py +++ b/examples/sb3/a2c_hyperparam_search.py @@ -9,13 +9,6 @@ SEP = os.path.sep -# Check if any args were passed to the script -if len(sys.argv) > 1: - # Get the index of the job array - experiment_index = int(sys.argv[1]) -else: - experiment_index = None - if __name__ == "__main__": """ This script runs a hyperparameter search for the A2C algorithm on the @@ -47,6 +40,13 @@ - SimpleEOS-v0 (not yet tested) - AgileEOS-v0 """ + # Check if any args were passed to the script + if len(sys.argv) > 1: + # Get the index of the job array + experiment_index = int(sys.argv[1]) + else: + experiment_index = None + max_steps = 90 n_its = 1 base_steps = 90 diff --git a/examples/sb3/dqn_hyperparam_search.py b/examples/sb3/dqn_hyperparam_search.py index 8e39838d..c63da826 100644 --- a/examples/sb3/dqn_hyperparam_search.py +++ b/examples/sb3/dqn_hyperparam_search.py @@ -7,13 +7,6 @@ SEP = os.path.sep -# Check if any args were passed to the script -if len(sys.argv) > 1: - # Get the index of the job array - experiment_index = int(sys.argv[1]) -else: - experiment_index = None - if __name__ == "__main__": """ This script runs a hyperparameter search for the DQN algorithm on the @@ -45,6 +38,13 @@ - AgileEOS-v0 """ + # Check if any args were passed to the script + if len(sys.argv) > 1: + # Get the index of the job array + experiment_index = int(sys.argv[1]) + else: + experiment_index = None + n_steps = 90 num_cores = cpu_count() - 2 n_its = 1 diff --git a/examples/sb3/ppo_hyperparam_search.py b/examples/sb3/ppo_hyperparam_search.py index 0a0e8de0..0bffef0a 100644 --- a/examples/sb3/ppo_hyperparam_search.py +++ b/examples/sb3/ppo_hyperparam_search.py @@ -9,13 +9,6 @@ SEP = os.path.sep -# Check if any args were passed to the script -if len(sys.argv) > 1: - # Get the index of the job array - experiment_index = int(sys.argv[1]) -else: - experiment_index = None - if __name__ == "__main__": """ This script runs a hyperparameter search for the PPO algorithm on the @@ -46,6 +39,13 @@ - SimpleEOS-v0 (not yet tested) - AgileEOS-v0 """ + # Check if any args were passed to the script + if len(sys.argv) > 1: + # Get the index of the job array + experiment_index = int(sys.argv[1]) + else: + experiment_index = None + # Define the environment parameters max_steps = 90 env_name = "MultiSensorEOS-v0" diff --git a/examples/sb3/sppo_hyperparam_search.py b/examples/sb3/sppo_hyperparam_search.py index 97425a0d..7253e4a5 100644 --- a/examples/sb3/sppo_hyperparam_search.py +++ b/examples/sb3/sppo_hyperparam_search.py @@ -9,12 +9,6 @@ SEP = os.path.sep -# Check if any args were passed to the script -if len(sys.argv) > 1: - # Get the index of the job array - experiment_index = int(sys.argv[1]) -else: - experiment_index = None if __name__ == "__main__": """ @@ -51,6 +45,13 @@ - SimpleEOS-v0 (not yet tested) - AgileEOS-v0 """ + # Check if any args were passed to the script + if len(sys.argv) > 1: + # Get the index of the job array + experiment_index = int(sys.argv[1]) + else: + experiment_index = None + # Define the environment parameters max_steps = 90 env_name = "MultiSensorEOS-v0" diff --git a/pyproject.toml b/pyproject.toml index 0829a54b..fff3c082 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,10 +28,11 @@ dependencies = [ "ruff>=0.1.9", "scikit-learn", "scipy", + "sphinx-rtd-theme", "stable-baselines3", "tensorflow", "torch", ] [project.scripts] -finish_install = "bsk_rl.finish_install:pck_install" +finish_install = "bsk_rl._finish_install:pck_install" diff --git a/src/bsk_rl/__init__.py b/src/bsk_rl/__init__.py index cd711e48..54d41e76 100644 --- a/src/bsk_rl/__init__.py +++ b/src/bsk_rl/__init__.py @@ -1,6 +1,6 @@ from gymnasium.envs.registration import register -from bsk_rl.check_bsk_version import check_bsk_version +from bsk_rl._check_bsk_version import _check_bsk_version register(id="SimpleEOS-v0", entry_point="bsk_rl.envs.simple_eos.gym_env:SimpleEOS") @@ -37,4 +37,4 @@ ) -check_bsk_version() +_check_bsk_version() diff --git a/src/bsk_rl/check_bsk_version.py b/src/bsk_rl/_check_bsk_version.py similarity index 96% rename from src/bsk_rl/check_bsk_version.py rename to src/bsk_rl/_check_bsk_version.py index d28a051c..70283c9b 100644 --- a/src/bsk_rl/check_bsk_version.py +++ b/src/bsk_rl/_check_bsk_version.py @@ -5,7 +5,7 @@ from packaging.version import parse as parse_version -def check_bsk_version(): +def _check_bsk_version(): f = open( os.path.join( os.path.dirname(os.path.realpath(__file__)), diff --git a/src/bsk_rl/finish_install.py b/src/bsk_rl/_finish_install.py similarity index 89% rename from src/bsk_rl/finish_install.py rename to src/bsk_rl/_finish_install.py index be45eaec..96fae5c9 100644 --- a/src/bsk_rl/finish_install.py +++ b/src/bsk_rl/_finish_install.py @@ -6,7 +6,7 @@ import requests -from bsk_rl.check_bsk_version import check_bsk_version +from bsk_rl._check_bsk_version import _check_bsk_version def pck_install(): @@ -32,4 +32,4 @@ def pck_install(): / "simplemaps_worldcities" ) - check_bsk_version() + _check_bsk_version() diff --git a/src/bsk_rl/agents/mcts.py b/src/bsk_rl/agents/mcts.py index 2c4a5ee5..6c48c6a2 100644 --- a/src/bsk_rl/agents/mcts.py +++ b/src/bsk_rl/agents/mcts.py @@ -14,26 +14,31 @@ class MCTS: If a heuristic rollout type is used, create the policy using the following two lines of code: - stateMachineMCTS = state_machine.StateMachine() - stateMachineMCTS.loadTransferConditions("agile_eos_ops.adv") - rollout_policy = AgileEOSRolloutPolicy(env=env, state_machine=stateMachineMCTS) + .. code-block:: python - Then, load the policy as the rollout_policy during initialization: + stateMachineMCTS = state_machine.StateMachine() + stateMachineMCTS.loadTransferConditions("agile_eos_ops.adv") + rollout_policy = AgileEOSRolloutPolicy(env=env, state_machine=stateMachineMCTS) - MCTS_Agent = MCTS(c=c, num_sims=num_sims, rollout_policy=rollout_policy) + Then, load the policy as the rollout_policy during initialization: - The parameter c controls the scaling of the exploration bonus + .. code-block:: python - The parameter num_sims determines the number of simulations per call of - selectAction() + MCTS_Agent = MCTS(c=c, num_sims=num_sims, rollout_policy=rollout_policy) The env and initial conditions must be loaded in after initialization. The algorithm will automatically restart the sim and step it forward to the last state. This is due to limitations in copying Basilisk: - MCTS_Agent.setEnv( - env_name, env.initial_conditions, max_steps=num_steps, max_length=t_final - ) + .. code-block:: python + + MCTS_Agent.setEnv( + env_name, env.initial_conditions, max_steps=num_steps, max_length=t_final + ) + + Args: + c: scaling of the exploration bonus + num_sims: number of simulations per call of selectAction() """ def __init__( diff --git a/src/bsk_rl/envs/agile_eos/bsk_sim.py b/src/bsk_rl/envs/agile_eos/bsk_sim.py index 47e34542..4ee1c7a5 100644 --- a/src/bsk_rl/envs/agile_eos/bsk_sim.py +++ b/src/bsk_rl/envs/agile_eos/bsk_sim.py @@ -38,7 +38,6 @@ unitTestSupport, vizSupport, ) -from numpy.random import uniform from bsk_rl.utilities.effector_primitives import actuator_primitives as ap from bsk_rl.utilities.initial_conditions import leo_orbit, sc_attitudes @@ -51,18 +50,21 @@ class AgileEOSSimulator(SimulationBaseClass.SimBaseClass): Simulates a spacecraft in LEO with atmospheric drag and J2. Dynamics Components - - Forces: J2, Atmospheric Drag - - Environment: Exponential density model; eclipse - - Actuators: ExternalForceTorque, reaction wheels - - Sensors: SimpleNav - - Power System: SimpleBattery, SimplePOwerSink, SimpleSolarPanel - - Data Management System: spaceToGroundTransmitter, simpleStorageUnit, - simpleInstrument + + * Forces: J2, Atmospheric Drag + * Environment: Exponential density model; eclipse + * Actuators: ExternalForceTorque, reaction wheels + * Sensors: SimpleNav + * Power System: SimpleBattery, SimplePOwerSink, SimpleSolarPanel + * Data Management System: spaceToGroundTransmitter, simpleStorageUnit, + simpleInstrument FSW Components: - - MRP Feedback controller - - locationPoint - targets, sun-pointing - - Desat + + * MRP Feedback controller + * locationPoint - targets, sun-pointing + * Desat + """ def __init__( @@ -196,7 +198,7 @@ def set_ICs(self): # Sample attitude and rates sigma_init, omega_init = sc_attitudes.random_tumble(maxSpinRate=0.00001) - wheel_speeds = uniform(-1500, 1500, 3) # RPMs + wheel_speeds = np.random.uniform(-1500, 1500, 3) # RPMs # Dict of initial conditions initial_conditions = { @@ -224,7 +226,7 @@ def set_ICs(self): "disturbance_magnitude": 4e-3, "disturbance_vector": np.random.standard_normal(3), # Reaction Wheel speeds - # "wheelSpeeds": uniform(-400,400,3), # RPM + # "wheelSpeeds": np.random.uniform(-400,400,3), # RPM "wheelSpeeds": wheel_speeds, # RPM "maxSpeed": 3000, # RPM # Solar Panel Parameters @@ -1858,12 +1860,14 @@ def close_gracefully(self): def compute_image_tuples(self, r_BN_N, v_BN_N): """ Computes the self.n_images image state tuples - 0-2: S/c Hill-Frame Position - 3: Priority - 4: Imaged? - 5: Downlinked? - :return: image state tuples (in a single np array) - normalized and - non-normalized + + * 0-2: S/c Hill-Frame Position + * 3: Priority + * 4: Imaged? + * 5: Downlinked? + + Returns: + image state tuples (in a single np array) - normalized and non-normalized """ # Initialize the image tuple array image_tuples = np.zeros(self.target_tuple_size * self.n_target_buffer) diff --git a/src/bsk_rl/envs/general_satellite_tasking/scenario/sat_observations.py b/src/bsk_rl/envs/general_satellite_tasking/scenario/sat_observations.py index 5280c6f4..31b6031e 100644 --- a/src/bsk_rl/envs/general_satellite_tasking/scenario/sat_observations.py +++ b/src/bsk_rl/envs/general_satellite_tasking/scenario/sat_observations.py @@ -92,8 +92,12 @@ def __init__( Args: obs_properties: List of properties that can be found in fsw or dynamics that are to be appended to the the observation. Properties are optionally - normalized by some factor. Specified in + normalized by some factor. Specified in the form + + :code-block: python + [dict(prop="prop_name", module="fsw"/"dynamics"/None, norm=1.0)] + If module is not specified or None, the source of the property is inferred. If norm is not specified, it is set to 1.0 (no normalization). args: Passed through to satellite @@ -184,11 +188,11 @@ def __init__( target_properties: List of properties to include in the observation in the format [dict(prop="prop_name", norm=norm)]. If norm is not specified, it is set to 1.0 (no normalization). Properties to choose from: - - priority - - location - - window_open - - window_mid - - window_close + * priority + * location + * window_open + * window_mid + * window_close args: Passed through to satellite kwargs: Passed through to satellite """ @@ -295,10 +299,10 @@ def __init__( in the format [dict(prop="prop_name", norm=norm)]. If norm is not specified, it is set to 1.0 (no normalization). Properties to choose from: - - location - - window_open - - window_mid - - window_close + * location + * window_open + * window_mid + * window_close args: Passed through to satellite kwargs: Passed through to satellite """ diff --git a/src/bsk_rl/envs/general_satellite_tasking/utils/orbital.py b/src/bsk_rl/envs/general_satellite_tasking/utils/orbital.py index 4ccc92b4..c23f7a99 100644 --- a/src/bsk_rl/envs/general_satellite_tasking/utils/orbital.py +++ b/src/bsk_rl/envs/general_satellite_tasking/utils/orbital.py @@ -61,7 +61,7 @@ def random_epoch(start: int = 2000, end: int = 2022): end: Final year. Returns: - Epoch in `YYYY MMM DD HH:MM:SS.SSS (UTC)` format + Epoch in ``YYYY MMM DD HH:MM:SS.SSS (UTC)`` format """ year = np.random.randint(start, end) month = np.random.choice( diff --git a/src/bsk_rl/envs/multisensor_eos/bsk_sim.py b/src/bsk_rl/envs/multisensor_eos/bsk_sim.py index f75da751..112d742e 100644 --- a/src/bsk_rl/envs/multisensor_eos/bsk_sim.py +++ b/src/bsk_rl/envs/multisensor_eos/bsk_sim.py @@ -38,15 +38,17 @@ class MultiSensorEOSSimulator(SimulationBaseClass.SimBaseClass): Simulates ground observations by a single spacecraft in LEO. Dynamics Components - - Forces: J2, Atmospheric Drag w/ COM offset - - Environment: Exponential density model; eclipse - - Actuators: ExternalForceTorque - - Sensors: SimpleNav - - Systems: SimpleBattery, SimpleSink, SimpleSolarPanel + + * Forces: J2, Atmospheric Drag w/ COM offset + * Environment: Exponential density model; eclipse + * Actuators: ExternalForceTorque + * Sensors: SimpleNav + * Systems: SimpleBattery, SimpleSink, SimpleSolarPanel FSW Components: - - mrpFeedback controller - - inertial3d (sun pointing), hillPoint (nadir pointing) + + * mrpFeedback controller + * inertial3d (sun pointing), hillPoint (nadir pointing) :return: """ @@ -134,11 +136,12 @@ def __init__( def set_env_dynamics(self): """ Sets up environmental dynamics for the sim, including: - - SPICE - - Eclipse - - Planetary atmosphere - - Gravity - - Spherical harmonics + + * SPICE + * Eclipse + * Planetary atmosphere + * Gravity + * Spherical harmonics """ # clear prior gravitational body and SPICE setup definitions self.gravFactory = simIncludeGravBody.gravBodyFactory() @@ -401,16 +404,19 @@ def setup_viz(self): def set_fsw(self): """ - Sets up the attitude guidance stack for the simulation. This simulator runs: - inertial3Dpoint - Sets the attitude guidance objective to point the main panel - at the sun. - hillPointTask: Sets the attitude guidance objective to point a "camera" - boresight towards nadir. - attitudeTrackingError: Computes the difference between estimated and guidance - attitudes - mrpFeedbackControl: Computes an appropriate control torque given an attitude - error - :return: + Sets up the attitude guidance stack for the simulation. + + This simulator runs: + + * inertial3Dpoint - Sets the attitude guidance objective to point the main panel + at the sun. + * hillPointTask: Sets the attitude guidance objective to point a "camera" + boresight towards nadir. + * attitudeTrackingError: Computes the difference between estimated and guidance + attitudes + * mrpFeedbackControl: Computes an appropriate control torque given an attitude + error + """ self.dyn_proc.addTask( self.CreateNewTask("sunPointTask", mc.sec2nano(self.fsw_step)), @@ -660,11 +666,14 @@ def zeroGateWayMsgs(self): def run_sim(self, action): """ Executes the sim for a specified duration given a mode command. - :param action: - 0 - Point solar panels at the sun - 1 - Desaturate reaction wheels - >1 - Image types - :return: + + Args: + action: + * 0 - Point solar panels at the sun + * 1 - Desaturate reaction wheels + * >1 - Image types + + Returns: sim_state - simulation states generated sim_over - episode over flag """ diff --git a/src/bsk_rl/envs/multisensor_eos/gym_env.py b/src/bsk_rl/envs/multisensor_eos/gym_env.py index fae74ae8..1ab32ef4 100644 --- a/src/bsk_rl/envs/multisensor_eos/gym_env.py +++ b/src/bsk_rl/envs/multisensor_eos/gym_env.py @@ -18,26 +18,29 @@ class MultiSensorEOS(gym.Env): results in full reward, other image types results in no reward. Action Space (discrete): - 0 - Points solar panels at the sun. - 1 - Desaturates the reaction wheels. - >1 - Orients the s/c towards the earth; takes image of type _. + + * 0 - Points solar panels at the sun. + * 1 - Desaturates the reaction wheels. + * >1 - Orients the s/c towards the earth; takes image of type _. Observation Space: - r_sc_I - float[3,] - spacecraft position. - v_sc - float[3,] - spacecraft velocity in PCPF. - |sigma_RB| - float [0,1] - norm of the spacecraft error MRP with respect to the - last reference frame specified. - |omega_BN| - float - norm of the total spacecraft bus rotational velocity with - respect to the inertial frame. - |omega_RW| - float - norm of the reaction wheel rotational velocities. - storedCharge - float [0,batCapacity] - indicates the s/c battery charge level in - W-s. - sun_indicator - float [0, 1] - indicates the flux mitigator due to eclipse. - access indicator - access to the next target - img_mode norm - float [0,1] - indicates the required imaging mode. + + * r_sc_I - float[3,] - spacecraft position. + * v_sc - float[3,] - spacecraft velocity in PCPF. + * sigma_RB - float [0,1] - norm of the spacecraft error MRP with respect to the + last reference frame specified. + * omega_BN - float - norm of the total spacecraft bus rotational velocity with + respect to the inertial frame. + * omega_RW - float - norm of the reaction wheel rotational velocities. + * storedCharge - float [0,batCapacity] - indicates the s/c battery charge level in + W-s. + * sun_indicator - float [0, 1] - indicates the flux mitigator due to eclipse. + * access indicator - access to the next target + * img_mode norm - float [0,1] - indicates the required imaging mode. Reward Function: r = 1/(1+ | sigma_RB|) if correct sensor + Intended to provide a rich reward in action 1 when the spacecraft is pointed towards the earth, decaying as sigma^2 as the pointing error increases. """ @@ -114,31 +117,27 @@ def step(self, action): The agent takes a step in the environment. Note that the simulator must be initialized - Parameters - ---------- - action : int - Returns - ------- - ob, reward, episode_over, truncated, info : tuple - ob (object) : - an environment-specific object representing your observation of - the environment. - reward (float) : - amount of reward achieved by the previous action. The scale - varies between environments, but the goal is always to increase - your total reward. - episode_over (bool) : - whether it's time to reset the environment again. Most (but not - all) tasks are divided up into well-defined episodes, and done - being True indicates the episode has terminated. (For example, - perhaps the pole tipped too far, or you lost your last life.) - truncated (truncated) : set to false. Gymnasium requirement. - info (dict) : - diagnostic information useful for debugging. It can sometimes - be useful for learning (for example, it might contain the raw - probabilities behind the environment's last state change). - However, official evaluations of your agent are not allowed to - use this for learning. + Args: + action: int + + Returns: + + * ob (object): an environment-specific object representing your observation of + the environment. + * reward (float): amount of reward achieved by the previous action. The scale + varies between environments, but the goal is always to increase + your total reward. + * episode_over (bool): whether it's time to reset the environment again. Most (but not + all) tasks are divided up into well-defined episodes, and done + being True indicates the episode has terminated. (For example, + perhaps the pole tipped too far, or you lost your last life.) + * truncated (truncated): set to false. Gymnasium requirement. + * info (dict): diagnostic information useful for debugging. It can sometimes + be useful for learning (for example, it might contain the raw + probabilities behind the environment's last state change). + However, official evaluations of your agent are not allowed to + use this for learning. + """ self.curr_step += 1 diff --git a/src/bsk_rl/envs/simple_eos/bsk_sim.py b/src/bsk_rl/envs/simple_eos/bsk_sim.py index 22418a74..1fae7ccc 100644 --- a/src/bsk_rl/envs/simple_eos/bsk_sim.py +++ b/src/bsk_rl/envs/simple_eos/bsk_sim.py @@ -40,7 +40,6 @@ unitTestSupport, vizSupport, ) -from numpy.random import uniform from bsk_rl.utilities.effector_primitives import actuator_primitives as ap from bsk_rl.utilities.initial_conditions import leo_orbit, sc_attitudes @@ -54,18 +53,21 @@ class SimpleEOSSimulator(SimulationBaseClass.SimBaseClass): the Earth. Dynamics Components - - Forces: J2, Atmospheric Drag w/ COM offset - - Environment: Exponential density model; eclipse - - Actuators: ExternalForceTorque - - Sensors: SimpleNav - - Power System: SimpleBattery, SimpleSink, SimpleSolarPanel - - Data Management System: spaceToGroundTransmitter, simpleStorageUnit, - simpleInstrument + + * Forces: J2, Atmospheric Drag w/ COM offset + * Environment: Exponential density model; eclipse + * Actuators: ExternalForceTorque + * Sensors: SimpleNav + * Power System: SimpleBattery, SimpleSink, SimpleSolarPanel + * Data Management System: spaceToGroundTransmitter, simpleStorageUnit, + simpleInstrument FSW Components: - - MRP Feedback controller - - inertial3d (sun pointing), hillPoint (nadir pointing) - - Desat + + * MRP Feedback controller + * inertial3d (sun pointing), hillPoint (nadir pointing) + * Desat + """ def __init__( @@ -188,7 +190,7 @@ def set_ICs(self): "disturbance_magnitude": 2e-3, "disturbance_vector": np.random.standard_normal(3), # Reaction Wheel speeds - "wheelSpeeds": uniform(-4000 * mc.RPM, 4000 * mc.RPM, 3), # rad/s + "wheelSpeeds": np.random.uniform(-4000 * mc.RPM, 4000 * mc.RPM, 3), # rad/s # Solar Panel Parameters "nHat_B": np.array([0, 1, 0]), "panelArea": 2 * 1.0 * 0.5, @@ -847,14 +849,15 @@ def set_dynamics(self): def set_fsw(self): """ Sets up the attitude guidance stack for the simulation. This simulator runs: - inertial3Dpoint - Sets the attitude guidance objective to point the main panel - at the sun. - hillPointTask: Sets the attitude guidance objective to point a "camera" angle - towards nadir. - attitudeTrackingError: Computes the difference between estimated and guidance - attitudes - mrpFeedbackControl: Computes an appropriate control torque given an attitude - error + + * inertial3Dpoint - Sets the attitude guidance objective to point the main panel + at the sun. + * hillPointTask: Sets the attitude guidance objective to point a "camera" angle + towards nadir. + * attitudeTrackingError: Computes the difference between estimated and guidance + attitudes + * mrpFeedbackControl: Computes an appropriate control torque given an attitude + error """ self.processName = self.DynamicsProcessName diff --git a/src/bsk_rl/envs/simple_eos/gym_env.py b/src/bsk_rl/envs/simple_eos/gym_env.py index d3b91f90..278bed02 100644 --- a/src/bsk_rl/envs/simple_eos/gym_env.py +++ b/src/bsk_rl/envs/simple_eos/gym_env.py @@ -14,27 +14,26 @@ class SimpleEOS(gym.Env): by nadir pointing. Specific imaging targets are not considered. Action Space (discrete, 0 or 1): - 0 - Imaging mode - 1 - Charging mode - 2 - Desat mode - 3 - Downlink mode + * 0 - Imaging mode + * 1 - Charging mode + * 2 - Desat mode + * 3 - Downlink mode Observation Space: - Inertial position and velocity - indices 0-5 - Attitude error and attitude rate - indices 6-7 - Reaction wheel speeds - indices 8-11 - Battery charge - indices 12 - Eclipse indicator - indices 13 - Stored data onboard spacecraft - indices 14 - Data transmitted over interval - indices 15 - Amount of time ground stations were accessible (s) - 16-22 - Percent through planning interval - 23 + * Inertial position and velocity - indices 0-5 + * Attitude error and attitude rate - indices 6-7 + * Reaction wheel speeds - indices 8-11 + * Battery charge - indices 12 + * Eclipse indicator - indices 13 + * Stored data onboard spacecraft - indices 14 + * Data transmitted over interval - indices 15 + * Amount of time ground stations were accessible (s) - 16-22 + * Percent through planning interval - 23 Reward Function: r = +1 for each MB downlinked and no failure r = +1 for each MB downlinked and no failure and +1 if t > t_max - r = - 1000 if failure - (battery drained, buffer overflow, reaction wheel speeds over max) + r = - 1000 if failure (battery drained, buffer overflow, reaction wheel speeds over max) """ def __init__(self): diff --git a/src/bsk_rl/envs/small_body_science/bsk_sim.py b/src/bsk_rl/envs/small_body_science/bsk_sim.py index 08ffb7f3..0c3d4af9 100644 --- a/src/bsk_rl/envs/small_body_science/bsk_sim.py +++ b/src/bsk_rl/envs/small_body_science/bsk_sim.py @@ -42,7 +42,6 @@ unitTestSupport, vizSupport, ) -from numpy.random import uniform from bsk_rl.utilities.effector_primitives import actuator_primitives as ap from bsk_rl.utilities.initial_conditions import sc_attitudes, small_body @@ -298,10 +297,10 @@ def set_ic(self): sigma_init, omega_init = sc_attitudes.random_tumble(maxSpinRate=0.00001) x_0_delta = np.zeros(12) - x_0_delta[0:3] = uniform(-50, 50.0, 3) # Relative s/c position - x_0_delta[3:6] = uniform(-0.1, 0.1, 3) # Relative s/c velocity - x_0_delta[6:9] = uniform(-0.1, 0.1, 3) # Small body attitude - x_0_delta[9:12] = uniform(-0.1, 0.1, 3) # Small body attitude rate + x_0_delta[0:3] = np.random.uniform(-50, 50.0, 3) # Relative s/c position + x_0_delta[3:6] = np.random.uniform(-0.1, 0.1, 3) # Relative s/c velocity + x_0_delta[6:9] = np.random.uniform(-0.1, 0.1, 3) # Small body attitude + x_0_delta[9:12] = np.random.uniform(-0.1, 0.1, 3) # Small body attitude rate mapping_points = small_body.generate_mapping_points( self.n_map_points, self.body_radius @@ -375,7 +374,7 @@ def set_ic(self): "sigma_init": sigma_init, "omega_init": omega_init, # Reaction Wheel speeds - "wheelSpeeds": uniform(-2000 * mc.RPM, 2000 * mc.RPM, 3), # rad/s + "wheelSpeeds": np.random.uniform(-2000 * mc.RPM, 2000 * mc.RPM, 3), # rad/s "max_dV": 40, # m/s # RW motor torque and thruster force mapping FSW config "controlAxes_B": [1, 0, 0, 0, 1, 0, 0, 0, 1], diff --git a/src/bsk_rl/envs/small_body_science/gym_env.py b/src/bsk_rl/envs/small_body_science/gym_env.py index 84e6414b..a12a0029 100644 --- a/src/bsk_rl/envs/small_body_science/gym_env.py +++ b/src/bsk_rl/envs/small_body_science/gym_env.py @@ -11,40 +11,40 @@ class SmallBodyScience(gym.Env): waypoints defined in the sun anti-momentum frame to image candidate landing sites or collect spectroscopy map data while avoiding resource constraint violations. Resource constraint violations include: - - Fuel - - Power - - Data storage - - Collision with the body (not necessarilly a resource, but considered a - failure condition) + * Fuel + * Power + * Data storage + * Collision with the body (not necessarilly a resource, but considered a + failure condition) Action Space (Discrete): - 0 - Charging Mode - 1 - 8 - Transition to waypoint 1-8 - 9 - Map - 10 - Downlink - 11 - Image + * 0 - Charging Mode + * 1 - 8 - Transition to waypoint 1-8 + * 9 - Map + * 10 - Downlink + * 11 - Image Observation Space (Box): - 0-2: Hill-frame position - 3-5: Hill-frame velocity - 6: Eclipse - 7: Data buffer storage - 8: Battery level - 9: dV consumed - 10: Downlink availability - 11-13: Current waypoint - 14-16: Last waypoint - 17: Imaged targets - 18: Downlinked targets - 19-21: Next closest unimaged target position in Hill frame - 22-30: Map regions collected + * 0-2: Hill-frame position + * 3-5: Hill-frame velocity + * 6: Eclipse + * 7: Data buffer storage + * 8: Battery level + * 9: dV consumed + * 10: Downlink availability + * 11-13: Current waypoint + * 14-16: Last waypoint + * 17: Imaged targets + * 18: Downlinked targets + * 19-21: Next closest unimaged target position in Hill frame + * 22-30: Map regions collected Reward Function: - r = +A each tgt downlinked for first time - r = +B for each tgt imaged for first time - r = +C for each map region downlinked for first time - r = +D for each map region collected for first time - r = -E for failure + * r = +A each tgt downlinked for first time + * r = +B for each tgt imaged for first time + * r = +C for each map region downlinked for first time + * r = +D for each map region collected for first time + * r = -E for failure """ def __init__( diff --git a/src/bsk_rl/envs/small_body_science_pomdp/gym_env.py b/src/bsk_rl/envs/small_body_science_pomdp/gym_env.py index 6bdcc47e..84cd8d3a 100644 --- a/src/bsk_rl/envs/small_body_science_pomdp/gym_env.py +++ b/src/bsk_rl/envs/small_body_science_pomdp/gym_env.py @@ -14,40 +14,41 @@ class SmallBodySciencePOMDP(SmallBodyScience): opposed to the SmallBodyScience environment, this environment is utilizes an EKF filter for the observation space to simulate a POMDP, which provides a belief state for the POMDP. + Resource constraint violations include: - - Fuel - - Power - - Data storage - - Collision with the body (not necessarilly a resource, but considered a - failure condition) + * Fuel + * Power + * Data storage + * Collision with the body (not necessarily a resource, but considered a + failure condition) Action Space (Discrete): - 0 - Charging Mode - 1 - 8 - Transition to waypoint 1-8 - 9 - Map - 10 - Downlink - 11 - Image - 12 - Navigation Mode + * 0 - Charging Mode + * 1 - 8 - Transition to waypoint 1-8 + * 9 - Map + * 10 - Downlink + * 11 - Image + * 12 - Navigation Mode Observation Space (Box): - 0-2: Hill-frame position - 3-5: Hill-frame velocity - 6: Eclipse - 7: Data buffer storage - 8: Battery level - 9: dV consumed - 10: Downlink availability - 11-13: Current waypoint - 14-16: Last waypoint - 17-20: Location of the next target for imaging - 20-26: Filter covariance diagonals + * 0-2: Hill-frame position + * 3-5: Hill-frame velocity + * 6: Eclipse + * 7: Data buffer storage + * 8: Battery level + * 9: dV consumed + * 10: Downlink availability + * 11-13: Current waypoint + * 14-16: Last waypoint + * 17-20: Location of the next target for imaging + * 20-26: Filter covariance diagonals Reward Function: - r = +A each tgt downlinked for first time - r = +B for each tgt imaged for first time - r = +C for each map region downlinked for first time - r = +D for each map region collected for first time - r = -E for failure + * r = +A each tgt downlinked for first time + * r = +B for each tgt imaged for first time + * r = +C for each map region downlinked for first time + * r = +D for each map region collected for first time + * r = -E for failure """ def __init__(self): diff --git a/src/bsk_rl/utilities/effector_primitives/actuator_primitives.py b/src/bsk_rl/utilities/effector_primitives/actuator_primitives.py index 8f704a72..0254c841 100644 --- a/src/bsk_rl/utilities/effector_primitives/actuator_primitives.py +++ b/src/bsk_rl/utilities/effector_primitives/actuator_primitives.py @@ -1,6 +1,6 @@ +import numpy as np from Basilisk.simulation import reactionWheelStateEffector, thrusterDynamicEffector from Basilisk.utilities import simIncludeRW, simIncludeThruster -from numpy.random import uniform def balancedHR16Triad( @@ -15,7 +15,7 @@ def balancedHR16Triad( """ rwFactory = simIncludeRW.rwFactory() if useRandom: - wheelSpeeds = uniform(randomBounds[0], randomBounds[1], 3) + wheelSpeeds = np.random.uniform(randomBounds[0], randomBounds[1], 3) rwFactory.create( "Honeywell_HR16", [1, 0, 0], maxMomentum=50.0, Omega=wheelSpeeds[0] # RPM diff --git a/src/bsk_rl/utilities/initial_conditions/leo_initial_conditions.py b/src/bsk_rl/utilities/initial_conditions/leo_initial_conditions.py index 5adb03c0..c452e96c 100644 --- a/src/bsk_rl/utilities/initial_conditions/leo_initial_conditions.py +++ b/src/bsk_rl/utilities/initial_conditions/leo_initial_conditions.py @@ -2,7 +2,6 @@ from Basilisk.utilities import astroFunctions from Basilisk.utilities import macros as mc from Basilisk.utilities import orbitalMotion -from numpy.random import uniform from bsk_rl.utilities.initial_conditions import leo_orbit, sc_attitudes @@ -37,7 +36,7 @@ def sampled_400km_leo_smallsat_tumble(): "disturbance_magnitude": 2e-4, "disturbance_vector": np.random.standard_normal(3), # Reaction Wheel speeds - "wheelSpeeds": uniform(-800, 800, 3), # RPM + "wheelSpeeds": np.random.uniform(-800, 800, 3), # RPM # Solar Panel Parameters "nHat_B": np.array([0, -1, 0]), "panelArea": 0.2 * 0.3, @@ -148,7 +147,7 @@ def walker_delta_single_sc_500_km(oe, sim_length, global_tgts, priorities): # Sample attitude and rates sigma_init, omega_init = sc_attitudes.random_tumble(maxSpinRate=0.00001) - wheel_speeds = uniform(-1500, 1500, 3) # RPMs + wheel_speeds = np.random.uniform(-1500, 1500, 3) # RPMs # Dict of initial conditions initial_conditions = { diff --git a/src/bsk_rl/utilities/initial_conditions/leo_orbit.py b/src/bsk_rl/utilities/initial_conditions/leo_orbit.py index d7129e41..76ec5124 100644 --- a/src/bsk_rl/utilities/initial_conditions/leo_orbit.py +++ b/src/bsk_rl/utilities/initial_conditions/leo_orbit.py @@ -7,7 +7,6 @@ from Basilisk.utilities import SimulationBaseClass from Basilisk.utilities import macros as mc from Basilisk.utilities import orbitalMotion, simIncludeGravBody -from numpy.random import uniform bskPath = __path__[0] @@ -40,7 +39,7 @@ def random_inclined_circular_300km(): """ oe = orbitalMotion.ClassicElements() - oe.a = 6371 * 1000.0 + uniform(290e3, 310e3) + oe.a = 6371 * 1000.0 + np.random.uniform(290e3, 310e3) oe.e = 0.0 oe.i = 45.0 * mc.D2R @@ -59,11 +58,11 @@ def sampled_400km(): """ oe = orbitalMotion.ClassicElements() oe.a = 6371 * 1000.0 + 400.0 * 1000 - oe.e = uniform(0, 0.001, 1) - oe.i = uniform(-90 * mc.D2R, 90 * mc.D2R, 1) - oe.Omega = uniform(0 * mc.D2R, 360 * mc.D2R, 1) - oe.omega = uniform(0 * mc.D2R, 360 * mc.D2R, 1) - oe.f = uniform(0 * mc.D2R, 360 * mc.D2R, 1) + oe.e = np.random.uniform(0, 0.001, 1) + oe.i = np.random.uniform(-90 * mc.D2R, 90 * mc.D2R, 1) + oe.Omega = np.random.uniform(0 * mc.D2R, 360 * mc.D2R, 1) + oe.omega = np.random.uniform(0 * mc.D2R, 360 * mc.D2R, 1) + oe.f = np.random.uniform(0 * mc.D2R, 360 * mc.D2R, 1) rN, vN = orbitalMotion.elem2rv(mu, oe) return oe, rN, vN @@ -78,12 +77,12 @@ def sampled_500km_boulder_gs(): mu = 0.3986004415e15 oe = orbitalMotion.ClassicElements() oe.a = 6371 * 1000.0 + 500.0 * 1000 - oe.e = uniform(0, 0.01, 1) - # oe.i = uniform(40*mc.D2R, 60*mc.D2R,1) - oe.i = uniform(40 * mc.D2R, 60 * mc.D2R, 1) - oe.Omega = uniform(0 * mc.D2R, 20 * mc.D2R, 1) - oe.omega = uniform(0 * mc.D2R, 20 * mc.D2R, 1) - oe.f = uniform(0 * mc.D2R, 360 * mc.D2R, 1) + oe.e = np.random.uniform(0, 0.01, 1) + # oe.i = np.random.uniform(40*mc.D2R, 60*mc.D2R,1) + oe.i = np.random.uniform(40 * mc.D2R, 60 * mc.D2R, 1) + oe.Omega = np.random.uniform(0 * mc.D2R, 20 * mc.D2R, 1) + oe.omega = np.random.uniform(0 * mc.D2R, 20 * mc.D2R, 1) + oe.f = np.random.uniform(0 * mc.D2R, 360 * mc.D2R, 1) rN, vN = orbitalMotion.elem2rv(mu, oe) return oe, rN, vN @@ -98,12 +97,12 @@ def sampled_boulder_gs(nominal_radius): mu = 0.3986004415e15 oe = orbitalMotion.ClassicElements() oe.a = nominal_radius - oe.e = uniform(0, 0.01, 1) - # oe.i = uniform(40*mc.D2R, 60*mc.D2R,1) - oe.i = uniform(40 * mc.D2R, 60 * mc.D2R, 1) - oe.Omega = uniform(0 * mc.D2R, 360 * mc.D2R, 1) - oe.omega = uniform(0 * mc.D2R, 360 * mc.D2R, 1) - oe.f = uniform(0 * mc.D2R, 360 * mc.D2R, 1) + oe.e = np.random.uniform(0, 0.01, 1) + # oe.i = np.random.uniform(40*mc.D2R, 60*mc.D2R,1) + oe.i = np.random.uniform(40 * mc.D2R, 60 * mc.D2R, 1) + oe.Omega = np.random.uniform(0 * mc.D2R, 360 * mc.D2R, 1) + oe.omega = np.random.uniform(0 * mc.D2R, 360 * mc.D2R, 1) + oe.f = np.random.uniform(0 * mc.D2R, 360 * mc.D2R, 1) rN, vN = orbitalMotion.elem2rv(mu, oe) return oe, rN, vN @@ -456,10 +455,13 @@ def distribute_tgts(rN, vN, sim_length, utc_init, global_tgts, dt=60.0): def elrange_req(sc_pos, tgt_pos): """ Determines if the spacecraft is within the elevation and range requirements of - a target - :param sc_pos: spacecraft position expressed in the ECEF frame - :param tgt_pos: tgt_pos expressed in the ECEF frame - :return within: T/F - within el, range requirements or not + a target + + Args: + sc_pos: spacecraft position expressed in the ECEF frame + tgt_pos: tgt_pos expressed in the ECEF frame + Returns: + T/F - within el, range requirements or not """ # Import relevant library