diff --git a/.flake8 b/.flake8 index c8dd7afed..95ce17390 100644 --- a/.flake8 +++ b/.flake8 @@ -1,61 +1,5 @@ -# flake8.pycqa.org/en/latest/user/configuration.html -# flake8.pycqa.org/en/latest/user/options.html [flake8] - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-count -count = True - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-exclude -# exclude = .svn,CVS,.bzr,.hg,.git,__pycache__,.tox,.nox,.eggs,*.egg - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-extend-exclude -extend-exclude = - # virtual environments - .venv/*,venv/*, - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-filename -filename = *.py - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-format -format = default - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-hang-closing -hang-closing = False - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-ignore -# ignore = E121,E123,E126,E226,E24,E704,W503,W504 - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-extend-ignore -extend-ignore = - # expected 2 blank lines - E302, - # line too long - E501, - # imported but unused - F401, - # doc line too long - W505, - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-max-line-length +ignore = E226,E302,E41,F401,E402,E501,E504 max-line-length = 120 - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-max-doc-length -max-doc-length = 120 - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-indent-size -indent-size = 4 - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-show-source -show-source = True - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-statistics -statistics = True - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-jobs -jobs = 8 - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-doctests -doctests = True - -# flake8.pycqa.org/en/latest/user/options.html#cmdoption-flake8-max-complexity -max-complexity = 12 +exclude = migrations/*,venv/*,docs/*,build/* +max-complexity = 10 diff --git a/.github/workflows/install-lint-test-on-mac.yml b/.github/workflows/install-lint-test-on-mac.yml deleted file mode 100644 index af2060990..000000000 --- a/.github/workflows/install-lint-test-on-mac.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Install Package, Lint Code & Run Tests on Mac - -on: - push: - branches: - - "*" - - pull_request: - branches: - - "*" - -jobs: - install-lint-test-on-mac: - runs-on: macos-latest - - strategy: - matrix: - python-version: - - '3.10' - - 3.11 - - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - - name: Set Up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Poetry - run: make get-poetry - - - name: Install Package & Dependencies - run: make install - - - name: Lint Code - run: make lint - - - name: Run Tests - run: make test - env: - LEPTON_API_KEY: ${{ secrets.LEPTON_API_KEY }} diff --git a/.github/workflows/install-lint-test-on-win.yml b/.github/workflows/install-lint-test-on-win.yml deleted file mode 100644 index e3073f126..000000000 --- a/.github/workflows/install-lint-test-on-win.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Install Package, Lint Code & Run Tests on Windows - -on: - push: - branches: - - "*" - - pull_request: - branches: - - "*" - -jobs: - install-lint-test-on-win: - runs-on: windows-latest - - strategy: - matrix: - python-version: - - '3.10' - - 3.11 - - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - - name: Set Up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install Poetry - run: .\make get-poetry - - - name: Install Package & Dependencies - run: .\make install - - - name: Lint Code - run: .\make lint - - - name: Run Tests - run: .\make test - env: - LEPTON_API_KEY: ${{ secrets.LEPTON_API_KEY }} diff --git a/.github/workflows/install-lint-test-on-ubuntu.yml b/.github/workflows/install-lint-test.yml similarity index 89% rename from .github/workflows/install-lint-test-on-ubuntu.yml rename to .github/workflows/install-lint-test.yml index cfeee6504..7fe3c1030 100644 --- a/.github/workflows/install-lint-test-on-ubuntu.yml +++ b/.github/workflows/install-lint-test.yml @@ -1,4 +1,4 @@ -name: Install Package, Lint Code & Run Tests on Ubuntu +name: Install Package, Lint Code & Run Tests on: push: @@ -10,7 +10,7 @@ on: - "*" jobs: - install-lint-test-on-ubuntu: + install-lint-test: runs-on: ubuntu-latest strategy: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 93768dfd2..b165e573c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,7 @@ jobs: run: make get-poetry - name: Build Distribution - run: make build + run: poetry build - name: Publish Distribution uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index 8289049d8..10ec8dcbc 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ poetry.lock __pycache__/ .pytest_cache/ .ropeproject/ -.venv/ # DOCUMENTATION diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000..b4300eeb7 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,15 @@ +[MESSAGES CONTROL] + +disable= + attribute-defined-outside-init, + duplicate-code, + fixme, + import-error, + line-too-long, + missing-class-docstring, + missing-function-docstring, + missing-module-docstring, + raw-checker-failed, + redefined-outer-name, + too-few-public-methods, + unnecessary-pass, diff --git a/.ruff.toml b/.ruff.toml index e5d1dc3f6..28137f5b7 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -1,126 +1,11 @@ -# docs.astral.sh/ruff/configuration - - -extend-exclude = [ -] - -line-length = 120 - - [format] exclude = [ - "*.py", - "*.toml", + '*.py', ] [lint] -select = ["ALL"] -extend-select = [ -] + ignore = [ - "A001", # variable ... is shadowing a Python builtin - "A003", # class attribute is shadowing a Python builtin - "ANN001", # missing type annotation for function argument - "ANN002", # missing type annotation for `*args` - "ANN003", # missing type annotation for `**kwargs` - "ANN101", # missing type annotation for `self` in method - "ANN102", # missing type annotation for `cls` in classmethod - "ANN201", # missing return type annotation for public function - "ANN202", # missing return type annotation for private function - "ANN204", # missing return type annotation for special method `__init__` - "ANN205", # missing return type annotation for staticmethod - "ANN206", # missing return type annotation for classmethod `setUpClass` - "ANN401", # dynamically typed expressions (typing.Any) are disallowed - "ARG002", # unused method argument: `storage_dir` - "B008", # do not perform function call in argument defaults; - # instead, perform the call within the function, or read the default from a module-level singleton variable - "B027", # ... is an empty method in an abstract base class, but has no abstract decorator - "B905", # `zip()` without an explicit `strict=` parameter - "COM812", # trailing comma missing - "CPY001", # missing copyright notice at top of file - "D100", # missing docstring in public module - "D101", # missing docstring in public class - "D102", # missing docstring in public method - "D103", # missing docstring in public function - "D104", # missing docstring in public package - "D107", # missing docstring in `__init__` - "D200", # one-line docstring should fit on one line - "D202", # no blank lines allowed after function docstring - "D204", # 1 blank line required after class docstring - "D205", # 1 blank line required between summary line and description - "D212", # multi-line docstring summary should start at the first line - "D213", # multi-line docstring summary should start at the second line - "D400", # first line should end with a period - "D401", # first line of docstring should be in imperative mood - "D404", # first word of the docstring should not be - "D407", # missing dashed underline after section - "D410", # Missing blank line after section - "D411", # missing blank line before section - "D415", # first line should end with a period, question mark, or exclamation point - "DTZ011", # the use of `datetime.date.today()` is not allowed, use `datetime.datetime.now(tz=).date()` instead - "E261", # insert at least two spaces before an inline comment - "E501", # line too long - "EM101", # exception must not use a string literal, assign to variable first - "EM102", # exception must not use an f-string literal, assign to variable first - "ERA001", # found commented-out code - "F401", # imported but unused - "FBT001", # boolean-typed positional argument in function definition - "FBT002", # boolean default positional argument in function definition - "FBT003", # boolean positional value in function call - "FIX002", # line contains TODO, consider resolving the issue - "I001", # import block is un-sorted or un-formatted - "INP001", # file is part of an implicit namespace package; add an `__init__.py` - "LOG009", # use of undocumented `logging.WARN` constant - "N811", # constant `SSM` imported as non-constant `LlamaIndexSSM` - "PERF401", # use a list comprehension to create a transformed list - "PGH003", # use specific rule codes when ignoring type issues - "PIE790", # unnecessary `pass` statement - "PTH100", # `os.path.abspath()` should be replaced by `Path.resolve()` - "PTH103", # `os.makedirs()` should be replaced by `Path.mkdir(parents=True)` - "PTH109", # `os.getcwd()` should be replaced by `Path.cwd()` - "PTH110", # `os.path.exists()` should be replaced by `Path.exists()` - "PTH112", # `os.path.isdir()` should be replaced by `Path.is_dir()` - "PTH118", # `os.path.join()` should be replaced by `Path` with `/` operator - "PTH119", # `os.path.basename()` should be replaced by `Path.name` - "PTH123", # `open()` should be replaced by `Path.open()` - "PLC0415", # `import` should be at the top-level of a file - "PLR0904", # too many public methods - "PLR0917", # too many positional arguments - "PLR0911", # too many return statements - "PLR0913", # too many arguments in function definition - "PLR2004", # magic value used in comparison, consider replacing with a constant variable - "PLR6301", # method could be a function, class method, or static method - "PT009", # use a regular `assert` instead of unittest-style `assertIsInstance` - "PT018", # assertion should be broken down into multiple parts - "PT027", # use `pytest.raises` instead of unittest-style `assertRaises` - "Q000", # single quotes found but double quotes preferred - "Q001", # single quote multiline found but double quotes preferred - "Q003", # change outer quotes to avoid escaping inner quotes - "RET504", # unnecessary assignment to `response` before `return` statement - "RUF001", # string contains ambiguous `’` (RIGHT SINGLE QUOTATION MARK); did you mean ``` (GRAVE ACCENT)? - "RUF002", # docstring contains ambiguous `’` (RIGHT SINGLE QUOTATION MARK); did you mean ``` (GRAVE ACCENT)? - "RUF003", # comment contains ambiguous `’` (RIGHT SINGLE QUOTATION MARK); did you mean ``` (GRAVE ACCENT)? - "RUF005", # consider `[..., *...]` instead of concatenation - "RUF013", # PEP 484 prohibits implicit `Optional` - "RUF017", # Avoid quadratic list summation" - "RUF018", # avoid assignment expressions in `assert` statements - "S101", # use of `assert` detected - "S106", # possible hardcoded password assigned to argument - "S605", # starting a process with a shell, possible injection detected - "SIM102", # use a single `if` statement instead of nested `if` statements - "SIM108", # use ternary operator `temp = temp["content"] if isinstance(temp, dict) else temp.content` instead of `if`-`else`-block - "SIM112", # use capitalized environment variable - "SIM401", # use `item.get("role", "assistant")` instead of an `if` block - "SLF001", # private member accessed - "T201", # `print` found - "TD002", # missing author in TODO; try: `# TODO(): ...` or `# TODO @: ...` - "TD003", # missing issue link on the line following this TODO - "TRY003", # avoid specifying long messages outside the exception class - "TRY004", # prefer `TypeError` exception for invalid type - "UP006", # use `list` instead of `List` for type annotation - "UP007", # use `X | Y` for type annotations - "UP035", # `typing.[X]` is deprecated, use `[x]` instead - "UP039", # unnecessary parentheses after class definition ] diff --git a/.vscode/extensions.json b/.vscode/extensions.json deleted file mode 100644 index 282a1b212..000000000 --- a/.vscode/extensions.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "recommendations": [ - "ms-python.flake8", // Flake8 (Microsoft) - "ms-python.pylint", // PyLint (Microsoft) - - "charliermarsh.ruff", // Ruff - - "garaioag.garaio-vscode-unwanted-recommendations" // Unwanted Recommendations - ], - - "unwantedRecommendations": [ - "ms-python.mypy-type-checker", // MyPy Type Checker (Microsoft) - - "matangover.mypy" // MyPy - ] -} diff --git a/.vscode/settings.json b/.vscode/settings.json index 7f8210683..86d4598e4 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,199 +2,18 @@ "editor.rulers": [120], "files.exclude": { - // Git - "**/.git": true, - - // Python & related "**/*.egg-info": true, + "**/.git": true, "**/.mypy_cache": true, "**/__pycache__": true, "**/.pytest_cache": true, - "**/.ropeproject": true, - "**/.ruff_cache": true + "**/.ropeproject": true }, - - // PYTHON - // ====== - "[python]": { "editor.formatOnSave": false }, "python.analysis.diagnosticSeverityOverrides": { - }, - - - // FLAKE8 - // ====== - - // Arguments passed to Flake8 for linting Python files. - // Each argument should be provided as a separate string in the array. - "flake8.args": [ - "--verbose", - "--color=always" - ], - - // Sets the current working directory used to lint Python files with Flake8. - // By default, it uses the root directory of the workspace ${workspaceFolder}. - // You can set it to ${fileDirname} to use the parent folder of the file being linted - // as the working directory for Flake8. - "flake8.cwd": "${workspaceFolder}", - - // Mapping of Flake8's message types to VS Code's diagnostic severity levels as displayed in the Problems window. - // You can also use it to override specific Flake8 error codes. - "flake8.severity": { - "convention": "Information", - "error": "Error", - "fatal": "Error", - "info": "Information", - "refactor": "Hint", - "warning": "Warning" - }, - - // Path or command to be used by the extension to lint Python files with Flake8. - // Accepts an array of a single or multiple strings. - // If passing a command, each argument should be provided as a separate string in the array. - // If set to ["flake8"], it will use the version of Flake8 available in the PATH environment variable. - // Note: Using this option may slowdown linting. - // "flake8.path": [], - - // Path to a Python executable or a command that will be used to launch the Flake8 server and any subprocess. - // Accepts an array of a single or multiple strings. - // When set to [], the extension will use the path to the selected Python interpreter. - // If passing a command, each argument should be provided as a separate string in the array. - "flake8.interpreter": [], - - // Defines which Flake8 binary to be used to lint Python files. - // When set to useBundled, the extension will use the Flake8 binary that is shipped with the extension. - // When set to fromEnvironment, the extension will attempt to use the Flake8 binary and all dependencies - // that are available in the currently selected environment. - // Note: If the extension can't find a valid Flake8 binary in the selected environment, - // it will fallback to using the Flake8 binary that is shipped with the extension. - // This setting will be overriden if flake8.path is set. - "flake8.importStrategy": "fromEnvironment", - - // Controls when notifications are shown by this extension. - // Accepted values are onError, onWarning, always and off. - "flake8.showNotification": "onWarning", - - // Enable linting Python files with Flake8 as you type. - "flake8.lintOnChange": true, - - // Configure glob patterns as supported by the fnmatch Python library (https://docs.python.org/3/library/fnmatch.html) - // to exclude files or folders from being linted with Flake8. - "flake8.ignorePatterns": [ - ".venv/*", "venv/*" - ], - - - // PYLINT - // ====== - - // Arguments passed to Pylint for linting Python files. - // Each argument should be provided as a separate string in the array. - "pylint.args": [ - ], - - // Sets the current working directory used to lint Python files with Pylint. - // By default, it uses the root directory of the workspace ${workspaceFolder}. - // You can set it to ${fileDirname} to use the parent folder of the file being linted - // as the working directory for Pylint. - "pylint.cwd": "${workspaceFolder}", - - // Enable/disable linting Python files with Pylint. - // This setting can be applied globally or at the workspace level. - // If disabled, the linting server itself will continue to be active and monitor read and write events, - // but it won't perform linting or expose code actions. - "pylint.enabled": true, - - // Mapping of Pylint's message types to VS Code's diagnostic severity levels as displayed in the Problems window. - // You can also use it to override specific Pylint error codes. - "pylint.severity": { - "convention": "Information", - "error": "Error", - "fatal": "Error", - "info": "Information", - "refactor": "Hint", - "warning": "Warning" - }, - - // Path or command to be used by the extension to lint Python files with Pylint. - // Accepts an array of a single or multiple strings. - // If passing a command, each argument should be provided as a separate string in the array. - // If set to ["pylint"], it will use the version of Pylint available in the PATH environment variable. - // Note: Using this option may slowdown linting. - // "pylint.path": [], - - // Path to a Python executable or a command that will be used to launch the Pylint server and any subprocess. - // Accepts an array of a single or multiple strings. - // When set to [], the extension will use the path to the selected Python interpreter. - // If passing a command, each argument should be provided as a separate string in the array. - "pylint.interpreter": [], - - // Defines which Pylint binary to be used to lint Python files. - // When set to useBundled, the extension will use the Pylint binary that is shipped with the extension. - // When set to fromEnvironment, the extension will attempt to use the Pylint binary and all dependencies - // that are available in the currently selected environment. - // Note: If the extension can't find a valid Pylint binary in the selected environment, - // it will fallback to using the Pylint binary that is shipped with the extension. - // This setting will be overriden if pylint.path is set. - "pylint.importStrategy": "fromEnvironment", - - // Controls when notifications are shown by this extension. - // Accepted values are onError, onWarning, always and off. - "pylint.showNotification": "onWarning", - - // Enable linting Python files with Pylint as you type. - "pylint.lintOnChange": true, - - // Configure glob patterns as supported by the fnmatch Python library - // to exclude files or folders from being linted with Pylint. - "pylint.ignorePatterns": [ - ".venv/*", "venv/*" - ], - - - // RUFF - // ==== - - // Additional command-line arguments to pass to ruff check, e.g., "args": ["--config=/path/to/pyproject.toml"]. - // Supports a subset of Ruff's command-line arguments, ignoring those that are required to operate the LSP, - // like --force-exclude and --verbose. - "ruff.lint.args": [ - ], - - // Path to a custom ruff executable, e.g., ["/path/to/ruff"]. - "ruff.path": [], - - // Path to a Python interpreter to use to run the linter server. - "ruff.interpreter": [], - - // Strategy for loading the ruff executable. - // fromEnvironment picks up Ruff from the environment, falling back to the bundled version if needed. - // useBundled uses the version bundled with the extension. - "ruff.importStrategy": "fromEnvironment", - - // Run Ruff on every keystroke (onType) or on save (onSave). - "ruff.lint.run": "onSave", - - // Whether to enable the Ruff extension. - // Modifying this setting requires restarting VS Code to take effect. - "ruff.enable": true, - - // Whether to register Ruff as capable of handling source.organizeImports actions. - "ruff.organizeImports": true, - - // Whether to register Ruff as capable of handling source.fixAll actions. - "ruff.fixAll": false, - - // Whether to display Quick Fix actions to autofix violations. - "ruff.codeAction.fixViolation.enable": true, - - // Whether to display Quick Fix actions to disable rules via noqa suppression comments. - "ruff.codeAction.disableRuleComment.enable": true, - - // Setting to control when a notification is shown: off, onError, onWarning, always. - "ruff.showNotification": "onWarning" + } } diff --git a/Makefile b/Makefile index 664dd2563..b11ba8418 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,22 @@ # DIRECTORY NAMES & PATHS # ======================= -LIB_DIR=openssa +PROJECT_DIR=$(PWD) +ROOT_DIR=$(PROJECT_DIR) -EXAMPLES_DIR=examples +LIB_DIR_NAME=openssa +LIB_DIR=$(PROJECT_DIR)/$(LIB_DIR_NAME) -TESTS_DIR=tests +EXAMPLES_DIR_NAME=examples +EXAMPLES_DIR=$(PROJECT_DIR)/$(EXAMPLES_DIR_NAME) + +TESTS_DIR_NAME=tests +TESTS_DIR=$(PROJECT_DIR)/$(TESTS_DIR_NAME) + +DOCS_DIR_NAME=docs +DOCS_DIR=$(PROJECT_DIR)/$(DOCS_DIR_NAME) -DOCS_DIR=docs DOCS_BUILD_DIR=$(DOCS_DIR)/_build + DOCS_SUBDIRS_TO_PUBLISH := _images _static @@ -35,32 +44,11 @@ install: poetry lock poetry install --extras=contrib --with=docs --with=lint --with=test -install-editable: - python3 -m pip install -e ".[contrib]" --upgrade --user - # LINTING # ======= -lint: lint-flake8 lint-pylint lint-ruff - -lint-flake8: - # flake8.pycqa.org/en/latest/user/invocation.html - # flake8.pycqa.org/en/latest/user/options.html - poetry run flake8 $(LIB_DIR) $(DOCS_DIR) $(EXAMPLES_DIR) $(TESTS_DIR) \ - --verbose --color always - -lint-pylint: - # pylint.readthedocs.io/en/latest/user_guide/usage/run.html - poetry run pylint $(LIB_DIR) $(DOCS_DIR) $(EXAMPLES_DIR) $(TESTS_DIR) - -lint-ruff: - # docs.astral.sh/ruff/linter - poetry run ruff check $(LIB_DIR) $(DOCS_DIR) $(EXAMPLES_DIR) $(TESTS_DIR) \ - --show-source \ - --output-format text \ - --target-version py310 \ - --preview \ - --respect-gitignore +lint: + poetry run pylint $(LIB_DIR_NAME) $(DOCS_DIR_NAME) $(EXAMPLES_DIR_NAME) $(TESTS_DIR_NAME) # TESTING @@ -76,7 +64,7 @@ pre-commit: lint test # DISTRIBUTION BUILDING & PYPI RELEASE # ==================================== -build: +dist: poetry build pypi-auth: @@ -114,7 +102,7 @@ docs-build-api: */contrib/streamlit_ssa_prob_solver/main.py */contrib/streamlit_ssa_prob_solver/pages # get rid of undocumented members - # sed -e /:undoc-members:/d -i .orig "$(DOCS_DIR)"/$(LIB_DIR)*.rst + # sed -e /:undoc-members:/d -i .orig "$(DOCS_DIR)"/$(LIB_DIR_NAME)*.rst # rm "$(DOCS_DIR)"/*.orig docs-build: docs-build-clean docs-build-api @@ -131,7 +119,7 @@ docs-deploy: docs-build rm *.html cp "$(DOCS_BUILD_DIR)"/*.html . git add --all "*.html" - git reset "$(DOCS_DIR)/*.html" + git reset "$(DOCS_DIR_NAME)/*.html" for docs_subdir_to_publish in $(DOCS_SUBDIRS_TO_PUBLISH) ; do \ echo "syncing $$docs_subdir_to_publish..." ; \ @@ -157,12 +145,4 @@ launch-solver: poetry run openssa launch solver public: - rsync . ../openssa/ \ - --archive \ - --delete \ - --exclude .git \ - --exclude __pycache__ --exclude .mypy_cache --exclude .pytest_cache --exclude .ruff_cache \ - --exclude .venv --exclude venv \ - --exclude *.rst --exclude "docs/_build" \ - --links \ - --verbose + rsync -av --delete --exclude .git --links . ../openssa/ diff --git a/README.md b/README.md index 21a42be01..5c2dd65c0 100644 --- a/README.md +++ b/README.md @@ -11,36 +11,18 @@ OpenSSA is an open-source framework for Small Specialist Agents (SSAs), problem- SSAs are light-weight, domain-focused and incorporate reasoning and planning capabilities. These characteristics make them ideal for complex hierarchical tasks typically found in industrial applications. +![OpenSSA One Pager](/docs/diagrams/openssa-onepager.png) ## Small Size, Specific-Domain Specialization The trend towards specialization in AI models is a clear trajectory seen by many in the field. ->   -> _Specialization is crucial for quality .. not general purpose Al models_ – Eric Schmidt, Schmidt Foundation ->   - ->   -> _.. small models .. for a specific task that are good_ – Matei Zaharia, Databricks ->   - ->   -> _.. small agents working together .. specific and best in their tasks_ – Harrison Chase, Langchain ->   - >   > _.. realize that smaller, cheaper, more specialized models make more sense for 99% of AI use-cases .._ – Clem Delangue, Hugging Face >   ->   -> _.. small but highly capable expert models_ – Andrej Karpathy, OpenAI ->   - ->   -> _.. small models are .. a massive paradigm shift .. about deploying AI models at scale_ – Rob Toews, Radical Ventures ->   -As predicted by Eric Schmidt and others, we will see “a rich ecosystem to emerge [of] high-value, specialized AI systems.” SSAs are the central part in the architecture of these systems. +As predicted by Clem Delangue and others, we will see “a rich ecosystem to emerge [of] high-value, specialized AI systems.” SSAs are the central part in the architecture of these systems. ## System-1 & System-2 Intelligence @@ -48,6 +30,9 @@ As predicted by Eric Schmidt and others, we will see “a rich ecosystem to emer In addition to information-retrieval and inferencing ("System-1 intelligence") capabilities, SSAs are additionally designed with hierachical reasoning and planning ("System-2 intelligence") capabilities. They can execute tasks following general-purpose problem-solving paradigms (such as OODA) and domain-specific expert heuristics, in order to solve a diverse variery of problems that are hard for System-1-only Large Language Models (LLMs) and traditional AI models. +System 2 thinking is often considered advantageous in certain contexts due to its deliberate, analytical nature. It excels in handling complex and novel situations, enabling individuals to engage in thoughtful reflection and make well-reasoned decisions. System 2 thinking is particularly valuable for risk assessment, mitigating impulsive judgments, and adapting mental models based on intentional learning. Moreover, it helps avoid cognitive biases and stereotypes by involving conscious, effortful processing. While System 1 thinking is valuable for quick and intuitive decision-making in familiar scenarios, System 2 thinking's strengths lie in its ability to navigate intricate situations, analyze information thoroughly, and make informed choices that consider long-term consequences. The effectiveness of each thinking system depends on the specific demands of the task at hand, with both contributing to the overall cognitive toolkit. + + ## SSA vs LLM Unlike LLMs, which are computationally intensive and generalized, SSAs are lean, efficient, and designed specifically for individual domains. This focus makes them an optimal choice for businesses, SMEs, researchers, and developers seeking specialized and robust AI solutions for industrial applications. @@ -75,14 +60,65 @@ Our primary audience includes: ## SSA Architecture +### OpenSSA Framework Library + +![OpenSSA Key Components](/docs/diagrams/ssm-key-components.drawio.png) + +### High-Level Class Diagram + +![OpenSSA High-Level Class Diagram](/docs/diagrams/ssm-class-diagram.drawio.png) + + +## Getting Started with OpenSSA + +### Who Are You? + +1. An end-user of OpenSSA-based applications + +2. A developer of applications or services using OpenSSA + +3. An aspiring contributor to OpenSSA + +4. A committer to OpenSSA + +### Getting Started as an End-User + + +### Getting Started as a Developer + +See some example user programs in the [examples/notebooks](./examples/notebooks) directory. For example, the see the sample use case on ALD semiconductor knowledge, do: + +```bash +% cd examples/notebooks +``` + +#### Common `make` targets for OpenSSM developers + +See [MAKEFILE](dev/makefile_info.md) for more details. + +```bash +% make clean +% make build +% make rebuild +% make test + +% make poetry-init +% make poetry-install +% make install # local installation of openssm + +% make pypi-auth # only for maintainers +% make publish # only for maintainers +``` +### Getting Started as an Aspiring Contributor -## Getting Started +OpenSSM is a community-driven initiative, and we warmly welcome contributions. Whether it's enhancing existing models, creating new SSMs for different industrial domains, or improving our documentation, every contribution counts. See our [Contribution Guide](../CONTRIBUTING.md) for more details. -See our [Getting Started Guide](docs/GETTING_STARTED.md) for more information. +You can begin contributing to the OpenSSM project in the `contrib/` directory. +### Getting Started as a Committer -## Roadmap +You already know what to do. ## Community @@ -104,3 +140,12 @@ OpenSSA is released under the [Apache 2.0 License](LICENSE.md). ## Note: Lepton API Key +Head to [Lepton](https://dashboard.lepton.ai/) to get your API key. +* Go to `Settings` +* Select `API tokens` +* Copy `` + +In terminal, run +```bash= +export LEPTON_API_KEY= +``` \ No newline at end of file diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 49bf33d37..36ae31321 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -1,4 +1,4 @@ -# Getting Started with OpenSSM +# Getting Started with OpenSSA ## Who Are You? @@ -14,12 +14,10 @@ ## Getting Started as a Developer -See some example user programs in the [examples](./examples) directory. For example, to run the `chatssm` example, do: +See some example user programs in the [examples/notebooks](./examples/notebooks) directory. For example, the see the sample use case on ALD semiconductor knowledge, do: ```bash -% cd examples/chatssm -% make clean -% make +% cd examples/notebooks ``` ### Common `make` targets for OpenSSM developers diff --git a/docs/diagrams/openssa-onepager.png b/docs/diagrams/openssa-onepager.png new file mode 100644 index 000000000..42440dbdf Binary files /dev/null and b/docs/diagrams/openssa-onepager.png differ diff --git a/make.bat b/make.bat deleted file mode 100644 index 66151f3e8..000000000 --- a/make.bat +++ /dev/null @@ -1,135 +0,0 @@ -@echo off - - -:: DIRECTORY NAMES & PATHS -:: ======================= -SET LIB_DIR=openssa - -SET EXAMPLES_DIR=examples - -SET TESTS_DIR=tests - -SET DOCS_DIR=docs -SET DOCS_BUILD_DIR=%DOCS_DIR%\_build - - -:: TARGETS -:: ======= -SET TARGET=%1 - -IF "%TARGET%"=="get-poetry" GOTO get-poetry - -IF "%TARGET%"=="install" GOTO install -IF "%TARGET%"=="install-editable" GOTO install-editable - -IF "%TARGET%"=="lint" GOTO lint -IF "%TARGET%"=="lint-flake8" GOTO lint-flake8 -IF "%TARGET%"=="lint-pylint" GOTO lint-pylint -IF "%TARGET%"=="lint-ruff" GOTO lint-ruff - -IF "%TARGET%"=="test" GOTO test - -IF "%TARGET%"=="pre-commit" GOTO pre-commit - -IF "%TARGET%"=="build" GOTO build -IF "%TARGET%"=="release" GOTO release - -IF "%TARGET%"=="version" GOTO version - -IF "%TARGET%"=="launch-solver" GOTO launch-solver - - -:: POETRY -:: ====== -:get-poetry - python3 -m pip install Poetry --upgrade --user - GOTO end - - -:: INSTALLATION -:: ============ -:install - poetry lock - poetry install --extras=contrib --with=docs --with=lint --with=test - GOTO end - -:install-editable - python3 -m pip install -e ".[contrib]" --upgrade --user - GOTO end - - -:: LINTING -:: ======= -:lint - GOTO lint-flake8 - GOTO lint-pylint - GOTO lint-ruff - GOTO end - -:lint-flake8 - :: flake8.pycqa.org/en/latest/user/invocation.html - :: flake8.pycqa.org/en/latest/user/options.html - poetry run flake8 %LIB_DIR% %DOCS_DIR% %EXAMPLES_DIR% %TESTS_DIR% ^ - --verbose --color always - GOTO end - -:lint-pylint - :: pylint.readthedocs.io/en/latest/user_guide/usage/run.html - poetry run pylint %LIB_DIR% %DOCS_DIR% %EXAMPLES_DIR% %TESTS_DIR% - GOTO end - -:lint-ruff - :: docs.astral.sh/ruff/linter - poetry run ruff check %LIB_DIR% %DOCS_DIR% %EXAMPLES_DIR% %TESTS_DIR% ^ - --show-source ^ - --output-format text ^ - --target-version py310 ^ - --preview ^ - --respect-gitignore - GOTO end - - -:: TESTING -:: ======= -:test - poetry run pytest - GOTO end - - -:: PRE-COMMIT LINTING & TESTING -:: ============================ -:pre-commit - GOTO lint - GOTO test - GOTO end - - -:: DISTRIBUTION BUILDING & PYPI RELEASE -:: ==================================== -:build - poetry build - GOTO end - -:release - GOTO build - poetry publish - GOTO end - - -:: VERSION MANAGEMENT -:: ================== -:version - poetry version %2 - GOTO end - - -:: MISC / OTHER -:: ============ -:launch-solver - poetry run openssa launch solver - GOTO end - - -:: END -:: === -:end diff --git a/openssa/core/ooda_rag/tools.py b/openssa/core/ooda_rag/tools.py index b69c2b761..0080d06b9 100644 --- a/openssa/core/ooda_rag/tools.py +++ b/openssa/core/ooda_rag/tools.py @@ -83,6 +83,7 @@ def __init__(self, agent: RAGSSM) -> None: super().__init__(description) self.agent = agent + def execute(self, question: str) -> str: """ Query a document base for factual information. @@ -90,6 +91,6 @@ def execute(self, question: str) -> str: :param question (str): The question to ask the document base. :return (str): The answer to the question. """ - response = self.agent.discuss(question) + response = self.agent.discuss(question) print(f"debug: {response}") return response diff --git a/openssa/integrations/llama_index/backend.py b/openssa/integrations/llama_index/backend.py index 73432c3ec..97fa6d24e 100644 --- a/openssa/integrations/llama_index/backend.py +++ b/openssa/integrations/llama_index/backend.py @@ -67,7 +67,8 @@ def query_engine(self) -> BaseQueryEngine: self._query_engine = self.index.as_query_engine( vector_store_query_mode="mmr", vector_store_kwargs={"mmr_threshold": self._relevance_threshold}, - service_context=self._service_context or ServiceContext.from_defaults(llm=self.llm), + service_context=self._service_context + or ServiceContext.from_defaults(llm=self.llm), similarity_top_k=self._similarity_top_k, ) return self._query_engine diff --git a/openssa/integrations/openai/ssm.py b/openssa/integrations/openai/ssm.py index a4fc2d445..3dfcc96aa 100644 --- a/openssa/integrations/openai/ssm.py +++ b/openssa/integrations/openai/ssm.py @@ -18,6 +18,7 @@ ) + # pylint: disable=too-many-instance-attributes class APIContext(AbstractAPIContext): @classmethod diff --git a/poetry.toml b/poetry.toml index cfd152fd3..3b549d6c8 100644 --- a/poetry.toml +++ b/poetry.toml @@ -1,3 +1,2 @@ [virtualenvs] create = true -in-project = true # important on Windows to avoid path lengths exceeding default max length of 260 diff --git a/pyproject.toml b/pyproject.toml index 047fab92b..9b5afbb0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "openssa" -version = "0.23.12.19" +version = "0.23.12.18" authors = [ "Aitomatic, Inc. ", @@ -40,20 +40,18 @@ packages = [ include = [ ] exclude = [ - "**/.openssa/", # OpenSSA cache data + "**/.openssa/" ] [tool.poetry.group.docs.dependencies] sphinx = ">=7.2" -sphinx-autobuild = ">=2021.3" +sphinx-autobuild = ">=2021.3.14" sphinx-press-theme = ">=0.8" myst-parser = ">=2.0" [tool.poetry.group.lint.dependencies] -flake8 = ">=6.1" pylint = ">=3.0" -ruff = ">=0.1" pydocstyle = ">=6.3" [tool.poetry.group.test.dependencies] @@ -61,11 +59,11 @@ pytest = ">=7.4" [tool.poetry.dependencies] python = ">=3.10,<3.12" -# OpenAI interface -openai = ">=1.6" +# OpenAI & related +openai = ">=1.4" # LlamaIndex & related -llama-index = ">=0.9.17" # should keep up-to-date with Llama-Index's minor releases (often backward-incompatible) -llama-hub = ">=0.0.60" +llama-index = ">=0.9.15" # should keep up-to-date with Llama-Index's minor releases (often backward-incompatible) +llama-hub = ">=0.0.59" docx2txt = ">=0.8" # for reading .docx files pypdf = ">=3.17" # for reading .pdf files pycryptodome = ">=3.19" # for reading .pdf files: PyCryptodome is required for AES algorithm @@ -105,25 +103,6 @@ requires = [ ] -# pylint.readthedocs.io/en/latest/user_guide/usage/run.html -# pylint.readthedocs.io/en/latest/user_guide/configuration/index.html -[tool.pylint."messages control"] -disable = [ - "attribute-defined-outside-init", - "duplicate-code", - "fixme", - "import-error", - "line-too-long", - "missing-class-docstring", - "missing-function-docstring", - "missing-module-docstring", - "raw-checker-failed", - "redefined-outer-name", - "too-few-public-methods", - "unnecessary-pass", -] - - [tool.pytest.ini_options] addopts = [ "--import-mode=importlib",