AppAPI 1.1.0, /init endpoint

.run/TalkBotAI (27).run.xml

@@ -0,0 +1,29 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="TalkBotAI (27)" type="PythonConfigurationType" factoryName="Python">
+    <module name="nc_py_api" />
+    <option name="INTERPRETER_OPTIONS" value="" />
+    <option name="PARENT_ENVS" value="true" />
+    <envs>
+      <env name="APP_ID" value="talk_bot_ai" />
+      <env name="APP_PORT" value="9034" />
+      <env name="APP_SECRET" value="12345" />
+      <env name="APP_VERSION" value="1.0.0" />
+      <env name="NEXTCLOUD_URL" value="http://stable27.local" />
+      <env name="PYTHONUNBUFFERED" value="1" />
+    </envs>
+    <option name="SDK_HOME" value="" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+    <option name="IS_MODULE_SDK" value="true" />
+    <option name="ADD_CONTENT_ROOTS" value="true" />
+    <option name="ADD_SOURCE_ROOTS" value="true" />
+    <EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
+    <option name="SCRIPT_NAME" value="$PROJECT_DIR$/examples/as_app/talk_bot_AI/src/main.py" />
+    <option name="PARAMETERS" value="" />
+    <option name="SHOW_COMMAND_LINE" value="false" />
+    <option name="EMULATE_TERMINAL" value="false" />
+    <option name="MODULE_MODE" value="false" />
+    <option name="REDIRECT_INPUT" value="false" />
+    <option name="INPUT_FILE" value="" />
+    <method v="2" />
+  </configuration>
+</component>

.run/TalkBotAI.run.xml → .run/TalkBotAI (28).run.xml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="TalkBotAI" type="PythonConfigurationType" factoryName="Python">
+  <configuration default="false" name="TalkBotAI (28)" type="PythonConfigurationType" factoryName="Python">
     <module name="nc_py_api" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />

.run/ToGif(28).run.xml → .run/ToGif (28).run.xml

@@ -1,15 +1,15 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="ToGif(28)" type="PythonConfigurationType" factoryName="Python">
+  <configuration default="false" name="ToGif (28)" type="PythonConfigurationType" factoryName="Python">
     <module name="nc_py_api" />
     <option name="INTERPRETER_OPTIONS" value="" />
     <option name="PARENT_ENVS" value="true" />
     <envs>
+      <env name="PYTHONUNBUFFERED" value="1" />
       <env name="APP_ID" value="to_gif" />
       <env name="APP_PORT" value="9031" />
       <env name="APP_SECRET" value="12345" />
       <env name="APP_VERSION" value="1.0.0" />
       <env name="NEXTCLOUD_URL" value="http://nextcloud.local" />
-      <env name="PYTHONUNBUFFERED" value="1" />
     </envs>
     <option name="SDK_HOME" value="" />
     <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.4.1 - 2023-10-17]
+
+### Added
+
+- Support for the new AppAPI endpoint `/init` and automatically downloading models from `huggingface`. #151
+
+### Changed
+
+- All examples were adjusted to changes in AppAPI.
+
 ## [0.4.0 - 2023-10-15]
 
 As the project moves closer to `beta`, final unification changes are being made.

examples/as_app/skeleton/Makefile

@@ -32,18 +32,18 @@ deploy:
 .PHONY: run28
 run28:
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent || true
-	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register skeleton docker_dev -e --force-scopes \
+	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register skeleton docker_dev --force-scopes \
 		--info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/skeleton/appinfo/info.xml
 
 .PHONY: run27
 run27:
 	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent || true
-	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register skeleton docker_dev -e --force-scopes \
+	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register skeleton docker_dev --force-scopes \
 		--info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/skeleton/appinfo/info.xml
 
 .PHONY: manual_register
 manual_register:
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister skeleton --silent || true
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register skeleton manual_install --json-info \
   "{\"appid\":\"skeleton\",\"name\":\"App Skeleton\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"host.docker.internal\",\"port\":9030,\"scopes\":{\"required\":[],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" \
-  -e --force-scopes
+  --force-scopes

examples/as_app/skeleton/requirements.txt

@@ -1 +1 @@
-nc_py_api[app]>=0.3.0
+nc_py_api[app]>=0.4.1

examples/as_app/talk_bot/Makefile

@@ -32,18 +32,18 @@ deploy:
 .PHONY: run28
 run28:
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot --silent || true
-	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot docker_dev -e --force-scopes \
+	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot docker_dev --force-scopes \
 		--info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot/appinfo/info.xml
 
 .PHONY: run27
 run27:
 	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister talk_bot --silent || true
-	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot docker_dev -e --force-scopes \
+	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot docker_dev --force-scopes \
 		--info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot/appinfo/info.xml
 
 .PHONY: manual_register
 manual_register:
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot --silent || true
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot manual_install --json-info \
   "{\"appid\":\"talk_bot\",\"name\":\"TalkBot\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"host.docker.internal\",\"port\":9032,\"scopes\":{\"required\":[\"TALK\", \"TALK_BOT\"],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" \
-  -e --force-scopes
+  --force-scopes

examples/as_app/talk_bot/requirements.txt

@@ -1 +1 @@
-nc_py_api[app]>=0.3.0
+nc_py_api[app]>=0.4.1

examples/as_app/talk_bot_ai/Makefile

@@ -17,7 +17,8 @@ help:
 	@echo "  For development of this example use PyCharm run configurations. Development is always set for last Nextcloud."
 	@echo "  First run 'TalkBotAI' and then 'make manual_register', after that you can use/debug/develop it and easy test."
 	@echo "  "
-	@echo "  manual_register   perform registration of running 'TalkBotAI' into the 'manual_install' deploy daemon."
+	@echo "  manual_register28 perform registration of running 'TalkBotAI' into the 'manual_install' deploy daemon."
+	@echo "  manual_register27 perform registration of running 'TalkBotAI' into the 'manual_install' deploy daemon."
 
 .PHONY: build-push
 build-push:
@@ -32,18 +33,25 @@ deploy:
 .PHONY: run28
 run28:
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent || true
-	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot_ai docker_dev -e --force-scopes \
+	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot_ai docker_dev --force-scopes \
 		--info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot_ai/appinfo/info.xml
 
+.PHONY: manual_register28
+manual_register28:
+	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent || true
+	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot_ai manual_install --json-info \
+  "{\"appid\":\"talk_bot_ai\",\"name\":\"TalkBotAI\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"host.docker.internal\",\"port\":9034,\"scopes\":{\"required\":[\"TALK\", \"TALK_BOT\"],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" \
+  --force-scopes
+
 .PHONY: run27
 run27:
 	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent || true
-	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot_ai docker_dev -e --force-scopes \
+	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot_ai docker_dev --force-scopes \
 		--info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot_ai/appinfo/info.xml
 
-.PHONY: manual_register
-manual_register:
-	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent || true
-	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot_ai manual_install --json-info \
+.PHONY: manual_register27
+manual_register27:
+	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister talk_bot_ai --silent || true
+	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot_ai manual_install --json-info \
   "{\"appid\":\"talk_bot_ai\",\"name\":\"TalkBotAI\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"host.docker.internal\",\"port\":9034,\"scopes\":{\"required\":[\"TALK\", \"TALK_BOT\"],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" \
-  -e --force-scopes
+  --force-scopes

examples/as_app/talk_bot_ai/requirements.txt

@@ -1,4 +1,4 @@
-nc_py_api[app]>=0.3.0
+nc_py_api[app]>=0.4.1
 transformers>=4.33
 torch
 torchvision

examples/as_app/talk_bot_ai/src/main.py

@@ -1,29 +1,25 @@
 """Example of an application that uses Python Transformers library with Talk Bot APIs."""
-
-# This line should be on top before any import of the "Transformers" library.
-from nc_py_api.ex_app import persist_transformers_cache  # noqa # isort:skip
 import re
-from threading import Thread
 from typing import Annotated
 
 import requests
 from fastapi import BackgroundTasks, Depends, FastAPI
+from huggingface_hub import snapshot_download
 from transformers import pipeline
 
 from nc_py_api import NextcloudApp, talk_bot
 from nc_py_api.ex_app import run_app, set_handlers, talk_bot_app
 
 APP = FastAPI()
 AI_BOT = talk_bot.TalkBot("/ai_talk_bot", "AI talk bot", "Usage: `@assistant What sounds do cats make?`")
-MODEL_NAME = "MBZUAI/LaMini-Flan-T5-77M"
-MODEL_INIT_THREAD = None
+MODEL_NAME = "MBZUAI/LaMini-Flan-T5-783M"
 
 
 def ai_talk_bot_process_request(message: talk_bot.TalkBotMessage):
     r = re.search(r"@assistant\s(.*)", message.object_content["message"], re.IGNORECASE)
     if r is None:
         return
-    model = pipeline("text2text-generation", model=MODEL_NAME)
+    model = pipeline("text2text-generation", model=snapshot_download(MODEL_NAME, local_files_only=True))
     response_text = model(r.group(1), max_length=64, do_sample=True)[0]["generated_text"]
     AI_BOT.send_message(response_text, message)
 
@@ -47,25 +43,9 @@ def enabled_handler(enabled: bool, nc: NextcloudApp) -> str:
     return ""
 
 
-def download_models():
-    pipeline("text2text-generation", model=MODEL_NAME)
-
-
-def heartbeat_handler() -> str:
-    global MODEL_INIT_THREAD
-    print("heartbeat_handler: called")
-    if MODEL_INIT_THREAD is None:
-        MODEL_INIT_THREAD = Thread(target=download_models)
-        MODEL_INIT_THREAD.start()
-        print("heartbeat_handler: started initialization thread")
-    r = "init" if MODEL_INIT_THREAD.is_alive() else "ok"
-    print(f"heartbeat_handler: result={r}")
-    return r
-
-
 @APP.on_event("startup")
 def initialization():
-    set_handlers(APP, enabled_handler, heartbeat_handler)
+    set_handlers(APP, enabled_handler, models_to_fetch=[MODEL_NAME])
 
 
 if __name__ == "__main__":

examples/as_app/talk_bot_multi/Makefile

@@ -32,18 +32,18 @@ deploy:
 .PHONY: run28
 run28:
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot_multi --silent || true
-	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot_multi docker_dev -e --force-scopes \
+	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot_multi docker_dev --force-scopes \
 		--info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot_multi/appinfo/info.xml
 
 .PHONY: run27
 run27:
 	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister talk_bot_multi --silent || true
-	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot_multi docker_dev -e --force-scopes \
+	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register talk_bot_multi docker_dev --force-scopes \
 		--info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/talk_bot_multi/appinfo/info.xml
 
 .PHONY: manual_register
 manual_register:
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister talk_bot_multi --silent || true
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register talk_bot_multi manual_install --json-info \
   "{\"appid\":\"talk_bot_multi\",\"name\":\"TalkBotMulti\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"host.docker.internal\",\"port\":9033,\"scopes\":{\"required\":[\"TALK\", \"TALK_BOT\"],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" \
-  -e --force-scopes
+  --force-scopes

examples/as_app/talk_bot_multi/requirements.txt

@@ -1 +1 @@
-nc_py_api[app]>=0.3.0
+nc_py_api[app]>=0.4.1

examples/as_app/to_gif/Makefile

@@ -33,25 +33,25 @@ deploy:
 .PHONY: run28
 run28:
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent || true
-	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register to_gif docker_dev -e --force-scopes \
+	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register to_gif docker_dev --force-scopes \
 		--info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/to_gif/appinfo/info.xml
 
 .PHONY: manual_register28
 manual_register28:
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent || true
 	docker exec master-nextcloud-1 sudo -u www-data php occ app_api:app:register to_gif manual_install --json-info \
   "{\"appid\":\"to_gif\",\"name\":\"to_gif\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"host.docker.internal\",\"port\":9031,\"scopes\":{\"required\":[\"FILES\", \"NOTIFICATIONS\"],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" \
-  -e --force-scopes
+  --force-scopes
 
 .PHONY: run27
 run27:
 	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent || true
-	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register to_gif docker_dev -e --force-scopes \
+	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register to_gif docker_dev --force-scopes \
 		--info-xml https://raw.githubusercontent.com/cloud-py-api/nc_py_api/main/examples/as_app/to_gif/appinfo/info.xml
 
 .PHONY: manual_register27
 manual_register27:
 	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:unregister to_gif --silent || true
 	docker exec master-stable27-1 sudo -u www-data php occ app_api:app:register to_gif manual_install --json-info \
   "{\"appid\":\"to_gif\",\"name\":\"to_gif\",\"daemon_config_name\":\"manual_install\",\"version\":\"1.0.0\",\"secret\":\"12345\",\"host\":\"host.docker.internal\",\"port\":9031,\"scopes\":{\"required\":[\"FILES\", \"NOTIFICATIONS\"],\"optional\":[]},\"protocol\":\"http\",\"system_app\":0}" \
-  -e --force-scopes
+  --force-scopes

examples/as_app/to_gif/requirements.txt

@@ -1,4 +1,4 @@
-nc_py_api[app]>=0.3.0
+nc_py_api[app]>=0.4.1
 pygifsicle
 imageio
 opencv-python

nc_py_api/_version.py

@@ -1,3 +1,3 @@
 """Version of nc_py_api."""
 
-__version__ = "0.4.0"
+__version__ = "0.4.1.dev0"

nc_py_api/ex_app/integration_fastapi.py

@@ -6,11 +6,20 @@
 import json
 import typing
 
-from fastapi import Depends, FastAPI, HTTPException, Request, responses, status
+from fastapi import (
+    BackgroundTasks,
+    Depends,
+    FastAPI,
+    HTTPException,
+    Request,
+    responses,
+    status,
+)
 
 from .._misc import get_username_secret_from_headers
 from ..nextcloud import NextcloudApp
 from ..talk_bot import TalkBotMessage, get_bot_secret
+from .misc import persistent_storage
 
 
 def nc_app(request: Request) -> NextcloudApp:
@@ -45,14 +54,50 @@ def set_handlers(
     fast_api_app: FastAPI,
     enabled_handler: typing.Callable[[bool, NextcloudApp], str],
     heartbeat_handler: typing.Optional[typing.Callable[[], str]] = None,
+    init_handler: typing.Optional[typing.Callable[[], None]] = None,
+    models_to_fetch: typing.Optional[list[str]] = None,
+    models_download_params: typing.Optional[dict] = None,
 ):
     """Defines handlers for the application.
 
     :param fast_api_app: FastAPI() call return value.
     :param enabled_handler: ``Required``, callback which will be called for `enabling`/`disabling` app event.
     :param heartbeat_handler: Optional, callback that will be called for the `heartbeat` deploy event.
+    :param init_handler: Optional, callback that will be called for the `init`  event.
+
+        .. note:: If ``init_handler`` is specified, it is up to a developer to set the application init progress status.
+            AppAPI will only call `enabled_handler` after it receives ``100`` as initialization status progress.
+
+    :param models_to_fetch: Dictionary describing which models should be downloaded during `init`.
+
+        .. note:: ```huggingface_hub`` package should be present for automatic models fetching.
+
+    :param models_download_params: Parameters to pass to ``snapshot_download`` function from **huggingface_hub**.
     """
 
+    def fetch_models_task(models: dict):
+        if models:
+            from huggingface_hub import snapshot_download  # noqa isort:skip pylint: disable=C0415 disable=E0401
+            from tqdm import tqdm  # noqa isort:skip pylint: disable=C0415 disable=E0401
+
+            class TqdmProgress(tqdm):
+                def display(self, msg=None, pos=None):
+                    if init_handler is None:
+                        NextcloudApp().set_init_status(min(int((self.n * 100 / self.total) / len(models)), 100))
+                    return super().display(msg, pos)
+
+            params = models_download_params if models_download_params else {}
+            if "max_workers" not in params:
+                params["max_workers"] = 2
+            if "cache_dir" not in params:
+                params["cache_dir"] = persistent_storage()
+            for model in models:
+                snapshot_download(model, tqdm_class=TqdmProgress, **params)  # noqa
+        if init_handler is None:
+            NextcloudApp().set_init_status(100)
+        else:
+            init_handler()
+
     @fast_api_app.put("/enabled")
     def enabled_callback(
         enabled: bool,
@@ -67,3 +112,8 @@ def heartbeat_callback():
         if heartbeat_handler is not None:
             return_status = heartbeat_handler()
         return responses.JSONResponse(content={"status": return_status}, status_code=200)
+
+    @fast_api_app.post("/init")
+    def init_callback(background_tasks: BackgroundTasks):
+        background_tasks.add_task(fetch_models_task, models_to_fetch if models_to_fetch else {})
+        return responses.JSONResponse(content={}, status_code=200)

nc_py_api/nextcloud.py

@@ -255,3 +255,20 @@ def request_sign_check(self, request: Request) -> bool:
             print(e)
             return False
         return True
+
+    def set_init_status(self, progress: int, error: str = "") -> None:
+        """Sets state of the app initialization.
+
+        :param progress: a number from ``0`` to ``100`` indicating the percentage of application readiness for work.
+            After sending ``100`` AppAPI will enable the application.
+        :param error: if non-empty, signals to AppAPI that the application cannot be initialized successfully.
+        """
+        self._session.ocs(
+            method="PUT",
+            path=f"/ocs/v1.php/apps/app_api/apps/status/{self._session.cfg.app_name}",
+            json={
+                "progress": progress,
+                "error": error,
+            },
+            not_parse=True,
+        )