Short-circuit API rate limiter for unauthenticated users (#607)

### Major
- Short-circuit API rate limiter for unauthenticated user
  Calls by unauthenticated users were failing at API rate limiter as it
  failed to access user info object. This is a bug.
  
  API rate limiter should short-circuit for unauthenicated users so a
  proper Forbidden response can be returned by API
  
  Add regression test to verify that unauthenticated users get 403
  response when calling the /chat API endpoint
  
### Minor
- Remove trailing slash to normalize khoj url in obsidian plugin settings
- Move used /api/config API controllers into separate module
- Delete unused /api/beta API endpoint
- Fix error message rendering in khoj.el, khoj obsidian chat
- Handle deprecation warnings for subscribe renew date, langchain, pydantic & logger.warn

pyproject.toml

@@ -55,7 +55,7 @@ dependencies = [
     "torch == 2.0.1",
     "uvicorn == 0.17.6",
     "aiohttp ~= 3.9.0",
-    "langchain >= 0.0.331",
+    "langchain <= 0.2.0",
     "requests >= 2.26.0",
     "bs4 >= 0.0.1",
     "anyio == 3.7.1",

src/interface/emacs/khoj.el

@@ -348,7 +348,7 @@ Auto invokes setup steps on calling main entrypoint."
       t
     ;; else general check via ping to khoj-server-url
     (if (ignore-errors
-          (url-retrieve-synchronously (format "%s/api/config/data/default" khoj-server-url)))
+          (url-retrieve-synchronously (format "%s/api/health" khoj-server-url)))
         ;; Successful ping to non-emacs khoj server indicates it is started and ready.
         ;; So update ready state tracker variable (and implicitly return true for started)
         (setq khoj--server-ready? t)
@@ -432,7 +432,7 @@ Auto invokes setup steps on calling main entrypoint."
                               (khoj--delete-open-network-connections-to-server)
                               (with-current-buffer (current-buffer)
                                 (search-forward "\n\n" nil t)
-                                (message "khoj.el: Failed to %supdate %s content index. Status: %s%s"
+                                (message "khoj.el: Failed to %supdate %scontent index. Status: %s%s"
                                          (if force "force " "")
                                          (if content-type (format "%s " content-type) "all")
                                          (string-trim (format "%s %s" (nth 1 (nth 1 status)) (nth 2 (nth 1 status))))
@@ -603,22 +603,6 @@ Use `BOUNDARY' to separate files. This is sent to Khoj server as a POST request.
 ;; --------------
 ;; Query Khoj API
 ;; --------------
-
-(defun khoj--post-new-config (config)
-  "Configure khoj server with provided CONFIG."
-  ;; POST provided config to khoj server
-  (let ((url-request-method "POST")
-        (url-request-extra-headers `(("Content-Type" . "application/json")
-                                     ("Authorization" . ,(format "Bearer %s" khoj-api-key))))
-        (url-request-data (encode-coding-string (json-encode-alist config) 'utf-8))
-        (config-url (format "%s/api/config/data" khoj-server-url)))
-    (with-current-buffer (url-retrieve-synchronously config-url)
-      (buffer-string)))
-  ;; Update index on khoj server after configuration update
-  (let ((khoj--server-ready? nil)
-        (url-request-extra-headers `(("Authorization" . ,(format "\"Bearer %s\"" khoj-api-key)))))
-    (url-retrieve (format "%s/api/update?client=emacs" khoj-server-url) #'identity)))
-
 (defun khoj--get-enabled-content-types ()
   "Get content types enabled for search from API."
   (let ((config-url (format "%s/api/config/types" khoj-server-url))

src/interface/obsidian/src/chat_modal.ts

@@ -245,7 +245,7 @@ export class KhojChatModal extends Modal {
             if (responseJson.detail) {
                 // If the server returns error details in response, render a setup hint.
                 let setupMsg = "Hi 👋🏾, to start chatting add available chat models options via [the Django Admin panel](/server/admin) on the Server";
-                this.renderMessage(chatBodyEl, setupMsg, "khoj", undefined, true);
+                this.renderMessage(chatBodyEl, setupMsg, "khoj", undefined);
 
                 return false;
             } else if (responseJson.response) {

src/interface/obsidian/src/settings.ts

@@ -42,7 +42,7 @@ export class KhojSettingTab extends PluginSettingTab {
             .addText(text => text
                 .setValue(`${this.plugin.settings.khojUrl}`)
                 .onChange(async (value) => {
-                    this.plugin.settings.khojUrl = value.trim();
+                    this.plugin.settings.khojUrl = value.trim().replace(/\/$/, '');
                     await this.plugin.saveSettings();
                     containerEl.firstElementChild?.setText(this.getBackendStatusMessage());
                 }));

src/khoj/configure.py

@@ -1,12 +1,14 @@
 import json
 import logging
 import os
+from datetime import datetime
 from enum import Enum
 from typing import Optional
 
 import openai
 import requests
 import schedule
+from django.utils.timezone import make_aware
 from starlette.authentication import (
     AuthCredentials,
     AuthenticationBackend,
@@ -59,7 +61,8 @@ def _initialize_default_user(self):
                 email="default@example.com",
                 password="default",
             )
-            Subscription.objects.create(user=default_user, type="standard", renewal_date="2100-04-01")
+            renewal_date = make_aware(datetime.strptime("2100-04-01", "%Y-%m-%d"))
+            Subscription.objects.create(user=default_user, type="standard", renewal_date=renewal_date)
 
     async def authenticate(self, request: HTTPConnection):
         current_user = request.session.get("user")
@@ -190,14 +193,14 @@ def initialize_content(regenerate: bool, search_type: Optional[SearchType] = Non
 def configure_routes(app):
     # Import APIs here to setup search types before while configuring server
     from khoj.routers.api import api
-    from khoj.routers.api_beta import api_beta
+    from khoj.routers.api_config import api_config
     from khoj.routers.auth import auth_router
     from khoj.routers.indexer import indexer
     from khoj.routers.subscription import subscription_router
     from khoj.routers.web_client import web_client
 
     app.include_router(api, prefix="/api")
-    app.include_router(api_beta, prefix="/api/beta")
+    app.include_router(api_config, prefix="/api/config")
     app.include_router(indexer, prefix="/api/v1/index")
     if state.billing_enabled:
         logger.info("💳 Enabled Billing")