Add Recap gateway

Recap now ships with a little HTTP/JSON gateway. The gateway has two paths:

- ls
- schema

These paths list subpaths and fetch schemas. For example:

```
GET http://127.0.0.1:8000/ls
["bq", "pg"]

GET http://127.0.0.1:8000/ls/pg
["postgres","template0","template1","testdb"]

GET http://127.0.0.1:8000/schema/pg/testdb/public/test_types
{"type": "struct", "fields": ... }
```

The gateway is configured using environment variableas and supports a `.env`
file (via `pydantic-settings`).

```bash
RECAP_SYSTEMS__BQ=bigquery://
RECAP_SYSTEMS__PG=postgresql://localhost:5432/testdb
```

In the future, Recap's CLI will use the same environment variables.

I'm leaving the gateway integration tests for a follow-on PR.

pyproject.toml

@@ -53,6 +53,12 @@ json = [
  "referencing>=0.30.0",
  "httpx>=0.24.1",
 ]
+gateway = [
+ "fastapi>=0.103.0",
+ "pydantic>=2.3.0",
+ "pydantic-settings>=2.0.3",
+ "uvicorn>=0.23.2",
+]
 
 [build-system]
 requires = ["pdm-backend"]
@@ -107,6 +113,7 @@ unit = "pytest tests/unit -vv"
 spec = "pytest tests/spec -vv"
 integration = "pytest tests/integration -vv"
 test = {composite = ["unit", "spec"]}
+serve ="uvicorn recap.gateway.app:app --reload"
 
 [tool.pytest.ini_options]
 addopts = [

recap/clients/dbapi.py

@@ -58,7 +58,7 @@ def ls_tables(self, catalog: str, schema: str) -> list[str]:
  )
  return [row[0] for row in cursor.fetchall()]
 
- def get_schema(self, table: str, schema: str, catalog: str) -> StructType:
+ def get_schema(self, catalog: str, schema: str, table: str) -> StructType:
  cursor = self.connection.cursor()
  cursor.execute(
  f"""

recap/gateway/app.py

@@ -0,0 +1,48 @@
+from fastapi import Depends, FastAPI
+
+from recap.clients import Client, create_client
+from recap.gateway.settings import RecapSettings
+from recap.types import to_dict
+
+app = FastAPI()
+settings = RecapSettings()
+
+
+async def get_reader(system_name: str | None = None):
+ if system_name and (url := settings.systems.get(system_name)):
+ with create_client(url.unicode_string()) as client:
+ yield client
+ else:
+ yield None
+
+
+@app.get("/ls")
+@app.get("/ls/{system_name}")
+@app.get("/ls/{system_name}/{path:path}")
+async def ls(
+ system_name: str | None = None,
+ path: str | None = None,
+ client: Client | None = Depends(get_reader),
+) -> list[str]:
+ if system_name is None:
+ return list(settings.systems.keys())
+ if client is None:
+ raise ValueError(f"Unknown system: {system_name}")
+ return client.ls(*_args(path))
+
+
+@app.get("/schema/{system_name}/{path:path}")
+async def schema(
+ path: str,
+ client: Client = Depends(get_reader),
+) -> dict:
+ print(path)
+ recap_struct = client.get_schema(*_args(path))
+ recap_dict = to_dict(recap_struct)
+ if not isinstance(recap_dict, dict):
+ raise ValueError(f"Expected dict, got {type(recap_dict)}")
+ return recap_dict
+
+
+def _args(path: str | None) -> list[str]:
+ return path.strip("/").split("/") if path else []

recap/gateway/settings.py

@@ -0,0 +1,11 @@
+from pydantic import AnyUrl, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class RecapSettings(BaseSettings):
+ systems: dict[str, AnyUrl] = Field(default_factory=dict)
+ model_config = SettingsConfigDict(
+ env_file=".env",
+ env_prefix="recap_",
+ env_nested_delimiter="__",
+ )

tests/integration/clients/test_mysql.py

@@ -77,7 +77,7 @@ def test_struct_method(self):
  client = MysqlClient(self.connection) # type: ignore
 
  # Test 'test_types' table. MySQL catalog is always 'def'.
- test_types_struct = client.get_schema("test_types", "testdb", "def")
+ test_types_struct = client.get_schema("def", "testdb", "test_types")
 
  # Define the expected output for 'test_types' table
  expected_fields = [

tests/integration/clients/test_postgresql.py

@@ -67,7 +67,7 @@ def test_struct_method(self):
  client = PostgresqlClient(self.connection) # type: ignore
 
  # Test 'test_types' table
- test_types_struct = client.get_schema("test_types", "public", "testdb")
+ test_types_struct = client.get_schema("testdb", "public", "test_types")
 
  # Define the expected output for 'test_types' table
  expected_fields = [

tests/unit/clients/test_snowflake.py

@@ -66,7 +66,7 @@ def setup_class(cls):
 
  def test_struct_method(self):
  client = SnowflakeClient(self.connection) # type: ignore
- test_types_struct = client.get_schema("TEST_TYPES", "PUBLIC", "TESTDB")
+ test_types_struct = client.get_schema("TESTDB", "PUBLIC", "TEST_TYPES")
  expected_fields = [
  UnionType(
  default=None,