滞在確率算出サーバと入退室時間のクラスタリングプログラム

.gitignore

@@ -226,5 +226,5 @@ backup/
 mysql/backup.sql
 mysql/test.sql
 
-
+.venv/
 __pycache__/

docker-compose.yml

@@ -19,6 +19,7 @@ services:
     depends_on:
       vol_mysql:
         condition: service_healthy
+
   vol_mysql:
     container_name: ${MYSQL_CONTAINER_NAME}
     build: ./mysql
@@ -38,19 +39,46 @@ services:
       interval: 10s
       timeout: 20s
       retries: 20
+
   vol_python:
     container_name: vol_python
     build: ./python
     volumes:
       - ./python/app:/app
     env_file:
       - .env
-    command: 
+    command:
       - "python3"
       - "main.py"
     depends_on:
       vol_mysql:
-        condition: service_healthy      
+        condition: service_healthy
+
+  vol_probability_api:
+    container_name: vol_probability_api
+    build:
+      context: .
+      dockerfile: ./probability/api/dockerfile
+    env_file:
+      - ./probability/.env
+    ports:
+      - "${PROBABILITY_PORT}:8090"
+    expose:
+      - 8090
+    depends_on:
+      - vol_mysql
+
+  vol_probability_clustering:
+    container_name: vol_probability_clustering
+    tty: true
+    build:
+      context: .
+      dockerfile: ./probability/clustering/dockerfile
+    env_file:
+      - ./probability/.env
+    depends_on:
+      - vol_mysql
+
 networks:
   default:
     external:

probability/api/controller/probability.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+import math
+from typing import List
+from pydantic import BaseModel
+from fastapi import APIRouter, Depends
+from fastapi.encoders import jsonable_encoder
+from fastapi.responses import ORJSONResponse
+from sqlalchemy.orm import Session
+from datetime import datetime, timedelta
+from models import cluster as cl, user as us, logs as lg
+from service import normal_distribution as nd
+from lib.mysql import get_db
+
+# レスポンス用のクラス
+class ProbabilityResponse(BaseModel):
+    userId: int
+    userName: str
+    probability: float
+
+router = APIRouter()
+
+# フロント側から現在の時刻を受け取ることとする（後で要変更)
+# 今日のある時間までに特定のユーザーが入退室する確率、もしくはある時間以降に入退室する確率を返す
+# 変数：user_id, true or false
+@router.get("/app/probability/{reporting}/{before}" , response_class=ORJSONResponse, response_model=ProbabilityResponse)
+async def get_probability_reporting_before(reporting:str, before:str, user_id:int = 0, date:str = '2024-1-1', time:str = "24:00:00", db: Session = Depends(get_db)):
+    r = True if reporting == "reporting" else False
+    b = True if before == "before" else False
+    date_object= datetime.strptime(date, '%Y-%m-%d') #今日の日付
+    seven_days_ago= date_object - timedelta(days=7)
+    clusters = cl.get_all_cluster_by_userId_and_date(db, user_id, seven_days_ago, r)
+    delta = abs(clusters[0].date - lg.get_oldest_log_by_userId(user_id).date + timedelta(days=1))
+    # 差分を週単位に変換
+    days_difference = math.floor(delta.days/7)
+    # ここでクラスタリングの結果を元に確率を計算する(bがTrueなら以前, Falseなら以降)
+    pr = nd.probability_from_normal_distribution(clusters, time, days_difference, b)
+    result = ProbabilityResponse(userId=user_id, userName=us.get_user_by_id(db, user_id).name, probability=pr)
+    result_json = jsonable_encoder(result)
+    return ORJSONResponse(result_json)
+
+# 全てのユーザがその日に入室する確率を返す
+@router.get("/app/probability/{community}/all", response_class=ORJSONResponse, response_model=List[ProbabilityResponse])
+async def get_probability_all(community:int, date:str = "2024-1-1", db: Session = Depends(get_db)):
+    date_object= datetime.strptime(date, '%Y-%m-%d')
+    seven_days_ago= date_object - timedelta(days=7)
+    users = us.get_all_users_by_community(community,db)
+    # 結果格納用のリスト
+    result: list[ProbabilityResponse] = []
+    # ユーザーごとに繰り返す
+    for user in users:
+        clusters = cl.get_all_cluster_by_userId_and_date(db, user.id, seven_days_ago, True)
+        delta = abs(clusters[0].date - cl.get_oldest_cluster_by_userId(db, user.id, True).date + timedelta(days=1))
+        # 差分を日単位に変換
+        days_difference = math.floor(delta.days/7)
+        # ここでクラスタリングの結果を元に確率を計算する(bがTrueなら以前, Falseなら以降)
+        pr = nd.probability_from_normal_distribution(clusters, "24:00:00", days_difference, True)
+        result.append(ProbabilityResponse(userId=user.id, userName=user.name, probability=pr))
+    # resultをjsonに変換
+    result_json = jsonable_encoder(result)
+    return ORJSONResponse(result_json)

probability/api/controller/root.py

@@ -0,0 +1,7 @@
+from fastapi import APIRouter
+
+router = APIRouter()
+
+@router.get("/")
+def Hello():
+    return {"Hello": "World"}

probability/api/dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.8.18
+
+WORKDIR /usr/src/probability
+
+RUN apt update && apt install -y \
+    libpq-dev \
+    gcc \
+    curl \
+    git
+
+COPY ./probability/api/ ./
+
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt
+
+EXPOSE 8090
+
+CMD ["python", "main.py"]

probability/api/lib/mysql.py

@@ -0,0 +1,22 @@
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+import os
+
+# 接続したいDBへの接続情報
+user_name = os.environ['USER_NAME']
+password = os.environ['PASSWORD']
+host = os.environ['HOST']
+port = os.environ['PORT']
+database = os.environ['DATABASE']
+
+SQLALCHEMY_DATABASE_URL = "mysql://" + user_name + ":" + password + "@" + host + ":" +port + "/" + database + "?charset=utf8&unix_socket=/var/run/mysqld/mysqld.sock"
+
+engine = create_engine(SQLALCHEMY_DATABASE_URL, echo=True)
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+
+def get_db() :
+    db = SessionLocal()
+    try:
+        yield db
+    finally:
+        db.close()

probability/api/main.py

@@ -0,0 +1,11 @@
+from fastapi import FastAPI
+from controller.root import router as root_router
+from controller.probability import router as probability_router
+
+app = FastAPI()
+app.include_router(root_router)
+app.include_router(probability_router)
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8090)

probability/api/models/cluster.py

@@ -0,0 +1,22 @@
+from __future__ import annotations
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+from . import struct as st
+
+# userIdから最新のクラスタリングを取得する
+def get_latest_cluster_by_userId(db: Session, userId, reporting:bool) -> st.Cluster | None:
+    # clustersを取得
+    cluster: st.Cluster | None = db.scalar(select(st.Cluster).where(st.Cluster.user_id == userId, st.Cluster.reporting == reporting).order_by(st.Cluster.date.desc()))
+    return cluster
+
+# userIdから最古のクラスタリングを取得する
+def get_oldest_cluster_by_userId(db: Session, userId, reporting:bool) -> st.Cluster | None:
+    # clustersを取得
+    cluster: st.Cluster | None = db.scalar(select(st.Cluster).where(st.Cluster.user_id == userId, st.Cluster.reporting == reporting).order_by(st.Cluster.date))
+    return cluster
+
+# 取得したclusterと同じuser_id、dateのclusterを全て取得する
+def get_all_cluster_by_userId_and_date(db: Session, userId, date, reporting:bool) -> list[st.Cluster]:
+    # clustersを取得
+    clusters: list[st.Cluster] = db.query(st.Cluster).where(st.Cluster.user_id == userId, st.Cluster.reporting == reporting, st.Cluster.date == date).all()
+    return clusters

probability/api/models/logs.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+from . import struct as st
+
+def get_oldest_log_by_userId(db: Session, userId) -> st.Logs:
+    # logsを取得
+    log: st.Logs = db.scalar(select(st.Logs).where(st.Logs.user_id == userId).order_by(st.Logs.time))
+    return log

probability/api/models/struct.py

@@ -0,0 +1,53 @@
+from sqlalchemy.orm import (
+    DeclarativeBase,
+    Mapped,
+    mapped_column
+)
+# datetime型をインポート
+import datetime
+
+# sqlalchemyのモデルを作成する
+# Baseクラスを作成
+class Base(DeclarativeBase):
+    pass
+
+# Baseクラスを継承したモデルを作成
+# # usersテーブルのモデルUsers
+# class Users(Base):
+#     __tablename__ = 'users'
+#     user_id = mapped_column(Integer, primary_key=True, autoincrement=True)
+#     uid = mapped_column(String(255), nullable=False)
+#     name = mapped_column(String(255), nullable=False)
+#     email = mapped_column(String(255), nullable=False)
+#     role = mapped_column(String(255), nullable=False)
+# logs(仮)テーブルのモデルLogs(仮)
+class Logs(Base):
+    __tablename__ = 'edited_logs'
+    id: Mapped[int] = mapped_column(primary_key=True, index=True)
+    user_id: Mapped[int] = mapped_column(nullable=False)
+    date:Mapped[datetime.date] = mapped_column(nullable=False)
+    reporting:Mapped[datetime.time] = mapped_column(nullable=False)
+    leave:Mapped[datetime.time] = mapped_column(nullable=False)
+# clusterテーブルのモデルCluster
+class Cluster(Base):
+    __tablename__ = 'clusters'
+    id: Mapped[int] = mapped_column(primary_key=True, index=True)
+    date: Mapped[datetime.date] = mapped_column(nullable=False)
+    reporting: Mapped[bool] = mapped_column(nullable=False)
+    average: Mapped[float] = mapped_column(nullable=False)
+    sd: Mapped[float] = mapped_column(nullable=False)
+    count: Mapped[int] = mapped_column(nullable=False)
+    user_id: Mapped[int] = mapped_column(nullable=False)
+
+class User(Base):
+    __tablename__ = 'users'
+    id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
+    created_at: Mapped[datetime.datetime] = mapped_column(nullable=False)
+    updated_at: Mapped[datetime.datetime] = mapped_column(nullable=False)
+    deleted_at: Mapped[datetime.datetime] = mapped_column(nullable=True)
+    uuid: Mapped[str] = mapped_column(nullable=False)
+    name: Mapped[str] = mapped_column(nullable=False)
+    email: Mapped[str] = mapped_column(nullable=False)
+    role: Mapped[int] = mapped_column(nullable=False)
+    beacon_id: Mapped[int] = mapped_column(nullable=False)
+    community_id: Mapped[int] = mapped_column(nullable=False)

probability/api/models/user.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+from . import struct as st
+
+# 全てのユーザを取得する
+def get_all_users_by_community(community:int, db: Session) -> list[st.User]:
+    users: list[st.User] = db.query(st.User).where(st.User.community_id == community).all()
+    return users
+
+# 特定のユーザを取得する
+def get_user_by_id(db: Session, user_id: int) -> st.User | None:
+    user: st.User | None = db.scalar(select(st.User).where(st.User.id == user_id))
+    return user

probability/api/requirements.txt

@@ -0,0 +1,26 @@
+annotated-types==0.6.0
+anyio==4.2.0
+click==8.1.7
+exceptiongroup==1.2.0
+fastapi==0.109.2
+h11==0.14.0
+idna==3.6
+mypy==1.9.0
+mypy-extensions==1.0.0
+mysqlclient==2.2.4
+numpy==1.24.4
+orjson==3.10.1
+pandas==2.0.3
+pydantic==2.6.0
+pydantic_core==2.16.1
+python-dateutil==2.9.0.post0
+pytz==2024.1
+scipy==1.10.1
+six==1.16.0
+sniffio==1.3.0
+SQLAlchemy @ git+https://github.com/sqlalchemy/sqlalchemy.git@a124a593c86325389a92903d2b61f40c34f6d6e2
+starlette==0.36.3
+tomli==2.0.1
+typing_extensions==4.9.0
+tzdata==2024.1
+uvicorn==0.27.0.post1

probability/api/service/date.py

@@ -0,0 +1,35 @@
+from datetime import timedelta
+
+def convert_seconds_to_hms(seconds):
+    hours, remainder = divmod(seconds, 3600)
+    minutes, seconds = divmod(remainder, 60)
+    return f"{int(hours):02d}:{int(minutes):02d}:{int(seconds):02d}"
+
+def number_days(start_date, end_date, weekday):
+    # 開始日と終了日をdatetimeオブジェクトに変換
+    # カウントを初期化
+    day_count = 0
+    # 開始日から終了日まで1日ずつ進みながら曜日を確認
+    current_date = start_date
+    while current_date <= end_date:
+        # 曜日が weekday であればカウントを増やす
+        if current_date.weekday() == weekday:
+            day_count += 1
+        # 次の日に進む
+        current_date += timedelta(days=1)
+    return day_count
+
+# def first_day_of_month(date):
+#     # 指定された日付をdatetimeオブジェクトに変換
+#     date_obj = datetime.strptime(date.values[0][0], "%Y-%m-%d")
+#     # 月の最初の日を取得
+#     first_date = date_obj.replace(day=1)
+#     return first_date
+
+# def last_day_of_month(date):
+#     # 指定された日付をdatetimeオブジェクトに変換
+#     date_obj = datetime.strptime(date.values[0][0], "%Y-%m-%d")
+#     # 月の最後の日を取得
+#     next_month = date_obj.replace(day=28) + timedelta(days=4)
+#     last_date = next_month - timedelta(days=next_month.day)
+#     return last_date