diff --git a/notebooks/package/pyproject.toml b/notebooks/package/pyproject.toml index 6f27cb2..9a37a4c 100644 --- a/notebooks/package/pyproject.toml +++ b/notebooks/package/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "smart_gestures" -version = "0.3.1" +version = "0.3.2" description = "Utilities and models for ASL alphabet training used in the SmartGlasses project" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.9, <3.13" diff --git a/notebooks/package/smart_gestures/alphabet/vgt_model/model.py b/notebooks/package/smart_gestures/alphabet/vgt_model/model.py index 5abeb30..c07f1f2 100644 --- a/notebooks/package/smart_gestures/alphabet/vgt_model/model.py +++ b/notebooks/package/smart_gestures/alphabet/vgt_model/model.py @@ -68,19 +68,13 @@ def create_model(num_classes: int, in_dim: int = IN_DIM) -> nn.Module: nn.Module: The created model. """ model = nn.Sequential( - nn.Linear(in_dim, 512), - nn.BatchNorm1d(512), + nn.Linear(in_dim, 256), nn.ReLU(inplace=True), - nn.Dropout(0.3), - nn.Linear(512, 512), - nn.BatchNorm1d(512), + nn.Dropout(0.2), + nn.Linear(256, 256), nn.ReLU(inplace=True), - nn.Dropout(0.3), - nn.Linear(512, 256), - nn.BatchNorm1d(256), - nn.ReLU(inplace=True), - nn.Dropout(0.3), - nn.Linear(256, num_classes) + nn.Dropout(0.2), + nn.Linear(256, num_classes), ) return model.to(DEVICE) diff --git a/notebooks/package/smart_gestures/gestures/lstm_model/model.py b/notebooks/package/smart_gestures/gestures/lstm_model/model.py index 7feb9db..9d531aa 100644 --- a/notebooks/package/smart_gestures/gestures/lstm_model/model.py +++ b/notebooks/package/smart_gestures/gestures/lstm_model/model.py @@ -17,8 +17,8 @@ # Paths THIS_DIR = Path(__file__).parent -DATA_DIR = THIS_DIR.parent.parent.parent / "data" -MODEL_DIR = THIS_DIR.parent.parent.parent / "models" +DATA_DIR = THIS_DIR / "data" +MODEL_DIR = THIS_DIR / "models" CLASSES_MAP_PATH = DATA_DIR / "gesture_map.json" MODEL_FILE = MODEL_DIR / "lstm_model.pth" @@ -70,6 +70,7 @@ def normalize_landmarks(sequence: np.ndarray) -> np.ndarray: class GestureLSTM(nn.Module): """LSTM-based Gesture Recognition Model""" + def __init__( self, in_dim: int, @@ -91,12 +92,12 @@ def __init__( self.dropout = nn.Dropout(dropout) def forward(self, x: torch.Tensor, sequence_lengths: torch.Tensor) -> torch.Tensor: - sorted_lengths, sorted_idx = sorted_lengths.sort(0, descending=True) + sorted_lengths, sorted_idx = sequence_lengths.sort(0, descending=True) x_sorted = x[sorted_idx] packed_input = pack_padded_sequence( x_sorted, - sequence_lengths.cpu(), + sorted_lengths.cpu(), batch_first=True, enforce_sorted=True, ) @@ -135,7 +136,7 @@ def create_model(num_classes: int) -> nn.Module: return model.to(DEVICE) -def get_classes() -> list[str]: +def get_classes() -> dict[str, int]: """ Load the class names from the gesture_map.json file. Returns: @@ -149,8 +150,10 @@ class LSTMModel: A high level class for the LSTM gesture recognition model. This class handles loading the model, preparing input data, and making predictions. """ + def __init__(self): self.classes = get_classes() + self.idx_to_class = {idx: name for name, idx in self.classes.items()} self.model = create_model(num_classes=len(self.classes)) state_dict = torch.load(MODEL_FILE, map_location=DEVICE) @@ -177,7 +180,7 @@ def prepare_input(self, sequence: np.ndarray) -> tuple[torch.Tensor, torch.Tenso return x_tensor, length_tensor - def predict(self, sequence: list[list[float]] | np.ndarray) -> str: + def predict(self, sequence: list[list[float]] | np.ndarray) -> tuple[str, float]: if not isinstance(sequence, np.ndarray): sequence = np.array(sequence, dtype=np.float32) @@ -192,7 +195,8 @@ def predict(self, sequence: list[list[float]] | np.ndarray) -> str: with torch.no_grad(): logits = self.model(x_tensor, length_tensor) pred_idx = int(torch.argmax(logits, dim=1).item()) - pred_name = self.classes[pred_idx] - confidence = float(torch.softmax(logits, dim=1)[0, pred_idx].item()) + pred_name = self.idx_to_class[pred_idx] + confidence = float(torch.softmax( + logits, dim=1)[0, pred_idx].item()) return pred_name, confidence diff --git a/server/pyproject.toml b/server/pyproject.toml index e506f5b..13b30b9 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "server" -version = "0.1.0" +version = "0.2.0" description = "Add your description here" readme = "README.md" requires-python = ">=3.12" @@ -13,7 +13,7 @@ dependencies = [ "mypy~=1.18.2", "pre-commit~=4.3.0", "typing-extensions~=4.15.0", - "smart_gestures==0.2.3", + "smart-gestures==0.3.1", ] @@ -47,3 +47,6 @@ strict = false # E501: Line too long (Black's formatting) extend-ignore = ["E203", "E501"] exclude = [".git", "__pycache__", "build", "dist"] + +[tool.uv.sources] +smart-gestures = { path = "../notebooks/package" } diff --git a/server/requirements.txt b/server/requirements.txt index 26f77af..bd53e41 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -5,4 +5,4 @@ typing_extensions~=4.15.0 tqdm mediapipe>=0.10.14 opencv-python-headless>=4.11.0.86 -smart_gestures==0.2.3 +smart_gestures==0.3.1 diff --git a/server/src/const.py b/server/src/const.py index a77824b..6dc14c0 100644 --- a/server/src/const.py +++ b/server/src/const.py @@ -13,14 +13,40 @@ class FastAPITags(Enum): ASL_MODEL = "ASL Model" VGT_MODEL = "VGT Model" KEYPOINTS = "Keypoints" + GESTURES = "Gestures" + LSTM_MODEL = "LSTM Model" IN_DIM = 63 # 21 landmarks * (x,y,z) NUM_POINTS = 21 # exact 21 punten +# LSTM Model Constants +LSTM_SEQUENCE_LENGTH = 40 # Number of frames required for LSTM prediction +LSTM_POSE_LANDMARKS = 33 # MediaPipe Pose landmarks +LSTM_HAND_LANDMARKS = 21 # MediaPipe Hand landmarks per hand + # ---- Data schema ---- class Landmark(BaseModel): + """Single landmark with x, y coordinates and optional z depth.""" + x: float y: float z: float = 0.0 + + +class PoseLandmark(BaseModel): + """Pose landmark with x, y, z coordinates and visibility score.""" + + x: float + y: float + z: float + visibility: float + + +class HandLandmark(BaseModel): + """Hand landmark with x, y, z coordinates.""" + + x: float + y: float + z: float diff --git a/server/src/main.py b/server/src/main.py index f37a9ed..0ee01be 100644 --- a/server/src/main.py +++ b/server/src/main.py @@ -53,4 +53,5 @@ app.include_router(routes.root.router) app.include_router(routes.ws.router) app.include_router(routes.alphabet.router) +app.include_router(routes.gestures.router) app.include_router(routes.keypoints.router) diff --git a/server/src/routes/__init__.py b/server/src/routes/__init__.py index ed3a648..0e79d36 100644 --- a/server/src/routes/__init__.py +++ b/server/src/routes/__init__.py @@ -1,8 +1,9 @@ -from . import alphabet, keypoints, root, ws +from . import alphabet, gestures, keypoints, root, ws __all__ = [ "root", "ws", "alphabet", + "gestures", "keypoints", ] diff --git a/server/src/routes/gestures/__init__.py b/server/src/routes/gestures/__init__.py new file mode 100644 index 0000000..7d58e54 --- /dev/null +++ b/server/src/routes/gestures/__init__.py @@ -0,0 +1,16 @@ +from fastapi import APIRouter + +from const import FastAPITags + +from . import lstm_model + +router = APIRouter( + prefix="/gestures", + tags=[FastAPITags.GESTURES], +) + +router.include_router(lstm_model.router) + +__all__ = [ + "router", +] diff --git a/server/src/routes/gestures/lstm_model/__init__.py b/server/src/routes/gestures/lstm_model/__init__.py new file mode 100644 index 0000000..e1f6587 --- /dev/null +++ b/server/src/routes/gestures/lstm_model/__init__.py @@ -0,0 +1,62 @@ +from fastapi import APIRouter, HTTPException +from smart_gestures.gestures.lstm_model import LSTMModel, get_classes + +from const import FastAPITags +from schemas import LSTMClassesResponse, LSTMPredictBody, LSTMPredictResponse + +classes = get_classes() + +model = LSTMModel() + +router = APIRouter( + prefix="/lstm", + tags=[FastAPITags.LSTM_MODEL], +) + + +@router.get("/classes") +async def lstm_model_classes() -> LSTMClassesResponse: + """ + Get the list of gesture classes that the LSTM model can predict. + + Returns: + LSTMClassesResponse: A list of class names. + """ + print(f"{classes=}") + return LSTMClassesResponse(classes=classes) + + +@router.post("/predict") +async def lstm_model_predict(body: LSTMPredictBody) -> LSTMPredictResponse: + """ + Predict a gesture from a sequence of keypoint frames. + + The LSTM model requires a sequence of frames, where each frame contains + 258 features: [POSE (33*4=132), Left Hand (21*3=63), Right Hand (21*3=63)]. + + Args: + body: LSTMPredictBody containing a sequence of keypoint frames. + + Returns: + LSTMPredictResponse: The predicted gesture name and confidence score. + + Raises: + HTTPException: If the input sequence is invalid or prediction fails. + """ + try: + # Convert the sequence to the format expected by the model + sequence = body.to_numpy_sequence() + + # Get prediction and confidence from the model + pred_name, confidence = model.predict(sequence) # type: ignore + + return LSTMPredictResponse(prediction=pred_name, confidence=confidence) + except ValueError as e: + raise HTTPException(status_code=400, detail=f"Invalid input: {str(e)}") + except Exception as e: + raise HTTPException(status_code=500, detail=f"Prediction error: {str(e)}") + + +__all__ = [ + "router", +] diff --git a/server/src/schemas/__init__.py b/server/src/schemas/__init__.py index fd8894a..f0aac73 100644 --- a/server/src/schemas/__init__.py +++ b/server/src/schemas/__init__.py @@ -1,12 +1,17 @@ -from .classes import ClassesResponse +from .classes import ClassesResponse, LSTMClassesResponse from .keypoints import KeypointsResponse +from .lstm_predict import LSTMFrame, LSTMPredictBody, LSTMPredictResponse from .predict import PredictBody, PredictResponse from .status import StatusResponse __all__ = [ "StatusResponse", "ClassesResponse", + "LSTMClassesResponse", "PredictBody", "PredictResponse", + "LSTMPredictBody", + "LSTMPredictResponse", + "LSTMFrame", "KeypointsResponse", ] diff --git a/server/src/schemas/classes.py b/server/src/schemas/classes.py index 84e2c45..c5b8545 100644 --- a/server/src/schemas/classes.py +++ b/server/src/schemas/classes.py @@ -3,3 +3,7 @@ class ClassesResponse(BaseModel): classes: list[str] = Field(..., description="List of class names") + + +class LSTMClassesResponse(BaseModel): + classes: dict[str, int] = Field(..., description="List of LSTM class names") diff --git a/server/src/schemas/lstm_predict.py b/server/src/schemas/lstm_predict.py new file mode 100644 index 0000000..409cc92 --- /dev/null +++ b/server/src/schemas/lstm_predict.py @@ -0,0 +1,119 @@ +import numpy as np +from numpy.typing import NDArray +from pydantic import BaseModel, Field, field_validator + +from const import ( + LSTM_HAND_LANDMARKS, + LSTM_POSE_LANDMARKS, + LSTM_SEQUENCE_LENGTH, + HandLandmark, + PoseLandmark, +) + + +class LSTMFrame(BaseModel): + """ + Single frame containing pose and hand landmarks for LSTM gesture recognition. + + Structure: + - pose: 33 pose landmarks (MediaPipe Pose) + - left_hand: 21 left hand landmarks (MediaPipe Hands) + - right_hand: 21 right hand landmarks (MediaPipe Hands) + + Each frame contains: + - 33 × 4 = 132 pose features (x, y, z, visibility) + - 21 × 3 = 63 left hand features (x, y, z) + - 21 × 3 = 63 right hand features (x, y, z) + Total: 258 features per frame + """ + + pose: list[PoseLandmark] = Field( + ..., + description="33 MediaPipe Pose landmarks with x, y, z, visibility", + min_length=LSTM_POSE_LANDMARKS, + max_length=LSTM_POSE_LANDMARKS, + ) + left_hand: list[HandLandmark] = Field( + ..., + description="21 MediaPipe left hand landmarks with x, y, z", + min_length=LSTM_HAND_LANDMARKS, + max_length=LSTM_HAND_LANDMARKS, + ) + right_hand: list[HandLandmark] = Field( + ..., + description="21 MediaPipe right hand landmarks with x, y, z", + min_length=LSTM_HAND_LANDMARKS, + max_length=LSTM_HAND_LANDMARKS, + ) + + def to_feature_array(self) -> NDArray[np.float32]: + """ + Convert frame to flat feature array of 258 values. + + Returns: + NDArray[np.float32]: Array of shape (258,) containing [pose, left_hand, right_hand] + """ + pose_features = np.array([[lm.x, lm.y, lm.z, lm.visibility] for lm in self.pose]).flatten() + left_hand_features = np.array([[lm.x, lm.y, lm.z] for lm in self.left_hand]).flatten() + right_hand_features = np.array([[lm.x, lm.y, lm.z] for lm in self.right_hand]).flatten() + + return np.concatenate([pose_features, left_hand_features, right_hand_features]) + + +class LSTMPredictBody(BaseModel): + """ + Request body for LSTM gesture prediction. + + The LSTM model requires EXACTLY 40 consecutive frames to make a prediction. + Each frame must contain: + - 33 pose landmarks (MediaPipe Pose) + - 21 left hand landmarks (MediaPipe Hands) + - 21 right hand landmarks (MediaPipe Hands) + + **Frontend Implementation Guide:** + 1. Capture 40 consecutive frames from the camera + 2. For each frame, extract landmarks using MediaPipe: + - Pose landmarks (33 points with x, y, z, visibility) + - Left hand landmarks (21 points with x, y, z) + - Right hand landmarks (21 points with x, y, z) + 3. Send all 40 frames in a single request + """ + + frames: list[LSTMFrame] = Field( + ..., + description=f"Sequence of exactly {LSTM_SEQUENCE_LENGTH} frames with pose and hand landmarks", + min_length=LSTM_SEQUENCE_LENGTH, + max_length=LSTM_SEQUENCE_LENGTH, + ) + + @field_validator("frames") + @classmethod + def validate_frames(cls, v: list[LSTMFrame]) -> list[LSTMFrame]: + """Validate that we have exactly 40 frames.""" + if len(v) != LSTM_SEQUENCE_LENGTH: + raise ValueError(f"LSTM model requires exactly {LSTM_SEQUENCE_LENGTH} frames, " f"received {len(v)} frames") + return v + + def to_numpy_sequence(self) -> NDArray[np.float32]: + """ + Convert the sequence of frames to a numpy array for model prediction. + + Returns: + NDArray[np.float32]: Array of shape (40, 258) containing all frame features + """ + return np.array([frame.to_feature_array() for frame in self.frames], dtype=np.float32) + + +class LSTMPredictResponse(BaseModel): + """Response containing the predicted gesture and confidence score.""" + + prediction: str = Field( + ..., + description="Predicted gesture class name", + ) + confidence: float = Field( + ..., + description="Confidence score of the prediction (0.0 to 1.0)", + ge=0.0, + le=1.0, + )