Update LSTM model implementation into the backend, enhance gesture pr…

notebooks/package/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "smart_gestures"
-version = "0.3.1"
+version = "0.3.2"
 description = "Utilities and models for ASL alphabet training used in the SmartGlasses project"
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.9, <3.13"

notebooks/package/smart_gestures/alphabet/vgt_model/model.py

@@ -68,19 +68,13 @@ def create_model(num_classes: int, in_dim: int = IN_DIM) -> nn.Module:
         nn.Module: The created model.
     """
     model = nn.Sequential(
-        nn.Linear(in_dim, 512),
-        nn.BatchNorm1d(512),
+        nn.Linear(in_dim, 256),
         nn.ReLU(inplace=True),
-        nn.Dropout(0.3),
-        nn.Linear(512, 512),
-        nn.BatchNorm1d(512),
+        nn.Dropout(0.2),
+        nn.Linear(256, 256),
         nn.ReLU(inplace=True),
-        nn.Dropout(0.3),
-        nn.Linear(512, 256),
-        nn.BatchNorm1d(256),
-        nn.ReLU(inplace=True),
-        nn.Dropout(0.3),
-        nn.Linear(256, num_classes)
+        nn.Dropout(0.2),
+        nn.Linear(256, num_classes),
     )
     return model.to(DEVICE)
 

notebooks/package/smart_gestures/gestures/lstm_model/model.py

@@ -17,8 +17,8 @@
 
 # Paths
 THIS_DIR = Path(__file__).parent
-DATA_DIR = THIS_DIR.parent.parent.parent / "data"
-MODEL_DIR = THIS_DIR.parent.parent.parent / "models"
+DATA_DIR = THIS_DIR / "data"
+MODEL_DIR = THIS_DIR / "models"
 
 CLASSES_MAP_PATH = DATA_DIR / "gesture_map.json"
 MODEL_FILE = MODEL_DIR / "lstm_model.pth"
@@ -70,6 +70,7 @@ def normalize_landmarks(sequence: np.ndarray) -> np.ndarray:
 
 class GestureLSTM(nn.Module):
     """LSTM-based Gesture Recognition Model"""
+
     def __init__(
         self,
         in_dim: int,
@@ -91,12 +92,12 @@ def __init__(
         self.dropout = nn.Dropout(dropout)
 
     def forward(self, x: torch.Tensor, sequence_lengths: torch.Tensor) -> torch.Tensor:
-        sorted_lengths, sorted_idx = sorted_lengths.sort(0, descending=True)
+        sorted_lengths, sorted_idx = sequence_lengths.sort(0, descending=True)
         x_sorted = x[sorted_idx]
 
         packed_input = pack_padded_sequence(
             x_sorted,
-            sequence_lengths.cpu(),
+            sorted_lengths.cpu(),
             batch_first=True,
             enforce_sorted=True,
         )
@@ -135,7 +136,7 @@ def create_model(num_classes: int) -> nn.Module:
     return model.to(DEVICE)
 
 
-def get_classes() -> list[str]:
+def get_classes() -> dict[str, int]:
     """
     Load the class names from the gesture_map.json file.
     Returns:
@@ -149,8 +150,10 @@ class LSTMModel:
     A high level class for the LSTM gesture recognition model.
     This class handles loading the model, preparing input data, and making predictions.
     """
+
     def __init__(self):
         self.classes = get_classes()
+        self.idx_to_class = {idx: name for name, idx in self.classes.items()}
         self.model = create_model(num_classes=len(self.classes))
 
         state_dict = torch.load(MODEL_FILE, map_location=DEVICE)
@@ -177,7 +180,7 @@ def prepare_input(self, sequence: np.ndarray) -> tuple[torch.Tensor, torch.Tenso
 
         return x_tensor, length_tensor
 
-    def predict(self, sequence: list[list[float]] | np.ndarray) -> str:
+    def predict(self, sequence: list[list[float]] | np.ndarray) -> tuple[str, float]:
         if not isinstance(sequence, np.ndarray):
             sequence = np.array(sequence, dtype=np.float32)
 
@@ -192,7 +195,8 @@ def predict(self, sequence: list[list[float]] | np.ndarray) -> str:
         with torch.no_grad():
             logits = self.model(x_tensor, length_tensor)
             pred_idx = int(torch.argmax(logits, dim=1).item())
-            pred_name = self.classes[pred_idx]
-            confidence = float(torch.softmax(logits, dim=1)[0, pred_idx].item())
+            pred_name = self.idx_to_class[pred_idx]
+            confidence = float(torch.softmax(
+                logits, dim=1)[0, pred_idx].item())
 
         return pred_name, confidence

server/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "server"
-version = "0.1.0"
+version = "0.2.0"
 description = "Add your description here"
 readme = "README.md"
 requires-python = ">=3.12"
@@ -13,7 +13,7 @@ dependencies = [
     "mypy~=1.18.2",
     "pre-commit~=4.3.0",
     "typing-extensions~=4.15.0",
-    "smart_gestures==0.2.3",
+    "smart-gestures==0.3.1",
 ]
 
 
@@ -47,3 +47,6 @@ strict = false
 # E501: Line too long (Black's formatting)
 extend-ignore = ["E203", "E501"]
 exclude = [".git", "__pycache__", "build", "dist"]
+
+[tool.uv.sources]
+smart-gestures = { path = "../notebooks/package" }

server/requirements.txt

@@ -5,4 +5,4 @@ typing_extensions~=4.15.0
 tqdm
 mediapipe>=0.10.14
 opencv-python-headless>=4.11.0.86
-smart_gestures==0.2.3
+smart_gestures==0.3.1

server/src/const.py

@@ -13,14 +13,40 @@ class FastAPITags(Enum):
     ASL_MODEL = "ASL Model"
     VGT_MODEL = "VGT Model"
     KEYPOINTS = "Keypoints"
+    GESTURES = "Gestures"
+    LSTM_MODEL = "LSTM Model"
 
 
 IN_DIM = 63  # 21 landmarks * (x,y,z)
 NUM_POINTS = 21  # exact 21 punten
 
+# LSTM Model Constants
+LSTM_SEQUENCE_LENGTH = 40  # Number of frames required for LSTM prediction
+LSTM_POSE_LANDMARKS = 33  # MediaPipe Pose landmarks
+LSTM_HAND_LANDMARKS = 21  # MediaPipe Hand landmarks per hand
+
 
 # ---- Data schema ----
 class Landmark(BaseModel):
+    """Single landmark with x, y coordinates and optional z depth."""
+
     x: float
     y: float
     z: float = 0.0
+
+
+class PoseLandmark(BaseModel):
+    """Pose landmark with x, y, z coordinates and visibility score."""
+
+    x: float
+    y: float
+    z: float
+    visibility: float
+
+
+class HandLandmark(BaseModel):
+    """Hand landmark with x, y, z coordinates."""
+
+    x: float
+    y: float
+    z: float

server/src/main.py

@@ -53,4 +53,5 @@
 app.include_router(routes.root.router)
 app.include_router(routes.ws.router)
 app.include_router(routes.alphabet.router)
+app.include_router(routes.gestures.router)
 app.include_router(routes.keypoints.router)

server/src/routes/__init__.py

@@ -1,8 +1,9 @@
-from . import alphabet, keypoints, root, ws
+from . import alphabet, gestures, keypoints, root, ws
 
 __all__ = [
     "root",
     "ws",
     "alphabet",
+    "gestures",
     "keypoints",
 ]

server/src/routes/gestures/__init__.py

@@ -0,0 +1,16 @@
+from fastapi import APIRouter
+
+from const import FastAPITags
+
+from . import lstm_model
+
+router = APIRouter(
+    prefix="/gestures",
+    tags=[FastAPITags.GESTURES],
+)
+
+router.include_router(lstm_model.router)
+
+__all__ = [
+    "router",
+]

server/src/routes/gestures/lstm_model/__init__.py

@@ -0,0 +1,62 @@
+from fastapi import APIRouter, HTTPException
+from smart_gestures.gestures.lstm_model import LSTMModel, get_classes
+
+from const import FastAPITags
+from schemas import LSTMClassesResponse, LSTMPredictBody, LSTMPredictResponse
+
+classes = get_classes()
+
+model = LSTMModel()
+
+router = APIRouter(
+    prefix="/lstm",
+    tags=[FastAPITags.LSTM_MODEL],
+)
+
+
+@router.get("/classes")
+async def lstm_model_classes() -> LSTMClassesResponse:
+    """
+    Get the list of gesture classes that the LSTM model can predict.
+
+    Returns:
+        LSTMClassesResponse: A list of class names.
+    """
+    print(f"{classes=}")
+    return LSTMClassesResponse(classes=classes)
+
+
+@router.post("/predict")
+async def lstm_model_predict(body: LSTMPredictBody) -> LSTMPredictResponse:
+    """
+    Predict a gesture from a sequence of keypoint frames.
+
+    The LSTM model requires a sequence of frames, where each frame contains
+    258 features: [POSE (33*4=132), Left Hand (21*3=63), Right Hand (21*3=63)].
+
+    Args:
+        body: LSTMPredictBody containing a sequence of keypoint frames.
+
+    Returns:
+        LSTMPredictResponse: The predicted gesture name and confidence score.
+
+    Raises:
+        HTTPException: If the input sequence is invalid or prediction fails.
+    """
+    try:
+        # Convert the sequence to the format expected by the model
+        sequence = body.to_numpy_sequence()
+
+        # Get prediction and confidence from the model
+        pred_name, confidence = model.predict(sequence)  # type: ignore
+
+        return LSTMPredictResponse(prediction=pred_name, confidence=confidence)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=f"Invalid input: {str(e)}")
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Prediction error: {str(e)}")
+
+
+__all__ = [
+    "router",
+]

server/src/schemas/__init__.py

@@ -1,12 +1,17 @@
-from .classes import ClassesResponse
+from .classes import ClassesResponse, LSTMClassesResponse
 from .keypoints import KeypointsResponse
+from .lstm_predict import LSTMFrame, LSTMPredictBody, LSTMPredictResponse
 from .predict import PredictBody, PredictResponse
 from .status import StatusResponse
 
 __all__ = [
     "StatusResponse",
     "ClassesResponse",
+    "LSTMClassesResponse",
     "PredictBody",
     "PredictResponse",
+    "LSTMPredictBody",
+    "LSTMPredictResponse",
+    "LSTMFrame",
     "KeypointsResponse",
 ]

server/src/schemas/classes.py

@@ -3,3 +3,7 @@
 
 class ClassesResponse(BaseModel):
     classes: list[str] = Field(..., description="List of class names")
+
+
+class LSTMClassesResponse(BaseModel):
+    classes: dict[str, int] = Field(..., description="List of LSTM class names")