human_pose_estimation.mbt

// 人体姿态估计与动作识别（v32.0.0）
// 关键点检测、骨架提取、姿态估计、动作识别、手势识别

///|
/// 姿态估计算法类型
pub(all) enum PoseEstimationAlgorithm {
  OpenPose
  MediaPipe
  AlphaPose
  HRNet
  SimpleBaseline
  DeepPose
  PoseNet
  MoveNet
  Unknown
} derive(Eq)

///|
/// 动作识别算法类型
pub(all) enum ActionRecognitionAlgorithm {
  C3D
  I3D
  TwoStream
  TSN
  TRN
  SlowFast
  TimeSformer
  VideoMAE
  Unknown
} derive(Eq)

///|
/// 人体关键点
pub(all) struct Keypoint {
  x : Double
  y : Double
  confidence : Double
  is_visible : Bool
  part_name : String
}

///|
/// 人体姿态
pub(all) struct HumanPose {
  keypoints : Array[Keypoint]
  num_keypoints : Int
  bounding_box : Array[Double]
  confidence : Double
  pose_id : Int
  is_tracked : Bool
}

///|
/// 骨架连接
pub(all) struct SkeletonConnection {
  joint1 : Int
  joint2 : Int
  confidence : Double
  length : Double
  angle : Double
}

///|
/// 动作识别结果
pub(all) struct ActionRecognitionResult {
  action_name : String
  confidence : Double
  action_id : Int
  duration_frames : Int
  start_frame : Int
  end_frame : Int
  is_completed : Bool
}

///|
/// 手势类型
pub(all) enum GestureType {
  OpenPalm
  Fist
  Pointing
  Victory
  ThumbsUp
  ThumbsDown
  OK
  Peace
  Rock
  Unknown
} derive(Eq)

///|
/// 手势识别结果
pub(all) struct GestureRecognitionResult {
  gesture_type : GestureType
  confidence : Double
  hand_keypoints : Array[Keypoint]
  is_left_hand : Bool
  bounding_box : Array[Double]
}

///|
/// 姿态估计配置
pub(all) struct PoseEstimationConfig {
  algorithm : PoseEstimationAlgorithm
  num_keypoints : Int
  confidence_threshold : Double
  nms_threshold : Double
  max_num_people : Int
  use_tracking : Bool
  use_smoothing : Bool
  smoothing_factor : Double
}

///|
/// 动作识别配置
pub(all) struct ActionRecognitionConfig {
  algorithm : ActionRecognitionAlgorithm
  num_classes : Int
  window_size : Int
  stride : Int
  confidence_threshold : Double
  use_temporal_smoothing : Bool
  smoothing_window : Int
}

///|
/// 创建默认姿态估计配置
pub fn create_default_pose_config() -> PoseEstimationConfig {
  PoseEstimationConfig::{
    algorithm: PoseEstimationAlgorithm::OpenPose,
    num_keypoints: 17,
    confidence_threshold: 0.5,
    nms_threshold: 0.6,
    max_num_people: 10,
    use_tracking: true,
    use_smoothing: true,
    smoothing_factor: 0.8,
  }
}

///|
/// 创建默认动作识别配置
pub fn create_default_action_config() -> ActionRecognitionConfig {
  ActionRecognitionConfig::{
    algorithm: ActionRecognitionAlgorithm::I3D,
    num_classes: 400,
    window_size: 16,
    stride: 8,
    confidence_threshold: 0.5,
    use_temporal_smoothing: true,
    smoothing_window: 5,
  }
}

///|
/// COCO 17 关键点名称
pub fn get_coco_keypoint_names() -> Array[String] {
  [
    "nose", "left_eye", "right_eye", "left_ear", "right_ear", "left_shoulder", "right_shoulder",
    "left_elbow", "right_elbow", "left_wrist", "right_wrist", "left_hip", "right_hip",
    "left_knee", "right_knee", "left_ankle", "right_ankle",
  ]
}

///|
/// COCO 骨架连接对
pub fn get_coco_skeleton_connections() -> Array[Array[Int]] {
  [
    [0, 1],
    [0, 2],
    [1, 3],
    [2, 4], // 头部
    [5, 6],
    [5, 7],
    [7, 9],
    [6, 8],
    [8, 10], // 手臂
    [5, 11],
    [6, 12],
    [11, 12], // 躯干
    [11, 13],
    [13, 15],
    [12, 14],
    [14, 16], // 腿部
  ]
}

///|
/// 检测人体关键点（简化版：基于图像特征的启发式检测）
pub fn detect_keypoints(
  img : @types.Image,
  config : PoseEstimationConfig,
) -> Array[HumanPose] {
  ignore(config)
  let w = img.width
  let h = img.height
  let poses : Array[HumanPose] = []

  // 简化版：检测图像中的高对比度区域作为人体
  // 实际实现应该使用深度学习模型

  // 计算图像梯度
  let mut max_grad = 0.0
  let mut center_x = w.to_double() / 2.0
  let mut center_y = h.to_double() / 2.0

  for y in 1..<(h - 1) {
    for x in 1..<(w - 1) {
      let idx = y * w + x
      let gx = img.data[idx + 1].to_double() - img.data[idx - 1].to_double()
      let gy = img.data[idx + w].to_double() - img.data[idx - w].to_double()
      let grad = (gx * gx + gy * gy).sqrt()
      if grad > max_grad {
        max_grad = grad
        center_x = x.to_double()
        center_y = y.to_double()
      }
    }
  }

  // 生成简化的人体姿态（17个关键点）
  let keypoint_names = get_coco_keypoint_names()
  let keypoints : Array[Keypoint] = []

  // 头部关键点
  keypoints.push(Keypoint::{
    x: center_x,
    y: center_y - h.to_double() * 0.3,
    confidence: 0.8,
    is_visible: true,
    part_name: keypoint_names[0],
  })
  keypoints.push(Keypoint::{
    x: center_x - 10.0,
    y: center_y - h.to_double() * 0.32,
    confidence: 0.7,
    is_visible: true,
    part_name: keypoint_names[1],
  })
  keypoints.push(Keypoint::{
    x: center_x + 10.0,
    y: center_y - h.to_double() * 0.32,
    confidence: 0.7,
    is_visible: true,
    part_name: keypoint_names[2],
  })
  keypoints.push(Keypoint::{
    x: center_x - 15.0,
    y: center_y - h.to_double() * 0.3,
    confidence: 0.6,
    is_visible: true,
    part_name: keypoint_names[3],
  })
  keypoints.push(Keypoint::{
    x: center_x + 15.0,
    y: center_y - h.to_double() * 0.3,
    confidence: 0.6,
    is_visible: true,
    part_name: keypoint_names[4],
  })

  // 躯干关键点
  keypoints.push(Keypoint::{
    x: center_x - 30.0,
    y: center_y - h.to_double() * 0.15,
    confidence: 0.8,
    is_visible: true,
    part_name: keypoint_names[5],
  })
  keypoints.push(Keypoint::{
    x: center_x + 30.0,
    y: center_y - h.to_double() * 0.15,
    confidence: 0.8,
    is_visible: true,
    part_name: keypoint_names[6],
  })

  // 手臂关键点
  keypoints.push(Keypoint::{
    x: center_x - 50.0,
    y: center_y,
    confidence: 0.7,
    is_visible: true,
    part_name: keypoint_names[7],
  })
  keypoints.push(Keypoint::{
    x: center_x + 50.0,
    y: center_y,
    confidence: 0.7,
    is_visible: true,
    part_name: keypoint_names[8],
  })
  keypoints.push(Keypoint::{
    x: center_x - 60.0,
    y: center_y + h.to_double() * 0.1,
    confidence: 0.6,
    is_visible: true,
    part_name: keypoint_names[9],
  })
  keypoints.push(Keypoint::{
    x: center_x + 60.0,
    y: center_y + h.to_double() * 0.1,
    confidence: 0.6,
    is_visible: true,
    part_name: keypoint_names[10],
  })

  // 髋部关键点
  keypoints.push(Keypoint::{
    x: center_x - 20.0,
    y: center_y + h.to_double() * 0.15,
    confidence: 0.8,
    is_visible: true,
    part_name: keypoint_names[11],
  })
  keypoints.push(Keypoint::{
    x: center_x + 20.0,
    y: center_y + h.to_double() * 0.15,
    confidence: 0.8,
    is_visible: true,
    part_name: keypoint_names[12],
  })

  // 腿部关键点
  keypoints.push(Keypoint::{
    x: center_x - 25.0,
    y: center_y + h.to_double() * 0.3,
    confidence: 0.7,
    is_visible: true,
    part_name: keypoint_names[13],
  })
  keypoints.push(Keypoint::{
    x: center_x + 25.0,
    y: center_y + h.to_double() * 0.3,
    confidence: 0.7,
    is_visible: true,
    part_name: keypoint_names[14],
  })
  keypoints.push(Keypoint::{
    x: center_x - 30.0,
    y: center_y + h.to_double() * 0.45,
    confidence: 0.6,
    is_visible: true,
    part_name: keypoint_names[15],
  })
  keypoints.push(Keypoint::{
    x: center_x + 30.0,
    y: center_y + h.to_double() * 0.45,
    confidence: 0.6,
    is_visible: true,
    part_name: keypoint_names[16],
  })

  let pose = HumanPose::{
    keypoints,
    num_keypoints: 17,
    bounding_box: [
      center_x - 80.0,
      center_y - h.to_double() * 0.4,
      160.0,
      h.to_double() * 0.9,
    ],
    confidence: 0.75,
    pose_id: 0,
    is_tracked: true,
  }

  poses.push(pose)
  poses
}

///|
/// 提取骨架连接
pub fn extract_skeleton(pose : HumanPose) -> Array[SkeletonConnection] {
  let connections = get_coco_skeleton_connections()
  let skeleton : Array[SkeletonConnection] = []

  for conn in connections {
    let j1 = conn[0]
    let j2 = conn[1]
    if j1 < pose.keypoints.length() && j2 < pose.keypoints.length() {
      let k1 = pose.keypoints[j1]
      let k2 = pose.keypoints[j2]
      let dx = k2.x - k1.x
      let dy = k2.y - k1.y
      let length = (dx * dx + dy * dy).sqrt()
      let angle = if length > 0.0 {
        let sin_val = dy / length
        // 简化的反正弦近似
        sin_val * 57.3
      } else {
        0.0
      }
      skeleton.push(SkeletonConnection::{
        joint1: j1,
        joint2: j2,
        confidence: (k1.confidence + k2.confidence) / 2.0,
        length,
        angle,
      })
    }
  }

  skeleton
}

///|
/// 计算关节角度
pub fn calculate_joint_angle(
  pose : HumanPose,
  joint1 : Int,
  joint_center : Int,
  joint2 : Int,
) -> Double {
  if joint1 < pose.keypoints.length() &&
    joint_center < pose.keypoints.length() &&
    joint2 < pose.keypoints.length() {
    let k1 = pose.keypoints[joint1]
    let kc = pose.keypoints[joint_center]
    let k2 = pose.keypoints[joint2]

    let v1x = k1.x - kc.x
    let v1y = k1.y - kc.y
    let v2x = k2.x - kc.x
    let v2y = k2.y - kc.y

    let dot = v1x * v2x + v1y * v2y
    let len1 = (v1x * v1x + v1y * v1y).sqrt()
    let len2 = (v2x * v2x + v2y * v2y).sqrt()

    if len1 > 0.0 && len2 > 0.0 {
      let cos_angle = dot / (len1 * len2)
      let clamped_cos = if cos_angle < -1.0 {
        -1.0
      } else if cos_angle > 1.0 {
        1.0
      } else {
        cos_angle
      }
      // 简化的反余弦
      (1.0 - clamped_cos) * 90.0
    } else {
      0.0
    }
  } else {
    0.0
  }
}

///|
/// 识别动作（简化版：基于关节角度变化）
pub fn recognize_action(
  poses : Array[HumanPose],
  config : ActionRecognitionConfig,
) -> ActionRecognitionResult {
  ignore(config)
  if poses.length() == 0 {
    return ActionRecognitionResult::{
      action_name: "unknown",
      confidence: 0.0,
      action_id: -1,
      duration_frames: 0,
      start_frame: 0,
      end_frame: 0,
      is_completed: false,
    }
  }

  let pose = poses[0]

  // 计算关键关节角度
  let left_elbow_angle = calculate_joint_angle(pose, 5, 7, 9) // 左肩-左肘-左腕
  let right_elbow_angle = calculate_joint_angle(pose, 6, 8, 10) // 右肩-右肘-右腕
  let left_knee_angle = calculate_joint_angle(pose, 11, 13, 15) // 左髋-左膝-左踝
  let right_knee_angle = calculate_joint_angle(pose, 12, 14, 16) // 右髋-右膝-右踝

  // 简化的动作分类
  let mut action_name = "walking"
  let mut confidence = 0.5

  if left_elbow_angle < 60.0 && right_elbow_angle < 60.0 {
    action_name = "arms_crossed"
    confidence = 0.7
  } else if left_elbow_angle > 150.0 && right_elbow_angle > 150.0 {
    action_name = "arms_raised"
    confidence = 0.75
  } else if left_knee_angle < 90.0 || right_knee_angle < 90.0 {
    action_name = "squatting"
    confidence = 0.65
  } else if left_knee_angle > 160.0 && right_knee_angle > 160.0 {
    action_name = "standing"
    confidence = 0.8
  } else {
    action_name = "walking"
    confidence = 0.5
  }

  ActionRecognitionResult::{
    action_name,
    confidence,
    action_id: 0,
    duration_frames: poses.length(),
    start_frame: 0,
    end_frame: poses.length() - 1,
    is_completed: true,
  }
}

///|
/// 识别手势（简化版：基于手部关键点）
pub fn recognize_gesture(pose : HumanPose) -> GestureRecognitionResult {
  // 简化版：基于手腕位置和手臂角度判断手势
  let left_wrist = if 9 < pose.keypoints.length() {
    pose.keypoints[9]
  } else {
    Keypoint::{
      x: 0.0,
      y: 0.0,
      confidence: 0.0,
      is_visible: false,
      part_name: "",
    }
  }
  let right_wrist = if 10 < pose.keypoints.length() {
    pose.keypoints[10]
  } else {
    Keypoint::{
      x: 0.0,
      y: 0.0,
      confidence: 0.0,
      is_visible: false,
      part_name: "",
    }
  }

  let left_shoulder = if 5 < pose.keypoints.length() {
    pose.keypoints[5]
  } else {
    Keypoint::{
      x: 0.0,
      y: 0.0,
      confidence: 0.0,
      is_visible: false,
      part_name: "",
    }
  }
  let right_shoulder = if 6 < pose.keypoints.length() {
    pose.keypoints[6]
  } else {
    Keypoint::{
      x: 0.0,
      y: 0.0,
      confidence: 0.0,
      is_visible: false,
      part_name: "",
    }
  }

  // 简化的手势分类
  let mut gesture_type = GestureType::Unknown
  let mut confidence = 0.3
  let mut is_left_hand = false

  let left_arm_up = left_wrist.y < left_shoulder.y
  let right_arm_up = right_wrist.y < right_shoulder.y

  if right_arm_up && !left_arm_up {
    gesture_type = GestureType::Pointing
    confidence = 0.6
    is_left_hand = false
  } else if left_arm_up && right_arm_up {
    gesture_type = GestureType::Victory
    confidence = 0.55
    is_left_hand = false
  } else if !left_arm_up && !right_arm_up {
    gesture_type = GestureType::OpenPalm
    confidence = 0.5
    is_left_hand = false
  } else {
    gesture_type = GestureType::Unknown
    confidence = 0.3
    is_left_hand = false
  }

  GestureRecognitionResult::{
    gesture_type,
    confidence,
    hand_keypoints: [],
    is_left_hand,
    bounding_box: [0.0, 0.0, 100.0, 100.0],
  }
}

///|
/// 可视化姿态（在图像上绘制骨架）
pub fn visualize_pose(img : @types.Image, pose : HumanPose) -> @types.Image {
  let w = img.width
  let h = img.height
  let total = w * h
  let out : Array[Byte] = []

  // 复制原图
  for i in 0..<total {
    out.push(img.data[i])
  }

  // 绘制关键点（红色圆点）
  for kp in pose.keypoints {
    if kp.confidence > 0.5 && kp.is_visible {
      let cx = kp.x.to_int()
      let cy = kp.y.to_int()
      for dy in -3..<4 {
        for dx in -3..<4 {
          let nx = cx + dx
          let ny = cy + dy
          if nx >= 0 && nx < w && ny >= 0 && ny < h {
            out[ny * w + nx] = b'\xFF' // 红色
          }
        }
      }
    }
  }

  // 绘制骨架连接（绿色线条）
  let skeleton = extract_skeleton(pose)
  for conn in skeleton {
    if conn.confidence > 0.5 {
      let k1 = pose.keypoints[conn.joint1]
      let k2 = pose.keypoints[conn.joint2]
      let x1 = k1.x.to_int()
      let y1 = k1.y.to_int()
      let x2 = k2.x.to_int()
      let y2 = k2.y.to_int()

      // 简单的线条绘制
      let steps = 20
      for i in 0..<steps {
        let t = i.to_double() / steps.to_double()
        let x = (x1.to_double() + (x2 - x1).to_double() * t).to_int()
        let y = (y1.to_double() + (y2 - y1).to_double() * t).to_int()
        if x >= 0 && x < w && y >= 0 && y < h {
          out[y * w + x] = b'\x00' // 绿色（简化版）
        }
      }
    }
  }

  @types.Image::{
    width: w,
    height: h,
    channels: img.channels,
    data: Bytes::from_array(out),
  }
}

///|
/// 生成姿态估计报告
pub fn pose_estimation_report_to_string(
  poses : Array[HumanPose],
  action_result : ActionRecognitionResult,
) -> String {
  let mut report = "## 人体姿态估计与动作识别报告\n\n"
  report = report + "### 检测结果\n\n"
  report = report + "| 指标 | 值 |\n|------|------|\n"
  report = report + "| 检测人数 | " + poses.length().to_string() + " |\n"

  if poses.length() > 0 {
    let pose = poses[0]
    report = report +
      "| 关键点数量 | " +
      pose.num_keypoints.to_string() +
      " |\n"
    report = report +
      "| 姿态置信度 | " +
      (pose.confidence * 100.0).to_string() +
      "% |\n"
    report = report +
      "| 跟踪状态 | " +
      (if pose.is_tracked { "跟踪中" } else { "未跟踪" }) +
      " |\n\n"

    // 关键点详情
    report = report + "### 关键点详情\n\n"
    report = report +
      "| 部位 | X | Y | 置信度 | 可见 |\n|------|------|------|------|------|\n"
    for kp in pose.keypoints {
      report = report +
        "| " +
        kp.part_name +
        " | " +
        kp.x.to_string() +
        " | " +
        kp.y.to_string() +
        " | " +
        (kp.confidence * 100.0).to_string() +
        "% | " +
        (if kp.is_visible { "是" } else { "否" }) +
        " |\n"
    }

    // 动作识别结果
    report = report + "\n### 动作识别结果\n\n"
    report = report + "| 指标 | 值 |\n|------|------|\n"
    report = report + "| 动作名称 | " + action_result.action_name + " |\n"
    report = report +
      "| 置信度 | " +
      (action_result.confidence * 100.0).to_string() +
      "% |\n"
    report = report +
      "| 持续帧数 | " +
      action_result.duration_frames.to_string() +
      " |\n"
    report = report +
      "| 完成状态 | " +
      (if action_result.is_completed { "已完成" } else { "进行中" }) +
      " |\n"
  }

  report
}
src\process\pose_estimation\human_pose_estimation.mbt

87.50%
