reinforcement_learning.mbt

// 强化学习与决策系统（v47.0.0）
// Q-Learning、DQN、PPO、A2C、策略梯度、强化学习环境

///|
/// 强化学习算法类型
pub(all) enum RLAlgorithm {
  QLearning
  DQN
  DoubleDQN
  DuelingDQN
  PPO
  A2C
  A3C
  REINFORCE
  DDPG
  SAC
  TD3
  Unknown
} derive(Eq)

///|
/// 探索策略类型
pub(all) enum ExplorationStrategy {
  EpsilonGreedy
  Boltzmann
  UCB
  ThompsonSampling
  NoisyNet
  ParameterNoise
  EntropyRegularization
  Unknown
} derive(Eq)

///|
/// 经验回放类型
pub(all) enum ReplayBufferType {
  Uniform
  Prioritized
  Hindsight
  Episodic
  Reservoir
  Unknown
} derive(Eq)

///|
/// 强化学习环境
pub(all) struct RLEnvironment {
  env_name : String
  num_states : Int
  num_actions : Int
  state_dim : Int
  action_dim : Int
  is_discrete : Bool
  max_steps : Int
  current_state : Array[Double]
  current_step : Int
  is_done : Bool
}

///|
/// 智能体
pub(all) struct RLAgent {
  algorithm : RLAlgorithm
  learning_rate : Double
  gamma : Double
  epsilon : Double
  epsilon_decay : Double
  epsilon_min : Double
  q_table : Array[Array[Double]]
  num_states : Int
  num_actions : Int
  total_reward : Double
  episodes_completed : Int
}

///|
/// 经验元组
pub(all) struct Experience {
  state : Array[Double]
  action : Int
  reward : Double
  next_state : Array[Double]
  is_done : Bool
  priority : Double
}

///|
/// 经验回放缓冲区
pub(all) struct ReplayBuffer {
  buffer : Array[Experience]
  buffer_size : Int
  current_size : Int
  buffer_type : ReplayBufferType
  batch_size : Int
  alpha : Double
  beta : Double
}

///|
/// 强化学习配置
pub(all) struct RLConfig {
  algorithm : RLAlgorithm
  exploration_strategy : ExplorationStrategy
  replay_buffer_type : ReplayBufferType
  num_episodes : Int
  max_steps_per_episode : Int
  learning_rate : Double
  gamma : Double
  epsilon_start : Double
  epsilon_end : Double
  epsilon_decay : Double
  batch_size : Int
  buffer_size : Int
  target_update_freq : Int
  use_soft_update : Bool
  tau : Double
  use_dueling : Bool
  use_double : Bool
  use_prioritized_replay : Bool
  clip_range : Double
  entropy_coef : Double
  value_coef : Double
  max_grad_norm : Double
}

///|
/// 训练结果
pub(all) struct RLTrainingResult {
  algorithm : RLAlgorithm
  num_episodes : Int
  total_reward : Double
  average_reward : Double
  final_epsilon : Double
  episodes_to_solve : Int
  reward_history : Array[Double]
  loss_history : Array[Double]
  epsilon_history : Array[Double]
  training_time_s : Double
  convergence_episode : Int
}

///|
/// 策略评估结果
pub(all) struct PolicyEvaluationResult {
  average_reward : Double
  std_reward : Double
  min_reward : Double
  max_reward : Double
  num_episodes : Int
  success_rate : Double
  average_steps : Double
}

///|
/// 创建默认强化学习配置
pub fn create_default_rl_config() -> RLConfig {
  RLConfig::{
    algorithm: RLAlgorithm::DQN,
    exploration_strategy: ExplorationStrategy::EpsilonGreedy,
    replay_buffer_type: ReplayBufferType::Prioritized,
    num_episodes: 1000,
    max_steps_per_episode: 200,
    learning_rate: 0.001,
    gamma: 0.99,
    epsilon_start: 1.0,
    epsilon_end: 0.01,
    epsilon_decay: 0.995,
    batch_size: 64,
    buffer_size: 10000,
    target_update_freq: 100,
    use_soft_update: false,
    tau: 0.005,
    use_dueling: false,
    use_double: false,
    use_prioritized_replay: true,
    clip_range: 0.2,
    entropy_coef: 0.01,
    value_coef: 0.5,
    max_grad_norm: 0.5,
  }
}

///|
/// 创建强化学习环境
pub fn create_rl_environment(
  env_name : String,
  num_states : Int,
  num_actions : Int,
  state_dim : Int,
  action_dim : Int,
  is_discrete : Bool,
  max_steps : Int,
) -> RLEnvironment {
  let initial_state : Array[Double] = []
  for _ in 0..<state_dim {
    initial_state.push(0.0)
  }

  RLEnvironment::{
    env_name,
    num_states,
    num_actions,
    state_dim,
    action_dim,
    is_discrete,
    max_steps,
    current_state: initial_state,
    current_step: 0,
    is_done: false,
  }
}

///|
/// 创建Q-Learning智能体
pub fn create_q_learning_agent(
  num_states : Int,
  num_actions : Int,
  learning_rate : Double,
  gamma : Double,
  epsilon : Double,
) -> RLAgent {
  let q_table : Array[Array[Double]] = []
  for _ in 0..<num_states {
    let row : Array[Double] = []
    for _ in 0..<num_actions {
      row.push(0.0)
    }
    q_table.push(row)
  }

  RLAgent::{
    algorithm: RLAlgorithm::QLearning,
    learning_rate,
    gamma,
    epsilon,
    epsilon_decay: 0.995,
    epsilon_min: 0.01,
    q_table,
    num_states,
    num_actions,
    total_reward: 0.0,
    episodes_completed: 0,
  }
}

///|
/// 创建经验回放缓冲区
pub fn create_replay_buffer(
  buffer_size : Int,
  batch_size : Int,
  buffer_type : ReplayBufferType,
) -> ReplayBuffer {
  let buffer : Array[Experience] = []

  ReplayBuffer::{
    buffer,
    buffer_size,
    current_size: 0,
    buffer_type,
    batch_size,
    alpha: 0.6,
    beta: 0.4,
  }
}

///|
/// 选择动作（epsilon-greedy）
pub fn select_action(agent : RLAgent, state_idx : Int) -> Int {
  let mut state = 42
  state = (state * 11035 + 12345) % 1000000
  let random_val = (state % 1000).to_double() / 1000.0

  if random_val < agent.epsilon {
    // 探索：随机选择
    state = (state * 11035 + 12345) % 1000000
    state % agent.num_actions
  } else {
    // 利用：选择Q值最大的动作
    let mut best_action = 0
    let mut best_q = -1000000000.0
    for action in 0..<agent.num_actions {
      if state_idx < agent.num_states && action < agent.num_actions {
        let q = agent.q_table[state_idx][action]
        if q > best_q {
          best_q = q
          best_action = action
        }
      }
    }
    best_action
  }
}

///|
/// Q-Learning更新
pub fn q_learning_update(
  agent : RLAgent,
  state_idx : Int,
  action : Int,
  reward : Double,
  next_state_idx : Int,
  is_done : Bool,
) -> RLAgent {
  let q_table = agent.q_table

  // 计算目标Q值
  let mut max_next_q = 0.0
  if !is_done && next_state_idx < agent.num_states {
    for a in 0..<agent.num_actions {
      if q_table[next_state_idx][a] > max_next_q {
        max_next_q = q_table[next_state_idx][a]
      }
    }
  }

  let target = reward + agent.gamma * max_next_q
  let current_q = q_table[state_idx][action]
  let new_q = current_q + agent.learning_rate * (target - current_q)
  q_table[state_idx][action] = new_q

  // 衰减epsilon
  let new_epsilon = if agent.epsilon > agent.epsilon_min {
    agent.epsilon * agent.epsilon_decay
  } else {
    agent.epsilon_min
  }

  RLAgent::{
    algorithm: agent.algorithm,
    learning_rate: agent.learning_rate,
    gamma: agent.gamma,
    epsilon: new_epsilon,
    epsilon_decay: agent.epsilon_decay,
    epsilon_min: agent.epsilon_min,
    q_table,
    num_states: agent.num_states,
    num_actions: agent.num_actions,
    total_reward: agent.total_reward + reward,
    episodes_completed: agent.episodes_completed,
  }
}

///|
/// 添加经验到回放缓冲区
pub fn add_experience(
  buffer : ReplayBuffer,
  experience : Experience,
) -> ReplayBuffer {
  if buffer.current_size < buffer.buffer_size {
    let new_buffer = buffer.buffer
    new_buffer.push(experience)
    ReplayBuffer::{
      buffer: new_buffer,
      buffer_size: buffer.buffer_size,
      current_size: buffer.current_size + 1,
      buffer_type: buffer.buffer_type,
      batch_size: buffer.batch_size,
      alpha: buffer.alpha,
      beta: buffer.beta,
    }
  } else {
    // 缓冲区满，替换最旧的
    buffer.buffer[0] = experience
    buffer
  }
}

///|
/// 从回放缓冲区采样
pub fn sample_experiences(
  buffer : ReplayBuffer,
  batch_size : Int,
) -> Array[Experience] {
  let sampled : Array[Experience] = []
  let actual_batch = if batch_size < buffer.current_size {
    batch_size
  } else {
    buffer.current_size
  }

  let mut state = 42
  for _ in 0..<actual_batch {
    state = (state * 11035 + 12345) % 1000000
    let idx = state % buffer.current_size
    sampled.push(buffer.buffer[idx])
  }

  sampled
}

///|
/// 模拟强化学习训练（简化版）
pub fn simulate_rl_training(
  config : RLConfig,
  env : RLEnvironment,
) -> RLTrainingResult {
  ignore(env)
  let reward_history : Array[Double] = []
  let loss_history : Array[Double] = []
  let epsilon_history : Array[Double] = []

  let mut epsilon = config.epsilon_start
  let mut total_reward = 0.0
  let mut convergence_episode = config.num_episodes

  for episode in 0..<config.num_episodes {
    // 模拟单回合奖励
    let progress = episode.to_double() / config.num_episodes.to_double()
    let episode_reward = -100.0 +
      progress * 200.0 +
      (episode % 10).to_double() * 5.0
    reward_history.push(episode_reward)
    total_reward += episode_reward

    // 模拟损失
    let loss = 1.0 * (1.0 - progress) + 0.1
    loss_history.push(loss)

    // 衰减epsilon
    epsilon = if epsilon > config.epsilon_end {
      epsilon * config.epsilon_decay
    } else {
      config.epsilon_end
    }
    epsilon_history.push(epsilon)

    // 检查收敛
    if episode_reward > 190.0 && convergence_episode == config.num_episodes {
      convergence_episode = episode
    }
  }

  let average_reward = total_reward / config.num_episodes.to_double()

  RLTrainingResult::{
    algorithm: config.algorithm,
    num_episodes: config.num_episodes,
    total_reward,
    average_reward,
    final_epsilon: epsilon,
    episodes_to_solve: convergence_episode,
    reward_history,
    loss_history,
    epsilon_history,
    training_time_s: config.num_episodes.to_double() * 0.1,
    convergence_episode,
  }
}

///|
/// 评估策略（简化版）
pub fn evaluate_policy(
  agent : RLAgent,
  env : RLEnvironment,
  num_episodes : Int,
) -> PolicyEvaluationResult {
  ignore(agent)
  ignore(env)
  let rewards : Array[Double] = []
  let mut total_reward = 0.0
  let mut success_count = 0
  let mut total_steps = 0

  for episode in 0..<num_episodes {
    // 模拟评估
    let reward = 150.0 + (episode % 20).to_double() * 2.5
    rewards.push(reward)
    total_reward += reward
    total_steps += 100

    if reward > 190.0 {
      success_count += 1
    }
  }

  let average_reward = total_reward / num_episodes.to_double()
  let mut variance = 0.0
  for r in rewards {
    variance += (r - average_reward) * (r - average_reward)
  }
  let std_reward = (variance / num_episodes.to_double()).sqrt()

  let mut min_reward = 1000000000.0
  let mut max_reward = -1000000000.0
  for r in rewards {
    if r < min_reward {
      min_reward = r
    }
    if r > max_reward {
      max_reward = r
    }
  }

  PolicyEvaluationResult::{
    average_reward,
    std_reward,
    min_reward,
    max_reward,
    num_episodes,
    success_rate: success_count.to_double() / num_episodes.to_double(),
    average_steps: total_steps.to_double() / num_episodes.to_double(),
  }
}

///|
/// 生成强化学习报告
pub fn reinforcement_learning_report_to_string(
  result : RLTrainingResult,
  config : RLConfig,
  env : RLEnvironment,
) -> String {
  let mut report = "## 强化学习与决策系统报告\n\n"

  report = report + "### 环境信息\n\n"
  report = report + "| 指标 | 值 |\n|------|------|\n"
  report = report + "| 环境名称 | " + env.env_name + " |\n"
  report = report + "| 状态数 | " + env.num_states.to_string() + " |\n"
  report = report + "| 动作数 | " + env.num_actions.to_string() + " |\n"
  report = report + "| 状态维度 | " + env.state_dim.to_string() + " |\n"
  report = report + "| 动作维度 | " + env.action_dim.to_string() + " |\n"
  report = report +
    "| 离散动作 | " +
    (if env.is_discrete { "是" } else { "否" }) +
    " |\n"
  report = report + "| 最大步数 | " + env.max_steps.to_string() + " |\n"

  report = report + "\n### 算法配置\n\n"
  report = report + "| 指标 | 值 |\n|------|------|\n"
  report = report +
    "| 算法 | " +
    (match config.algorithm {
      RLAlgorithm::QLearning => "Q-Learning"
      RLAlgorithm::DQN => "DQN"
      RLAlgorithm::DoubleDQN => "Double DQN"
      RLAlgorithm::DuelingDQN => "Dueling DQN"
      RLAlgorithm::PPO => "PPO"
      RLAlgorithm::A2C => "A2C"
      RLAlgorithm::DDPG => "DDPG"
      RLAlgorithm::SAC => "SAC"
      _ => "未知"
    }) +
    " |\n"
  report = report +
    "| 探索策略 | " +
    (match config.exploration_strategy {
      ExplorationStrategy::EpsilonGreedy => "Epsilon-Greedy"
      ExplorationStrategy::Boltzmann => "Boltzmann"
      ExplorationStrategy::UCB => "UCB"
      _ => "未知"
    }) +
    " |\n"
  report = report +
    "| 训练回合数 | " +
    config.num_episodes.to_string() +
    " |\n"
  report = report + "| 学习率 | " + config.learning_rate.to_string() + " |\n"
  report = report + "| 折扣因子 | " + config.gamma.to_string() + " |\n"
  report = report +
    "| 初始Epsilon | " +
    config.epsilon_start.to_string() +
    " |\n"
  report = report +
    "| 最终Epsilon | " +
    config.epsilon_end.to_string() +
    " |\n"
  report = report + "| 批次大小 | " + config.batch_size.to_string() + " |\n"
  report = report +
    "| 缓冲区大小 | " +
    config.buffer_size.to_string() +
    " |\n"

  report = report + "\n### 训练结果\n\n"
  report = report + "| 指标 | 值 |\n|------|------|\n"
  report = report + "| 总奖励 | " + result.total_reward.to_string() + " |\n"
  report = report +
    "| 平均奖励 | " +
    result.average_reward.to_string() +
    " |\n"
  report = report +
    "| 最终Epsilon | " +
    result.final_epsilon.to_string() +
    " |\n"
  report = report +
    "| 收敛回合 | " +
    result.convergence_episode.to_string() +
    " |\n"
  report = report +
    "| 训练时间 | " +
    result.training_time_s.to_string() +
    " s |\n"

  report
}
src\process\reinforcement_learning\reinforcement_learning.mbt

80.41%
