{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "37aee204",
   "metadata": {},
   "source": [
    "# =============================================================================\n",
    "# Quantitative Trading — Data Pipeline Demo\n",
    "\n",
    "# =============================================================================\n",
    "# Topics covered:\n",
    "#   1. Price Adjustment for Corporate Actions (splits & dividends)\n",
    "#   2. Simple Returns vs Log Returns\n",
    "#   3. Multi-stock Panel & Missing Value Handling\n",
    "#   4. Outlier Detection & Treatment (Z-score, MAD, Winsorize)\n",
    "#   4b. Circuit Breakers & Price Limit Flags (涨跌停) [A-shares]\n",
    "#   5. Trading Calendar & Cross-market Alignment\n",
    "#   6. End-to-End DataPipeline Class\n",
    "\n",
    "# ============================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "32497e8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "环境准备完成!\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "# 使用我们在 Docker 中配置的字体\n",
    "plt.rcParams['font.sans-serif'] = ['WenQuanYi Zen Hei', 'Arial Unicode MS', 'SimHei', 'DejaVu Sans']\n",
    "plt.rcParams['axes.unicode_minus'] = False\n",
    "plt.rcParams['figure.figsize'] = (12, 6)\n",
    "plt.rcParams['figure.dpi'] = 100\n",
    "\n",
    "np.random.seed(42)\n",
    "print(\"环境准备完成!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "998e880d",
   "metadata": {},
   "source": [
    "# =============================================================================\n",
    "# TOPIC 1: Price Adjustment for Corporate Actions (价格复权)\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# Raw stock prices have artificial jumps caused by stock splits and cash\n",
    "# dividends. These must be adjusted before computing returns or signals,\n",
    "# otherwise your algorithm will see fake crashes/spikes.\n",
    "#\n",
    "# Key concepts:\n",
    "#   - Unadjusted price (未复权): raw market price, has visible jumps\n",
    "#   - Forward-adjusted (前复权): history scaled to today's price level\n",
    "#   - Backward-adjusted (后复权): history kept real, recent prices scaled up\n",
    "#   - Adjustment factor (复权因子): multiplier that accounts for all events\n",
    "\n",
    "# ============================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ed3f078e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------adj factors------\n",
      "2020-01-02    1.0\n",
      "2020-01-03    1.0\n",
      "2020-01-06    1.0\n",
      "2020-01-07    1.0\n",
      "2020-01-08    1.0\n",
      "             ... \n",
      "2023-10-26    1.0\n",
      "2023-10-27    1.0\n",
      "2023-10-30    1.0\n",
      "2023-10-31    1.0\n",
      "2023-11-01    1.0\n",
      "Freq: B, Length: 1000, dtype: float64\n",
      "-------end of adj factors------\n",
      "公司行为事件:\n",
      "      date     type  ratio  amount\n",
      "2021-02-25    split    2.0     NaN\n",
      "2020-03-26 dividend    NaN     0.5\n",
      "2020-09-17 dividend    NaN     0.5\n",
      "2021-03-11 dividend    NaN     0.5\n",
      "2021-09-02 dividend    NaN     0.5\n",
      "2022-02-24 dividend    NaN     0.5\n",
      "2022-08-18 dividend    NaN     0.5\n",
      "2023-02-09 dividend    NaN     0.5\n",
      "2023-08-03 dividend    NaN     0.5\n",
      "\n",
      "数据形状: (1000, 7)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 213068 characters]",
      "text/plain": [
       "<Figure size 1400x1000 with 4 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "收益率验证 (拆股日前后):\n",
      "拆股日期: 2021-02-25\n",
      "未复权收益率: -0.5063\n",
      "前复权收益率: -0.0126\n",
      "真实收益率:   -0.0126\n"
     ]
    }
   ],
   "source": [
    "def generate_raw_stock_data(n_days=1000, seed=42):\n",
    "    \"\"\"生成包含拆股和分红事件的原始股票数据\"\"\"\n",
    "    np.random.seed(seed)\n",
    "    dates = pd.bdate_range(start='2020-01-02', periods=n_days)\n",
    "\n",
    "    # 生成\"真实\"价格序列 (几何布朗运动)\n",
    "    mu, sigma = 0.12, 0.25\n",
    "    dt = 1 / 252\n",
    "    log_ret = (mu - 0.5 * sigma**2) * dt + sigma * np.sqrt(dt) * np.random.randn(n_days)\n",
    "    true_prices = 100 * np.exp(np.cumsum(log_ret))\n",
    "\n",
    "    # 模拟成交量\n",
    "    volume = np.random.lognormal(mean=15, sigma=0.5, size=n_days).astype(int)\n",
    "\n",
    "    # 构建DataFrame\n",
    "    df = pd.DataFrame({\n",
    "        'date': dates,\n",
    "        'close': true_prices,\n",
    "        'volume': volume\n",
    "    }).set_index('date')\n",
    "\n",
    "    # 添加开高低价\n",
    "    daily_range = df['close'] * np.abs(np.random.randn(n_days)) * 0.02\n",
    "    df['open'] = df['close'] + np.random.randn(n_days) * daily_range * 0.5\n",
    "    df['high'] = np.maximum(df['close'], df['open']) + np.abs(np.random.randn(n_days)) * daily_range * 0.3\n",
    "    df['low'] = np.minimum(df['close'], df['open']) - np.abs(np.random.randn(n_days)) * daily_range * 0.3\n",
    "\n",
    "    # 定义公司行为事件\n",
    "    events = []\n",
    "\n",
    "    # 1:2拆股, 在第300个交易日\n",
    "    split_idx = 300\n",
    "    split_date = dates[split_idx]\n",
    "    events.append({'date': split_date, 'type': 'split', 'ratio': 2})\n",
    "\n",
    "    # 现金分红, 每季度一次\n",
    "    for q in [60, 185, 310, 435, 560, 685, 810, 935]:\n",
    "        if q < n_days:\n",
    "            events.append({'date': dates[q], 'type': 'dividend', 'amount': 0.5})\n",
    "\n",
    "    # 应用公司行为到\"未复权\"价格\n",
    "    unadj_close = df['close'].copy()\n",
    "    adj_factors = pd.Series(1.0, index=dates)\n",
    "\n",
    "    print('-------adj factors------')\n",
    "    print(adj_factors)\n",
    "    print('-------end of adj factors------')\n",
    "\n",
    "\n",
    "    for event in sorted(events, key=lambda x: x['date']):\n",
    "        idx = dates.get_loc(event['date'])\n",
    "        if event['type'] == 'split':\n",
    "            # 拆股\n",
    "            unadj_close.iloc[idx:] = unadj_close.iloc[idx:] / event['ratio']\n",
    "            adj_factors.iloc[idx:] = adj_factors.iloc[idx:] * event['ratio']\n",
    "        elif event['type'] == 'dividend':\n",
    "            # 分红\n",
    "            price_before = unadj_close.iloc[idx - 1] if idx > 0 else unadj_close.iloc[idx]\n",
    "            div_ratio = 1 - event['amount'] / price_before\n",
    "            unadj_close.iloc[idx:] = unadj_close.iloc[idx:] * div_ratio\n",
    "            adj_factors.iloc[:idx] = adj_factors.iloc[:idx] * div_ratio\n",
    "\n",
    "    df['unadj_close'] = unadj_close\n",
    "    df['adj_factor'] = adj_factors\n",
    "\n",
    "    events_df = pd.DataFrame(events)\n",
    "\n",
    "    return df, events_df\n",
    "\n",
    "def forward_adjust(unadj_prices, adj_factor):\n",
    "    \"\"\"前复权：以最新价格为基准，向前调整历史价格\"\"\"\n",
    "    normalized_factor = adj_factor / adj_factor.iloc[-1]\n",
    "    return unadj_prices * normalized_factor\n",
    "\n",
    "def backward_adjust(unadj_prices, adj_factor):\n",
    "    \"\"\"后复权：以最早价格为基准，向后调整\"\"\"\n",
    "    normalized_factor = adj_factor / adj_factor.iloc[0]\n",
    "    return unadj_prices * normalized_factor\n",
    "\n",
    "# 生成数据\n",
    "stock_data, events = generate_raw_stock_data()\n",
    "\n",
    "print(\"公司行为事件:\")\n",
    "print(events.to_string(index=False))\n",
    "print(f\"\\n数据形状: {stock_data.shape}\")\n",
    "\n",
    "# 计算复权价格\n",
    "stock_data['fwd_adj_close'] = forward_adjust(stock_data['unadj_close'], stock_data['adj_factor'])\n",
    "stock_data['bwd_adj_close'] = backward_adjust(stock_data['unadj_close'], stock_data['adj_factor'])\n",
    "\n",
    "# 可视化对比\n",
    "fig, axes = plt.subplots(2, 2, figsize=(14, 10))\n",
    "\n",
    "# 1. 未复权价格\n",
    "ax = axes[0, 0]\n",
    "ax.plot(stock_data['unadj_close'], color='blue', linewidth=1)\n",
    "ax.set_title('未复权价格 (可见拆股/分红跳跃)', fontsize=13)\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "for _, event in events.iterrows():\n",
    "    ax.axvline(x=event['date'], color='red', linestyle='--', alpha=0.5)\n",
    "\n",
    "# 2. 前复权价格\n",
    "ax = axes[0, 1]\n",
    "ax.plot(stock_data['fwd_adj_close'], color='green', linewidth=1)\n",
    "ax.set_title('前复权价格 (当前价格真实)', fontsize=13)\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "# 3. 后复权价格\n",
    "ax = axes[1, 0]\n",
    "ax.plot(stock_data['bwd_adj_close'], color='orange', linewidth=1)\n",
    "ax.set_title('后复权价格 (历史价格真实)', fontsize=13)\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "# 4. \"真实\"价格对比\n",
    "ax = axes[1, 1]\n",
    "ax.plot(stock_data['close'], label='真实连续价格', color='black', linewidth=1.5)\n",
    "ax.plot(stock_data['fwd_adj_close'], label='前复权价格', color='green', linewidth=1, alpha=0.7)\n",
    "ax.set_title('真实价格 vs 前复权价格', fontsize=13)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "plt.suptitle('复权价格对比', fontsize=15, y=1.02)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# 验证收益率\n",
    "print(\"\\n收益率验证 (拆股日前后):\")\n",
    "split_date = events[events['type'] == 'split']['date'].iloc[0]\n",
    "split_idx = stock_data.index.get_loc(split_date)\n",
    "\n",
    "print(f\"拆股日期: {split_date.strftime('%Y-%m-%d')}\")\n",
    "print(f\"未复权收益率: {stock_data['unadj_close'].iloc[split_idx] / stock_data['unadj_close'].iloc[split_idx-1] - 1:.4f}\")\n",
    "print(f\"前复权收益率: {stock_data['fwd_adj_close'].iloc[split_idx] / stock_data['fwd_adj_close'].iloc[split_idx-1] - 1:.4f}\")\n",
    "print(f\"真实收益率:   {stock_data['close'].iloc[split_idx] / stock_data['close'].iloc[split_idx-1] - 1:.4f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "632a08a5",
   "metadata": {},
   "source": [
    "# =============================================================================\n",
    "# TOPIC 2: Simple Returns vs Log Returns (简单收益率 vs 对数收益率)\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# Two ways to measure how much a price changed:\n",
    "#   - Simple return:  r = (P_t / P_{t-1}) - 1\n",
    "#   - Log return:     r = ln(P_t / P_{t-1})\n",
    "#\n",
    "# Why log returns are preferred in quant research:\n",
    "#   - Time-additive: sum daily log returns to get total log return\n",
    "#   - Better statistical properties (closer to normal distribution)\n",
    "#   - Numerically stable for long compounding periods\n",
    "#\n",
    "# At daily frequency the difference is tiny (~1bp), but it grows at\n",
    "# monthly/annual frequency — never mix the two in the same calculation.\n",
    "\n",
    "# =============================================================================\n",
    "\n",
    "# 使用前复权价格计算收益率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50ad6628",
   "metadata": {},
   "outputs": [],
   "source": [
    "prices = stock_data['fwd_adj_close']\n",
    "\n",
    "# 简单收益率\n",
    "simple_returns = prices.pct_change().dropna()\n",
    "\n",
    "# 对数收益率\n",
    "log_returns = np.log(prices / prices.shift(1)).dropna()\n",
    "\n",
    "# 对比\n",
    "comparison = pd.DataFrame({\n",
    "    '简单收益率': simple_returns,\n",
    "    '对数收益率': log_returns,\n",
    "    '差异': simple_returns - log_returns,\n",
    "    '差异(bp)': (simple_returns - log_returns) * 10000  # 基点\n",
    "})\n",
    "\n",
    "print(\"简单收益率 vs 对数收益率统计:\")\n",
    "print(f\"平均绝对差异:  {comparison['差异'].abs().mean():.6f} ({comparison['差异(bp)'].abs().mean():.6f})\")\n",
    "print(f\"最大差异:      {comparison['差异'].abs().max():.6f} ({comparison['差异(bp)'].abs().max():.6f})\")\n",
    "print(f\"\\n日频数据下两者差异很小，但月频或年频时差异会显著增大。\")\n",
    "\n",
    "# 验证时间可加性\n",
    "print(\"\\n=== 时间可加性验证 ===\")\n",
    "# 对数收益率：时间上直接求和\n",
    "total_log_return = log_returns.sum()\n",
    "actual_total = np.log(prices.iloc[-1] / prices.iloc[0])\n",
    "print(f\"对数收益率求和: {total_log_return:.6f}\")\n",
    "print(f\"实际总对数收益: {actual_total:.6f}\")\n",
    "print(f\"差异: {abs(total_log_return - actual_total):.10f} (几乎为零)\")\n",
    "\n",
    "print(\"\\n简单收益率需要连乘:\")\n",
    "total_simple_product = (1 + simple_returns).prod() - 1\n",
    "actual_simple = prices.iloc[-1] / prices.iloc[0] - 1\n",
    "print(f\"简单收益率连乘: {total_simple_product:.6f}\")\n",
    "print(f\"实际总简单收益: {actual_simple:.6f}\")\n",
    "print(f\"差异: {abs(total_simple_product - actual_simple):.10f}\")\n",
    "\n",
    "# 可视化\n",
    "fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n",
    "\n",
    "ax = axes[0]\n",
    "ax.scatter(simple_returns, log_returns, alpha=0.3, s=5)\n",
    "ax.plot([-0.1, 0.1], [-0.1, 0.1], 'r--', linewidth=1, label='y=x')\n",
    "ax.set_xlabel('简单收益率')\n",
    "ax.set_ylabel('对数收益率')\n",
    "ax.set_title('简单收益率 vs 对数收益率 (日频) ', fontsize=13)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "ax = axes[1]\n",
    "# 月频对比\n",
    "monthly_simple = prices.resample('ME').last().pct_change().dropna()\n",
    "monthly_log = np.log(prices.resample('ME').last() / prices.resample('ME').last().shift(1)).dropna()\n",
    "ax.scatter(monthly_simple, monthly_log, alpha=0.5, s=20)\n",
    "ax.plot([-0.2, 0.2], [-0.2, 0.2], 'r--', linewidth=1, label='y=x')\n",
    "ax.set_xlabel('简单收益率')\n",
    "ax.set_ylabel('对数收益率')\n",
    "ax.set_title('简单收益率 vs 对数收益率 (月频, 差异更明显) ', fontsize=13)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a171952c",
   "metadata": {},
   "source": [
    "# =============================================================================\n",
    "# TOPIC 3: Multi-stock Panel & Missing Value Handling (多标的面板与缺失值处理)\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# Real market data panels have two main sources of missing values:\n",
    "#   - Suspensions (停牌): a stock halts trading for days/weeks\n",
    "#   - Data feed gaps: vendor errors, holidays, late reporting\n",
    "#\n",
    "# Strategies (choice depends on context):\n",
    "#   - ffill (forward fill):    use last known price — standard for suspensions\n",
    "#   - Linear interpolation:    smooth gaps — OK for short random gaps\n",
    "#   - ffill with limit:        ffill but cap at N days — conservative choice\n",
    "#   - Drop:                    remove rows/columns — loses cross-sectional data\n",
    "#\n",
    "# Rule of thumb: if missing% > 10%, consider excluding the stock entirely.\n",
    "\n",
    "# ============================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ba3c95d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_multi_stock_panel(n_stocks=5, n_days=500, seed=42):\n",
    "    \"\"\"生成多标的日线数据面板，包含停牌和缺失值\"\"\"\n",
    "    np.random.seed(seed)\n",
    "    dates = pd.bdate_range(start='2022-01-03', periods=n_days)\n",
    "    \n",
    "    stock_names = [f'Stock_{chr(65+i)}' for i in range(n_stocks)]  # Stock_A, Stock_B, ...\n",
    "    \n",
    "    all_data = {}\n",
    "    for i, name in enumerate(stock_names):\n",
    "        mu = np.random.uniform(0.05, 0.20)\n",
    "        sigma = np.random.uniform(0.15, 0.35)\n",
    "        S0 = np.random.uniform(20, 200)\n",
    "        \n",
    "        dt = 1 / 252\n",
    "        log_ret = (mu - 0.5 * sigma**2) * dt + sigma * np.sqrt(dt) * np.random.randn(n_days)\n",
    "        prices = S0 * np.exp(np.cumsum(log_ret))\n",
    "        volume = np.random.lognormal(mean=14, sigma=0.8, size=n_days).astype(int)\n",
    "        \n",
    "        # 模拟停牌 (随机选择一些连续日期设为NaN)\n",
    "        suspension_starts = np.random.choice(range(50, n_days - 30), size=2 + i % 3, replace=False)\n",
    "        for start in suspension_starts:\n",
    "            duration = np.random.randint(1, 10)\n",
    "            end = min(start + duration, n_days)\n",
    "            prices[start:end] = np.nan\n",
    "            volume[start:end] = 0\n",
    "            \n",
    "        # 模拟随机缺失\n",
    "        random_missing = np.random.choice(range(n_days), size=5, replace=False)\n",
    "        prices[random_missing] = np.nan\n",
    "        \n",
    "        all_data[name] = pd.DataFrame({\n",
    "            'close': prices,\n",
    "            'volume': volume\n",
    "        }, index=dates)\n",
    "        \n",
    "    # 构建面板\n",
    "    price_panel = pd.DataFrame({name: data['close'] for name, data in all_data.items()})\n",
    "    volume_panel = pd.DataFrame({name: data['volume'] for name, data in all_data.items()})\n",
    "    \n",
    "    return price_panel, volume_panel\n",
    "\n",
    "# 生成多标的面板\n",
    "price_panel, volume_panel = generate_multi_stock_panel(n_stocks=5, n_days=500)\n",
    "\n",
    "# 缺失值统计\n",
    "missing_stats = pd.DataFrame({\n",
    "    '总天数': len(price_panel),\n",
    "    '缺失天数': price_panel.isna().sum(),\n",
    "    '缺失比例': price_panel.isna().mean(),\n",
    "    '最长连续缺失': [price_panel[col].isna().astype(int).groupby(\n",
    "        price_panel[col].notna().astype(int).cumsum()).sum().max()\n",
    "        for col in price_panel.columns]\n",
    "})\n",
    "\n",
    "print(\"缺失值统计:\")\n",
    "print(missing_stats)\n",
    "print(f\"\\n面板形状: {price_panel.shape}\")\n",
    "\n",
    "# 缺失值处理方法对比\n",
    "def handle_missing_values(prices, method='ffill'):\n",
    "    \"\"\"处理缺失值的不同方法\"\"\"\n",
    "    if method == 'ffill':\n",
    "        # ffill = forward fill (前向填充): 用前一个有效值填补缺失值\n",
    "        # 最常用于停牌场景: 假设停牌期间价格保持不变\n",
    "        return prices.ffill()\n",
    "    elif method == 'linear':\n",
    "        # 线性插值\n",
    "        return prices.interpolate(method='linear')\n",
    "    elif method == 'drop':\n",
    "        # 直接删除\n",
    "        return prices.dropna()\n",
    "    elif method == 'ffill_limit':\n",
    "        # 有限制的向前填充 (最多填充5天)\n",
    "        return prices.ffill(limit=5)\n",
    "    else:\n",
    "        raise ValueError(f\"Unknown method: {method}\")\n",
    "\n",
    "# 对Stock_A展示不同方法的效果\n",
    "stock_a = price_panel['Stock_A'].copy()\n",
    "\n",
    "fig, axes = plt.subplots(2, 2, figsize=(14, 10))\n",
    "\n",
    "methods = {\n",
    "    '原始数据 (含缺失值) ': stock_a,\n",
    "    '向前填充 (ffill)': handle_missing_values(stock_a, 'ffill'),\n",
    "    '线性插值 (interpolate)': handle_missing_values(stock_a, 'linear'),\n",
    "    '有限填充 (ffill, limit=5)': handle_missing_values(stock_a, 'ffill_limit')\n",
    "}\n",
    "\n",
    "for ax, (title, data) in zip(axes.flat, methods.items()):\n",
    "    ax.plot(data, linewidth=1)\n",
    "    # 标记原始缺失位置\n",
    "    missing_mask = stock_a.isna()\n",
    "    if not missing_mask.all() and title != '原始数据 (含缺失值) ':\n",
    "        filled_values = data[missing_mask].dropna()\n",
    "        if len(filled_values) > 0:\n",
    "            ax.scatter(filled_values.index, filled_values.values,\n",
    "                       color='red', s=15, zorder=5, label='填充值')\n",
    "            ax.legend(fontsize=9)\n",
    "    ax.set_title(title, fontsize=12)\n",
    "    ax.grid(alpha=0.3)\n",
    "\n",
    "plt.suptitle('缺失值处理方法对比 (Stock_A)', fontsize=14, y=1.02)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"\\n实践建议:\")\n",
    "print(\"- 停牌: 使用ffill (假设停牌期间价格不变) \")\n",
    "print(\"- 短期缺失(<3天): 可用线性插值\")\n",
    "print(\"- 长期缺失(>10天): 考虑从分析中排除该标的\")\n",
    "print(\"- 计算收益率前: 先填充缺失值, 再计算\")\n",
    "\n",
    "\n",
    "# 构建干净的多标的数据面板\n",
    "# Steps: ffill → drop stocks/rows still NaN → compute returns\n",
    "#\n",
    "# ⚠️  Look-Ahead Bias Warning: DO NOT use bfill() here.\n",
    "#     bfill() fills early NaN rows by looking at future prices, which means\n",
    "#     your \"day 1\" price would be sourced from a future observation.\n",
    "#     This is data leakage — it would inflate backtest performance.\n",
    "#     Safe rule: only ffill (past → present), then drop what remains NaN.\n",
    "\n",
    "clean_prices = price_panel.ffill()          # only look backward\n",
    "clean_prices = clean_prices.dropna(how='any')  # drop rows where any stock still NaN\n",
    "\n",
    "print(f\"清洗前缺失值: {price_panel.isna().sum().sum()}\")\n",
    "print(f\"清洗后缺失值: {clean_prices.isna().sum().sum()}\")\n",
    "print(f\"(dropped {len(price_panel) - len(clean_prices)} leading rows to avoid look-ahead bias)\")\n",
    "\n",
    "# 计算收益率面板\n",
    "returns_panel = clean_prices.pct_change().dropna()\n",
    "\n",
    "# 可视化归一化价格\n",
    "fig, ax = plt.subplots(figsize=(14, 6))\n",
    "normalized = clean_prices / clean_prices.iloc[0] * 100\n",
    "for col in normalized.columns:\n",
    "    ax.plot(normalized[col], label=col, linewidth=1.2)\n",
    "\n",
    "ax.set_title('多标的归一化价格走势 (基期=100) ', fontsize=14)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# 相关性矩阵\n",
    "corr_matrix = returns_panel.corr()\n",
    "fig, ax = plt.subplots(figsize=(8, 6))\n",
    "im = ax.imshow(corr_matrix.values, cmap='RdBu_r', vmin=-1, vmax=1)\n",
    "ax.set_xticks(range(len(corr_matrix)))\n",
    "ax.set_xticklabels(corr_matrix.columns, rotation=45)\n",
    "ax.set_yticks(range(len(corr_matrix)))\n",
    "ax.set_yticklabels(corr_matrix.columns)\n",
    "for i in range(len(corr_matrix)):\n",
    "    for j in range(len(corr_matrix)):\n",
    "        ax.text(j, i, f'{corr_matrix.iloc[i,j]:.2f}', ha='center', va='center', fontsize=11)\n",
    "plt.colorbar(im, ax=ax)\n",
    "ax.set_title('收益率相关性矩阵', fontsize=14)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "689e9f4b",
   "metadata": {},
   "source": [
    "# =============================================================================\n",
    "# TOPIC 4: Outlier Detection & Treatment (异常值检测与处理)\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# Financial return series contain extreme values from real events (crashes,\n",
    "# halts, data errors). How you handle them affects every downstream signal.\n",
    "#\n",
    "# Detection methods:\n",
    "#   - Z-score:   |z| = |(x - mean) / std| > 3  — simple, but mean/std are\n",
    "#                themselves pulled by outliers (not robust)\n",
    "#   - MAD:       uses median instead of mean — robust to extreme values,\n",
    "#                preferred for financial data\n",
    "#\n",
    "# Treatment methods:\n",
    "#   - Remove:    delete the outlier row — reduces sample size\n",
    "#   - Winsorize: clip to [p1, p99] boundary — keeps all data, limits impact\n",
    "#                standard practice before factor construction\n",
    "#\n",
    "# Q-Q plot: visualizes how far returns deviate from a normal distribution.\n",
    "# Fat tails (leptokurtosis) are a universal property of financial returns.\n",
    "\n",
    "# ============================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8fd49375",
   "metadata": {},
   "outputs": [],
   "source": [
    "def detect_outliers_zscore(series, threshold=3.0):\n",
    "    \"\"\"Z-score (标准分数) 方法检测异常值\n",
    "    \n",
    "    Z = (x - mean) / std\n",
    "    即：这个值偏离平均值多少个标准差？\n",
    "    |Z| > threshold (默认3) 视为异常值\n",
    "    \n",
    "    通俗理解：如果大家平均考70分，标准差10分，你考了100分，\n",
    "    Z = (100-70)/10 = 3，说明你偏离平均值3个标准差，属于异常值。\n",
    "    \n",
    "    缺点：均值和标准差本身受异常值影响\n",
    "    \"\"\"\n",
    "    z_scores = (series - series.mean()) / series.std()\n",
    "    outlier_mask = z_scores.abs() > threshold\n",
    "    return outlier_mask, z_scores\n",
    "\n",
    "\n",
    "def detect_outliers_mad(series, threshold=3.0):\n",
    "    \"\"\"MAD (中位数绝对偏差, Median Absolute Deviation) 方法检测异常值\n",
    "    \n",
    "    MAD = median(|x - median(x)|)  即所有值与中位数的距离的中位数\n",
    "    Modified Z = 0.6745 * (x - median) / MAD\n",
    "    \n",
    "    优点：用中位数替代均值，比Z-score更稳健，不受极端值影响\n",
    "    通俗理解：Z-score用\"平均值\"做参考，容易被极端值带偏；\n",
    "    MAD用\"中位数\"做参考，即使有极端值也不受影响。\n",
    "    \"\"\"\n",
    "    median = series.median()\n",
    "    mad = (series - median).abs().median()\n",
    "    if mad == 0:\n",
    "        mad = 1e-10 # 避免除零\n",
    "    modified_z = 0.6745 * (series - median) / mad\n",
    "    outlier_mask = modified_z.abs() > threshold\n",
    "    return outlier_mask, modified_z\n",
    "\n",
    "\n",
    "def winsorize(series, lower_pct=0.01, upper_pct=0.99):\n",
    "    \"\"\"Winsorize (缩尾处理): 将超出分位数的值截断到分位数边界\n",
    "    \n",
    "    不同于删除异常值，Winsorize保留了所有数据点，只是限制了极端值。\n",
    "    比如设 lower_pct=0.01, upper_pct=0.99:\n",
    "    - 低于1%分位数的值 -> 拉回到1%分位数\n",
    "    - 高于99%分位数的值 -> 拉回到99%分位数\n",
    "    \"\"\"\n",
    "    lower = series.quantile(lower_pct)\n",
    "    upper = series.quantile(upper_pct)\n",
    "    return series.clip(lower=lower, upper=upper)\n",
    "\n",
    "\n",
    "# 使用Stock_A的收益率进行演示\n",
    "ret_a = returns_panel['Stock_A'].copy()\n",
    "\n",
    "# 人为注入一些异常值以演示效果\n",
    "np.random.seed(123)\n",
    "outlier_indices = np.random.choice(ret_a.index, size=5, replace=False)\n",
    "ret_a_with_outliers = ret_a.copy()\n",
    "for idx in outlier_indices:\n",
    "    ret_a_with_outliers[idx] = np.random.choice([-1, 1]) * np.random.uniform(0.15, 0.30)\n",
    "\n",
    "# 检测异常值\n",
    "zscore_mask, z_scores = detect_outliers_zscore(ret_a_with_outliers, threshold=3.0)\n",
    "mad_mask, mad_scores = detect_outliers_mad(ret_a_with_outliers, threshold=3.0)\n",
    "\n",
    "print(\"异常值检测结果:\")\n",
    "print(f\"Z-score方法 (|Z|>3): 检测到 {zscore_mask.sum()} 个异常值\")\n",
    "print(f\"MAD方法 (|Modified Z|>3): 检测到 {mad_mask.sum()} 个异常值\")\n",
    "\n",
    "# Winsorize处理\n",
    "ret_winsorized = winsorize(ret_a_with_outliers, lower_pct=0.01, upper_pct=0.99)\n",
    "\n",
    "print(f\"\\nWinsorize前 - 范围: [{ret_a_with_outliers.min():.4f}, {ret_a_with_outliers.max():.4f}]\")\n",
    "print(f\"Winsorize后 - 范围: [{ret_winsorized.min():.4f}, {ret_winsorized.max():.4f}]\")\n",
    "\n",
    "\n",
    "# 可视化异常值检测\n",
    "fig, axes = plt.subplots(2, 2, figsize=(14, 10))\n",
    "\n",
    "# 1. 收益率时序与异常值标记\n",
    "ax = axes[0, 0]\n",
    "ax.plot(ret_a_with_outliers, color='blue', linewidth=0.8, alpha=0.7)\n",
    "outliers = ret_a_with_outliers[zscore_mask]\n",
    "ax.scatter(outliers.index, outliers.values, color='red', s=40, zorder=5, label=f'异常值 ({len(outliers)}个)')\n",
    "ax.set_title('Z-score异常值检测 (|Z|>3)', fontsize=13)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "# 2. MAD方法\n",
    "ax = axes[0, 1]\n",
    "ax.plot(ret_a_with_outliers, color='blue', linewidth=0.8, alpha=0.7)\n",
    "outliers_mad = ret_a_with_outliers[mad_mask]\n",
    "ax.scatter(outliers_mad.index, outliers_mad.values, color='orange', s=40, zorder=5, label=f'异常值 ({len(outliers_mad)}个)')\n",
    "ax.set_title('MAD异常值检测 (|Modified Z|>3)', fontsize=13)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "# 3. Winsorize前后分布对比\n",
    "ax = axes[1, 0]\n",
    "ax.hist(ret_a_with_outliers, bins=60, alpha=0.5, label='原始', color='blue', density=True)\n",
    "ax.hist(ret_winsorized, bins=60, alpha=0.5, label='Winsorize后', color='green', density=True)\n",
    "ax.set_title('Winsorize前后收益率分布', fontsize=13)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "# 4. Q-Q图\n",
    "ax = axes[1, 1]\n",
    "sorted_data = np.sort(ret_a_with_outliers.dropna())\n",
    "n = len(sorted_data)\n",
    "theoretical = stats.norm.ppf(np.arange(1, n + 1) / (n + 1))\n",
    "ax.scatter(theoretical, sorted_data, alpha=0.5, s=10)\n",
    "ax.plot([theoretical.min(), theoretical.max()], \n",
    "        [theoretical.min() * ret_a_with_outliers.std() + ret_a_with_outliers.mean(),\n",
    "         theoretical.max() * ret_a_with_outliers.std() + ret_a_with_outliers.mean()],\n",
    "         'r--', linewidth=1, label='正态分布参考线')\n",
    "ax.set_xlabel('理论分位数')\n",
    "ax.set_ylabel('样本分位数')\n",
    "ax.set_title('Q-Q图 (检验正态性) ', fontsize=13)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"\\n方法选择建议:\")\n",
    "print(\"- Z-score: 简单快速，但受极端值影响 (不够稳健) \")\n",
    "print(\"- MAD: 更稳健，推荐用于金融数据\")\n",
    "print(\"- Winsorize: 保留数据量，适合构建因子时使用\")\n",
    "print(\"- 实际中常用: Winsorize(1%, 99%) + MAD检测\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e71339e7",
   "metadata": {},
   "source": [
    "# =============================================================================\n",
    "# TOPIC 4b: Circuit Breakers & Price Limit Flags (涨跌停标记)\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# In Chinese A-shares, stocks have a ±10% daily price limit (±5% for ST stocks).\n",
    "# When a stock hits the limit, it is NOT a data error — it is real market data.\n",
    "# However, it signals a critical data quality issue for downstream use:\n",
    "#\n",
    "#   - The price IS the true closing price\n",
    "#   - But the stock may have been UNTRADEABLE for most of the day (zero liquidity)\n",
    "#   - A return of exactly +10% or -10% means you likely CANNOT execute at that price\n",
    "#\n",
    "# What to do in data prep (we flag, not remove):\n",
    "#   - Add a boolean column: `is_limit_up`, `is_limit_down`\n",
    "#   - Downstream strategy code can then decide to skip, weight-down, or\n",
    "#     exclude limit-hit days from signal calculations\n",
    "#\n",
    "# ⚠️  Survivorship Bias Note (out of scope for data prep, but important to know):\n",
    "#     This demo only generates stocks that \"survive\" the full period. In reality,\n",
    "#     stocks get delisted (bankruptcy, M&A, regulatory removal). If you only\n",
    "#     include currently-alive stocks in your historical dataset, you are\n",
    "#     implicitly selecting winners — your backtest returns will be inflated.\n",
    "#     Fix: source historical data that includes ALL stocks, including delisted ones.\n",
    "#     This is a DATA SOURCING problem, handled before data ever enters this pipeline.\n",
    "\n",
    "# ============================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f31d3fff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def flag_price_limits(prices, limit_pct=0.10):\n",
    "    \"\"\"Flag days where a stock hit the daily price limit (涨跌停).\n",
    "\n",
    "    Returns a DataFrame of the same shape with values:\n",
    "        +1  = limit up   (涨停)\n",
    "        -1  = limit down (跌停)\n",
    "         0  = normal day\n",
    "\n",
    "    A small tolerance (0.5%) is used because real closing prices may be\n",
    "    rounded and not land exactly on the theoretical limit price.\n",
    "    \"\"\"\n",
    "    returns = prices.pct_change()\n",
    "    tolerance = 0.005  # 0.5% buffer for rounding\n",
    "\n",
    "    limit_up   = (returns >= limit_pct - tolerance).astype(int)\n",
    "    limit_down = (returns <= -limit_pct + tolerance).astype(int) * -1\n",
    "    flags = limit_up + limit_down\n",
    "    return flags\n",
    "\n",
    "\n",
    "# Inject realistic limit-hit events into the panel for demonstration\n",
    "np.random.seed(99)\n",
    "demo_prices = clean_prices.copy()\n",
    "for col in demo_prices.columns:\n",
    "    # Randomly inject 3 limit-up and 3 limit-down days per stock\n",
    "    up_days   = np.random.choice(range(1, len(demo_prices) - 1), size=3, replace=False)\n",
    "    down_days = np.random.choice(range(1, len(demo_prices) - 1), size=3, replace=False)\n",
    "    for d in up_days:\n",
    "        demo_prices.iloc[d][col] = demo_prices.iloc[d - 1][col] * 1.10  # exactly +10%\n",
    "    for d in down_days:\n",
    "        demo_prices.iloc[d][col] = demo_prices.iloc[d - 1][col] * 0.90  # exactly -10%\n",
    "\n",
    "limit_flags = flag_price_limits(demo_prices, limit_pct=0.10)\n",
    "\n",
    "# Summary\n",
    "total_limit_up   = (limit_flags == 1).sum().sum()\n",
    "total_limit_down = (limit_flags == -1).sum().sum()\n",
    "print(f\"\\n涨跌停统计:\")\n",
    "print(f\"  涨停天数 (limit up):   {total_limit_up}\")\n",
    "print(f\"  跌停天数 (limit down): {total_limit_down}\")\n",
    "print(f\"\\n各标的涨停次数:\")\n",
    "print((limit_flags == 1).sum())\n",
    "print(f\"\\n各标的跌停次数:\")\n",
    "print((limit_flags == -1).sum())\n",
    "\n",
    "# Visualize limit flags on Stock_A\n",
    "fig, axes = plt.subplots(2, 1, figsize=(14, 8), sharex=True)\n",
    "\n",
    "ax = axes[0]\n",
    "ax.plot(demo_prices['Stock_A'], color='steelblue', linewidth=1, label='Stock_A Price')\n",
    "limit_up_dates   = limit_flags.index[limit_flags['Stock_A'] == 1]\n",
    "limit_down_dates = limit_flags.index[limit_flags['Stock_A'] == -1]\n",
    "ax.scatter(limit_up_dates,   demo_prices.loc[limit_up_dates,   'Stock_A'],\n",
    "           color='red',   s=60, zorder=5, label='涨停 (limit up)',   marker='^')\n",
    "ax.scatter(limit_down_dates, demo_prices.loc[limit_down_dates, 'Stock_A'],\n",
    "           color='green', s=60, zorder=5, label='跌停 (limit down)', marker='v')\n",
    "ax.set_title('价格序列与涨跌停标记 (Stock_A)', fontsize=13)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "ax = axes[1]\n",
    "ax.bar(limit_flags.index, limit_flags['Stock_A'],\n",
    "       color=limit_flags['Stock_A'].map({1: 'red', -1: 'green', 0: 'lightgray'}),\n",
    "       width=1)\n",
    "ax.set_title('涨跌停标志 (+1=涨停, -1=跌停, 0=正常)', fontsize=13)\n",
    "ax.set_ylim(-1.5, 1.5)\n",
    "ax.set_yticks([-1, 0, 1])\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"\\n实践建议:\")\n",
    "print(\"- 涨跌停日: 保留价格数据，但添加 is_limit_up / is_limit_down 标记列\")\n",
    "print(\"- 策略层: 遇到涨跌停标记时跳过信号，因无法成交\")\n",
    "print(\"- 统计分析: 计算波动率/相关性时可选择排除涨跌停日，避免失真\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "32e68310",
   "metadata": {},
   "source": [
    "# =============================================================================\n",
    "# TOPIC 5: Trading Calendar & Cross-market Alignment (交易日历与跨市场对齐)\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# Different markets trade on different days (national holidays, weekends).\n",
    "# When combining data from multiple markets, you must decide how to align\n",
    "# the time axis — the wrong choice distorts correlation calculations.\n",
    "#\n",
    "# Two alignment strategies:\n",
    "#   - Intersection (取交集): keep only dates both markets are open.\n",
    "#     Best for: correlation, regression, signal comparison.\n",
    "#     Drawback: loses data on single-market trading days.\n",
    "#\n",
    "#   - Forward fill (向前填充): fill non-trading days with last known price.\n",
    "#     Best for: portfolio construction, continuous NAV calculation.\n",
    "#     Drawback: introduces artificial zero-return days — inflates Sharpe,\n",
    "#               understates volatility. Must be accounted for in analysis.\n",
    "#\n",
    "# Note: this demo uses simplified hardcoded holiday lists. Production systems\n",
    "# should use exchange_calendars or pandas_market_calendars libraries.\n",
    "\n",
    "# ============================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cc3eb282",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_trading_calendars():\n",
    "    \"\"\"创建不同市场的交易日历示例\n",
    "    \n",
    "    不同市场有不同的假日和交易时间，跨市场策略必须正确处理。\n",
    "    \"\"\"\n",
    "    # 生成2024年的工作日\n",
    "    all_bdays = pd.bdate_range(start='2024-01-01', end='2024-12-31')\n",
    "    \n",
    "    # 美国市场假日 (简化版)\n",
    "    us_holidays = pd.to_datetime([\n",
    "        '2024-01-01',  # 新年\n",
    "        '2024-01-15',  # MLK Day\n",
    "        '2024-02-19',  # Presidents Day\n",
    "        '2024-03-29',  # Good Friday\n",
    "        '2024-05-27',  # Memorial Day\n",
    "        '2024-06-19',  # Juneteenth\n",
    "        '2024-07-04',  # Independence Day\n",
    "        '2024-09-02',  # Labor Day\n",
    "        '2024-11-28',  # Thanksgiving\n",
    "        '2024-12-25',  # Christmas\n",
    "    ])\n",
    "    \n",
    "    # 中国A股假日 (简化版)\n",
    "    cn_holidays = pd.to_datetime([\n",
    "        '2024-01-01',  # 元旦\n",
    "        '2024-02-09', '2024-02-12', '2024-02-13', '2024-02-14',\n",
    "        '2024-02-15', '2024-02-16',  # 春节\n",
    "        '2024-04-04', '2024-04-05',  # 清明\n",
    "        '2024-05-01', '2024-05-02', '2024-05-03',  # 劳动节\n",
    "        '2024-06-10',  # 端午\n",
    "        '2024-09-16', '2024-09-17',  # 中秋\n",
    "        '2024-10-01', '2024-10-02', '2024-10-03', '2024-10-04',\n",
    "        '2024-10-07',  # 国庆\n",
    "    ])\n",
    "    \n",
    "    us_trading_days = all_bdays[~all_bdays.isin(us_holidays)]\n",
    "    cn_trading_days = all_bdays[~all_bdays.isin(cn_holidays)]\n",
    "    \n",
    "    return us_trading_days, cn_trading_days\n",
    "\n",
    "us_calendar, cn_calendar = create_trading_calendars()\n",
    "\n",
    "print(f\"2024年美股交易日: {len(us_calendar)}天\")\n",
    "print(f\"2024年A股交易日: {len(cn_calendar)}天\")\n",
    "print(f\"共同交易日: {len(us_calendar.intersection(cn_calendar))}天\")\n",
    "print(f\"仅美股交易日: {len(us_calendar.difference(cn_calendar))}天\")\n",
    "print(f\"仅A股交易日: {len(cn_calendar.difference(us_calendar))}天\")\n",
    "\n",
    "# 模拟跨市场数据对齐\n",
    "np.random.seed(42)\n",
    "\n",
    "# 生成美股和A股数据\n",
    "us_prices = pd.Series(\n",
    "    100 * np.exp(np.cumsum(np.random.randn(len(us_calendar)) * 0.01)),\n",
    "    index=us_calendar, name='US_ETF'\n",
    ")\n",
    "cn_prices = pd.Series(\n",
    "    50 * np.exp(np.cumsum(np.random.randn(len(cn_calendar)) * 0.015)),\n",
    "    index=cn_calendar, name='CN_ETF'\n",
    ")\n",
    "\n",
    "# 直接合并会有大量NaN\n",
    "combined_raw = pd.DataFrame({'US_ETF': us_prices, 'CN_ETF': cn_prices})\n",
    "print(f\"直接合并后:\")\n",
    "print(f\"  总行数: {len(combined_raw)}\")\n",
    "print(f\"  US_ETF缺失: {combined_raw['US_ETF'].isna().sum()}\")\n",
    "print(f\"  CN_ETF缺失: {combined_raw['CN_ETF'].isna().sum()}\")\n",
    "\n",
    "# 方法1: 取交集 (只保留双方都交易的日期)\n",
    "common_dates = us_calendar.intersection(cn_calendar)\n",
    "combined_inner = combined_raw.loc[common_dates].dropna()\n",
    "\n",
    "# 方法2: 向前填充 (用最近的收盘价填充)\n",
    "combined_ffill = combined_raw.ffill().dropna()\n",
    "\n",
    "print(f\"\\n方法1 (取交集): {len(combined_inner)}行\")\n",
    "print(f\"方法2 (ffill): {len(combined_ffill)}行\")\n",
    "\n",
    "# 可视化\n",
    "fig, axes = plt.subplots(1, 2, figsize=(14, 5))\n",
    "\n",
    "ax = axes[0]\n",
    "ax.plot(combined_inner.index, combined_inner['US_ETF'] / combined_inner['US_ETF'].iloc[0],\n",
    "        label='US_ETF', linewidth=1.2)\n",
    "ax.plot(combined_inner.index, combined_inner['CN_ETF'] / combined_inner['CN_ETF'].iloc[0],\n",
    "        label='CN_ETF', linewidth=1.2)\n",
    "ax.set_title('方法1: 取交集日期', fontsize=13)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "ax = axes[1]\n",
    "ax.plot(combined_ffill.index, combined_ffill['US_ETF'] / combined_ffill['US_ETF'].iloc[0],\n",
    "        label='US_ETF', linewidth=1.2)\n",
    "ax.plot(combined_ffill.index, combined_ffill['CN_ETF'] / combined_ffill['CN_ETF'].iloc[0],\n",
    "        label='CN_ETF', linewidth=1.2)\n",
    "ax.set_title('方法2: 向前填充', fontsize=13)\n",
    "ax.legend()\n",
    "ax.grid(alpha=0.3)\n",
    "\n",
    "plt.suptitle('跨市场数据对齐方法对比', fontsize=14, y=1.02)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print(\"\\n实践建议:\")\n",
    "print(\"- 计算相关性、回归: 使用交集日期，确保数据同步\")\n",
    "print(\"- 构建组合/计算市值: 使用ffill，避免数据断裂\")\n",
    "print(\"- 重要: ffill会引入假的'零收益日'，计算波动率时需注意\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "625e9a2a",
   "metadata": {},
   "source": [
    "# =============================================================================\n",
    "# TOPIC 6: End-to-End DataPipeline Class (完整数据清洗管道)\n",
    "\n",
    "# -----------------------------------------------------------------------------\n",
    "# Combines all prior topics into a reusable, sequential pipeline:\n",
    "#   Step 1 — Filter stocks with too many missing values\n",
    "#   Step 2 — Fill remaining missing values (ffill only — no bfill/look-ahead)\n",
    "#   Step 3 — Compute return series\n",
    "#   Step 4 — Detect outliers (MAD) and apply Winsorize\n",
    "#   Step 5 — Generate a data quality report per stock\n",
    "#\n",
    "# This pattern (class-based pipeline with a .run() method) mirrors how\n",
    "# production quant systems structure data ingestion — each step is auditable,\n",
    "# configurable, and produces diagnostic output.\n",
    "\n",
    "# ============================================================================="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab7f3e0b",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataPipeline:\n",
    "    \"\"\"简单的数据清洗管道\n",
    "    \n",
    "    实现了从原始数据到干净数据面板的完整流程：\n",
    "    1. 缺失值处理\n",
    "    2. 异常值处理\n",
    "    3. 收益率计算\n",
    "    4. 数据质量报告\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, prices, max_missing_pct=0.1, winsorize_pct=(0.01, 0.99)):\n",
    "        self.raw_prices = prices.copy()\n",
    "        self.max_missing_pct = max_missing_pct\n",
    "        self.winsorize_pct = winsorize_pct\n",
    "        self.report = {}\n",
    "        \n",
    "    def run(self):\n",
    "        \"\"\"运行完整的数据清洗管道\"\"\"\n",
    "        print(\"=\" * 60)\n",
    "        print(\"数据清洗管道运行中...\")\n",
    "        print(\"=\" * 60)\n",
    "        \n",
    "        # Step 1: 过滤缺失值过多的标的\n",
    "        missing_pct = self.raw_prices.isna().mean()\n",
    "        valid_cols = missing_pct[missing_pct <= self.max_missing_pct].index.tolist()\n",
    "        dropped_cols = missing_pct[missing_pct > self.max_missing_pct].index.tolist()\n",
    "        \n",
    "        print(f\"\\n[Step 1] 过滤缺失值 > {self.max_missing_pct:.0%} 的标的\")\n",
    "        print(f\"  保留: {len(valid_cols)}个标的, 删除: {len(dropped_cols)}个标的\")\n",
    "        if dropped_cols:\n",
    "            print(f\"  被删除的标的: {dropped_cols}\")\n",
    "            \n",
    "        prices = self.raw_prices[valid_cols].copy()\n",
    "        \n",
    "        # Step 2: 填充缺失值\n",
    "        # ⚠️  Only ffill (backward-looking). bfill would use future prices to\n",
    "        #     fill early NaNs — that is look-ahead bias / data leakage.\n",
    "        #     Any rows still NaN after ffill are dropped (typically only the\n",
    "        #     very first row if the series starts with a missing value).\n",
    "        print(f\"\\n[Step 2] 填充缺失值 (ffill only, no bfill)\")\n",
    "        n_missing_before = prices.isna().sum().sum()\n",
    "        prices = prices.ffill().dropna(how='any')\n",
    "        n_missing_after = prices.isna().sum().sum()\n",
    "        print(f\"  填充前: {n_missing_before}个NaN -> 填充后: {n_missing_after}个NaN\")\n",
    "        \n",
    "        # Step 3: 计算收益率\n",
    "        print(f\"\\n[Step 3] 计算收益率\")\n",
    "        returns = prices.pct_change().iloc[1:]  # 删除第一行NaN\n",
    "        print(f\"  收益率面板: {returns.shape[0]}天 x {returns.shape[1]}标的\")\n",
    "        \n",
    "        # Step 4: 检测和处理异常值\n",
    "        print(f\"\\n[Step 4] 异常值检测与Winsorize\")\n",
    "        n_outliers_total = 0\n",
    "        returns_clean = returns.copy()\n",
    "        for col in returns_clean.columns:\n",
    "            mask, _ = detect_outliers_mad(returns_clean[col], threshold=5.0)\n",
    "            n_outliers = mask.sum()\n",
    "            n_outliers_total += n_outliers\n",
    "            # Winsorize\n",
    "            returns_clean[col] = winsorize(returns_clean[col], \n",
    "                                           self.winsorize_pct[0], self.winsorize_pct[1])\n",
    "            \n",
    "        print(f\"  检测到 {n_outliers_total} 个异常值 (MAD, |Z|>5)\")\n",
    "        print(f\"  已进行Winsorize处理 ({self.winsorize_pct[0]:.0%}, {self.winsorize_pct[1]:.0%})\")\n",
    "        \n",
    "        # Step 5: 数据质量报告\n",
    "        print(f\"\\n[Step 5] 数据质量报告\")\n",
    "        quality = pd.DataFrame({\n",
    "            '标的': valid_cols,\n",
    "            '日均收益率(bp)': (returns_clean.mean() * 10000).round(2),\n",
    "            '日波动率(%)': (returns_clean.std() * 100).round(3),\n",
    "            '年化收益率(%)': (returns_clean.mean() * 252 * 100).round(2),\n",
    "            '年化波动率(%)': (returns_clean.std() * np.sqrt(252) * 100).round(2),\n",
    "            '偏度': returns_clean.skew().round(3),\n",
    "            '峰度': returns_clean.kurtosis().round(3),\n",
    "        }).set_index('标的')\n",
    "        print(quality)\n",
    "        \n",
    "        self.clean_prices = prices\n",
    "        self.clean_returns = returns_clean\n",
    "        self.quality_report = quality\n",
    "        \n",
    "        print(f\"\\n管道运行完成！\")\n",
    "        return returns_clean\n",
    "\n",
    "# 运行管道\n",
    "pipeline = DataPipeline(price_panel, max_missing_pct=0.15)\n",
    "clean_returns = pipeline.run()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:trading] *",
   "language": "python",
   "name": "conda-env-trading-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
