# 自然写教学数据分析与学情诊断系统软件 V1.0
# etl/flink_processor.py - Flink ETL实时数据处理管道

import logging
import json
import hashlib
from typing import Any, Dict, List, Optional, Tuple
from datetime import datetime, timedelta
from dataclasses import dataclass, field, asdict
from enum import Enum

logger = logging.getLogger("writech.analytics.etl")


# ============================================================
#  ETL数据模型
# ============================================================

class EventType(str, Enum):
    """数据事件类型"""
    STROKE_RAW = "stroke_raw"           # 原始笔迹数据
    GRADE_RESULT = "grade_result"       # 批改结果
    HOMEWORK_SUBMIT = "homework_submit" # 作业提交
    OCR_RESULT = "ocr_result"           # OCR识别结果
    STROKE_ORDER = "stroke_order"       # 笔顺评分结果
    WRITING_QUALITY = "writing_quality" # 书写质量评分
    EXAM_SCORE = "exam_score"           # 考试成绩
    LOGIN_EVENT = "login_event"         # 登录事件


@dataclass
class RawEvent:
    """原始事件数据"""
    event_id: str
    event_type: EventType
    student_id: str
    class_id: str
    school_id: str
    timestamp: str
    payload: Dict[str, Any]
    source: str = ""  # 事件来源（pad/mobile/pc/board）


@dataclass
class AggregatedMetric:
    """聚合指标数据（写入ClickHouse）"""
    metric_id: str
    student_id: str
    class_id: str
    school_id: str
    subject: str
    metric_type: str  # 指标类型
    metric_value: float
    dimension: str = ""  # 维度（如knowledge_id）
    period: str = "daily"  # 聚合周期
    period_start: str = ""
    period_end: str = ""
    created_at: str = ""


@dataclass
class StudentDailyStats:
    """学生每日统计汇总"""
    student_id: str
    date: str
    subject: str
    # 作业维度
    homework_count: int = 0
    homework_completed: int = 0
    homework_avg_score: float = 0.0
    # 练习维度
    practice_count: int = 0
    practice_total_chars: int = 0
    practice_avg_score: float = 0.0
    # 书写维度
    writing_quality_avg: float = 0.0
    stroke_order_accuracy: float = 0.0
    writing_speed_avg: float = 0.0
    # 错题维度
    error_count: int = 0
    error_knowledge_points: List[str] = field(default_factory=list)
    # 时间维度
    study_duration_minutes: int = 0


# ============================================================
#  Flink ETL处理管道
# ============================================================

class FlinkETLProcessor:
    """
    Flink实时ETL处理器
    
    数据流:
      原始笔迹/批改数据 → Kafka → Flink实时计算 → 
      聚合指标写入ClickHouse → 定时生成诊断报告
    
    处理阶段:
      1. 数据采集（Kafka Source）
      2. 数据清洗与标准化
      3. 实时聚合计算
      4. 窗口统计
      5. 写入ClickHouse（Sink）
    """

    def __init__(self, config: Dict[str, Any]):
        """初始化ETL处理器"""
        self.kafka_brokers = config.get("kafka_brokers", "localhost:9092")
        self.kafka_topics = config.get("kafka_topics", [])
        self.clickhouse_config = config.get("clickhouse", {})
        self.batch_size = config.get("batch_size", 100)
        self.window_size_seconds = config.get("window_size", 60)

        # 内存中的聚合缓冲区
        self._daily_stats_buffer: Dict[str, StudentDailyStats] = {}
        self._metric_buffer: List[AggregatedMetric] = []
        self._error_records_buffer: List[Dict[str, Any]] = []

        # 数据质量统计
        self._processed_count = 0
        self._error_count = 0
        self._dropped_count = 0

        logger.info(
            "FlinkETL初始化: brokers=%s, topics=%s, batch=%d",
            self.kafka_brokers,
            self.kafka_topics,
            self.batch_size,
        )

    def start_pipeline(self) -> None:
        """启动ETL处理管道"""
        logger.info("启动Flink ETL处理管道...")

        # 配置Flink执行环境
        # env = StreamExecutionEnvironment.get_execution_environment()
        # env.set_parallelism(4)
        # env.enable_checkpointing(60000)  # 60秒checkpoint

        # 定义Kafka数据源
        # kafka_source = KafkaSource.builder() \
        #     .set_bootstrap_servers(self.kafka_brokers) \
        #     .set_topics(self.kafka_topics) \
        #     .set_group_id("analytics-etl") \
        #     .set_starting_offsets(KafkaOffsetsInitializer.latest()) \
        #     .set_value_only_deserializer(SimpleStringSchema()) \
        #     .build()

        # 创建数据流
        # stream = env.from_source(kafka_source, ...)

        # 数据处理链
        # stream \
        #     .map(self._parse_event) \
        #     .filter(self._validate_event) \
        #     .key_by(lambda e: e.student_id) \
        #     .window(TumblingEventTimeWindows.of(Time.minutes(1))) \
        #     .process(self._aggregate_window) \
        #     .add_sink(clickhouse_sink)

        # env.execute("WritechAnalyticsETL")

        logger.info("ETL管道已启动")

    def _parse_event(self, raw_json: str) -> Optional[RawEvent]:
        """
        解析原始JSON消息为RawEvent对象
        
        数据清洗规则:
        - 必须包含event_type, student_id, timestamp字段
        - timestamp格式校验（ISO 8601）
        - 过滤空payload
        """
        try:
            data = json.loads(raw_json)

            # 字段完整性校验
            required_fields = ["event_type", "student_id", "timestamp"]
            for field_name in required_fields:
                if field_name not in data or not data[field_name]:
                    self._dropped_count += 1
                    logger.debug("丢弃不完整事件: 缺少%s", field_name)
                    return None

            # 事件类型校验
            try:
                event_type = EventType(data["event_type"])
            except ValueError:
                self._dropped_count += 1
                logger.debug("丢弃未知事件类型: %s", data["event_type"])
                return None

            # 时间戳校验
            try:
                datetime.fromisoformat(
                    data["timestamp"].replace("Z", "+00:00")
                )
            except (ValueError, AttributeError):
                self._dropped_count += 1
                return None

            # 生成唯一事件ID（去重用）
            event_id = hashlib.md5(
                f"{data['student_id']}_{data['timestamp']}_{raw_json[:50]}"
                .encode()
            ).hexdigest()

            event = RawEvent(
                event_id=event_id,
                event_type=event_type,
                student_id=data["student_id"],
                class_id=data.get("class_id", ""),
                school_id=data.get("school_id", ""),
                timestamp=data["timestamp"],
                payload=data.get("payload", {}),
                source=data.get("source", ""),
            )

            self._processed_count += 1
            return event

        except json.JSONDecodeError as e:
            self._error_count += 1
            logger.warning("JSON解析失败: %s", str(e))
            return None
        except Exception as e:
            self._error_count += 1
            logger.error("事件解析异常: %s", str(e))
            return None

    def _validate_event(self, event: Optional[RawEvent]) -> bool:
        """事件有效性过滤"""
        if event is None:
            return False

        # 过滤过旧的数据（超过7天不处理）
        try:
            event_time = datetime.fromisoformat(
                event.timestamp.replace("Z", "+00:00")
            )
            if datetime.now(event_time.tzinfo) - event_time > timedelta(days=7):
                self._dropped_count += 1
                return False
        except Exception:
            return False

        return True

    def process_event(self, event: RawEvent) -> None:
        """
        根据事件类型分发处理
        
        不同事件类型有不同的聚合逻辑:
        - stroke_raw: 累计书写笔迹量
        - grade_result: 更新作业得分统计
        - stroke_order: 更新笔顺准确率
        - writing_quality: 更新书写质量评分
        """
        handler_map = {
            EventType.STROKE_RAW: self._process_stroke,
            EventType.GRADE_RESULT: self._process_grade,
            EventType.HOMEWORK_SUBMIT: self._process_homework,
            EventType.OCR_RESULT: self._process_ocr,
            EventType.STROKE_ORDER: self._process_stroke_order,
            EventType.WRITING_QUALITY: self._process_writing_quality,
            EventType.EXAM_SCORE: self._process_exam_score,
        }

        handler = handler_map.get(event.event_type)
        if handler:
            handler(event)
        else:
            logger.debug("未处理的事件类型: %s", event.event_type)

    def _get_daily_stats(
        self, student_id: str, date_str: str, subject: str
    ) -> StudentDailyStats:
        """获取或创建学生每日统计缓冲"""
        key = f"{student_id}_{date_str}_{subject}"
        if key not in self._daily_stats_buffer:
            self._daily_stats_buffer[key] = StudentDailyStats(
                student_id=student_id,
                date=date_str,
                subject=subject,
            )
        return self._daily_stats_buffer[key]

    def _process_stroke(self, event: RawEvent) -> None:
        """处理原始笔迹数据事件"""
        payload = event.payload
        stroke_count = payload.get("stroke_count", 0)
        page_id = payload.get("page_id", "")

        # 累计笔迹量到每日统计
        date_str = event.timestamp[:10]
        subject = payload.get("subject", "unknown")
        stats = self._get_daily_stats(event.student_id, date_str, subject)
        stats.practice_total_chars += stroke_count

        # 生成笔迹量聚合指标
        metric = AggregatedMetric(
            metric_id=event.event_id,
            student_id=event.student_id,
            class_id=event.class_id,
            school_id=event.school_id,
            subject=subject,
            metric_type="stroke_count",
            metric_value=float(stroke_count),
            dimension=page_id,
            period_start=date_str,
            created_at=event.timestamp,
        )
        self._metric_buffer.append(metric)

    def _process_grade(self, event: RawEvent) -> None:
        """处理批改结果事件"""
        payload = event.payload
        score = payload.get("score", 0)
        total_score = payload.get("total_score", 100)
        subject = payload.get("subject", "unknown")
        homework_id = payload.get("homework_id", "")

        date_str = event.timestamp[:10]
        stats = self._get_daily_stats(event.student_id, date_str, subject)
        stats.homework_count += 1
        stats.homework_completed += 1

        # 增量更新平均分
        n = stats.homework_completed
        stats.homework_avg_score = (
            stats.homework_avg_score * (n - 1) + score
        ) / n

        # 处理错题记录
        errors = payload.get("errors", [])
        for error in errors:
            knowledge_point = error.get("knowledge_point", "")
            if knowledge_point:
                stats.error_count += 1
                if knowledge_point not in stats.error_knowledge_points:
                    stats.error_knowledge_points.append(knowledge_point)

                # 错题写入MySQL
                self._error_records_buffer.append({
                    "student_id": event.student_id,
                    "homework_id": homework_id,
                    "question_id": error.get("question_id", ""),
                    "subject": subject,
                    "knowledge_point": knowledge_point,
                    "error_type": error.get("error_type", ""),
                    "created_at": event.timestamp,
                })

    def _process_homework(self, event: RawEvent) -> None:
        """处理作业提交事件"""
        payload = event.payload
        subject = payload.get("subject", "unknown")
        time_cost = payload.get("time_cost_minutes", 0)

        date_str = event.timestamp[:10]
        stats = self._get_daily_stats(event.student_id, date_str, subject)
        stats.study_duration_minutes += time_cost

    def _process_ocr(self, event: RawEvent) -> None:
        """处理OCR识别结果事件"""
        payload = event.payload
        confidence = payload.get("confidence", 0.0)
        char_count = payload.get("char_count", 0)

        # OCR识别结果用于辅助计算书写清晰度指标
        metric = AggregatedMetric(
            metric_id=event.event_id,
            student_id=event.student_id,
            class_id=event.class_id,
            school_id=event.school_id,
            subject="chinese",
            metric_type="ocr_confidence",
            metric_value=confidence,
            created_at=event.timestamp,
        )
        self._metric_buffer.append(metric)

    def _process_stroke_order(self, event: RawEvent) -> None:
        """处理笔顺评分结果事件"""
        payload = event.payload
        score = payload.get("score", 0.0)
        character = payload.get("character", "")

        date_str = event.timestamp[:10]
        stats = self._get_daily_stats(event.student_id, date_str, "chinese")

        # 增量更新笔顺准确率
        stats.practice_count += 1
        n = stats.practice_count
        stats.stroke_order_accuracy = (
            stats.stroke_order_accuracy * (n - 1) + score
        ) / n

    def _process_writing_quality(self, event: RawEvent) -> None:
        """处理书写质量评分事件"""
        payload = event.payload
        quality_score = payload.get("quality_score", 0.0)
        speed = payload.get("speed", 0.0)

        date_str = event.timestamp[:10]
        stats = self._get_daily_stats(event.student_id, date_str, "chinese")

        # 更新书写质量指标
        count = max(stats.practice_count, 1)
        stats.writing_quality_avg = (
            stats.writing_quality_avg * (count - 1) + quality_score
        ) / count
        stats.writing_speed_avg = (
            stats.writing_speed_avg * (count - 1) + speed
        ) / count

    def _process_exam_score(self, event: RawEvent) -> None:
        """处理考试成绩事件"""
        payload = event.payload
        subject = payload.get("subject", "unknown")
        score = payload.get("score", 0)
        total = payload.get("total_score", 100)

        metric = AggregatedMetric(
            metric_id=event.event_id,
            student_id=event.student_id,
            class_id=event.class_id,
            school_id=event.school_id,
            subject=subject,
            metric_type="exam_score",
            metric_value=float(score),
            dimension=payload.get("exam_id", ""),
            created_at=event.timestamp,
        )
        self._metric_buffer.append(metric)

    def flush_to_clickhouse(self) -> int:
        """
        将缓冲区的聚合指标批量写入ClickHouse
        
        使用ClickHouse的INSERT批量写入提高性能。
        写入后清空缓冲区。
        返回写入的记录数。
        """
        if not self._metric_buffer and not self._daily_stats_buffer:
            return 0

        total_written = 0

        # 写入聚合指标
        if self._metric_buffer:
            metrics = [asdict(m) for m in self._metric_buffer]
            # clickhouse_client.execute(
            #     "INSERT INTO analytics_metrics VALUES",
            #     metrics,
            # )
            total_written += len(metrics)
            logger.info("写入%d条聚合指标到ClickHouse", len(metrics))
            self._metric_buffer.clear()

        # 写入每日统计
        if self._daily_stats_buffer:
            daily_stats = [
                asdict(s) for s in self._daily_stats_buffer.values()
            ]
            # clickhouse_client.execute(
            #     "INSERT INTO student_daily_stats VALUES",
            #     daily_stats,
            # )
            total_written += len(daily_stats)
            logger.info("写入%d条每日统计到ClickHouse", len(daily_stats))
            self._daily_stats_buffer.clear()

        # 写入错题记录到MySQL
        if self._error_records_buffer:
            # mysql_batch_insert("error_record", self._error_records_buffer)
            total_written += len(self._error_records_buffer)
            logger.info(
                "写入%d条错题记录到MySQL", len(self._error_records_buffer)
            )
            self._error_records_buffer.clear()

        return total_written

    def get_pipeline_stats(self) -> Dict[str, int]:
        """获取管道处理统计"""
        return {
            "processed": self._processed_count,
            "errors": self._error_count,
            "dropped": self._dropped_count,
            "buffer_metrics": len(self._metric_buffer),
            "buffer_daily": len(self._daily_stats_buffer),
            "buffer_errors": len(self._error_records_buffer),
        }

    def stop_pipeline(self) -> None:
        """停止ETL管道，刷新所有缓冲区"""
        logger.info("正在停止ETL管道...")
        self.flush_to_clickhouse()
        logger.info(
            "ETL管道已停止. 统计: %s", self.get_pipeline_stats()
        )
