- 优化 `OcrBuilder` 重名名为 `OcrPredictor` - 优化 `OcrPredictor` 的 `preprocess_image` 支持多种图像管道。 - 修复 `OcrPredictor` 引发的并发BUG。
397 lines
16 KiB
Rust
397 lines
16 KiB
Rust
use crate::charset::{TokenFilter, ValidationCtx};
|
||
use crate::model_metadata::{ModelMetadata, Resize};
|
||
use crate::models::base::ModelArgs;
|
||
use crate::models::loader::{ModelLoader, ModelSession, ModelType};
|
||
use crate::utils::color_filter::{ColorFilter, HsvRange, filter_image};
|
||
use crate::utils::image_io::png_rgba_white_preprocess;
|
||
use crate::utils::image_processor::{convert_to_grayscale, resize_image};
|
||
use anyhow::Context;
|
||
use anyhow::{Result, anyhow};
|
||
use image::{DynamicImage, ImageBuffer, Rgb};
|
||
use std::borrow::Cow;
|
||
use std::collections::HashSet;
|
||
use tract_onnx::prelude::tract_ndarray::{s, ArrayView2};
|
||
use tract_onnx::prelude::{
|
||
DatumType, Graph, IntoTensor, RunnableModel, Tensor, TypedFact, TypedOp, tract_ndarray, tvec,
|
||
};
|
||
// 引入 cv_ops 模块中的 OpenCV HSV 转换算子
|
||
use crate::utils::cv_ops::rgb_to_opencv_hsv;
|
||
|
||
|
||
pub struct Ocr {
|
||
pub session: RunnableModel<TypedFact, Box<dyn TypedOp>, Graph<TypedFact, Box<dyn TypedOp>>>,
|
||
pub model_metadata: ModelMetadata,
|
||
}
|
||
impl ModelSession for Ocr {
|
||
fn get_model_type(&self) -> ModelType {
|
||
todo!()
|
||
}
|
||
fn desc(&self) -> String {
|
||
"Ocr Model 加载成功".to_string()
|
||
}
|
||
}
|
||
impl Ocr {
|
||
pub fn new(model_path: String, model_metadata: ModelMetadata) -> Result<Self, anyhow::Error> {
|
||
let session = ModelLoader::load_model(&model_path)?.session;
|
||
Ok(Self {
|
||
session,
|
||
model_metadata,
|
||
})
|
||
}
|
||
/// 对应 Python 的 _inference
|
||
fn inference(&self, tensor: Tensor) -> anyhow::Result<Tensor> {
|
||
// tract 的 run 会返回一个 Vec<TValue>,我们通常只需要第一个输出
|
||
// let result = self.session.run(tvec!(tensor.into()))?;
|
||
let mut result = self
|
||
.session
|
||
.run(tvec!(tensor.into()))
|
||
.context("执行模型推理失败")?;
|
||
println!("模型输出原始数据: {:?}", result);
|
||
Ok(result.swap_remove(0).into_tensor())
|
||
}
|
||
|
||
/// 核心解析逻辑:将模型输出的各种维度/类型的 Tensor 转为字符索引序列
|
||
fn extract_indices_from_tensor(&self, raw_tensor: &Tensor) -> anyhow::Result<Vec<i64>> {
|
||
let shape = raw_tensor.shape();
|
||
println!("模型输出shape数据: {:?}", shape);
|
||
let datum_type = raw_tensor.datum_type();
|
||
println!("模型输出datum_type数据: {:?}", datum_type);
|
||
|
||
match datum_type {
|
||
// 情况 1: huashi666 式模型,直接输出 i64 索引 (通常是模型内部做好了 Argmax)
|
||
DatumType::I64 => {
|
||
let view = raw_tensor.to_array_view::<i64>()?;
|
||
|
||
Ok(view.iter().cloned().collect())
|
||
}
|
||
|
||
// 情况 2: sml2h3 原版模型,输出 F32 概率矩阵
|
||
DatumType::F32 => {
|
||
let view = raw_tensor.to_array_view::<f32>()?;
|
||
let (steps, classes, data_view) = match shape.len() {
|
||
3 => {
|
||
if shape[1] == 1 {
|
||
// 形状: [Steps, 1, Classes] -> 你的原有逻辑
|
||
(shape[0], shape[2], view.into_dyn())
|
||
} else if shape[0] == 1 {
|
||
// 形状: [1, Steps, Classes] -> 另一种常见导出格式
|
||
(shape[1], shape[2], view.into_dyn())
|
||
} else {
|
||
// 默认取第一个 batch: [Batch, Steps, Classes]
|
||
// 使用 slice 对应 Python 的 output[0, :, :]
|
||
let sliced = view.slice(s![0, .., ..]);
|
||
(shape[1], shape[2], sliced.into_dyn())
|
||
}
|
||
}
|
||
2 => {
|
||
// 形状: [Steps, Classes] -> 已经剥离了 Batch 维度
|
||
(shape[0], shape[1], view.into_dyn())
|
||
}
|
||
// 形状: [Classes] -> 单字符输出(对应 Python 的 ndim == 0 保护逻辑)
|
||
// 我们把它虚构成一个 [1, Classes] 的 2D 矩阵来复用后面的 argmax 逻辑
|
||
1 => (1, shape[0], view.into_dyn()),
|
||
_ => return Err(anyhow::anyhow!("不支持的输出维度: {:?}", shape)),
|
||
};
|
||
let array_2d = data_view.to_shape((steps, classes))?;
|
||
//
|
||
// 对每一行执行 Argmax (寻找概率最大的字符索引)
|
||
let indices = array_2d
|
||
.outer_iter()
|
||
.map(|row| {
|
||
row.iter()
|
||
.enumerate()
|
||
.max_by(|(_, a), (_, b)| {
|
||
a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)
|
||
})
|
||
.map(|(idx, _)| idx as i64)
|
||
.unwrap_or(0)
|
||
})
|
||
.collect();
|
||
Ok(indices)
|
||
}
|
||
_ => Err(anyhow::anyhow!(
|
||
"不支持的模型输出数据类型: {:?}",
|
||
datum_type
|
||
)),
|
||
}
|
||
}
|
||
/// 管道 2:纯文本解码流水线 (高性能版:免去 Softmax 计算)
|
||
fn process_text_pipeline(&self, matrix_view: ArrayView2<f32>) -> anyhow::Result<String> {
|
||
// 直接在原始分值(Logits)上进行 Argmax,数学结果与 Softmax 后完全一致
|
||
let indices: Vec<i64> = matrix_view
|
||
.outer_iter()
|
||
.map(|row| {
|
||
row.iter()
|
||
.enumerate()
|
||
.max_by(|(_, a), (_, b)| a.total_cmp(b))
|
||
.map(|(idx, _)| idx as i64)
|
||
.unwrap_or(0)
|
||
})
|
||
.collect();
|
||
|
||
// 丢给现有的 CTC 解码器去重并映射成字符串
|
||
Ok(self.ctc_decode_to_string(&indices))
|
||
}
|
||
|
||
pub fn predictor(&'_ self) -> OcrPredictor<'_> {
|
||
OcrPredictor::new(self)
|
||
}
|
||
}
|
||
|
||
pub struct OcrPredictor<'a> {
|
||
ocr: &'a Ocr,
|
||
// image: &'a DynamicImage,
|
||
/// 是否修复PNG格式问题
|
||
png_fix: bool,
|
||
/// 是否返回概率信息
|
||
#[allow(dead_code)]
|
||
probability: bool,
|
||
/// 颜色过滤:保留的颜色列表
|
||
color_filter: Result<Option<Vec<HsvRange>>, String>,
|
||
|
||
/// 字符集范围
|
||
charset_restrict: Option<Vec<usize>>,
|
||
}
|
||
|
||
impl<'a> OcrPredictor<'a> {
|
||
// 初始化任务,设置默认参数
|
||
pub fn new(ocr: &'a Ocr) -> Self {
|
||
Self {
|
||
ocr,
|
||
// image,
|
||
png_fix: false, // 默认值
|
||
probability: false,
|
||
color_filter: Ok(None),
|
||
charset_restrict: None,
|
||
}
|
||
}
|
||
pub fn png_fix(mut self, value: bool) -> Self {
|
||
self.png_fix = value;
|
||
self
|
||
}
|
||
|
||
// 反复调用color_filter怎么处理?
|
||
pub fn color_filter(mut self, filter: &dyn ColorFilter) -> Self {
|
||
// self.color_filter = Some(value);
|
||
|
||
// 一句话把活全包了!错误信息无缝传递,完美熔断
|
||
match filter.collect_to_vec() {
|
||
Ok(new_ranges) => self.color_filter = Ok(new_ranges),
|
||
Err(err_msg) => self.color_filter = Err(err_msg), // 校验失败,Builder 正式中毒
|
||
}
|
||
|
||
self
|
||
}
|
||
|
||
pub fn charset_restrict(mut self, restrict: &dyn TokenFilter) -> Self {
|
||
let charset = &self.ocr.model_metadata.charset;
|
||
let tokens = &charset.tokens;
|
||
// let mut temp_indices = Vec::new();
|
||
self.charset_restrict = restrict.apply_to_charset(tokens);
|
||
self
|
||
}
|
||
}
|
||
impl<'a> OcrPredictor<'a> {
|
||
pub fn predict(self, image: &DynamicImage) -> anyhow::Result<String> {
|
||
println!("当前颜色过滤器状态: {:?}", self.color_filter);
|
||
// =====================================================================
|
||
// 管道节点 1: 颜色过滤流水线
|
||
// 使用 Cow (Copy-On-Write) 智能指针。
|
||
// 如果未开启过滤,img_cow 内部只是持有原图的【只读借用】,发生【零内存分配】!
|
||
// =====================================================================
|
||
let img_cow = match &self.color_filter {
|
||
Err(err_msg) => {
|
||
return Err(anyhow::anyhow!(
|
||
"颜色过滤器初始化失败,全链路短路: {}",
|
||
err_msg
|
||
));
|
||
}
|
||
Ok(None) => {
|
||
// 核心优化点:直接借用原图,不发生任何克隆
|
||
Cow::Borrowed(image)
|
||
}
|
||
Ok(Some(ranges)) => {
|
||
// 只有真正需要过滤时,才在内部提取像素并生成清洗后的 Owned 新图
|
||
let filtered_img = filter_image(image, ranges)?;
|
||
Cow::Owned(filtered_img)
|
||
}
|
||
};
|
||
let tensor = self.preprocess_image(&img_cow)?;
|
||
|
||
let raw_tensor = self.ocr.inference(tensor)?;
|
||
let raw_indices = self.ocr.extract_indices_from_tensor(&raw_tensor)?;
|
||
// 步骤 2: 将索引切片 `&[i64]` 传给解码器进行 CTC 去重和字符映射
|
||
let final_text = self.ctc_decode_to_string(&raw_indices);
|
||
|
||
println!("最终识别出的验证码是: {}", final_text);
|
||
Ok(final_text)
|
||
}
|
||
/// 对应 Python 的 _preprocess_image
|
||
/// 负责:透明背景修复 -> 灰度化 -> 按比例 Resize -> 归一化 -> 4维张量转换
|
||
fn preprocess_image(&self, img: &DynamicImage) -> anyhow::Result<Tensor> {
|
||
// 1. 获取模型元数据配置
|
||
let meta = &self.ocr.model_metadata;
|
||
|
||
// A. 修复 PNG 透明背景 (内部逻辑你之前已实现)
|
||
let current_img = if self.png_fix && img.color().has_alpha() {
|
||
// 只有满足条件才去触发分配,生成新图
|
||
Cow::Owned(png_rgba_white_preprocess(img))
|
||
} else {
|
||
// 正常情况下,仅仅是再次安全借用,无开销
|
||
Cow::Borrowed(img)
|
||
};
|
||
|
||
// 3. 管道节点 2: 根据 Resize 策略计算目标宽高并进行缩放
|
||
let (target_w, target_h) = match meta.resize {
|
||
Resize::Fixed(w, h) => (w, h),
|
||
Resize::DynamicWidth(h) => {
|
||
// 高度固定,宽度根据原始比例动态计算:W_target = W_orig * (H_target / H_orig)
|
||
let w = (current_img.width() as f32 * (h as f32 / current_img.height() as f32)) as u32;
|
||
(w, h)
|
||
}
|
||
Resize::Square(size) => {
|
||
// 单字识别模型,直接缩放为正方形
|
||
(size, size)
|
||
}
|
||
};
|
||
// 执行缩放
|
||
let resized_img = resize_image(¤t_img, target_w, target_h);
|
||
|
||
// 4. 管道节点 3: 颜色通道转换(单通道灰度 vs 三通道 RGB)与 4D 张量填充
|
||
let tensor = match meta.channel {
|
||
// --- 情况 A: 单通道(灰度图),对应 Python 的 len(shape) == 2 展开 ---
|
||
1 => {
|
||
let gray_img = convert_to_grayscale(&resized_img);
|
||
|
||
let array = tract_ndarray::Array4::from_shape_fn(
|
||
(1, 1, target_h as usize, target_w as usize),
|
||
|(_, _, y, x)| {
|
||
let pixel = gray_img.get_pixel(x as u32, y as u32)[0] as f32;
|
||
pixel / 255.0 // 严格对齐 Python 归一化 [0.0, 1.0]
|
||
},
|
||
);
|
||
Tensor::from(array)
|
||
}
|
||
|
||
// --- 情况 B: 三通道(RGB),对应 Python 的 transpose(2, 0, 1) 的 CHW 布局 ---
|
||
3 => {
|
||
let rgb_img = resized_img.to_rgb8();
|
||
|
||
let array = tract_ndarray::Array4::from_shape_fn(
|
||
(1, 3, target_h as usize, target_w as usize),
|
||
|(_, c, y, x)| {
|
||
let pixel = rgb_img.get_pixel(x as u32, y as u32)[c] as f32;
|
||
pixel / 255.0 // 严格对齐 Python 归一化 [0.0, 1.0]
|
||
},
|
||
);
|
||
Tensor::from(array)
|
||
}
|
||
|
||
_ => return Err(anyhow::anyhow!("不支持的通道数配置: {}", meta.channel)),
|
||
};
|
||
|
||
Ok(tensor)
|
||
|
||
|
||
|
||
// let h = 64u32;
|
||
// let w = (current_img.width() as f32 * (h as f32 / current_img.height() as f32)) as u32;
|
||
// let gray_img = convert_to_grayscale(¤t_img);
|
||
// let resized = resize_image(&gray_img, w, h);
|
||
// // resized.save("debug_preprocessed.png").unwrap();
|
||
// // 1. 预处理:转灰度 -> Resize -> 归一化
|
||
// // let resized = img.resize_exact(w, h, FilterType::Lanczos3).to_luma8();
|
||
//
|
||
// // 使用 tract_ndarray 构造,避免版本冲突
|
||
// let array =
|
||
// tract_ndarray::Array4::from_shape_fn((1, 1, h as usize, w as usize), |(_, _, y, x)| {
|
||
// let pixel = resized.get_pixel(x as u32, y as u32)[0] as f32;
|
||
// (pixel / 255.0 - 0.5) / 0.5
|
||
// });
|
||
//
|
||
// let tensor = Tensor::from(array);
|
||
//
|
||
// Ok(tensor)
|
||
}
|
||
}
|
||
impl<'a> OcrPredictor<'a> {
|
||
// pub fn get_valid_indices(&self) -> HashSet<usize> {
|
||
// match &self.charset_restrict {
|
||
// Some(indices) => indices.iter().cloned().collect(),
|
||
// // 如果是 None,现场映射出全量索引集给外部
|
||
// None => (0..self.ocr.model_metadata.charset.tokens.len()).collect(),
|
||
// }
|
||
// }
|
||
// compute_valid_indices
|
||
// fn valid_indices(&self) -> (bool, HashSet<usize>) {
|
||
// let charset = &self.ocr.model_metadata.charset;
|
||
|
||
/// 【按需延迟打印】:当用户真的需要“知道当前有哪些限制字符”时,一秒反查并打印
|
||
/// 这里的 &str 完美借用了自 tokens,依然是彻底的零拷贝!
|
||
pub fn valid_tokens(&self) -> Vec<&str> {
|
||
let charset = &self.ocr.model_metadata.charset;
|
||
let tokens = &charset.tokens;
|
||
match &self.charset_restrict {
|
||
Some(indices) => indices
|
||
.iter()
|
||
.filter_map(|&idx| tokens.get(idx).map(|cow| cow.as_ref()))
|
||
.collect(),
|
||
// 如果是 None,现场映射出全量 Token 视图给外部
|
||
None => tokens.iter().map(|cow| cow.as_ref()).collect(),
|
||
}
|
||
}
|
||
pub fn valid_size(&self) -> usize {
|
||
match &self.charset_restrict {
|
||
Some(indices) => indices.len(),
|
||
None => self.ocr.model_metadata.charset.tokens.len(),
|
||
}
|
||
}
|
||
|
||
/// 获取有效字符索引列表 (用于外部验证或过滤)
|
||
fn ctc_decode_to_string(&self, predicted_indices: &[i64]) -> String {
|
||
println!("indices模型输出原始数据: {:?}", predicted_indices);
|
||
let charset = &self.ocr.model_metadata.charset;
|
||
let tokens = &charset.tokens;
|
||
// let valid_indices = &charset.valid_indices;
|
||
|
||
// 对应 _ctc_decode_indices 的逻辑:去重、去 blank (0)
|
||
let mut res = String::new();
|
||
let mut prev_idx: i64 = -1;
|
||
|
||
for &idx in predicted_indices {
|
||
// 1. CTC 去重:如果是连续重复的,直接跳过
|
||
if idx == prev_idx {
|
||
continue;
|
||
}
|
||
// 【关键核心】只要不是连续重复,立刻更新 prev_idx 状态,绝对不能被后续的过滤短路!
|
||
prev_idx = idx;
|
||
|
||
// 2. CTC 过滤 Blank (0)
|
||
if idx == 0 {
|
||
continue;
|
||
}
|
||
// 3. 类型安全转换
|
||
let u_idx = match usize::try_from(idx) {
|
||
Ok(u) => u,
|
||
Err(_) => continue,
|
||
};
|
||
|
||
// 史诗级加速点:如果是 None,说明没限制,根本不进入分支,直接放行!
|
||
// 只有当有具体限制(Some)时,才去跑 4-5 次 CPU 寄存器级别的二分查找
|
||
if let Some(ref indices) = self.charset_restrict {
|
||
if indices.binary_search(&u_idx).is_err() {
|
||
continue;
|
||
}
|
||
}
|
||
|
||
// 5. 字符映射
|
||
if let Some(char_str) = tokens.get(u_idx) {
|
||
res.push_str(char_str);
|
||
} else {
|
||
eprintln!("警告: 预测索引 {} 超出字符集范围", u_idx);
|
||
}
|
||
}
|
||
res
|
||
}
|
||
}
|