feat: 优化 项目目录结构
This commit is contained in:
40
src/models/base.rs
Normal file
40
src/models/base.rs
Normal file
@@ -0,0 +1,40 @@
|
||||
pub trait ModelArgs {
|
||||
// 获取模型路径
|
||||
fn model_path(&self) -> &str;
|
||||
|
||||
// 获取字符集(由于 Det 没有,所以返回 Option)
|
||||
fn charset(&self) -> Option<&str>;
|
||||
}
|
||||
|
||||
pub struct HasCharset {
|
||||
pub charset: String,
|
||||
} // 给 Ocr 和 Custom 用
|
||||
pub struct NoCharset; // 给 Det 用
|
||||
|
||||
pub struct Model<T> {
|
||||
pub path: String,
|
||||
pub metadata: T,
|
||||
}
|
||||
// 针对有字符集的模型 (Ocr / Custom)
|
||||
impl ModelArgs for Model<HasCharset> {
|
||||
fn model_path(&self) -> &str {
|
||||
&self.path
|
||||
}
|
||||
fn charset(&self) -> Option<&str> {
|
||||
Some(&self.metadata.charset)
|
||||
}
|
||||
}
|
||||
|
||||
// 针对没有字符集的模型 (Det)
|
||||
impl ModelArgs for Model<NoCharset> {
|
||||
fn model_path(&self) -> &str {
|
||||
&self.path
|
||||
}
|
||||
fn charset(&self) -> Option<&str> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
pub type OcrModel = Model<HasCharset>;
|
||||
pub type DetModel = Model<NoCharset>;
|
||||
pub type CustomModel = Model<HasCharset>; // Ocr 和 Custom 逻辑一致,可以复用
|
||||
263
src/models/det.rs
Normal file
263
src/models/det.rs
Normal file
@@ -0,0 +1,263 @@
|
||||
use crate::models::loader::{ModelLoader, ModelSession, ModelType};
|
||||
use anyhow::{Context, Result};
|
||||
use image::{DynamicImage, GenericImageView, imageops::FilterType};
|
||||
use tract_onnx::prelude::tract_ndarray::{Array2, Array3, Array4, Axis, prelude::*, s};
|
||||
use tract_onnx::prelude::{Graph, RunnableModel, Tensor, TypedFact, TypedOp, tvec};
|
||||
|
||||
pub struct Det {
|
||||
session: RunnableModel<TypedFact, Box<dyn TypedOp>, Graph<TypedFact, Box<dyn TypedOp>>>,
|
||||
}
|
||||
impl ModelSession for Det {
|
||||
fn get_model_type(&self) -> ModelType {
|
||||
todo!()
|
||||
}
|
||||
fn desc(&self) -> String {
|
||||
"Detection Model 加载成功".to_string()
|
||||
}
|
||||
}
|
||||
impl Det {
|
||||
pub fn new(model_path: String) -> Result<Self, anyhow::Error> {
|
||||
let session = ModelLoader::load_model(&model_path)?.session;
|
||||
Ok(Self { session })
|
||||
}
|
||||
pub fn predict(&self, image_bytes: &[u8]) -> Result<Vec<Vec<i32>>> {
|
||||
// Rust 中通常在调用层处理文件/PIL转换,这里直接进入核心逻辑
|
||||
self.get_bbox(image_bytes)
|
||||
}
|
||||
/// 2. preproc: 纯 Rust 实现 (替代 OpenCV)
|
||||
fn preproc(&self, img: &DynamicImage, input_size: (u32, u32)) -> Result<(Tensor, f32)> {
|
||||
let (target_h, target_w) = input_size;
|
||||
let (img_w, img_h) = img.dimensions();
|
||||
|
||||
// 计算缩放比例 (Letterbox)
|
||||
let r = (target_h as f32 / img_h as f32).min(target_w as f32 / img_w as f32);
|
||||
let new_h = (img_h as f32 * r) as u32;
|
||||
let new_w = (img_w as f32 * r) as u32;
|
||||
|
||||
// Resize 图像
|
||||
let resized = img.resize_exact(new_w, new_h, FilterType::Triangle);
|
||||
// 2. 关键:将 DynamicImage 显式转换为 RgbImage (Rgb<u8>)
|
||||
let resized_rgb = resized.to_rgb8();
|
||||
// 创建 114 灰度填充的背景
|
||||
let mut base_img =
|
||||
image::ImageBuffer::from_pixel(target_w, target_h, image::Rgb([114u8, 114, 114]));
|
||||
|
||||
// 将 resize 后的图像覆盖到左上角 (类似于原始代码中的 padded_img[:h, :w])
|
||||
image::imageops::overlay(&mut base_img, &resized_rgb, 0, 0);
|
||||
|
||||
// 构造 NCHW Tensor
|
||||
let mut array = Array4::<f32>::zeros((1, 3, target_h as usize, target_w as usize));
|
||||
for (x, y, pixel) in base_img.enumerate_pixels() {
|
||||
let x = x as usize;
|
||||
let y = y as usize;
|
||||
// 核心对标 Python 的 BGR 逻辑:
|
||||
// pixel[0] 是 R, pixel[1] 是 G, pixel[2] 是 B
|
||||
// 如果模型需要 BGR:
|
||||
// array[[0, 0, y as usize, x as usize]] = pixel[0] as f32;
|
||||
// array[[0, 1, y as usize, x as usize]] = pixel[1] as f32;
|
||||
// array[[0, 2, y as usize, x as usize]] = pixel[2] as f32;
|
||||
array[[0, 0, y, x]] = pixel[2] as f32; // B
|
||||
array[[0, 1, y, x]] = pixel[1] as f32; // G
|
||||
array[[0, 2, y, x]] = pixel[0] as f32; // R
|
||||
}
|
||||
|
||||
Ok((array.into(), r))
|
||||
}
|
||||
|
||||
/// 3. demo_postprocess (逻辑与 Python 一致)
|
||||
fn demo_postprocess(&self, mut outputs: Array3<f32>, img_size: (i32, i32)) -> Array3<f32> {
|
||||
let strides = [8, 16, 32];
|
||||
|
||||
// 遍历每一个 Batch(支持动态 Batch 推理)
|
||||
for mut batch in outputs.axis_iter_mut(Axis(0)) {
|
||||
let mut offset = 0;
|
||||
|
||||
for &stride in &strides {
|
||||
// 计算当前特征图的尺寸
|
||||
let h = img_size.0 / stride;
|
||||
let w = img_size.1 / stride;
|
||||
let f_stride = stride as f32;
|
||||
|
||||
for y in 0..h {
|
||||
for x in 0..w {
|
||||
// 计算当前格子在 25200 个锚点中的线性索引
|
||||
let idx = offset + (y * w + x) as usize;
|
||||
// 1. 还原中心点坐标 (cx, cy)
|
||||
// 公式: (output + grid_offset) * stride
|
||||
batch[[idx, 0]] = (batch[[idx, 0]] + x as f32) * f_stride;
|
||||
batch[[idx, 1]] = (batch[[idx, 1]] + y as f32) * f_stride;
|
||||
|
||||
// 2. 还原宽高 (w, h)
|
||||
// 公式: exp(output) * stride
|
||||
batch[[idx, 2]] = batch[[idx, 2]].exp() * f_stride;
|
||||
batch[[idx, 3]] = batch[[idx, 3]].exp() * f_stride;
|
||||
}
|
||||
}
|
||||
// 移动到下一个步长的起始位置
|
||||
offset += (h * w) as usize;
|
||||
}
|
||||
}
|
||||
outputs
|
||||
}
|
||||
|
||||
/// 4. nms
|
||||
fn nms(&self, boxes: &Array2<f32>, scores: &Array1<f32>, nms_thr: f32) -> Vec<usize> {
|
||||
let mut keep = Vec::new();
|
||||
let x1 = boxes.column(0);
|
||||
let y1 = boxes.column(1);
|
||||
let x2 = boxes.column(2);
|
||||
let y2 = boxes.column(3);
|
||||
// 在每一项前加上 &,并确保括号内的计算顺序
|
||||
// 注意:ndarray 的 View 运算需要 &view1 - &view2
|
||||
let areas = (&x2 - &x1 + 1.0) * (&y2 - &y1 + 1.0);
|
||||
|
||||
// 初始排序索引
|
||||
let mut v: Vec<usize> = (0..scores.len()).collect();
|
||||
v.sort_unstable_by(|&i, &j| {
|
||||
scores[j]
|
||||
.partial_cmp(&scores[i])
|
||||
.unwrap_or(std::cmp::Ordering::Equal)
|
||||
});
|
||||
// 我们不使用 v.remove(0),而是直接通过索引池操作
|
||||
let mut active_indices = v;
|
||||
|
||||
while !active_indices.is_empty() {
|
||||
// 取出当前池子中得分最高的框(即第一个元素)
|
||||
let i = active_indices[0];
|
||||
keep.push(i);
|
||||
|
||||
// 如果池子里只剩一个了,直接结束
|
||||
if active_indices.len() == 1 {
|
||||
break;
|
||||
}
|
||||
|
||||
// 5. 核心逻辑:使用 retain 一次性过滤掉:
|
||||
// (a) 当前框自己 (idx == i)
|
||||
// (b) 与当前框重叠度过高的框 (iou > nms_thr)
|
||||
active_indices.retain(|&idx| {
|
||||
// 如果是当前正在处理的框,不保留(因为它已经进入 keep 了)
|
||||
if idx == i {
|
||||
return false;
|
||||
}
|
||||
|
||||
// 计算 IoU
|
||||
let xx1 = x1[i].max(x1[idx]);
|
||||
let yy1 = y1[i].max(y1[idx]);
|
||||
let xx2 = x2[i].min(x2[idx]);
|
||||
let yy2 = y2[i].min(y2[idx]);
|
||||
|
||||
let w = (xx2 - xx1 + 1.0).max(0.0);
|
||||
let h = (yy2 - yy1 + 1.0).max(0.0);
|
||||
let inter = w * h;
|
||||
|
||||
let iou = inter / (areas[i] + areas[idx] - inter);
|
||||
|
||||
// 只保留 IoU 小于阈值的框
|
||||
iou <= nms_thr
|
||||
});
|
||||
}
|
||||
|
||||
keep
|
||||
}
|
||||
|
||||
/// 5. multiclass_nms
|
||||
pub fn multiclass_nms(
|
||||
&self,
|
||||
boxes: &Array2<f32>, // [25200, 4] -> xyxy 格式
|
||||
scores: &Array2<f32>, // [25200, 80] -> 已经乘以 objectness 的得分
|
||||
nms_thr: f32,
|
||||
score_thr: f32,
|
||||
) -> Vec<Vec<f32>> {
|
||||
let mut candidates = Vec::new();
|
||||
|
||||
// 1. 筛选高分框 (单次遍历完成 Argmax 和 Threshold 过滤)
|
||||
for i in 0..scores.nrows() {
|
||||
let row = scores.row(i);
|
||||
|
||||
// 找到当前行(即当前锚点)得分最高的类别
|
||||
let mut max_score = 0.0;
|
||||
let mut cls_id = 0;
|
||||
for (j, &s) in row.iter().enumerate() {
|
||||
if s > max_score {
|
||||
max_score = s;
|
||||
cls_id = j;
|
||||
}
|
||||
}
|
||||
|
||||
// 仅保留超过阈值的候选框
|
||||
if max_score > score_thr {
|
||||
// 暂时存储索引和元数据,避免频繁创建大数组
|
||||
candidates.push((i, max_score, cls_id));
|
||||
}
|
||||
}
|
||||
|
||||
if candidates.is_empty() {
|
||||
return vec![];
|
||||
}
|
||||
|
||||
// 2. 准备 NMS 输入
|
||||
// 构造 NMS 需要的子集数组
|
||||
let mut b_subset = Array2::<f32>::zeros((candidates.len(), 4));
|
||||
let mut s_subset = Array1::<f32>::zeros(candidates.len());
|
||||
|
||||
for (new_idx, &(orig_idx, score, _)) in candidates.iter().enumerate() {
|
||||
b_subset.row_mut(new_idx).assign(&boxes.row(orig_idx));
|
||||
s_subset[new_idx] = score;
|
||||
}
|
||||
|
||||
// 3. 执行 NMS (返回保留下来的子集索引)
|
||||
let keep = self.nms(&b_subset, &s_subset, nms_thr);
|
||||
|
||||
// 4. 组装最终结果 [x1, y1, x2, y2, score, class_id]
|
||||
keep.into_iter()
|
||||
.map(|k_idx| {
|
||||
let (orig_idx, score, cls_id) = candidates[k_idx];
|
||||
let b = boxes.row(orig_idx);
|
||||
vec![b[0], b[1], b[2], b[3], score, cls_id as f32]
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
/// 6. get_bbox (完全解耦 OpenCV)
|
||||
pub fn get_bbox(&self, image_bytes: &[u8]) -> Result<Vec<Vec<i32>>> {
|
||||
// 使用 utils crate 解码
|
||||
let dynamic_img = image::load_from_memory(image_bytes).context("Failed to decode utils")?;
|
||||
let (orig_w, orig_h) = dynamic_img.dimensions();
|
||||
|
||||
let (input_tensor, ratio) = self.preproc(&dynamic_img, (416, 416))?;
|
||||
|
||||
// tract 推理
|
||||
let outputs = self.session.run(tvec!(input_tensor.into()))?;
|
||||
let output_array = outputs[0]
|
||||
.to_array_view::<f32>()?
|
||||
.to_owned()
|
||||
.into_dimensionality::<Ix3>()?;
|
||||
|
||||
let predictions = self.demo_postprocess(output_array, (416, 416));
|
||||
let pred = predictions.slice(s![0, .., ..]);
|
||||
|
||||
let boxes = pred.slice(s![.., 0..4]);
|
||||
let scores = &pred.slice(s![.., 4..5]) * &pred.slice(s![.., 5..]);
|
||||
|
||||
let mut boxes_xyxy = Array2::<f32>::zeros(boxes.raw_dim());
|
||||
for i in 0..boxes.nrows() {
|
||||
boxes_xyxy[[i, 0]] = (boxes[[i, 0]] - boxes[[i, 2]] / 2.0) / ratio;
|
||||
boxes_xyxy[[i, 1]] = (boxes[[i, 1]] - boxes[[i, 3]] / 2.0) / ratio;
|
||||
boxes_xyxy[[i, 2]] = (boxes[[i, 0]] + boxes[[i, 2]] / 2.0) / ratio;
|
||||
boxes_xyxy[[i, 3]] = (boxes[[i, 1]] + boxes[[i, 3]] / 2.0) / ratio;
|
||||
}
|
||||
|
||||
let detections = self.multiclass_nms(&boxes_xyxy, &scores, 0.45, 0.1);
|
||||
|
||||
Ok(detections
|
||||
.into_iter()
|
||||
.map(|d| {
|
||||
vec![
|
||||
(d[0] as i32).max(0).min(orig_w as i32),
|
||||
(d[1] as i32).max(0).min(orig_h as i32),
|
||||
(d[2] as i32).max(0).min(orig_w as i32),
|
||||
(d[3] as i32).max(0).min(orig_h as i32),
|
||||
]
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
}
|
||||
40
src/models/loader.rs
Normal file
40
src/models/loader.rs
Normal file
@@ -0,0 +1,40 @@
|
||||
use anyhow::Context;
|
||||
use image::DynamicImage;
|
||||
use tract_onnx::onnx;
|
||||
use tract_onnx::prelude::*;
|
||||
// 关键点:直接使用 tract 重导出的 ndarray
|
||||
use crate::utils::image_io::png_rgba_white_preprocess;
|
||||
use crate::utils::image_processor::{convert_to_grayscale, resize_image};
|
||||
use std::collections::HashMap;
|
||||
use tract_onnx::prelude::tract_ndarray::s;
|
||||
|
||||
/// OCR 模型:包含路径和字符集
|
||||
|
||||
pub enum ModelType {
|
||||
Ocr,
|
||||
Det,
|
||||
Custom,
|
||||
}
|
||||
// 定义统一的 trait
|
||||
pub trait ModelSession {
|
||||
fn get_model_type(&self) -> ModelType;
|
||||
fn desc(&self) -> String;
|
||||
}
|
||||
|
||||
pub struct ModelLoader {
|
||||
pub session: RunnableModel<TypedFact, Box<dyn TypedOp>, Graph<TypedFact, Box<dyn TypedOp>>>,
|
||||
}
|
||||
|
||||
impl ModelLoader {
|
||||
pub fn load_model<P>(model_path: P) -> anyhow::Result<Self>
|
||||
where
|
||||
P: AsRef<std::path::Path>,
|
||||
{
|
||||
let session = onnx()
|
||||
.model_for_path(model_path)
|
||||
.with_context(|| "加载 ONNX 模型失败,请检查路径是否正确")?
|
||||
.into_optimized()?
|
||||
.into_runnable()?;
|
||||
Ok(Self { session })
|
||||
}
|
||||
}
|
||||
5
src/models/mod.rs
Normal file
5
src/models/mod.rs
Normal file
@@ -0,0 +1,5 @@
|
||||
pub mod base;
|
||||
pub mod loader;
|
||||
pub mod ocr;
|
||||
pub mod det;
|
||||
pub mod slide;
|
||||
251
src/models/ocr.rs
Normal file
251
src/models/ocr.rs
Normal file
@@ -0,0 +1,251 @@
|
||||
use crate::models::base::ModelArgs;
|
||||
use crate::utils::image_io::png_rgba_white_preprocess;
|
||||
use crate::utils::image_processor::{convert_to_grayscale, resize_image};
|
||||
use crate::models::loader::{ModelLoader, ModelSession, ModelType};
|
||||
use anyhow::Context;
|
||||
use image::DynamicImage;
|
||||
use tract_onnx::prelude::tract_ndarray::s;
|
||||
use tract_onnx::prelude::{
|
||||
DatumType, Graph, IntoTensor, RunnableModel, Tensor, TypedFact, TypedOp, tract_ndarray, tvec,
|
||||
};
|
||||
|
||||
// 颜色过滤的自定义范围:(低值RGB, 高值RGB)
|
||||
pub type ColorRange = ((u8, u8, u8), (u8, u8, u8));
|
||||
|
||||
// 字符集范围类型
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum CharsetRange {
|
||||
All, // 所有字符
|
||||
Digit, // 数字
|
||||
Letter, // 字母
|
||||
Alphanumeric, // 字母数字
|
||||
Single(String), // 单字符串
|
||||
Multiple(Vec<String>), // 多个字符串
|
||||
Range(char, char), // 字符范围
|
||||
Custom(Vec<char>), // 自定义字符列表
|
||||
}
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PredictArgs {
|
||||
/// 是否修复PNG格式问题
|
||||
pub png_fix: bool,
|
||||
/// 是否返回概率信息
|
||||
pub probability: bool,
|
||||
/// 颜色过滤:保留的颜色列表
|
||||
pub color_filter_colors: Option<Vec<String>>,
|
||||
/// 颜色过滤:自定义RGB范围
|
||||
pub color_filter_custom_ranges: Option<Vec<ColorRange>>,
|
||||
/// 字符集范围
|
||||
pub charset_range: Option<CharsetRange>,
|
||||
}
|
||||
|
||||
impl Default for PredictArgs {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
png_fix: false,
|
||||
probability: false,
|
||||
color_filter_colors: None,
|
||||
color_filter_custom_ranges: None,
|
||||
charset_range: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PredictArgs {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
// Builder 模式方法
|
||||
pub fn png_fix(mut self, enabled: bool) -> Self {
|
||||
self.png_fix = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn probability(mut self, enabled: bool) -> Self {
|
||||
self.probability = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn color_filter_colors(mut self, colors: Vec<String>) -> Self {
|
||||
self.color_filter_colors = Some(colors);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn color_filter_custom_ranges(mut self, ranges: Vec<ColorRange>) -> Self {
|
||||
self.color_filter_custom_ranges = Some(ranges);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn charset_range(mut self, range: CharsetRange) -> Self {
|
||||
self.charset_range = Some(range);
|
||||
self
|
||||
}
|
||||
|
||||
// 便捷构造方法
|
||||
pub fn quick() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
pub fn with_probability() -> Self {
|
||||
Self::default().probability(true)
|
||||
}
|
||||
|
||||
pub fn with_png_fix() -> Self {
|
||||
Self::default().png_fix(true)
|
||||
}
|
||||
}
|
||||
pub struct Ocr {
|
||||
session: RunnableModel<TypedFact, Box<dyn TypedOp>, Graph<TypedFact, Box<dyn TypedOp>>>,
|
||||
charset: Vec<String>,
|
||||
}
|
||||
impl ModelSession for Ocr {
|
||||
fn get_model_type(&self) -> ModelType {
|
||||
todo!()
|
||||
}
|
||||
fn desc(&self) -> String {
|
||||
"Ocr Model 加载成功".to_string()
|
||||
}
|
||||
}
|
||||
impl Ocr {
|
||||
pub fn new(model_path: String, charset: Vec<String>) -> Result<Self, anyhow::Error> {
|
||||
let session = ModelLoader::load_model(&model_path)?.session;
|
||||
Ok(Self { session, charset })
|
||||
}
|
||||
pub fn predict(&self, image: &DynamicImage, png_fix: bool) -> Result<String, anyhow::Error> {
|
||||
let tensor = self.preprocess_image(image, png_fix)?;
|
||||
//
|
||||
// let result = self.session.run(tvec!(tensor.into()))?;
|
||||
// // 3. 解析结果
|
||||
// // let output = result[0].to_array_view::<i64>()?;
|
||||
let output = self.inference(tensor)?;
|
||||
let output2 = self.process_text_output(&output)?;
|
||||
Ok(self.ctc_decode_indices(&output2))
|
||||
// Ok("ocr result".to_string())
|
||||
}
|
||||
/// 对应 Python 的 _preprocess_image
|
||||
/// 负责:透明背景修复 -> 灰度化 -> 按比例 Resize -> 归一化 -> 4维张量转换
|
||||
fn preprocess_image(&self, img: &DynamicImage, png_fix: bool) -> anyhow::Result<Tensor> {
|
||||
// A. 修复 PNG 透明背景 (内部逻辑你之前已实现)
|
||||
let _ = if png_fix && img.color().has_alpha() {
|
||||
png_rgba_white_preprocess(img)
|
||||
} else {
|
||||
img.clone()
|
||||
};
|
||||
|
||||
let h = 64u32;
|
||||
let w = (img.width() as f32 * (h as f32 / img.height() as f32)) as u32;
|
||||
let gray_img = convert_to_grayscale(img);
|
||||
let resized = resize_image(&gray_img, w, h);
|
||||
// resized.save("debug_preprocessed.png").unwrap();
|
||||
// 1. 预处理:转灰度 -> Resize -> 归一化
|
||||
// let resized = img.resize_exact(w, h, FilterType::Lanczos3).to_luma8();
|
||||
|
||||
// 使用 tract_ndarray 构造,避免版本冲突
|
||||
let array =
|
||||
tract_ndarray::Array4::from_shape_fn((1, 1, h as usize, w as usize), |(_, _, y, x)| {
|
||||
let pixel = resized.get_pixel(x as u32, y as u32)[0] as f32;
|
||||
(pixel / 255.0 - 0.5) / 0.5
|
||||
});
|
||||
|
||||
let tensor = Tensor::from(array);
|
||||
|
||||
Ok(tensor)
|
||||
}
|
||||
/// 对应 Python 的 _inference
|
||||
fn inference(&self, tensor: Tensor) -> anyhow::Result<Tensor> {
|
||||
// tract 的 run 会返回一个 Vec<TValue>,我们通常只需要第一个输出
|
||||
// let result = self.session.run(tvec!(tensor.into()))?;
|
||||
let mut result = self
|
||||
.session
|
||||
.run(tvec!(tensor.into()))
|
||||
.context("执行模型推理失败")?;
|
||||
println!("模型输出原始数据: {:?}", result);
|
||||
Ok(result.remove(0).into_tensor())
|
||||
}
|
||||
/// 核心解析逻辑:将模型输出的各种维度/类型的 Tensor 转为字符索引序列
|
||||
fn process_text_output(&self, raw_tensor: &Tensor) -> anyhow::Result<Vec<i64>> {
|
||||
let shape = raw_tensor.shape();
|
||||
println!("模型输出shape数据: {:?}", shape);
|
||||
let datum_type = raw_tensor.datum_type();
|
||||
println!("模型输出datum_type数据: {:?}", datum_type);
|
||||
|
||||
match raw_tensor.datum_type() {
|
||||
// 情况 1: huashi666 式模型,直接输出 i64 索引 (通常是模型内部做好了 Argmax)
|
||||
DatumType::I64 => {
|
||||
let view = raw_tensor.to_array_view::<i64>()?;
|
||||
Ok(view.iter().cloned().collect())
|
||||
}
|
||||
|
||||
// 情况 2: sml2h3 原版模型,输出 F32 概率矩阵
|
||||
DatumType::F32 => {
|
||||
let view = raw_tensor.to_array_view::<f32>()?;
|
||||
let (steps, classes, data_view) = match shape.len() {
|
||||
3 => {
|
||||
if shape[1] == 1 {
|
||||
// 形状: [Steps, 1, Classes] -> 你的原有逻辑
|
||||
(shape[0], shape[2], view.into_dyn())
|
||||
} else if shape[0] == 1 {
|
||||
// 形状: [1, Steps, Classes] -> 另一种常见导出格式
|
||||
(shape[1], shape[2], view.into_dyn())
|
||||
} else {
|
||||
// 默认取第一个 batch: [Batch, Steps, Classes]
|
||||
// 使用 slice 对应 Python 的 output[0, :, :]
|
||||
let sliced = view.slice(s![0, .., ..]);
|
||||
(shape[1], shape[2], sliced.into_dyn())
|
||||
}
|
||||
}
|
||||
2 => {
|
||||
// 形状: [Steps, Classes] -> 已经剥离了 Batch 维度
|
||||
(shape[0], shape[1], view.into_dyn())
|
||||
}
|
||||
_ => return Err(anyhow::anyhow!("不支持的输出维度: {:?}", shape)),
|
||||
};
|
||||
let array_2d = data_view.to_shape((steps, classes))?;
|
||||
//
|
||||
// 对每一行执行 Argmax (寻找概率最大的字符索引)
|
||||
let indices = array_2d
|
||||
.outer_iter()
|
||||
.map(|row| {
|
||||
row.iter()
|
||||
.enumerate()
|
||||
.max_by(|(_, a), (_, b)| {
|
||||
a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)
|
||||
})
|
||||
.map(|(idx, _)| idx as i64)
|
||||
.unwrap_or(0)
|
||||
})
|
||||
.collect();
|
||||
Ok(indices)
|
||||
}
|
||||
_ => Err(anyhow::anyhow!(
|
||||
"不支持的模型输出数据类型: {:?}",
|
||||
raw_tensor.datum_type()
|
||||
)),
|
||||
}
|
||||
}
|
||||
fn ctc_decode_indices(&self, predicted_indices: &[i64]) -> String {
|
||||
println!("indices模型输出原始数据: {:?}", predicted_indices);
|
||||
|
||||
// 对应 _ctc_decode_indices 的逻辑:去重、去 blank (0)
|
||||
let mut res = String::new();
|
||||
let mut prev_idx: i64 = -1;
|
||||
|
||||
for &idx in predicted_indices {
|
||||
// 1. 跳过连续重复的索引
|
||||
// 2. 跳过 blank 字符 (假设索引 0 是 blank)
|
||||
if idx != prev_idx && idx != 0 {
|
||||
if let Ok(u_idx) = usize::try_from(idx) {
|
||||
if let Some(char_str) = self.charset.get(u_idx) {
|
||||
res.push_str(char_str);
|
||||
} else {
|
||||
// 保护逻辑:如果模型预测的索引超出了字符集范围
|
||||
eprintln!("警告: 预测索引 {} 超出字符集范围", u_idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
prev_idx = idx;
|
||||
}
|
||||
println!("最终识别出的验证码是: {}", res);
|
||||
res
|
||||
}
|
||||
}
|
||||
276
src/models/slide.rs
Normal file
276
src/models/slide.rs
Normal file
@@ -0,0 +1,276 @@
|
||||
use crate::utils::cv_ops::{min_max_loc, rgb_to_gray, ndarray_to_luma8, abs_diff};
|
||||
use crate::utils::image_io::image_to_ndarray;
|
||||
use anyhow::{Context, Result, anyhow};
|
||||
use image::{DynamicImage, GenericImageView};
|
||||
use image::{ImageBuffer, Luma};
|
||||
use imageproc::distance_transform::Norm;
|
||||
use imageproc::edges::canny;
|
||||
use imageproc::morphology::{close, open};
|
||||
use imageproc::region_labelling::{Connectivity, connected_components};
|
||||
use imageproc::template_matching::{MatchTemplateMethod, match_template};
|
||||
use std::cmp::{max, min};
|
||||
use imageproc::contrast::{threshold, ThresholdType};
|
||||
use tract_onnx::prelude::tract_ndarray::{Array2, Array3, ArrayView2, ArrayView3, Axis, s};
|
||||
|
||||
pub struct SlideResult {
|
||||
pub target: [i32; 2],
|
||||
pub target_x: i32,
|
||||
pub target_y: i32,
|
||||
pub confidence: f64,
|
||||
}
|
||||
|
||||
pub struct Slide;
|
||||
|
||||
impl Slide {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
|
||||
/// 对应 Python: slide_match
|
||||
pub fn slide_match(
|
||||
&self,
|
||||
target_image: &DynamicImage,
|
||||
background_image: &DynamicImage,
|
||||
simple_target: bool,
|
||||
) -> Result<SlideResult> {
|
||||
let target_array = image_to_ndarray(target_image);
|
||||
let background_array = image_to_ndarray(background_image);
|
||||
|
||||
self.perform_slide_match(target_array.view(), background_array.view(), simple_target)
|
||||
.map_err(|e| anyhow!("滑块匹配失败: {}", e))
|
||||
}
|
||||
/// 对应 Python: slide_comparison
|
||||
/// 用于比较带坑位的图片与原始背景图,定位差异点
|
||||
pub fn slide_comparison(
|
||||
&self,
|
||||
target_image: &DynamicImage,
|
||||
background_image: &DynamicImage,
|
||||
) -> Result<SlideResult> {
|
||||
// 1. 转换为 ndarray (HWC RGB)
|
||||
let target_array = image_to_ndarray(target_image);
|
||||
let background_array = image_to_ndarray(background_image);
|
||||
|
||||
// 2. 执行比较逻辑 (对应 _perform_slide_comparison)
|
||||
self.perform_slide_comparison(target_array.view(), background_array.view())
|
||||
.map_err(|e| anyhow!("滑块比较执行失败: {}", e))
|
||||
}
|
||||
/// 对应 Python: _perform_slide_comparison
|
||||
pub fn perform_slide_comparison(
|
||||
&self,
|
||||
target: ArrayView3<u8>,
|
||||
background: ArrayView3<u8>,
|
||||
) -> Result<SlideResult> {
|
||||
let (h, w, _) = target.dim();
|
||||
|
||||
// 1. 计算图像差异并灰度化 (对应 cv2.absdiff + cv2.cvtColor)
|
||||
// 使用 OpenCV 标准权重公式:0.299R + 0.587G + 0.114B
|
||||
// let mut diff_buffer = ImageBuffer::new(w as u32, h as u32);
|
||||
// for y in 0..h {
|
||||
// for x in 0..w {
|
||||
// let r_diff = (target[[y, x, 0]] as i16 - background[[y, x, 0]] as i16).abs() as f32;
|
||||
// let g_diff = (target[[y, x, 1]] as i16 - background[[y, x, 1]] as i16).abs() as f32;
|
||||
// let b_diff = (target[[y, x, 2]] as i16 - background[[y, x, 2]] as i16).abs() as f32;
|
||||
//
|
||||
// let gray_diff = (0.299 * r_diff + 0.587 * g_diff + 0.114 * b_diff) as u8;
|
||||
// diff_buffer.put_pixel(x as u32, y as u32, Luma([gray_diff]));
|
||||
// }
|
||||
// }
|
||||
// 1. 计算差异数组 (复用 cv2::absdiff)
|
||||
let diff_array = abs_diff(&target, &background);
|
||||
|
||||
// 2. 转换为灰度数组 (复用你的 cv2::rgb_to_gray)
|
||||
let gray_array = rgb_to_gray(diff_array.view());
|
||||
// 3. 转为 ImageBuffer 以使用 imageproc 的高级功能
|
||||
let gray_buffer = ndarray_to_luma8(gray_array.view());
|
||||
|
||||
// 2. 二值化 (对应 cv2.threshold(..., 30, 255, cv2.THRESH_BINARY))
|
||||
// let mut binary = ImageBuffer::new(w as u32, h as u32);
|
||||
// for (x, y, pixel) in diff_buffer.enumerate_pixels() {
|
||||
// let val = if pixel.0[0] > 30 { 255u8 } else { 0u8 };
|
||||
// binary.put_pixel(x, y, Luma([val]));
|
||||
// }
|
||||
let binary = threshold(&gray_buffer, 30, ThresholdType::Binary);
|
||||
// 3. 形态学操作去噪 (对应 cv2.morphologyEx)
|
||||
// 闭运算 (Close): 先膨胀后腐蚀,用于填补缺口内的细小黑色空洞
|
||||
// 开运算 (Open): 先腐蚀后膨胀,用于消除背景中的白色噪点点
|
||||
let norm = Norm::LInf; // 对应 3x3 的矩形内核
|
||||
let radius = 1u8; // 1 表示 3x3 的范围,2 表示 5x5 的范围
|
||||
let closed = close(&binary, norm, radius);
|
||||
let cleaned = open(&closed, norm, radius);
|
||||
|
||||
// 4. 寻找最大连通区域 (对应 findContours + max area)
|
||||
// connected_components 会给每个独立的白色区域打上不同的标签 (ID)
|
||||
let background_label = Luma([0u8]);
|
||||
let labelled = connected_components(&cleaned, Connectivity::Eight, background_label);
|
||||
|
||||
// 统计每个标签出现的频率(即面积)
|
||||
let mut max_label = 0;
|
||||
let mut max_area = 0;
|
||||
let mut areas = std::collections::HashMap::new();
|
||||
|
||||
for pixel in labelled.pixels() {
|
||||
let label = pixel.0[0];
|
||||
if label == 0 {
|
||||
continue;
|
||||
} // 跳过背景
|
||||
let count = areas.entry(label).or_insert(0);
|
||||
*count += 1;
|
||||
if *count > max_area {
|
||||
max_area = *count;
|
||||
max_label = label;
|
||||
}
|
||||
}
|
||||
|
||||
if max_label == 0 {
|
||||
return Ok(SlideResult {
|
||||
target: [0, 0],
|
||||
target_x: 0,
|
||||
target_y: 0,
|
||||
confidence: 0.0,
|
||||
});
|
||||
}
|
||||
|
||||
// 5. 计算最大区域的边界框 (对应 cv2.boundingRect)
|
||||
let mut min_x = w as u32;
|
||||
let mut max_x = 0;
|
||||
let mut min_y = h as u32;
|
||||
let mut max_y = 0;
|
||||
|
||||
for (x, y, pixel) in labelled.enumerate_pixels() {
|
||||
if pixel.0[0] == max_label {
|
||||
min_x = min(min_x, x);
|
||||
max_x = max(max_x, x);
|
||||
min_y = min(min_y, y);
|
||||
max_y = max(max_y, y);
|
||||
}
|
||||
}
|
||||
|
||||
// 6. 计算中心点
|
||||
let rect_w = max_x - min_x;
|
||||
let rect_h = max_y - min_y;
|
||||
let center_x = (min_x + rect_w / 2) as i32;
|
||||
let center_y = (min_y + rect_h / 2) as i32;
|
||||
|
||||
Ok(SlideResult {
|
||||
target: [center_x, center_y],
|
||||
target_x: center_x,
|
||||
target_y: center_y,
|
||||
confidence: 1.0, // Comparison 模式下通常认为找到即为 1.0
|
||||
})
|
||||
}
|
||||
|
||||
/// 对应 Python: _perform_slide_match
|
||||
// 在 SlideEngine 中修改此入口进行测试
|
||||
fn perform_slide_match(
|
||||
&self,
|
||||
target: ArrayView3<u8>,
|
||||
background: ArrayView3<u8>,
|
||||
simple_target: bool, // 增加这个参数
|
||||
) -> Result<SlideResult> {
|
||||
// 1. 统一灰度化
|
||||
let target_gray = rgb_to_gray(target);
|
||||
let background_gray = rgb_to_gray(background);
|
||||
|
||||
if simple_target {
|
||||
// 2a. 简单模式:直接在灰度图上匹配
|
||||
self.simple_template_match(target_gray.view(), background_gray.view())
|
||||
} else {
|
||||
// 2b. 复杂模式:先提取边缘,再匹配
|
||||
|
||||
self.edge_based_match(target_gray.view(), background_gray.view())
|
||||
}
|
||||
}
|
||||
/// 对应 Python: _simple_template_match
|
||||
/// 使用 SAD (Sum of Absolute Differences) 算法
|
||||
/// 核心模板匹配:SAD + 有效像素过滤
|
||||
fn simple_template_match(
|
||||
&self,
|
||||
target: ArrayView2<u8>,
|
||||
background: ArrayView2<u8>,
|
||||
) -> Result<SlideResult> {
|
||||
// 1. 将 ndarray 转换为 imageproc 需要的 ImageBuffer (无拷贝或轻量转换)
|
||||
|
||||
// let (bh, bw) = background.dim();
|
||||
|
||||
// 转换逻辑 (假设你已经有方法转回 ImageBuffer)
|
||||
let t_buf = ndarray_to_luma8(target);
|
||||
let b_buf = ndarray_to_luma8(background);
|
||||
// t_buf.save("debug_rust_target.png").unwrap();
|
||||
|
||||
// 2. 调用 imageproc 的 NCC 算法 (等价于 cv2.TM_CCOEFF_NORMED)
|
||||
// 模板匹配 (完全对齐 cv2.matchTemplate(..., cv2.TM_CCOEFF_NORMED))
|
||||
let result = match_template(
|
||||
&b_buf,
|
||||
&t_buf,
|
||||
MatchTemplateMethod::CrossCorrelationNormalized,
|
||||
);
|
||||
// save_rust_result(&result, "debug_rust_target2.png");
|
||||
// 3. 寻找最大值 (等价于 cv2.minMaxLoc)
|
||||
let (max_val, max_loc) = min_max_loc(&result);
|
||||
|
||||
// 4. 计算中心点 (与 Python 逻辑完全一致)
|
||||
let (th, tw) = target.dim();
|
||||
let center_x = max_loc.0 as i32 + (tw as i32 / 2);
|
||||
let center_y = max_loc.1 as i32 + (th as i32 / 2);
|
||||
// println!("Rust Target Width (tw): {}", tw);
|
||||
// println!("Rust Best Max Loc X: {}", max_loc.0);
|
||||
// println!("Rust Final Center X: {}", center_x);
|
||||
Ok(SlideResult {
|
||||
target: [center_x, center_y],
|
||||
target_x: center_x,
|
||||
target_y: center_y,
|
||||
confidence: max_val as f64,
|
||||
})
|
||||
}
|
||||
|
||||
/// 对应 Python: _edge_based_match
|
||||
/// 基于边缘检测的滑块匹配 (对齐 Python _edge_based_match)
|
||||
pub fn edge_based_match(
|
||||
&self,
|
||||
target: ArrayView2<u8>,
|
||||
background: ArrayView2<u8>,
|
||||
) -> Result<SlideResult> {
|
||||
// 1. 将 ndarray 转换为 ImageBuffer
|
||||
// 注意:Canny 和 match_template 需要 ImageBuffer 格式
|
||||
let t_buf = ndarray_to_luma8(target);
|
||||
let b_buf = ndarray_to_luma8(background);
|
||||
|
||||
// 2. 边缘检测 (完全对齐 cv2.Canny(50, 150))
|
||||
// 这步会生成黑底白线的二值化边缘图
|
||||
let target_edges = canny(&t_buf, 50.0, 150.0);
|
||||
let background_edges = canny(&b_buf, 50.0, 150.0);
|
||||
|
||||
// target_edges.save("debug_target_edges.png").ok();
|
||||
// background_edges.save("debug_bg_edges.png").ok();
|
||||
|
||||
// 3. 模板匹配 (完全对齐 cv2.matchTemplate(..., cv2.TM_CCOEFF_NORMED))
|
||||
// 在边缘图上计算归一化互相关系数
|
||||
let result = match_template(
|
||||
&background_edges,
|
||||
&target_edges,
|
||||
MatchTemplateMethod::CrossCorrelationNormalized,
|
||||
);
|
||||
|
||||
// 4. 找到最佳匹配位置 (对齐 cv2.minMaxLoc)
|
||||
let (max_val, max_loc) = min_max_loc(&result);
|
||||
// 5. 计算中心位置 (对齐 Python 逻辑)
|
||||
// target_w, target_h 来自输入数组的维度
|
||||
let (th, tw) = target.dim();
|
||||
let center_x = max_loc.0 as i32 + (tw as i32 / 2);
|
||||
let center_y = max_loc.1 as i32 + (th as i32 / 2);
|
||||
|
||||
// 打印调试信息,方便与 Python 对比
|
||||
// println!("Edge Match: max_val: {}, max_loc: {:?}", max_val, max_loc);
|
||||
println!("-Rust Target Width (tw): {}", tw);
|
||||
println!("-Rust Best Max Loc X: {}", max_loc.0);
|
||||
println!("-Rust Final Center X: {}", center_x);
|
||||
Ok(SlideResult {
|
||||
target: [center_x, center_y],
|
||||
target_x: center_x,
|
||||
target_y: center_y,
|
||||
confidence: max_val as f64,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user