VideoStitching APAP视频拼接算法

#include "stdafx.h" #include <iostream> #include <fstream> #include <string> #include <time.h> #include <Windows.h> #include "composer_gpu.h" #include "video_stitcher.h" #include "apap.h" #include "gutil.h" MyVideoStitcher::MyVideoStitcher() { is_preview_ = true; is_save_video_ = false; start_frame_index_ = 0; end_frame_index_ = -1; is_try_gpu_ = false; is_debug_ = false; trim_type_ = MyVideoStitcher::TRIM_NO; work_megapix_ = 1.0;//-1;// seam_megapix_ = 0.2;//-1;// is_prepared_ = false; conf_thresh_ = 1.f; features_type_ = "orb";//"surf";// ba_cost_func_ = "ray"; ba_refine_mask_ = "xxxxx"; is_do_wave_correct_ = true; wave_correct_ = detail::WAVE_CORRECT_HORIZ; is_save_graph_ = false; warp_type_ = "cylindrical";//"plane";//"apap";//"paniniA2B1";//"transverseMercator";//"spherical";// expos_comp_type_ = ExposureCompensator::GAIN_BLOCKS;//ExposureCompensator::GAIN;// match_conf_ = 0.3f; seam_find_type_ = "gc_color";//"voronoi";// blend_type_ = Blender::FEATHER;//Blender::MULTI_BAND;//Blender::NO;// blend_strength_ = 5; // 获取当前系统的核数 SYSTEM_INFO sys_info; GetSystemInfo(&sys_info); parallel_num_ = sys_info.dwNumberOfProcessors; } int MyVideoStitcher::stitch( vector<VideoCapture> &captures, string &writer_file_name ) { int video_num = captures.size(); vector<Mat> src(video_num); Mat frame, dst, show_dst; // Debug用信息 bool is_save_input_frames = false; bool is_save_output_frames = true; double fps = captures[0].get(CV_CAP_PROP_FPS); // skip some frames for(int j = 0; j < video_num; j++) for(int i = 0; i < start_frame_index_; i++) captures[j].read(frame); // 第一帧，做一些初始化，并且确定结果视频的分辨率 for(int j = 0; j < video_num; j++) { if( !captures[j].read(frame)) return -1; frame.copyTo(src[j]); if(is_debug_) { char img_save_name[100]; sprintf(img_save_name, "/%d.jpg", j+1); imwrite(debug_dir_path_ + img_save_name, src[j]); } } long prepare_start_clock = clock(); int prepare_status = Prepare(src); // 先用ORB特征测试，错误的话再使用SURF，仍然错误则报错，输入视频不符合条件 if( prepare_status == STITCH_CONFIG_ERROR ) { cout << "video stitch config error!" << endl; return -1; } if( prepare_status != STITCH_SUCCESS) { features_type_ = "surf"; cout << "video stitch first try failed, second try ... " << endl; if( Prepare(src) != STITCH_SUCCESS) { cout << "videos input are invalid. Initialization failed." << endl; return -1; } } long prepare_end_clock = clock(); cout << "\tprepare time: " << prepare_end_clock - prepare_start_clock << "ms" << endl; StitchFrame(src, dst); if(is_debug_) //保存第一帧拼接结果和mask { imwrite(debug_dir_path_ + "/res.jpg", dst); vector<Mat> img_masks(video_num); for(int i = 0; i < video_num; i++) { img_masks[i].create(src[i].rows, src[i].cols, CV_8UC3); img_masks[i].setTo(Scalar::all(255)); } Mat dst_mask; StitchFrame(img_masks, dst_mask); imwrite(debug_dir_path_ + "/mask.jpg", dst_mask); } // 创建结果视频 VideoWriter writer; if(is_save_video_) { writer.open(writer_file_name, CV_FOURCC('D', 'I','V', '3'), 20, Size(dst.cols, dst.rows)); writer.write(dst); } // 开始拼接 double stitch_time = 0; FrameInfo frame_info; frame_info.src.resize(video_num); int frameidx = 1; cout << "Stitching..." << endl; string window_name = "视频拼接"; if(is_preview_) namedWindow(window_name); double show_scale = 1.0, scale_interval = 0.03; int frame_show_interval = cvFloor(1000 / fps); int failed_frame_count = 0; char log_string[1000]; char log_file_name[200]; SYSTEMTIME sys_time = {0}; GetLocalTime(&sys_time); sprintf(log_file_name, "%d%02d%02d-%02d%02d%02d.log", sys_time.wYear, sys_time.wMonth, sys_time.wDay, sys_time.wHour, sys_time.wMinute, sys_time.wSecond); ofstream log_file; if(is_debug_) log_file.open(debug_dir_path_ + log_file_name); long long startTime = clock(); while(true) { long frame_time = 0; // 采集 long cap_start_clock = clock(); int j; for(j = 0; j < video_num; j++) { if( !captures[j].read(frame)) break; frame.copyTo(frame_info.src[j]); } frame_info.frame_idx = frameidx; frameidx++; if(j != video_num || (end_frame_index_ >= 0 && frameidx >= end_frame_index_)) //有一个视频源结束，则停止拼接 break; // 拼接 long stitch_start_clock = clock(); frame_info.stitch_status = StitchFrame(frame_info.src, frame_info.dst); long stitch_clock = clock(); sprintf(log_string, "\tframe %d: stitch(%dms), capture(%dms)", frame_info.frame_idx, stitch_clock - stitch_start_clock, stitch_start_clock - cap_start_clock); printf("%s", log_string); if(is_debug_) log_file << log_string << endl; stitch_time += stitch_clock - stitch_start_clock; frame_time += stitch_clock - cap_start_clock; // 拼接失败 if(frame_info.stitch_status != 0) { cout << "failed\n"; if(is_debug_) log_file << "failed" << endl; failed_frame_count++; break; } // 保存视频 if(is_save_video_) { cout << ", write("; if(is_save_output_frames) { char img_save_name[100]; sprintf(img_save_name, "/images/%d.jpg", frame_info.frame_idx); imwrite(debug_dir_path_ + img_save_name, frame_info.dst); } long write_start_clock = clock(); writer.write(frame_info.dst); long write_clock = clock(); cout << write_clock - write_start_clock << "ms)"; frame_time += write_clock - write_start_clock; } cout << endl; // 显示--- if(is_preview_) { int key = waitKey(std::max(1, (int)(frame_show_interval - frame_time))); if(key == 27) // ESC break; else if(key == 61 || key == 43) // + show_scale += scale_interval; else if(key == 45) // - if(show_scale >= scale_interval) show_scale -= scale_interval; resize(frame_info.dst, show_dst, Size(show_scale * dst.cols, show_scale * dst.rows)); imshow(window_name, show_dst); } } long long endTime = clock(); cout << "test " << endTime - startTime << endl; cout << "\nStitch over" << endl; cout << failed_frame_count << " frames failed." << endl; cout << "\tfull view angle is " << cvRound(view_angle_) << "°" << endl; if(is_debug_) log_file << "\tfull view angle is " << cvRound(view_angle_) << "°" << endl; writer.release(); cout << "\ton average: stitch time = " << stitch_time / (frameidx-1) << "ms" << endl; cout << "\tcenter: (" << -dst_roi_.x << ", " << -dst_roi_.y << ")" << endl; if(is_debug_) { log_file << "\ton average: stitch time = " << stitch_time / (frameidx-1) << "ms" << endl; log_file << "\tcenter: (" << -dst_roi_.x << ", " << -dst_roi_.y << ")" << endl; log_file.close(); } return 0; } void MyVideoStitcher::InitMembers(int num_images) { } /* * 初始图像可能分辨率很高，先做一步降采样，可以提高时间效率 */ void MyVideoStitcher::SetScales(vector<Mat> &src) { if(work_megapix_ < 0) work_scale_ = 1.0; else work_scale_ = min(1.0, sqrt(work_megapix_ * 1e6 / src[0].size().area())); if(seam_megapix_ < 0) seam_scale_ = 1.0; else seam_scale_ = min(1.0, sqrt(seam_megapix_ * 1e6 / src[0].size().area())); } /* * 特征提取，支持SURF和ORB */ int MyVideoStitcher::FindFeatures(vector<Mat> &src, vector<ImageFeatures> &features) { Ptr<FeaturesFinder> finder; if (features_type_ == "surf") { #ifdef HAVE_OPENCV_GPU if (is_try_gpu_ && gpu::getCudaEnabledDeviceCount() > 0) finder = new SurfFeaturesFinderGpu(); else #endif finder = new SurfFeaturesFinder(); } else if (features_type_ == "orb") { finder = new OrbFeaturesFinder();//Size(3,1), 1500, 1.3, 5); } else { cout << "Unknown 2D features type: '" << features_type_ << "'.\n"; return STITCH_CONFIG_ERROR; } int num_images = static_cast<int>(src.size()); Mat full_img, img; for (int i = 0; i < num_images; ++i) { full_img = src[i].clone();// if (work_megapix_ < 0) img = full_img; else resize(full_img, img, Size(), work_scale_, work_scale_); (*finder)(img, features[i]); //LOGLN("Features in image #" << i+1 <<