基于多模态的信息理解与抽取的方法（python）

// Copyright (c) Facebook, Inc. and its affiliates. #include "cocoeval.h" #include <time.h> #include <algorithm> #include <cstdint> #include <numeric> using namespace pybind11::literals; namespace detectron2 { namespace COCOeval { // Sort detections from highest score to lowest, such that // detection_instances[detection_sorted_indices[t]] >= // detection_instances[detection_sorted_indices[t+1]]. Use stable_sort to match // original COCO API void SortInstancesByDetectionScore( const std::vector<InstanceAnnotation>& detection_instances, std::vector<uint64_t>* detection_sorted_indices) { detection_sorted_indices->resize(detection_instances.size()); std::iota( detection_sorted_indices->begin(), detection_sorted_indices->end(), 0); std::stable_sort( detection_sorted_indices->begin(), detection_sorted_indices->end(), [&detection_instances](size_t j1, size_t j2) { return detection_instances[j1].score > detection_instances[j2].score; }); } // Partition the ground truth objects based on whether or not to ignore them // based on area void SortInstancesByIgnore( const std::array<double, 2>& area_range, const std::vector<InstanceAnnotation>& ground_truth_instances, std::vector<uint64_t>* ground_truth_sorted_indices, std::vector<bool>* ignores) { ignores->clear(); ignores->reserve(ground_truth_instances.size()); for (auto o : ground_truth_instances) { ignores->push_back( o.ignore || o.area < area_range[0] || o.area > area_range[1]); } ground_truth_sorted_indices->resize(ground_truth_instances.size()); std::iota( ground_truth_sorted_indices->begin(), ground_truth_sorted_indices->end(), 0); std::stable_sort( ground_truth_sorted_indices->begin(), ground_truth_sorted_indices->end(), [&ignores](size_t j1, size_t j2) { return (int)(*ignores)[j1] < (int)(*ignores)[j2]; }); } // For each IOU threshold, greedily match each detected instance to a ground // truth instance (if possible) and store the results void MatchDetectionsToGroundTruth( const std::vector<InstanceAnnotation>& detection_instances, const std::vector<uint64_t>& detection_sorted_indices, const std::vector<InstanceAnnotation>& ground_truth_instances, const std::vector<uint64_t>& ground_truth_sorted_indices, const std::vector<bool>& ignores, const std::vector<std::vector<double>>& ious, const std::vector<double>& iou_thresholds, const std::array<double, 2>& area_range, ImageEvaluation* results) { // Initialize memory to store return data matches and ignore const int num_iou_thresholds = iou_thresholds.size(); const int num_ground_truth = ground_truth_sorted_indices.size(); const int num_detections = detection_sorted_indices.size(); std::vector<uint64_t> ground_truth_matches( num_iou_thresholds * num_ground_truth, 0); std::vector<uint64_t>& detection_matches = results->detection_matches; std::vector<bool>& detection_ignores = results->detection_ignores; std::vector<bool>& ground_truth_ignores = results->ground_truth_ignores; detection_matches.resize(num_iou_thresholds * num_detections, 0); detection_ignores.resize(num_iou_thresholds * num_detections, false); ground_truth_ignores.resize(num_ground_truth); for (auto g = 0; g < num_ground_truth; ++g) { ground_truth_ignores[g] = ignores[ground_truth_sorted_indices[g]]; } for (auto t = 0; t < num_iou_thresholds; ++t) { for (auto d = 0; d < num_detections; ++d) { // information about best match so far (match=-1 -> unmatched) double best_iou = std::min(iou_thresholds[t], 1 - 1e-10); int match = -1; for (auto g = 0; g < num_ground_truth; ++g) { // if this ground truth instance is already matched and not a // crowd, it cannot be matched to another detection if (ground_truth_matches[t * num_ground_truth + g] > 0 && !ground_truth_instances[ground_truth_sorted_indices[g]].is_crowd) { continue; } // if detected instance matched to a regular ground truth // instance, we can break on the first ground truth instance // tagged as ignore (because they are sorted by the ignore tag) if (match >= 0 && !ground_truth_ignores[match] && ground_truth_ignores[g]) { break; } // if IOU overlap is the best so far, store the match appropriately if (ious[d][ground_truth_sorted_indices[g]] >= best_iou) { best_iou = ious[d][ground_truth_sorted_indices[g]]; match = g; } } // if match was made, store id of match for both detection and // ground truth if (match >= 0) { detection_ignores[t * num_detections + d] = ground_truth_ignores[match]; detection_matches[t * num_detections + d] = ground_truth_instances[ground_truth_sorted_indices[match]].id; ground_truth_matches[t * num_ground_truth + match] = detection_instances[detection_sorted_indices[d]].id; } // set unmatched detections outside of area range to ignore const InstanceAnnotation& detection = detection_instances[detection_sorted_indices[d]]; detection_ignores[t * num_detections + d] = detection_ignores[t * num_detections + d] || (detection_matches[t * num_detections + d] == 0 && (detection.area < area_range[0] || detection.area > area_range[1])); } } // store detection score results results->detection_scores.resize(detection_sorted_indices.size()); for (size_t d = 0; d < detection_sorted_indices.size(); ++d) { results->detection_scores[d] = detection_instances[detection_sorted_indices[d]].score; } } std::vector<ImageEvaluation> EvaluateImages( const std::vector<std::array<double, 2>>& area_ranges, int max_detections, const std::vector<double>& iou_thresholds, const ImageCategoryInstances<std::vector<double>>& image_category_ious, const ImageCategoryInstances<InstanceAnnotation>& image_category_ground_truth_instances, const ImageCategoryInstances<InstanceAnnotation>& image_category_detection_instances) { const int num_area_ranges = area_ranges.size(); const int num_images = image_category_ground_truth_instances.size(); const int num_categories = image_category_ious.size() > 0 ? image_category_ious[0].size() : 0; std::vector<uint64_t> detection_sorted_indices; std::vector<uint64_t> ground_truth_sorted_indices; std::vector<bool> ignores; std::vector<ImageEvaluation> results_all( num_images * num_area_ranges * num_categories); // Store results for each image, category, and area range combination. Results // for each IOU threshold are packed into the same ImageEvaluation object for (auto i = 0; i < num_images; ++i) { for (auto c = 0; c < num_categories; ++c) { const std::vector<InstanceAnnotation>& ground_truth_instances = image_category_ground_truth_instances[i][c]; const std::vector<InstanceAnnotation>& detection_instances = image_category_detection_instances[i][c]; SortInstancesByDetectionScore( detection_instances, &detection_sorted_indices); if ((int)detection_sorted_indices.size() > max_detections) { detection_sorted_indices.resize(max_detections); } for (size_t a = 0; a < area_ranges.size(); ++a) { SortInstancesByIgnore( area_ranges[a], ground_truth_instances, &ground_truth_sorted_indices, &ignores); MatchDetectionsToGroundTruth( detection_instances, detection_sorted_indices, ground_truth_instances, ground_truth_sorted_indices, ignores, image_category_ious[i][c], iou_thresholds, area_ranges[a], &results_all [c * num_area_ranges *