workflows.ensemble_selection
1import logging 2import time 3 4import numpy as np 5from fiftyone import ViewField as F 6from torch.utils.tensorboard import SummaryWriter 7from tqdm import tqdm 8 9 10def calculate_iou(box1, box2): 11 """Calculate the Intersection over Union (IoU) of two bounding boxes.""" 12 # box format: [minx, miny, width, height] normalized between [0,1] 13 x1_min, y1_min, w1, h1 = box1 14 x2_min, y2_min, w2, h2 = box2 15 x1_max, y1_max = x1_min + w1, y1_min + h1 16 x2_max, y2_max = x2_min + w2, y2_min + h2 17 18 # Calculate the intersection box 19 inter_x_min = max(x1_min, x2_min) 20 inter_y_min = max(y1_min, y2_min) 21 inter_x_max = min(x1_max, x2_max) 22 inter_y_max = min(y1_max, y2_max) 23 24 if inter_x_min < inter_x_max and inter_y_min < inter_y_max: 25 inter_area = (inter_x_max - inter_x_min) * (inter_y_max - inter_y_min) 26 else: 27 inter_area = 0 28 29 box1_area = w1 * h1 30 box2_area = w2 * h2 31 union_area = box1_area + box2_area - inter_area 32 33 return inter_area / union_area if union_area > 0 else 0 34 35 36def calculcate_bbox_size(box): 37 """Calculate the size of a bounding box.""" 38 return box[2] * box[3] 39 40 41class EnsembleSelection: 42 """Performs ensemble selection to identify overlapping detections from multiple models based on IoU and agreement thresholds.""" 43 44 def __init__(self, dataset, config): 45 """Initializes ensemble selection with dataset and configuration for detection agreement analysis.""" 46 self.dataset = dataset 47 self.config = config 48 self.positive_classes = config["positive_classes"] 49 self.agreement_threshold = config["agreement_threshold"] 50 self.iou_threshold = config["iou_threshold"] 51 self.max_bbox_size = config["max_bbox_size"] 52 pred_prefix = config["field_includes"] 53 self.v51_agreement_tag = "detections_overlap" 54 self.n_unique_field = "n_unique_ensemble_selection" 55 self.n_unique_id_field = "n_unique_id_ensemble_selection" 56 57 logging.info( 58 f"Collecting detections of fields with prefix '{pred_prefix}'. Successful detections will be tagged with '{self.v51_agreement_tag}'." 59 ) 60 61 # Get V51 fields that store detection results 62 self.v51_detection_fields = [] 63 dataset_schema = self.dataset.get_field_schema() 64 for field in tqdm( 65 dataset_schema, desc="Collecting dataset fields with detections" 66 ): 67 if pred_prefix in field: 68 self.v51_detection_fields.append(field) 69 70 # Make sure that some classes in 'positive_classes' are part of the classes used for detections 71 detection_classes = self.dataset.distinct("%s.detections.label" % field) 72 pos_set = set(self.positive_classes) 73 det_set = set(detection_classes) 74 common_classes = pos_set & det_set 75 if common_classes: 76 logging.info( 77 f"The classes {common_classes} are shared between the list of positive classes and detections in '{field}'." 78 ) 79 else: 80 logging.warning( 81 f"No classes in the list of positive classes {pos_set} are part of the classes used for detections in '{field}'." 82 ) 83 84 if len(self.v51_detection_fields) < self.agreement_threshold: 85 logging.error( 86 f"Number of detection models used ({len(self.v51_detection_fields)}) is less than the agreement threshold ({self.agreement_threshold}). No agreements will be possible. Detections are expected in the field {pred_prefix}. Detections can be generated with the workflow `auto_labeling_zero_shot`" 87 ) 88 89 # Get filtered V51 view for faster processing 90 conditions = [ 91 (F(f"{field}") != None) # Field exists 92 & (F(f"{field}.detections") != []) # Field has detections 93 & F(f"{field}.detections.label").contains( 94 self.positive_classes 95 ) # Detections include "cat" or "dog" 96 for field in self.v51_detection_fields 97 ] 98 99 self.view = self.dataset.match(F.any(conditions)) 100 101 for field in tqdm( 102 self.v51_detection_fields, 103 desc="Generating filtered Voxel51 view for fast processing", 104 ): 105 self.view = self.view.filter_labels( 106 f"{field}.detections", 107 F("label").is_in(self.positive_classes), 108 only_matches=False, 109 ) 110 self.n_samples = len(self.view) 111 112 def ensemble_selection(self): 113 """Selects and tags overlapping detections from multiple models using IoU-based ensemble method, tracking unique VRU detections.""" 114 115 writer = SummaryWriter(log_dir="logs/tensorboard/ensemble_selection") 116 117 # Get detections from V51 with efficient "values" method 118 samples_detections = [] # List of lists of list [model][sample][detections] 119 for field in tqdm( 120 self.v51_detection_fields, desc="Collecting model detections" 121 ): 122 field_detections = self.view.values( 123 f"{field}.detections" 124 ) # list of lists of detections per sample 125 samples_detections.append(field_detections) 126 127 # Cleaning up tags from previous runs 128 for i in tqdm(range(self.n_samples), desc="Cleaning up tags"): 129 for j in range(len(self.v51_detection_fields)): 130 detections = samples_detections[j][i] 131 if detections: 132 for k in range(len(detections)): 133 samples_detections[j][i][k].tags = [ 134 x 135 for x in samples_detections[j][i][k].tags 136 if x != self.v51_agreement_tag 137 ] 138 139 # Counting variables 140 n_bboxes_agreed = 0 141 n_samples_agreed = 0 142 n_unique_vru = 0 143 144 # Iterate over all samples and check overlapping detections 145 for step, sample_index in enumerate( 146 tqdm(range(self.n_samples), desc="Finding detection agreements in samples") 147 ): 148 start_time = time.time() 149 unique_vru_detections_set = set() 150 all_bboxes = [] # List of all bounding box detections per sample 151 bbox_model_indices = [] # Track which model each bounding box belongs to 152 bbox_detection_indices = ( 153 [] 154 ) # Track the index of each detection in the model's list 155 156 for model_index, model_detections in enumerate(samples_detections): 157 for detection_index, det in enumerate(model_detections[sample_index]): 158 all_bboxes.append(det.bounding_box) 159 bbox_model_indices.append(model_index) 160 bbox_detection_indices.append(detection_index) 161 162 n_bboxes = len(all_bboxes) 163 if n_bboxes == 0: 164 logging.warning(f"No detections found in sample {sample_index}.") 165 continue 166 167 # Compute IoU between all bounding boxes and store additional information 168 involved_models_matrix = np.full((n_bboxes, n_bboxes), -1) 169 bbox_ids_matrix = np.full((n_bboxes, n_bboxes), -1) 170 for a in range(n_bboxes): 171 for b in range(a + 1, n_bboxes): 172 iou = calculate_iou(all_bboxes[a], all_bboxes[b]) 173 # Only compare detections of small bounding boxes 174 if ( 175 self.max_bbox_size 176 and calculcate_bbox_size(all_bboxes[a]) <= self.max_bbox_size 177 and calculcate_bbox_size(all_bboxes[b]) <= self.max_bbox_size 178 ): 179 # Only compare detections of high IoU scores 180 if iou > self.iou_threshold: 181 involved_models_matrix[a, b] = bbox_model_indices[ 182 b 183 ] # Store model index that was compared to 184 involved_models_matrix[b, a] = bbox_model_indices[a] 185 involved_models_matrix[a, a] = bbox_model_indices[ 186 a 187 ] # Trick to also store the model itself, as the diagonal is not used (b = a+1, symmetry) 188 involved_models_matrix[b, b] = bbox_model_indices[b] 189 190 bbox_ids_matrix[a, b] = ( 191 b # Store detection indices to get involved bounding_boxes 192 ) 193 bbox_ids_matrix[b, a] = a 194 bbox_ids_matrix[a, a] = a 195 bbox_ids_matrix[b, b] = b 196 197 # Get number of involved models by finding unique values in rows 198 # "-1" ist not an involved model 199 involved_models = [np.unique(row) for row in involved_models_matrix] 200 for index in range(len(involved_models)): 201 involved_models[index] = involved_models[index][ 202 involved_models[index] != -1 203 ] 204 205 # Get list of involved bounding box indices 206 involved_bboxes = [np.unique(row) for row in bbox_ids_matrix] 207 for index in range(len(bbox_ids_matrix)): 208 involved_bboxes[index] = involved_bboxes[index][ 209 involved_bboxes[index] != -1 210 ] 211 212 # Checking that all arrays have the same lengths 213 if not ( 214 len(all_bboxes) 215 == len(bbox_model_indices) 216 == len(bbox_detection_indices) 217 == len(involved_models) 218 ): 219 logging.error( 220 "Array lengths mismatch: all_bboxes(%d), bbox_model_indices(%d), bbox_detection_indices(%d), involved_models(%d)", 221 len(all_bboxes), 222 len(bbox_model_indices), 223 len(bbox_detection_indices), 224 len(involved_models), 225 ) 226 227 # Check bbox detections for agreements 228 for index in range(n_bboxes): 229 model_indices = involved_models[index] 230 bbox_indices = involved_bboxes[index] 231 all_connected_boxes = bbox_indices 232 233 if len(model_indices) >= self.agreement_threshold: 234 # Get all involved bounding boxe indices 235 for bbox_index in bbox_indices: 236 connected_bboxes = involved_bboxes[bbox_index] 237 all_connected_boxes = np.unique( 238 np.concatenate((all_connected_boxes, connected_bboxes)) 239 ) 240 unique_detection_id = np.min(all_connected_boxes) 241 242 # If bounding box has not been processed yet 243 if unique_detection_id not in unique_vru_detections_set: 244 unique_vru_detections_set.add(unique_detection_id) 245 # Set V51 tag to all connected boxes 246 for bbox_index in all_connected_boxes: 247 model_index = bbox_model_indices[bbox_index] 248 det_index = bbox_detection_indices[bbox_index] 249 samples_detections[model_index][sample_index][ 250 det_index 251 ].tags.append(self.v51_agreement_tag) 252 samples_detections[model_index][sample_index][det_index][ 253 self.n_unique_id_field 254 ] = unique_detection_id 255 n_bboxes_agreed += 1 256 257 if len(unique_vru_detections_set) > 0: 258 n_samples_agreed += 1 259 n_unique_vru += len(unique_vru_detections_set) 260 261 # Log inference performance 262 end_time = time.time() 263 sample_duration = end_time - start_time 264 frames_per_second = 1 / sample_duration 265 writer.add_scalar("inference/frames_per_second", frames_per_second, step) 266 267 # Save results to dataset 268 for field, field_detections in tqdm( 269 zip(self.v51_detection_fields, samples_detections), 270 desc="Saving results to dataset", 271 total=len(self.v51_detection_fields), 272 ): 273 self.view.set_values(field + ".detections", field_detections) 274 275 logging.info("Calculate number of unique detections per sample") 276 try: 277 self.dataset.delete_sample_field(self.n_unique_field) 278 except: 279 pass 280 view_tagged = self.view.select_labels(tags=self.v51_agreement_tag) 281 for sample in view_tagged.iter_samples(progress=True, autosave=True): 282 n_unique_set = set() 283 for field in self.v51_detection_fields: 284 detections = sample[field].detections 285 n_unique_set.update(d[self.n_unique_id_field] for d in detections) 286 sample[self.n_unique_field] = len(n_unique_set) 287 288 logging.info( 289 f"Found {n_unique_vru} unique detections in {n_samples_agreed} samples. Based on {n_bboxes_agreed} total detections with {self.agreement_threshold} or more overlapping detections." 290 )
def
calculate_iou(box1, box2):
11def calculate_iou(box1, box2): 12 """Calculate the Intersection over Union (IoU) of two bounding boxes.""" 13 # box format: [minx, miny, width, height] normalized between [0,1] 14 x1_min, y1_min, w1, h1 = box1 15 x2_min, y2_min, w2, h2 = box2 16 x1_max, y1_max = x1_min + w1, y1_min + h1 17 x2_max, y2_max = x2_min + w2, y2_min + h2 18 19 # Calculate the intersection box 20 inter_x_min = max(x1_min, x2_min) 21 inter_y_min = max(y1_min, y2_min) 22 inter_x_max = min(x1_max, x2_max) 23 inter_y_max = min(y1_max, y2_max) 24 25 if inter_x_min < inter_x_max and inter_y_min < inter_y_max: 26 inter_area = (inter_x_max - inter_x_min) * (inter_y_max - inter_y_min) 27 else: 28 inter_area = 0 29 30 box1_area = w1 * h1 31 box2_area = w2 * h2 32 union_area = box1_area + box2_area - inter_area 33 34 return inter_area / union_area if union_area > 0 else 0
Calculate the Intersection over Union (IoU) of two bounding boxes.
def
calculcate_bbox_size(box):
37def calculcate_bbox_size(box): 38 """Calculate the size of a bounding box.""" 39 return box[2] * box[3]
Calculate the size of a bounding box.
class
EnsembleSelection:
42class EnsembleSelection: 43 """Performs ensemble selection to identify overlapping detections from multiple models based on IoU and agreement thresholds.""" 44 45 def __init__(self, dataset, config): 46 """Initializes ensemble selection with dataset and configuration for detection agreement analysis.""" 47 self.dataset = dataset 48 self.config = config 49 self.positive_classes = config["positive_classes"] 50 self.agreement_threshold = config["agreement_threshold"] 51 self.iou_threshold = config["iou_threshold"] 52 self.max_bbox_size = config["max_bbox_size"] 53 pred_prefix = config["field_includes"] 54 self.v51_agreement_tag = "detections_overlap" 55 self.n_unique_field = "n_unique_ensemble_selection" 56 self.n_unique_id_field = "n_unique_id_ensemble_selection" 57 58 logging.info( 59 f"Collecting detections of fields with prefix '{pred_prefix}'. Successful detections will be tagged with '{self.v51_agreement_tag}'." 60 ) 61 62 # Get V51 fields that store detection results 63 self.v51_detection_fields = [] 64 dataset_schema = self.dataset.get_field_schema() 65 for field in tqdm( 66 dataset_schema, desc="Collecting dataset fields with detections" 67 ): 68 if pred_prefix in field: 69 self.v51_detection_fields.append(field) 70 71 # Make sure that some classes in 'positive_classes' are part of the classes used for detections 72 detection_classes = self.dataset.distinct("%s.detections.label" % field) 73 pos_set = set(self.positive_classes) 74 det_set = set(detection_classes) 75 common_classes = pos_set & det_set 76 if common_classes: 77 logging.info( 78 f"The classes {common_classes} are shared between the list of positive classes and detections in '{field}'." 79 ) 80 else: 81 logging.warning( 82 f"No classes in the list of positive classes {pos_set} are part of the classes used for detections in '{field}'." 83 ) 84 85 if len(self.v51_detection_fields) < self.agreement_threshold: 86 logging.error( 87 f"Number of detection models used ({len(self.v51_detection_fields)}) is less than the agreement threshold ({self.agreement_threshold}). No agreements will be possible. Detections are expected in the field {pred_prefix}. Detections can be generated with the workflow `auto_labeling_zero_shot`" 88 ) 89 90 # Get filtered V51 view for faster processing 91 conditions = [ 92 (F(f"{field}") != None) # Field exists 93 & (F(f"{field}.detections") != []) # Field has detections 94 & F(f"{field}.detections.label").contains( 95 self.positive_classes 96 ) # Detections include "cat" or "dog" 97 for field in self.v51_detection_fields 98 ] 99 100 self.view = self.dataset.match(F.any(conditions)) 101 102 for field in tqdm( 103 self.v51_detection_fields, 104 desc="Generating filtered Voxel51 view for fast processing", 105 ): 106 self.view = self.view.filter_labels( 107 f"{field}.detections", 108 F("label").is_in(self.positive_classes), 109 only_matches=False, 110 ) 111 self.n_samples = len(self.view) 112 113 def ensemble_selection(self): 114 """Selects and tags overlapping detections from multiple models using IoU-based ensemble method, tracking unique VRU detections.""" 115 116 writer = SummaryWriter(log_dir="logs/tensorboard/ensemble_selection") 117 118 # Get detections from V51 with efficient "values" method 119 samples_detections = [] # List of lists of list [model][sample][detections] 120 for field in tqdm( 121 self.v51_detection_fields, desc="Collecting model detections" 122 ): 123 field_detections = self.view.values( 124 f"{field}.detections" 125 ) # list of lists of detections per sample 126 samples_detections.append(field_detections) 127 128 # Cleaning up tags from previous runs 129 for i in tqdm(range(self.n_samples), desc="Cleaning up tags"): 130 for j in range(len(self.v51_detection_fields)): 131 detections = samples_detections[j][i] 132 if detections: 133 for k in range(len(detections)): 134 samples_detections[j][i][k].tags = [ 135 x 136 for x in samples_detections[j][i][k].tags 137 if x != self.v51_agreement_tag 138 ] 139 140 # Counting variables 141 n_bboxes_agreed = 0 142 n_samples_agreed = 0 143 n_unique_vru = 0 144 145 # Iterate over all samples and check overlapping detections 146 for step, sample_index in enumerate( 147 tqdm(range(self.n_samples), desc="Finding detection agreements in samples") 148 ): 149 start_time = time.time() 150 unique_vru_detections_set = set() 151 all_bboxes = [] # List of all bounding box detections per sample 152 bbox_model_indices = [] # Track which model each bounding box belongs to 153 bbox_detection_indices = ( 154 [] 155 ) # Track the index of each detection in the model's list 156 157 for model_index, model_detections in enumerate(samples_detections): 158 for detection_index, det in enumerate(model_detections[sample_index]): 159 all_bboxes.append(det.bounding_box) 160 bbox_model_indices.append(model_index) 161 bbox_detection_indices.append(detection_index) 162 163 n_bboxes = len(all_bboxes) 164 if n_bboxes == 0: 165 logging.warning(f"No detections found in sample {sample_index}.") 166 continue 167 168 # Compute IoU between all bounding boxes and store additional information 169 involved_models_matrix = np.full((n_bboxes, n_bboxes), -1) 170 bbox_ids_matrix = np.full((n_bboxes, n_bboxes), -1) 171 for a in range(n_bboxes): 172 for b in range(a + 1, n_bboxes): 173 iou = calculate_iou(all_bboxes[a], all_bboxes[b]) 174 # Only compare detections of small bounding boxes 175 if ( 176 self.max_bbox_size 177 and calculcate_bbox_size(all_bboxes[a]) <= self.max_bbox_size 178 and calculcate_bbox_size(all_bboxes[b]) <= self.max_bbox_size 179 ): 180 # Only compare detections of high IoU scores 181 if iou > self.iou_threshold: 182 involved_models_matrix[a, b] = bbox_model_indices[ 183 b 184 ] # Store model index that was compared to 185 involved_models_matrix[b, a] = bbox_model_indices[a] 186 involved_models_matrix[a, a] = bbox_model_indices[ 187 a 188 ] # Trick to also store the model itself, as the diagonal is not used (b = a+1, symmetry) 189 involved_models_matrix[b, b] = bbox_model_indices[b] 190 191 bbox_ids_matrix[a, b] = ( 192 b # Store detection indices to get involved bounding_boxes 193 ) 194 bbox_ids_matrix[b, a] = a 195 bbox_ids_matrix[a, a] = a 196 bbox_ids_matrix[b, b] = b 197 198 # Get number of involved models by finding unique values in rows 199 # "-1" ist not an involved model 200 involved_models = [np.unique(row) for row in involved_models_matrix] 201 for index in range(len(involved_models)): 202 involved_models[index] = involved_models[index][ 203 involved_models[index] != -1 204 ] 205 206 # Get list of involved bounding box indices 207 involved_bboxes = [np.unique(row) for row in bbox_ids_matrix] 208 for index in range(len(bbox_ids_matrix)): 209 involved_bboxes[index] = involved_bboxes[index][ 210 involved_bboxes[index] != -1 211 ] 212 213 # Checking that all arrays have the same lengths 214 if not ( 215 len(all_bboxes) 216 == len(bbox_model_indices) 217 == len(bbox_detection_indices) 218 == len(involved_models) 219 ): 220 logging.error( 221 "Array lengths mismatch: all_bboxes(%d), bbox_model_indices(%d), bbox_detection_indices(%d), involved_models(%d)", 222 len(all_bboxes), 223 len(bbox_model_indices), 224 len(bbox_detection_indices), 225 len(involved_models), 226 ) 227 228 # Check bbox detections for agreements 229 for index in range(n_bboxes): 230 model_indices = involved_models[index] 231 bbox_indices = involved_bboxes[index] 232 all_connected_boxes = bbox_indices 233 234 if len(model_indices) >= self.agreement_threshold: 235 # Get all involved bounding boxe indices 236 for bbox_index in bbox_indices: 237 connected_bboxes = involved_bboxes[bbox_index] 238 all_connected_boxes = np.unique( 239 np.concatenate((all_connected_boxes, connected_bboxes)) 240 ) 241 unique_detection_id = np.min(all_connected_boxes) 242 243 # If bounding box has not been processed yet 244 if unique_detection_id not in unique_vru_detections_set: 245 unique_vru_detections_set.add(unique_detection_id) 246 # Set V51 tag to all connected boxes 247 for bbox_index in all_connected_boxes: 248 model_index = bbox_model_indices[bbox_index] 249 det_index = bbox_detection_indices[bbox_index] 250 samples_detections[model_index][sample_index][ 251 det_index 252 ].tags.append(self.v51_agreement_tag) 253 samples_detections[model_index][sample_index][det_index][ 254 self.n_unique_id_field 255 ] = unique_detection_id 256 n_bboxes_agreed += 1 257 258 if len(unique_vru_detections_set) > 0: 259 n_samples_agreed += 1 260 n_unique_vru += len(unique_vru_detections_set) 261 262 # Log inference performance 263 end_time = time.time() 264 sample_duration = end_time - start_time 265 frames_per_second = 1 / sample_duration 266 writer.add_scalar("inference/frames_per_second", frames_per_second, step) 267 268 # Save results to dataset 269 for field, field_detections in tqdm( 270 zip(self.v51_detection_fields, samples_detections), 271 desc="Saving results to dataset", 272 total=len(self.v51_detection_fields), 273 ): 274 self.view.set_values(field + ".detections", field_detections) 275 276 logging.info("Calculate number of unique detections per sample") 277 try: 278 self.dataset.delete_sample_field(self.n_unique_field) 279 except: 280 pass 281 view_tagged = self.view.select_labels(tags=self.v51_agreement_tag) 282 for sample in view_tagged.iter_samples(progress=True, autosave=True): 283 n_unique_set = set() 284 for field in self.v51_detection_fields: 285 detections = sample[field].detections 286 n_unique_set.update(d[self.n_unique_id_field] for d in detections) 287 sample[self.n_unique_field] = len(n_unique_set) 288 289 logging.info( 290 f"Found {n_unique_vru} unique detections in {n_samples_agreed} samples. Based on {n_bboxes_agreed} total detections with {self.agreement_threshold} or more overlapping detections." 291 )
Performs ensemble selection to identify overlapping detections from multiple models based on IoU and agreement thresholds.
EnsembleSelection(dataset, config)
45 def __init__(self, dataset, config): 46 """Initializes ensemble selection with dataset and configuration for detection agreement analysis.""" 47 self.dataset = dataset 48 self.config = config 49 self.positive_classes = config["positive_classes"] 50 self.agreement_threshold = config["agreement_threshold"] 51 self.iou_threshold = config["iou_threshold"] 52 self.max_bbox_size = config["max_bbox_size"] 53 pred_prefix = config["field_includes"] 54 self.v51_agreement_tag = "detections_overlap" 55 self.n_unique_field = "n_unique_ensemble_selection" 56 self.n_unique_id_field = "n_unique_id_ensemble_selection" 57 58 logging.info( 59 f"Collecting detections of fields with prefix '{pred_prefix}'. Successful detections will be tagged with '{self.v51_agreement_tag}'." 60 ) 61 62 # Get V51 fields that store detection results 63 self.v51_detection_fields = [] 64 dataset_schema = self.dataset.get_field_schema() 65 for field in tqdm( 66 dataset_schema, desc="Collecting dataset fields with detections" 67 ): 68 if pred_prefix in field: 69 self.v51_detection_fields.append(field) 70 71 # Make sure that some classes in 'positive_classes' are part of the classes used for detections 72 detection_classes = self.dataset.distinct("%s.detections.label" % field) 73 pos_set = set(self.positive_classes) 74 det_set = set(detection_classes) 75 common_classes = pos_set & det_set 76 if common_classes: 77 logging.info( 78 f"The classes {common_classes} are shared between the list of positive classes and detections in '{field}'." 79 ) 80 else: 81 logging.warning( 82 f"No classes in the list of positive classes {pos_set} are part of the classes used for detections in '{field}'." 83 ) 84 85 if len(self.v51_detection_fields) < self.agreement_threshold: 86 logging.error( 87 f"Number of detection models used ({len(self.v51_detection_fields)}) is less than the agreement threshold ({self.agreement_threshold}). No agreements will be possible. Detections are expected in the field {pred_prefix}. Detections can be generated with the workflow `auto_labeling_zero_shot`" 88 ) 89 90 # Get filtered V51 view for faster processing 91 conditions = [ 92 (F(f"{field}") != None) # Field exists 93 & (F(f"{field}.detections") != []) # Field has detections 94 & F(f"{field}.detections.label").contains( 95 self.positive_classes 96 ) # Detections include "cat" or "dog" 97 for field in self.v51_detection_fields 98 ] 99 100 self.view = self.dataset.match(F.any(conditions)) 101 102 for field in tqdm( 103 self.v51_detection_fields, 104 desc="Generating filtered Voxel51 view for fast processing", 105 ): 106 self.view = self.view.filter_labels( 107 f"{field}.detections", 108 F("label").is_in(self.positive_classes), 109 only_matches=False, 110 ) 111 self.n_samples = len(self.view)
Initializes ensemble selection with dataset and configuration for detection agreement analysis.
def
ensemble_selection(self):
113 def ensemble_selection(self): 114 """Selects and tags overlapping detections from multiple models using IoU-based ensemble method, tracking unique VRU detections.""" 115 116 writer = SummaryWriter(log_dir="logs/tensorboard/ensemble_selection") 117 118 # Get detections from V51 with efficient "values" method 119 samples_detections = [] # List of lists of list [model][sample][detections] 120 for field in tqdm( 121 self.v51_detection_fields, desc="Collecting model detections" 122 ): 123 field_detections = self.view.values( 124 f"{field}.detections" 125 ) # list of lists of detections per sample 126 samples_detections.append(field_detections) 127 128 # Cleaning up tags from previous runs 129 for i in tqdm(range(self.n_samples), desc="Cleaning up tags"): 130 for j in range(len(self.v51_detection_fields)): 131 detections = samples_detections[j][i] 132 if detections: 133 for k in range(len(detections)): 134 samples_detections[j][i][k].tags = [ 135 x 136 for x in samples_detections[j][i][k].tags 137 if x != self.v51_agreement_tag 138 ] 139 140 # Counting variables 141 n_bboxes_agreed = 0 142 n_samples_agreed = 0 143 n_unique_vru = 0 144 145 # Iterate over all samples and check overlapping detections 146 for step, sample_index in enumerate( 147 tqdm(range(self.n_samples), desc="Finding detection agreements in samples") 148 ): 149 start_time = time.time() 150 unique_vru_detections_set = set() 151 all_bboxes = [] # List of all bounding box detections per sample 152 bbox_model_indices = [] # Track which model each bounding box belongs to 153 bbox_detection_indices = ( 154 [] 155 ) # Track the index of each detection in the model's list 156 157 for model_index, model_detections in enumerate(samples_detections): 158 for detection_index, det in enumerate(model_detections[sample_index]): 159 all_bboxes.append(det.bounding_box) 160 bbox_model_indices.append(model_index) 161 bbox_detection_indices.append(detection_index) 162 163 n_bboxes = len(all_bboxes) 164 if n_bboxes == 0: 165 logging.warning(f"No detections found in sample {sample_index}.") 166 continue 167 168 # Compute IoU between all bounding boxes and store additional information 169 involved_models_matrix = np.full((n_bboxes, n_bboxes), -1) 170 bbox_ids_matrix = np.full((n_bboxes, n_bboxes), -1) 171 for a in range(n_bboxes): 172 for b in range(a + 1, n_bboxes): 173 iou = calculate_iou(all_bboxes[a], all_bboxes[b]) 174 # Only compare detections of small bounding boxes 175 if ( 176 self.max_bbox_size 177 and calculcate_bbox_size(all_bboxes[a]) <= self.max_bbox_size 178 and calculcate_bbox_size(all_bboxes[b]) <= self.max_bbox_size 179 ): 180 # Only compare detections of high IoU scores 181 if iou > self.iou_threshold: 182 involved_models_matrix[a, b] = bbox_model_indices[ 183 b 184 ] # Store model index that was compared to 185 involved_models_matrix[b, a] = bbox_model_indices[a] 186 involved_models_matrix[a, a] = bbox_model_indices[ 187 a 188 ] # Trick to also store the model itself, as the diagonal is not used (b = a+1, symmetry) 189 involved_models_matrix[b, b] = bbox_model_indices[b] 190 191 bbox_ids_matrix[a, b] = ( 192 b # Store detection indices to get involved bounding_boxes 193 ) 194 bbox_ids_matrix[b, a] = a 195 bbox_ids_matrix[a, a] = a 196 bbox_ids_matrix[b, b] = b 197 198 # Get number of involved models by finding unique values in rows 199 # "-1" ist not an involved model 200 involved_models = [np.unique(row) for row in involved_models_matrix] 201 for index in range(len(involved_models)): 202 involved_models[index] = involved_models[index][ 203 involved_models[index] != -1 204 ] 205 206 # Get list of involved bounding box indices 207 involved_bboxes = [np.unique(row) for row in bbox_ids_matrix] 208 for index in range(len(bbox_ids_matrix)): 209 involved_bboxes[index] = involved_bboxes[index][ 210 involved_bboxes[index] != -1 211 ] 212 213 # Checking that all arrays have the same lengths 214 if not ( 215 len(all_bboxes) 216 == len(bbox_model_indices) 217 == len(bbox_detection_indices) 218 == len(involved_models) 219 ): 220 logging.error( 221 "Array lengths mismatch: all_bboxes(%d), bbox_model_indices(%d), bbox_detection_indices(%d), involved_models(%d)", 222 len(all_bboxes), 223 len(bbox_model_indices), 224 len(bbox_detection_indices), 225 len(involved_models), 226 ) 227 228 # Check bbox detections for agreements 229 for index in range(n_bboxes): 230 model_indices = involved_models[index] 231 bbox_indices = involved_bboxes[index] 232 all_connected_boxes = bbox_indices 233 234 if len(model_indices) >= self.agreement_threshold: 235 # Get all involved bounding boxe indices 236 for bbox_index in bbox_indices: 237 connected_bboxes = involved_bboxes[bbox_index] 238 all_connected_boxes = np.unique( 239 np.concatenate((all_connected_boxes, connected_bboxes)) 240 ) 241 unique_detection_id = np.min(all_connected_boxes) 242 243 # If bounding box has not been processed yet 244 if unique_detection_id not in unique_vru_detections_set: 245 unique_vru_detections_set.add(unique_detection_id) 246 # Set V51 tag to all connected boxes 247 for bbox_index in all_connected_boxes: 248 model_index = bbox_model_indices[bbox_index] 249 det_index = bbox_detection_indices[bbox_index] 250 samples_detections[model_index][sample_index][ 251 det_index 252 ].tags.append(self.v51_agreement_tag) 253 samples_detections[model_index][sample_index][det_index][ 254 self.n_unique_id_field 255 ] = unique_detection_id 256 n_bboxes_agreed += 1 257 258 if len(unique_vru_detections_set) > 0: 259 n_samples_agreed += 1 260 n_unique_vru += len(unique_vru_detections_set) 261 262 # Log inference performance 263 end_time = time.time() 264 sample_duration = end_time - start_time 265 frames_per_second = 1 / sample_duration 266 writer.add_scalar("inference/frames_per_second", frames_per_second, step) 267 268 # Save results to dataset 269 for field, field_detections in tqdm( 270 zip(self.v51_detection_fields, samples_detections), 271 desc="Saving results to dataset", 272 total=len(self.v51_detection_fields), 273 ): 274 self.view.set_values(field + ".detections", field_detections) 275 276 logging.info("Calculate number of unique detections per sample") 277 try: 278 self.dataset.delete_sample_field(self.n_unique_field) 279 except: 280 pass 281 view_tagged = self.view.select_labels(tags=self.v51_agreement_tag) 282 for sample in view_tagged.iter_samples(progress=True, autosave=True): 283 n_unique_set = set() 284 for field in self.v51_detection_fields: 285 detections = sample[field].detections 286 n_unique_set.update(d[self.n_unique_id_field] for d in detections) 287 sample[self.n_unique_field] = len(n_unique_set) 288 289 logging.info( 290 f"Found {n_unique_vru} unique detections in {n_samples_agreed} samples. Based on {n_bboxes_agreed} total detections with {self.agreement_threshold} or more overlapping detections." 291 )
Selects and tags overlapping detections from multiple models using IoU-based ensemble method, tracking unique VRU detections.