================================================ FILE: ultralytics/tracker/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from .track import register_tracker from .trackers import BOTSORT, BYTETracker __all__ = 'register_tracker', 'BOTSORT', 'BYTETracker' # allow simpler import ================================================ FILE: ultralytics/tracker/cfg/botsort.yaml ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license # Default YOLO tracker settings for BoT-SORT tracker https://github.com/NirAharon/BoT-SORT tracker_type: botsort # tracker type, ['botsort', 'bytetrack'] track_high_thresh: 0.5 # threshold for the first association track_low_thresh: 0.1 # threshold for the second association new_track_thresh: 0.6 # threshold for init new track if the detection does not match any tracks track_buffer: 30 # buffer to calculate the time when to remove tracks match_thresh: 0.8 # threshold for matching tracks # min_box_area: 10 # threshold for min box areas(for tracker evaluation, not used for now) # mot20: False # for tracker evaluation(not used for now) # BoT-SORT settings cmc_method: sparseOptFlow # method of global motion compensation # ReID model related thresh (not supported yet) proximity_thresh: 0.5 appearance_thresh: 0.25 with_reid: False ================================================ FILE: ultralytics/tracker/cfg/bytetrack.yaml ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license # Default YOLO tracker settings for ByteTrack tracker https://github.com/ifzhang/ByteTrack tracker_type: bytetrack # tracker type, ['botsort', 'bytetrack'] track_high_thresh: 0.5 # threshold for the first association track_low_thresh: 0.1 # threshold for the second association new_track_thresh: 0.6 # threshold for init new track if the detection does not match any tracks track_buffer: 30 # buffer to calculate the time when to remove tracks match_thresh: 0.8 # threshold for matching tracks # min_box_area: 10 # threshold for min box areas(for tracker evaluation, not used for now) # mot20: False # for tracker evaluation(not used for now) ================================================ FILE: ultralytics/tracker/track.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from functools import partial import torch from ultralytics.yolo.utils import IterableSimpleNamespace, yaml_load from ultralytics.yolo.utils.checks import check_yaml from .trackers import BOTSORT, BYTETracker TRACKER_MAP = {'bytetrack': BYTETracker, 'botsort': BOTSORT} def on_predict_start(predictor, persist=False): """ Initialize trackers for object tracking during prediction. Args: predictor (object): The predictor object to initialize trackers for. persist (bool, optional): Whether to persist the trackers if they already exist. Defaults to False. Raises: AssertionError: If the tracker_type is not 'bytetrack' or 'botsort'. """ if hasattr(predictor, 'trackers') and persist: return tracker = check_yaml(predictor.args.tracker) cfg = IterableSimpleNamespace(**yaml_load(tracker)) assert cfg.tracker_type in ['bytetrack', 'botsort'], \ f"Only support 'bytetrack' and 'botsort' for now, but got '{cfg.tracker_type}'" trackers = [] for _ in range(predictor.dataset.bs): tracker = TRACKER_MAP[cfg.tracker_type](args=cfg, frame_rate=30) trackers.append(tracker) predictor.trackers = trackers def on_predict_postprocess_end(predictor): """Postprocess detected boxes and update with object tracking.""" bs = predictor.dataset.bs im0s = predictor.batch[1] for i in range(bs): det = predictor.results[i].boxes.cpu().numpy() if len(det) == 0: continue tracks = predictor.trackers[i].update(det, im0s[i]) if len(tracks) == 0: continue idx = tracks[:, -1].astype(int) predictor.results[i] = predictor.results[i][idx] predictor.results[i].update(boxes=torch.as_tensor(tracks[:, :-1])) def register_tracker(model, persist): """ Register tracking callbacks to the model for object tracking during prediction. Args: model (object): The model object to register tracking callbacks for. persist (bool): Whether to persist the trackers if they already exist. """ model.add_callback('on_predict_start', partial(on_predict_start, persist=persist)) model.add_callback('on_predict_postprocess_end', on_predict_postprocess_end) ================================================ FILE: ultralytics/tracker/trackers/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from .bot_sort import BOTSORT from .byte_tracker import BYTETracker __all__ = 'BOTSORT', 'BYTETracker' # allow simpler import ================================================ FILE: ultralytics/tracker/trackers/basetrack.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from collections import OrderedDict import numpy as np class TrackState: """Enumeration of possible object tracking states.""" New = 0 Tracked = 1 Lost = 2 Removed = 3 class BaseTrack: """Base class for object tracking, handling basic track attributes and operations.""" _count = 0 track_id = 0 is_activated = False state = TrackState.New history = OrderedDict() features = [] curr_feature = None score = 0 start_frame = 0 frame_id = 0 time_since_update = 0 # Multi-camera location = (np.inf, np.inf) @property def end_frame(self): """Return the last frame ID of the track.""" return self.frame_id @staticmethod def next_id(): """Increment and return the global track ID counter.""" BaseTrack._count += 1 return BaseTrack._count def activate(self, *args): """Activate the track with the provided arguments.""" raise NotImplementedError def predict(self): """Predict the next state of the track.""" raise NotImplementedError def update(self, *args, **kwargs): """Update the track with new observations.""" raise NotImplementedError def mark_lost(self): """Mark the track as lost.""" self.state = TrackState.Lost def mark_removed(self): """Mark the track as removed.""" self.state = TrackState.Removed @staticmethod def reset_id(): """Reset the global track ID counter.""" BaseTrack._count = 0 ================================================ FILE: ultralytics/tracker/trackers/bot_sort.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from collections import deque import numpy as np from ..utils import matching from ..utils.gmc import GMC from ..utils.kalman_filter import KalmanFilterXYWH from .basetrack import TrackState from .byte_tracker import BYTETracker, STrack class BOTrack(STrack): shared_kalman = KalmanFilterXYWH() def __init__(self, tlwh, score, cls, feat=None, feat_history=50): """Initialize YOLOv8 object with temporal parameters, such as feature history, alpha and current features.""" super().__init__(tlwh, score, cls) self.smooth_feat = None self.curr_feat = None if feat is not None: self.update_features(feat) self.features = deque([], maxlen=feat_history) self.alpha = 0.9 def update_features(self, feat): """Update features vector and smooth it using exponential moving average.""" feat /= np.linalg.norm(feat) self.curr_feat = feat if self.smooth_feat is None: self.smooth_feat = feat else: self.smooth_feat = self.alpha * self.smooth_feat + (1 - self.alpha) * feat self.features.append(feat) self.smooth_feat /= np.linalg.norm(self.smooth_feat) def predict(self): """Predicts the mean and covariance using Kalman filter.""" mean_state = self.mean.copy() if self.state != TrackState.Tracked: mean_state[6] = 0 mean_state[7] = 0 self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) def re_activate(self, new_track, frame_id, new_id=False): """Reactivates a track with updated features and optionally assigns a new ID.""" if new_track.curr_feat is not None: self.update_features(new_track.curr_feat) super().re_activate(new_track, frame_id, new_id) def update(self, new_track, frame_id): """Update the YOLOv8 instance with new track and frame ID.""" if new_track.curr_feat is not None: self.update_features(new_track.curr_feat) super().update(new_track, frame_id) @property def tlwh(self): """Get current position in bounding box format `(top left x, top left y, width, height)`. """ if self.mean is None: return self._tlwh.copy() ret = self.mean[:4].copy() ret[:2] -= ret[2:] / 2 return ret @staticmethod def multi_predict(stracks): """Predicts the mean and covariance of multiple object tracks using shared Kalman filter.""" if len(stracks) <= 0: return multi_mean = np.asarray([st.mean.copy() for st in stracks]) multi_covariance = np.asarray([st.covariance for st in stracks]) for i, st in enumerate(stracks): if st.state != TrackState.Tracked: multi_mean[i][6] = 0 multi_mean[i][7] = 0 multi_mean, multi_covariance = BOTrack.shared_kalman.multi_predict(multi_mean, multi_covariance) for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): stracks[i].mean = mean stracks[i].covariance = cov def convert_coords(self, tlwh): """Converts Top-Left-Width-Height bounding box coordinates to X-Y-Width-Height format.""" return self.tlwh_to_xywh(tlwh) @staticmethod def tlwh_to_xywh(tlwh): """Convert bounding box to format `(center x, center y, width, height)`. """ ret = np.asarray(tlwh).copy() ret[:2] += ret[2:] / 2 return ret class BOTSORT(BYTETracker): def __init__(self, args, frame_rate=30): """Initialize YOLOv8 object with ReID module and GMC algorithm.""" super().__init__(args, frame_rate) # ReID module self.proximity_thresh = args.proximity_thresh self.appearance_thresh = args.appearance_thresh if args.with_reid: # Haven't supported BoT-SORT(reid) yet self.encoder = None # self.gmc = GMC(method=args.cmc_method, verbose=[args.name, args.ablation]) self.gmc = GMC(method=args.cmc_method) def get_kalmanfilter(self): """Returns an instance of KalmanFilterXYWH for object tracking.""" return KalmanFilterXYWH() def init_track(self, dets, scores, cls, img=None): """Initialize track with detections, scores, and classes.""" if len(dets) == 0: return [] if self.args.with_reid and self.encoder is not None: features_keep = self.encoder.inference(img, dets) return [BOTrack(xyxy, s, c, f) for (xyxy, s, c, f) in zip(dets, scores, cls, features_keep)] # detections else: return [BOTrack(xyxy, s, c) for (xyxy, s, c) in zip(dets, scores, cls)] # detections def get_dists(self, tracks, detections): """Get distances between tracks and detections using IoU and (optionally) ReID embeddings.""" dists = matching.iou_distance(tracks, detections) dists_mask = (dists > self.proximity_thresh) # TODO: mot20 # if not self.args.mot20: dists = matching.fuse_score(dists, detections) if self.args.with_reid and self.encoder is not None: emb_dists = matching.embedding_distance(tracks, detections) / 2.0 emb_dists[emb_dists > self.appearance_thresh] = 1.0 emb_dists[dists_mask] = 1.0 dists = np.minimum(dists, emb_dists) return dists def multi_predict(self, tracks): """Predict and track multiple objects with YOLOv8 model.""" BOTrack.multi_predict(tracks) ================================================ FILE: ultralytics/tracker/trackers/byte_tracker.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import numpy as np from ..utils import matching from ..utils.kalman_filter import KalmanFilterXYAH from .basetrack import BaseTrack, TrackState class STrack(BaseTrack): shared_kalman = KalmanFilterXYAH() def __init__(self, tlwh, score, cls): """wait activate.""" self._tlwh = np.asarray(self.tlbr_to_tlwh(tlwh[:-1]), dtype=np.float32) self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False self.score = score self.tracklet_len = 0 self.cls = cls self.idx = tlwh[-1] def predict(self): """Predicts mean and covariance using Kalman filter.""" mean_state = self.mean.copy() if self.state != TrackState.Tracked: mean_state[7] = 0 self.mean, self.covariance = self.kalman_filter.predict(mean_state, self.covariance) @staticmethod def multi_predict(stracks): """Perform multi-object predictive tracking using Kalman filter for given stracks.""" if len(stracks) <= 0: return multi_mean = np.asarray([st.mean.copy() for st in stracks]) multi_covariance = np.asarray([st.covariance for st in stracks]) for i, st in enumerate(stracks): if st.state != TrackState.Tracked: multi_mean[i][7] = 0 multi_mean, multi_covariance = STrack.shared_kalman.multi_predict(multi_mean, multi_covariance) for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): stracks[i].mean = mean stracks[i].covariance = cov @staticmethod def multi_gmc(stracks, H=np.eye(2, 3)): """Update state tracks positions and covariances using a homography matrix.""" if len(stracks) > 0: multi_mean = np.asarray([st.mean.copy() for st in stracks]) multi_covariance = np.asarray([st.covariance for st in stracks]) R = H[:2, :2] R8x8 = np.kron(np.eye(4, dtype=float), R) t = H[:2, 2] for i, (mean, cov) in enumerate(zip(multi_mean, multi_covariance)): mean = R8x8.dot(mean) mean[:2] += t cov = R8x8.dot(cov).dot(R8x8.transpose()) stracks[i].mean = mean stracks[i].covariance = cov def activate(self, kalman_filter, frame_id): """Start a new tracklet.""" self.kalman_filter = kalman_filter self.track_id = self.next_id() self.mean, self.covariance = self.kalman_filter.initiate(self.convert_coords(self._tlwh)) self.tracklet_len = 0 self.state = TrackState.Tracked if frame_id == 1: self.is_activated = True self.frame_id = frame_id self.start_frame = frame_id def re_activate(self, new_track, frame_id, new_id=False): """Reactivates a previously lost track with a new detection.""" self.mean, self.covariance = self.kalman_filter.update(self.mean, self.covariance, self.convert_coords(new_track.tlwh)) self.tracklet_len = 0 self.state = TrackState.Tracked self.is_activated = True self.frame_id = frame_id if new_id: self.track_id = self.next_id() self.score = new_track.score self.cls = new_track.cls self.idx = new_track.idx def update(self, new_track, frame_id): """ Update a matched track :type new_track: STrack :type frame_id: int :return: """ self.frame_id = frame_id self.tracklet_len += 1 new_tlwh = new_track.tlwh self.mean, self.covariance = self.kalman_filter.update(self.mean, self.covariance, self.convert_coords(new_tlwh)) self.state = TrackState.Tracked self.is_activated = True self.score = new_track.score self.cls = new_track.cls self.idx = new_track.idx def convert_coords(self, tlwh): """Convert a bounding box's top-left-width-height format to its x-y-angle-height equivalent.""" return self.tlwh_to_xyah(tlwh) @property def tlwh(self): """Get current position in bounding box format `(top left x, top left y, width, height)`. """ if self.mean is None: return self._tlwh.copy() ret = self.mean[:4].copy() ret[2] *= ret[3] ret[:2] -= ret[2:] / 2 return ret @property def tlbr(self): """Convert bounding box to format `(min x, min y, max x, max y)`, i.e., `(top left, bottom right)`. """ ret = self.tlwh.copy() ret[2:] += ret[:2] return ret @staticmethod def tlwh_to_xyah(tlwh): """Convert bounding box to format `(center x, center y, aspect ratio, height)`, where the aspect ratio is `width / height`. """ ret = np.asarray(tlwh).copy() ret[:2] += ret[2:] / 2 ret[2] /= ret[3] return ret @staticmethod def tlbr_to_tlwh(tlbr): """Converts top-left bottom-right format to top-left width height format.""" ret = np.asarray(tlbr).copy() ret[2:] -= ret[:2] return ret @staticmethod def tlwh_to_tlbr(tlwh): """Converts tlwh bounding box format to tlbr format.""" ret = np.asarray(tlwh).copy() ret[2:] += ret[:2] return ret def __repr__(self): """Return a string representation of the BYTETracker object with start and end frames and track ID.""" return f'OT_{self.track_id}_({self.start_frame}-{self.end_frame})' class BYTETracker: def __init__(self, args, frame_rate=30): """Initialize a YOLOv8 object to track objects with given arguments and frame rate.""" self.tracked_stracks = [] # type: list[STrack] self.lost_stracks = [] # type: list[STrack] self.removed_stracks = [] # type: list[STrack] self.frame_id = 0 self.args = args self.max_time_lost = int(frame_rate / 30.0 * args.track_buffer) self.kalman_filter = self.get_kalmanfilter() self.reset_id() def update(self, results, img=None): """Updates object tracker with new detections and returns tracked object bounding boxes.""" self.frame_id += 1 activated_stracks = [] refind_stracks = [] lost_stracks = [] removed_stracks = [] scores = results.conf bboxes = results.xyxy # Add index bboxes = np.concatenate([bboxes, np.arange(len(bboxes)).reshape(-1, 1)], axis=-1) cls = results.cls remain_inds = scores > self.args.track_high_thresh inds_low = scores > self.args.track_low_thresh inds_high = scores < self.args.track_high_thresh inds_second = np.logical_and(inds_low, inds_high) dets_second = bboxes[inds_second] dets = bboxes[remain_inds] scores_keep = scores[remain_inds] scores_second = scores[inds_second] cls_keep = cls[remain_inds] cls_second = cls[inds_second] detections = self.init_track(dets, scores_keep, cls_keep, img) # Add newly detected tracklets to tracked_stracks unconfirmed = [] tracked_stracks = [] # type: list[STrack] for track in self.tracked_stracks: if not track.is_activated: unconfirmed.append(track) else: tracked_stracks.append(track) # Step 2: First association, with high score detection boxes strack_pool = self.joint_stracks(tracked_stracks, self.lost_stracks) # Predict the current location with KF self.multi_predict(strack_pool) if hasattr(self, 'gmc') and img is not None: warp = self.gmc.apply(img, dets) STrack.multi_gmc(strack_pool, warp) STrack.multi_gmc(unconfirmed, warp) dists = self.get_dists(strack_pool, detections) matches, u_track, u_detection = matching.linear_assignment(dists, thresh=self.args.match_thresh) for itracked, idet in matches: track = strack_pool[itracked] det = detections[idet] if track.state == TrackState.Tracked: track.update(det, self.frame_id) activated_stracks.append(track) else: track.re_activate(det, self.frame_id, new_id=False) refind_stracks.append(track) # Step 3: Second association, with low score detection boxes # association the untrack to the low score detections detections_second = self.init_track(dets_second, scores_second, cls_second, img) r_tracked_stracks = [strack_pool[i] for i in u_track if strack_pool[i].state == TrackState.Tracked] # TODO dists = matching.iou_distance(r_tracked_stracks, detections_second) matches, u_track, u_detection_second = matching.linear_assignment(dists, thresh=0.5) for itracked, idet in matches: track = r_tracked_stracks[itracked] det = detections_second[idet] if track.state == TrackState.Tracked: track.update(det, self.frame_id) activated_stracks.append(track) else: track.re_activate(det, self.frame_id, new_id=False) refind_stracks.append(track) for it in u_track: track = r_tracked_stracks[it] if track.state != TrackState.Lost: track.mark_lost() lost_stracks.append(track) # Deal with unconfirmed tracks, usually tracks with only one beginning frame detections = [detections[i] for i in u_detection] dists = self.get_dists(unconfirmed, detections) matches, u_unconfirmed, u_detection = matching.linear_assignment(dists, thresh=0.7) for itracked, idet in matches: unconfirmed[itracked].update(detections[idet], self.frame_id) activated_stracks.append(unconfirmed[itracked]) for it in u_unconfirmed: track = unconfirmed[it] track.mark_removed() removed_stracks.append(track) # Step 4: Init new stracks for inew in u_detection: track = detections[inew] if track.score < self.args.new_track_thresh: continue track.activate(self.kalman_filter, self.frame_id) activated_stracks.append(track) # Step 5: Update state for track in self.lost_stracks: if self.frame_id - track.end_frame > self.max_time_lost: track.mark_removed() removed_stracks.append(track) self.tracked_stracks = [t for t in self.tracked_stracks if t.state == TrackState.Tracked] self.tracked_stracks = self.joint_stracks(self.tracked_stracks, activated_stracks) self.tracked_stracks = self.joint_stracks(self.tracked_stracks, refind_stracks) self.lost_stracks = self.sub_stracks(self.lost_stracks, self.tracked_stracks) self.lost_stracks.extend(lost_stracks) self.lost_stracks = self.sub_stracks(self.lost_stracks, self.removed_stracks) self.tracked_stracks, self.lost_stracks = self.remove_duplicate_stracks(self.tracked_stracks, self.lost_stracks) self.removed_stracks.extend(removed_stracks) if len(self.removed_stracks) > 1000: self.removed_stracks = self.removed_stracks[-999:] # clip remove stracks to 1000 maximum return np.asarray( [x.tlbr.tolist() + [x.track_id, x.score, x.cls, x.idx] for x in self.tracked_stracks if x.is_activated], dtype=np.float32) def get_kalmanfilter(self): """Returns a Kalman filter object for tracking bounding boxes.""" return KalmanFilterXYAH() def init_track(self, dets, scores, cls, img=None): """Initialize object tracking with detections and scores using STrack algorithm.""" return [STrack(xyxy, s, c) for (xyxy, s, c) in zip(dets, scores, cls)] if len(dets) else [] # detections def get_dists(self, tracks, detections): """Calculates the distance between tracks and detections using IOU and fuses scores.""" dists = matching.iou_distance(tracks, detections) # TODO: mot20 # if not self.args.mot20: dists = matching.fuse_score(dists, detections) return dists def multi_predict(self, tracks): """Returns the predicted tracks using the YOLOv8 network.""" STrack.multi_predict(tracks) def reset_id(self): """Resets the ID counter of STrack.""" STrack.reset_id() @staticmethod def joint_stracks(tlista, tlistb): """Combine two lists of stracks into a single one.""" exists = {} res = [] for t in tlista: exists[t.track_id] = 1 res.append(t) for t in tlistb: tid = t.track_id if not exists.get(tid, 0): exists[tid] = 1 res.append(t) return res @staticmethod def sub_stracks(tlista, tlistb): """DEPRECATED CODE in https://github.com/ultralytics/ultralytics/pull/1890/ stracks = {t.track_id: t for t in tlista} for t in tlistb: tid = t.track_id if stracks.get(tid, 0): del stracks[tid] return list(stracks.values()) """ track_ids_b = {t.track_id for t in tlistb} return [t for t in tlista if t.track_id not in track_ids_b] @staticmethod def remove_duplicate_stracks(stracksa, stracksb): """Remove duplicate stracks with non-maximum IOU distance.""" pdist = matching.iou_distance(stracksa, stracksb) pairs = np.where(pdist < 0.15) dupa, dupb = [], [] for p, q in zip(*pairs): timep = stracksa[p].frame_id - stracksa[p].start_frame timeq = stracksb[q].frame_id - stracksb[q].start_frame if timep > timeq: dupb.append(q) else: dupa.append(p) resa = [t for i, t in enumerate(stracksa) if i not in dupa] resb = [t for i, t in enumerate(stracksb) if i not in dupb] return resa, resb ================================================ FILE: ultralytics/tracker/utils/__init__.py ================================================ ================================================ FILE: ultralytics/tracker/utils/gmc.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import copy import cv2 import numpy as np from ultralytics.yolo.utils import LOGGER class GMC: def __init__(self, method='sparseOptFlow', downscale=2, verbose=None): """Initialize a video tracker with specified parameters.""" super().__init__() self.method = method self.downscale = max(1, int(downscale)) if self.method == 'orb': self.detector = cv2.FastFeatureDetector_create(20) self.extractor = cv2.ORB_create() self.matcher = cv2.BFMatcher(cv2.NORM_HAMMING) elif self.method == 'sift': self.detector = cv2.SIFT_create(nOctaveLayers=3, contrastThreshold=0.02, edgeThreshold=20) self.extractor = cv2.SIFT_create(nOctaveLayers=3, contrastThreshold=0.02, edgeThreshold=20) self.matcher = cv2.BFMatcher(cv2.NORM_L2) elif self.method == 'ecc': number_of_iterations = 5000 termination_eps = 1e-6 self.warp_mode = cv2.MOTION_EUCLIDEAN self.criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, number_of_iterations, termination_eps) elif self.method == 'sparseOptFlow': self.feature_params = dict(maxCorners=1000, qualityLevel=0.01, minDistance=1, blockSize=3, useHarrisDetector=False, k=0.04) # self.gmc_file = open('GMC_results.txt', 'w') elif self.method in ['file', 'files']: seqName = verbose[0] ablation = verbose[1] if ablation: filePath = r'tracker/GMC_files/MOT17_ablation' else: filePath = r'tracker/GMC_files/MOTChallenge' if '-FRCNN' in seqName: seqName = seqName[:-6] elif '-DPM' in seqName or '-SDP' in seqName: seqName = seqName[:-4] self.gmcFile = open(f'{filePath}/GMC-{seqName}.txt') if self.gmcFile is None: raise ValueError(f'Error: Unable to open GMC file in directory:{filePath}') elif self.method in ['none', 'None']: self.method = 'none' else: raise ValueError(f'Error: Unknown CMC method:{method}') self.prevFrame = None self.prevKeyPoints = None self.prevDescriptors = None self.initializedFirstFrame = False def apply(self, raw_frame, detections=None): """Apply object detection on a raw frame using specified method.""" if self.method in ['orb', 'sift']: return self.applyFeatures(raw_frame, detections) elif self.method == 'ecc': return self.applyEcc(raw_frame, detections) elif self.method == 'sparseOptFlow': return self.applySparseOptFlow(raw_frame, detections) elif self.method == 'file': return self.applyFile(raw_frame, detections) elif self.method == 'none': return np.eye(2, 3) else: return np.eye(2, 3) def applyEcc(self, raw_frame, detections=None): """Initialize.""" height, width, _ = raw_frame.shape frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) H = np.eye(2, 3, dtype=np.float32) # Downscale image (TODO: consider using pyramids) if self.downscale > 1.0: frame = cv2.GaussianBlur(frame, (3, 3), 1.5) frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) width = width // self.downscale height = height // self.downscale # Handle first frame if not self.initializedFirstFrame: # Initialize data self.prevFrame = frame.copy() # Initialization done self.initializedFirstFrame = True return H # Run the ECC algorithm. The results are stored in warp_matrix. # (cc, H) = cv2.findTransformECC(self.prevFrame, frame, H, self.warp_mode, self.criteria) try: (cc, H) = cv2.findTransformECC(self.prevFrame, frame, H, self.warp_mode, self.criteria, None, 1) except Exception as e: LOGGER.warning(f'WARNING: find transform failed. Set warp as identity {e}') return H def applyFeatures(self, raw_frame, detections=None): """Initialize.""" height, width, _ = raw_frame.shape frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) H = np.eye(2, 3) # Downscale image (TODO: consider using pyramids) if self.downscale > 1.0: # frame = cv2.GaussianBlur(frame, (3, 3), 1.5) frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) width = width // self.downscale height = height // self.downscale # Find the keypoints mask = np.zeros_like(frame) # mask[int(0.05 * height): int(0.95 * height), int(0.05 * width): int(0.95 * width)] = 255 mask[int(0.02 * height):int(0.98 * height), int(0.02 * width):int(0.98 * width)] = 255 if detections is not None: for det in detections: tlbr = (det[:4] / self.downscale).astype(np.int_) mask[tlbr[1]:tlbr[3], tlbr[0]:tlbr[2]] = 0 keypoints = self.detector.detect(frame, mask) # Compute the descriptors keypoints, descriptors = self.extractor.compute(frame, keypoints) # Handle first frame if not self.initializedFirstFrame: # Initialize data self.prevFrame = frame.copy() self.prevKeyPoints = copy.copy(keypoints) self.prevDescriptors = copy.copy(descriptors) # Initialization done self.initializedFirstFrame = True return H # Match descriptors. knnMatches = self.matcher.knnMatch(self.prevDescriptors, descriptors, 2) # Filtered matches based on smallest spatial distance matches = [] spatialDistances = [] maxSpatialDistance = 0.25 * np.array([width, height]) # Handle empty matches case if len(knnMatches) == 0: # Store to next iteration self.prevFrame = frame.copy() self.prevKeyPoints = copy.copy(keypoints) self.prevDescriptors = copy.copy(descriptors) return H for m, n in knnMatches: if m.distance < 0.9 * n.distance: prevKeyPointLocation = self.prevKeyPoints[m.queryIdx].pt currKeyPointLocation = keypoints[m.trainIdx].pt spatialDistance = (prevKeyPointLocation[0] - currKeyPointLocation[0], prevKeyPointLocation[1] - currKeyPointLocation[1]) if (np.abs(spatialDistance[0]) < maxSpatialDistance[0]) and \ (np.abs(spatialDistance[1]) < maxSpatialDistance[1]): spatialDistances.append(spatialDistance) matches.append(m) meanSpatialDistances = np.mean(spatialDistances, 0) stdSpatialDistances = np.std(spatialDistances, 0) inliers = (spatialDistances - meanSpatialDistances) < 2.5 * stdSpatialDistances goodMatches = [] prevPoints = [] currPoints = [] for i in range(len(matches)): if inliers[i, 0] and inliers[i, 1]: goodMatches.append(matches[i]) prevPoints.append(self.prevKeyPoints[matches[i].queryIdx].pt) currPoints.append(keypoints[matches[i].trainIdx].pt) prevPoints = np.array(prevPoints) currPoints = np.array(currPoints) # Draw the keypoint matches on the output image # if False: # import matplotlib.pyplot as plt # matches_img = np.hstack((self.prevFrame, frame)) # matches_img = cv2.cvtColor(matches_img, cv2.COLOR_GRAY2BGR) # W = np.size(self.prevFrame, 1) # for m in goodMatches: # prev_pt = np.array(self.prevKeyPoints[m.queryIdx].pt, dtype=np.int_) # curr_pt = np.array(keypoints[m.trainIdx].pt, dtype=np.int_) # curr_pt[0] += W # color = np.random.randint(0, 255, 3) # color = (int(color[0]), int(color[1]), int(color[2])) # # matches_img = cv2.line(matches_img, prev_pt, curr_pt, tuple(color), 1, cv2.LINE_AA) # matches_img = cv2.circle(matches_img, prev_pt, 2, tuple(color), -1) # matches_img = cv2.circle(matches_img, curr_pt, 2, tuple(color), -1) # # plt.figure() # plt.imshow(matches_img) # plt.show() # Find rigid matrix if (np.size(prevPoints, 0) > 4) and (np.size(prevPoints, 0) == np.size(prevPoints, 0)): H, inliers = cv2.estimateAffinePartial2D(prevPoints, currPoints, cv2.RANSAC) # Handle downscale if self.downscale > 1.0: H[0, 2] *= self.downscale H[1, 2] *= self.downscale else: LOGGER.warning('WARNING: not enough matching points') # Store to next iteration self.prevFrame = frame.copy() self.prevKeyPoints = copy.copy(keypoints) self.prevDescriptors = copy.copy(descriptors) return H def applySparseOptFlow(self, raw_frame, detections=None): """Initialize.""" # t0 = time.time() height, width, _ = raw_frame.shape frame = cv2.cvtColor(raw_frame, cv2.COLOR_BGR2GRAY) H = np.eye(2, 3) # Downscale image if self.downscale > 1.0: # frame = cv2.GaussianBlur(frame, (3, 3), 1.5) frame = cv2.resize(frame, (width // self.downscale, height // self.downscale)) # Find the keypoints keypoints = cv2.goodFeaturesToTrack(frame, mask=None, **self.feature_params) # Handle first frame if not self.initializedFirstFrame: # Initialize data self.prevFrame = frame.copy() self.prevKeyPoints = copy.copy(keypoints) # Initialization done self.initializedFirstFrame = True return H # Find correspondences matchedKeypoints, status, err = cv2.calcOpticalFlowPyrLK(self.prevFrame, frame, self.prevKeyPoints, None) # Leave good correspondences only prevPoints = [] currPoints = [] for i in range(len(status)): if status[i]: prevPoints.append(self.prevKeyPoints[i]) currPoints.append(matchedKeypoints[i]) prevPoints = np.array(prevPoints) currPoints = np.array(currPoints) # Find rigid matrix if (np.size(prevPoints, 0) > 4) and (np.size(prevPoints, 0) == np.size(prevPoints, 0)): H, inliers = cv2.estimateAffinePartial2D(prevPoints, currPoints, cv2.RANSAC) # Handle downscale if self.downscale > 1.0: H[0, 2] *= self.downscale H[1, 2] *= self.downscale else: LOGGER.warning('WARNING: not enough matching points') # Store to next iteration self.prevFrame = frame.copy() self.prevKeyPoints = copy.copy(keypoints) # gmc_line = str(1000 * (time.time() - t0)) + "\t" + str(H[0, 0]) + "\t" + str(H[0, 1]) + "\t" + str( # H[0, 2]) + "\t" + str(H[1, 0]) + "\t" + str(H[1, 1]) + "\t" + str(H[1, 2]) + "\n" # self.gmc_file.write(gmc_line) return H def applyFile(self, raw_frame, detections=None): """Return the homography matrix based on the GCPs in the next line of the input GMC file.""" line = self.gmcFile.readline() tokens = line.split('\t') H = np.eye(2, 3, dtype=np.float_) H[0, 0] = float(tokens[1]) H[0, 1] = float(tokens[2]) H[0, 2] = float(tokens[3]) H[1, 0] = float(tokens[4]) H[1, 1] = float(tokens[5]) H[1, 2] = float(tokens[6]) return H ================================================ FILE: ultralytics/tracker/utils/kalman_filter.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import numpy as np import scipy.linalg # Table for the 0.95 quantile of the chi-square distribution with N degrees of freedom (contains values for N=1, ..., 9) # Taken from MATLAB/Octave's chi2inv function and used as Mahalanobis gating threshold. chi2inv95 = {1: 3.8415, 2: 5.9915, 3: 7.8147, 4: 9.4877, 5: 11.070, 6: 12.592, 7: 14.067, 8: 15.507, 9: 16.919} class KalmanFilterXYAH: """ For bytetrack A simple Kalman filter for tracking bounding boxes in image space. The 8-dimensional state space x, y, a, h, vx, vy, va, vh contains the bounding box center position (x, y), aspect ratio a, height h, and their respective velocities. Object motion follows a constant velocity model. The bounding box location (x, y, a, h) is taken as direct observation of the state space (linear observation model). """ def __init__(self): """Initialize Kalman filter model matrices with motion and observation uncertainty weights.""" ndim, dt = 4, 1. # Create Kalman filter model matrices. self._motion_mat = np.eye(2 * ndim, 2 * ndim) for i in range(ndim): self._motion_mat[i, ndim + i] = dt self._update_mat = np.eye(ndim, 2 * ndim) # Motion and observation uncertainty are chosen relative to the current # state estimate. These weights control the amount of uncertainty in # the model. This is a bit hacky. self._std_weight_position = 1. / 20 self._std_weight_velocity = 1. / 160 def initiate(self, measurement): """Create track from unassociated measurement. Parameters ---------- measurement : ndarray Bounding box coordinates (x, y, a, h) with center position (x, y), aspect ratio a, and height h. Returns ------- (ndarray, ndarray) Returns the mean vector (8 dimensional) and covariance matrix (8x8 dimensional) of the new track. Unobserved velocities are initialized to 0 mean. """ mean_pos = measurement mean_vel = np.zeros_like(mean_pos) mean = np.r_[mean_pos, mean_vel] std = [ 2 * self._std_weight_position * measurement[3], 2 * self._std_weight_position * measurement[3], 1e-2, 2 * self._std_weight_position * measurement[3], 10 * self._std_weight_velocity * measurement[3], 10 * self._std_weight_velocity * measurement[3], 1e-5, 10 * self._std_weight_velocity * measurement[3]] covariance = np.diag(np.square(std)) return mean, covariance def predict(self, mean, covariance): """Run Kalman filter prediction step. Parameters ---------- mean : ndarray The 8 dimensional mean vector of the object state at the previous time step. covariance : ndarray The 8x8 dimensional covariance matrix of the object state at the previous time step. Returns ------- (ndarray, ndarray) Returns the mean vector and covariance matrix of the predicted state. Unobserved velocities are initialized to 0 mean. """ std_pos = [ self._std_weight_position * mean[3], self._std_weight_position * mean[3], 1e-2, self._std_weight_position * mean[3]] std_vel = [ self._std_weight_velocity * mean[3], self._std_weight_velocity * mean[3], 1e-5, self._std_weight_velocity * mean[3]] motion_cov = np.diag(np.square(np.r_[std_pos, std_vel])) # mean = np.dot(self._motion_mat, mean) mean = np.dot(mean, self._motion_mat.T) covariance = np.linalg.multi_dot((self._motion_mat, covariance, self._motion_mat.T)) + motion_cov return mean, covariance def project(self, mean, covariance): """Project state distribution to measurement space. Parameters ---------- mean : ndarray The state's mean vector (8 dimensional array). covariance : ndarray The state's covariance matrix (8x8 dimensional). Returns ------- (ndarray, ndarray) Returns the projected mean and covariance matrix of the given state estimate. """ std = [ self._std_weight_position * mean[3], self._std_weight_position * mean[3], 1e-1, self._std_weight_position * mean[3]] innovation_cov = np.diag(np.square(std)) mean = np.dot(self._update_mat, mean) covariance = np.linalg.multi_dot((self._update_mat, covariance, self._update_mat.T)) return mean, covariance + innovation_cov def multi_predict(self, mean, covariance): """Run Kalman filter prediction step (Vectorized version). Parameters ---------- mean : ndarray The Nx8 dimensional mean matrix of the object states at the previous time step. covariance : ndarray The Nx8x8 dimensional covariance matrix of the object states at the previous time step. Returns ------- (ndarray, ndarray) Returns the mean vector and covariance matrix of the predicted state. Unobserved velocities are initialized to 0 mean. """ std_pos = [ self._std_weight_position * mean[:, 3], self._std_weight_position * mean[:, 3], 1e-2 * np.ones_like(mean[:, 3]), self._std_weight_position * mean[:, 3]] std_vel = [ self._std_weight_velocity * mean[:, 3], self._std_weight_velocity * mean[:, 3], 1e-5 * np.ones_like(mean[:, 3]), self._std_weight_velocity * mean[:, 3]] sqr = np.square(np.r_[std_pos, std_vel]).T motion_cov = [np.diag(sqr[i]) for i in range(len(mean))] motion_cov = np.asarray(motion_cov) mean = np.dot(mean, self._motion_mat.T) left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2)) covariance = np.dot(left, self._motion_mat.T) + motion_cov return mean, covariance def update(self, mean, covariance, measurement): """Run Kalman filter correction step. Parameters ---------- mean : ndarray The predicted state's mean vector (8 dimensional). covariance : ndarray The state's covariance matrix (8x8 dimensional). measurement : ndarray The 4 dimensional measurement vector (x, y, a, h), where (x, y) is the center position, a the aspect ratio, and h the height of the bounding box. Returns ------- (ndarray, ndarray) Returns the measurement-corrected state distribution. """ projected_mean, projected_cov = self.project(mean, covariance) chol_factor, lower = scipy.linalg.cho_factor(projected_cov, lower=True, check_finite=False) kalman_gain = scipy.linalg.cho_solve((chol_factor, lower), np.dot(covariance, self._update_mat.T).T, check_finite=False).T innovation = measurement - projected_mean new_mean = mean + np.dot(innovation, kalman_gain.T) new_covariance = covariance - np.linalg.multi_dot((kalman_gain, projected_cov, kalman_gain.T)) return new_mean, new_covariance def gating_distance(self, mean, covariance, measurements, only_position=False, metric='maha'): """Compute gating distance between state distribution and measurements. A suitable distance threshold can be obtained from `chi2inv95`. If `only_position` is False, the chi-square distribution has 4 degrees of freedom, otherwise 2. Parameters ---------- mean : ndarray Mean vector over the state distribution (8 dimensional). covariance : ndarray Covariance of the state distribution (8x8 dimensional). measurements : ndarray An Nx4 dimensional matrix of N measurements, each in format (x, y, a, h) where (x, y) is the bounding box center position, a the aspect ratio, and h the height. only_position : Optional[bool] If True, distance computation is done with respect to the bounding box center position only. Returns ------- ndarray Returns an array of length N, where the i-th element contains the squared Mahalanobis distance between (mean, covariance) and `measurements[i]`. """ mean, covariance = self.project(mean, covariance) if only_position: mean, covariance = mean[:2], covariance[:2, :2] measurements = measurements[:, :2] d = measurements - mean if metric == 'gaussian': return np.sum(d * d, axis=1) elif metric == 'maha': cholesky_factor = np.linalg.cholesky(covariance) z = scipy.linalg.solve_triangular(cholesky_factor, d.T, lower=True, check_finite=False, overwrite_b=True) return np.sum(z * z, axis=0) # square maha else: raise ValueError('invalid distance metric') class KalmanFilterXYWH: """ For BoT-SORT A simple Kalman filter for tracking bounding boxes in image space. The 8-dimensional state space x, y, w, h, vx, vy, vw, vh contains the bounding box center position (x, y), width w, height h, and their respective velocities. Object motion follows a constant velocity model. The bounding box location (x, y, w, h) is taken as direct observation of the state space (linear observation model). """ def __init__(self): """Initialize Kalman filter model matrices with motion and observation uncertainties.""" ndim, dt = 4, 1. # Create Kalman filter model matrices. self._motion_mat = np.eye(2 * ndim, 2 * ndim) for i in range(ndim): self._motion_mat[i, ndim + i] = dt self._update_mat = np.eye(ndim, 2 * ndim) # Motion and observation uncertainty are chosen relative to the current # state estimate. These weights control the amount of uncertainty in # the model. This is a bit hacky. self._std_weight_position = 1. / 20 self._std_weight_velocity = 1. / 160 def initiate(self, measurement): """Create track from unassociated measurement. Parameters ---------- measurement : ndarray Bounding box coordinates (x, y, w, h) with center position (x, y), width w, and height h. Returns ------- (ndarray, ndarray) Returns the mean vector (8 dimensional) and covariance matrix (8x8 dimensional) of the new track. Unobserved velocities are initialized to 0 mean. """ mean_pos = measurement mean_vel = np.zeros_like(mean_pos) mean = np.r_[mean_pos, mean_vel] std = [ 2 * self._std_weight_position * measurement[2], 2 * self._std_weight_position * measurement[3], 2 * self._std_weight_position * measurement[2], 2 * self._std_weight_position * measurement[3], 10 * self._std_weight_velocity * measurement[2], 10 * self._std_weight_velocity * measurement[3], 10 * self._std_weight_velocity * measurement[2], 10 * self._std_weight_velocity * measurement[3]] covariance = np.diag(np.square(std)) return mean, covariance def predict(self, mean, covariance): """Run Kalman filter prediction step. Parameters ---------- mean : ndarray The 8 dimensional mean vector of the object state at the previous time step. covariance : ndarray The 8x8 dimensional covariance matrix of the object state at the previous time step. Returns ------- (ndarray, ndarray) Returns the mean vector and covariance matrix of the predicted state. Unobserved velocities are initialized to 0 mean. """ std_pos = [ self._std_weight_position * mean[2], self._std_weight_position * mean[3], self._std_weight_position * mean[2], self._std_weight_position * mean[3]] std_vel = [ self._std_weight_velocity * mean[2], self._std_weight_velocity * mean[3], self._std_weight_velocity * mean[2], self._std_weight_velocity * mean[3]] motion_cov = np.diag(np.square(np.r_[std_pos, std_vel])) mean = np.dot(mean, self._motion_mat.T) covariance = np.linalg.multi_dot((self._motion_mat, covariance, self._motion_mat.T)) + motion_cov return mean, covariance def project(self, mean, covariance): """Project state distribution to measurement space. Parameters ---------- mean : ndarray The state's mean vector (8 dimensional array). covariance : ndarray The state's covariance matrix (8x8 dimensional). Returns ------- (ndarray, ndarray) Returns the projected mean and covariance matrix of the given state estimate. """ std = [ self._std_weight_position * mean[2], self._std_weight_position * mean[3], self._std_weight_position * mean[2], self._std_weight_position * mean[3]] innovation_cov = np.diag(np.square(std)) mean = np.dot(self._update_mat, mean) covariance = np.linalg.multi_dot((self._update_mat, covariance, self._update_mat.T)) return mean, covariance + innovation_cov def multi_predict(self, mean, covariance): """Run Kalman filter prediction step (Vectorized version). Parameters ---------- mean : ndarray The Nx8 dimensional mean matrix of the object states at the previous time step. covariance : ndarray The Nx8x8 dimensional covariance matrix of the object states at the previous time step. Returns ------- (ndarray, ndarray) Returns the mean vector and covariance matrix of the predicted state. Unobserved velocities are initialized to 0 mean. """ std_pos = [ self._std_weight_position * mean[:, 2], self._std_weight_position * mean[:, 3], self._std_weight_position * mean[:, 2], self._std_weight_position * mean[:, 3]] std_vel = [ self._std_weight_velocity * mean[:, 2], self._std_weight_velocity * mean[:, 3], self._std_weight_velocity * mean[:, 2], self._std_weight_velocity * mean[:, 3]] sqr = np.square(np.r_[std_pos, std_vel]).T motion_cov = [np.diag(sqr[i]) for i in range(len(mean))] motion_cov = np.asarray(motion_cov) mean = np.dot(mean, self._motion_mat.T) left = np.dot(self._motion_mat, covariance).transpose((1, 0, 2)) covariance = np.dot(left, self._motion_mat.T) + motion_cov return mean, covariance def update(self, mean, covariance, measurement): """Run Kalman filter correction step. Parameters ---------- mean : ndarray The predicted state's mean vector (8 dimensional). covariance : ndarray The state's covariance matrix (8x8 dimensional). measurement : ndarray The 4 dimensional measurement vector (x, y, w, h), where (x, y) is the center position, w the width, and h the height of the bounding box. Returns ------- (ndarray, ndarray) Returns the measurement-corrected state distribution. """ projected_mean, projected_cov = self.project(mean, covariance) chol_factor, lower = scipy.linalg.cho_factor(projected_cov, lower=True, check_finite=False) kalman_gain = scipy.linalg.cho_solve((chol_factor, lower), np.dot(covariance, self._update_mat.T).T, check_finite=False).T innovation = measurement - projected_mean new_mean = mean + np.dot(innovation, kalman_gain.T) new_covariance = covariance - np.linalg.multi_dot((kalman_gain, projected_cov, kalman_gain.T)) return new_mean, new_covariance def gating_distance(self, mean, covariance, measurements, only_position=False, metric='maha'): """Compute gating distance between state distribution and measurements. A suitable distance threshold can be obtained from `chi2inv95`. If `only_position` is False, the chi-square distribution has 4 degrees of freedom, otherwise 2. Parameters ---------- mean : ndarray Mean vector over the state distribution (8 dimensional). covariance : ndarray Covariance of the state distribution (8x8 dimensional). measurements : ndarray An Nx4 dimensional matrix of N measurements, each in format (x, y, a, h) where (x, y) is the bounding box center position, a the aspect ratio, and h the height. only_position : Optional[bool] If True, distance computation is done with respect to the bounding box center position only. Returns ------- ndarray Returns an array of length N, where the i-th element contains the squared Mahalanobis distance between (mean, covariance) and `measurements[i]`. """ mean, covariance = self.project(mean, covariance) if only_position: mean, covariance = mean[:2], covariance[:2, :2] measurements = measurements[:, :2] d = measurements - mean if metric == 'gaussian': return np.sum(d * d, axis=1) elif metric == 'maha': cholesky_factor = np.linalg.cholesky(covariance) z = scipy.linalg.solve_triangular(cholesky_factor, d.T, lower=True, check_finite=False, overwrite_b=True) return np.sum(z * z, axis=0) # square maha else: raise ValueError('invalid distance metric') ================================================ FILE: ultralytics/tracker/utils/matching.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import numpy as np import scipy from scipy.spatial.distance import cdist from .kalman_filter import chi2inv95 try: import lap # for linear_assignment assert lap.__version__ # verify package is not directory except (ImportError, AssertionError, AttributeError): from ultralytics.yolo.utils.checks import check_requirements check_requirements('lap>=0.4') # install import lap def merge_matches(m1, m2, shape): """Merge two sets of matches and return matched and unmatched indices.""" O, P, Q = shape m1 = np.asarray(m1) m2 = np.asarray(m2) M1 = scipy.sparse.coo_matrix((np.ones(len(m1)), (m1[:, 0], m1[:, 1])), shape=(O, P)) M2 = scipy.sparse.coo_matrix((np.ones(len(m2)), (m2[:, 0], m2[:, 1])), shape=(P, Q)) mask = M1 * M2 match = mask.nonzero() match = list(zip(match[0], match[1])) unmatched_O = tuple(set(range(O)) - {i for i, j in match}) unmatched_Q = tuple(set(range(Q)) - {j for i, j in match}) return match, unmatched_O, unmatched_Q def _indices_to_matches(cost_matrix, indices, thresh): """_indices_to_matches: Return matched and unmatched indices given a cost matrix, indices, and a threshold.""" matched_cost = cost_matrix[tuple(zip(*indices))] matched_mask = (matched_cost <= thresh) matches = indices[matched_mask] unmatched_a = tuple(set(range(cost_matrix.shape[0])) - set(matches[:, 0])) unmatched_b = tuple(set(range(cost_matrix.shape[1])) - set(matches[:, 1])) return matches, unmatched_a, unmatched_b def linear_assignment(cost_matrix, thresh, use_lap=True): """Linear assignment implementations with scipy and lap.lapjv.""" if cost_matrix.size == 0: return np.empty((0, 2), dtype=int), tuple(range(cost_matrix.shape[0])), tuple(range(cost_matrix.shape[1])) if use_lap: _, x, y = lap.lapjv(cost_matrix, extend_cost=True, cost_limit=thresh) matches = [[ix, mx] for ix, mx in enumerate(x) if mx >= 0] unmatched_a = np.where(x < 0)[0] unmatched_b = np.where(y < 0)[0] else: # Scipy linear sum assignment is NOT working correctly, DO NOT USE y, x = scipy.optimize.linear_sum_assignment(cost_matrix) # row y, col x matches = np.asarray([[i, x] for i, x in enumerate(x) if cost_matrix[i, x] <= thresh]) unmatched = np.ones(cost_matrix.shape) for i, xi in matches: unmatched[i, xi] = 0.0 unmatched_a = np.where(unmatched.all(1))[0] unmatched_b = np.where(unmatched.all(0))[0] return matches, unmatched_a, unmatched_b def ious(atlbrs, btlbrs): """ Compute cost based on IoU :type atlbrs: list[tlbr] | np.ndarray :type atlbrs: list[tlbr] | np.ndarray :rtype ious np.ndarray """ ious = np.zeros((len(atlbrs), len(btlbrs)), dtype=np.float32) if ious.size == 0: return ious ious = bbox_ious(np.ascontiguousarray(atlbrs, dtype=np.float32), np.ascontiguousarray(btlbrs, dtype=np.float32)) return ious def iou_distance(atracks, btracks): """ Compute cost based on IoU :type atracks: list[STrack] :type btracks: list[STrack] :rtype cost_matrix np.ndarray """ if (len(atracks) > 0 and isinstance(atracks[0], np.ndarray)) \ or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)): atlbrs = atracks btlbrs = btracks else: atlbrs = [track.tlbr for track in atracks] btlbrs = [track.tlbr for track in btracks] _ious = ious(atlbrs, btlbrs) return 1 - _ious # cost matrix def v_iou_distance(atracks, btracks): """ Compute cost based on IoU :type atracks: list[STrack] :type btracks: list[STrack] :rtype cost_matrix np.ndarray """ if (len(atracks) > 0 and isinstance(atracks[0], np.ndarray)) \ or (len(btracks) > 0 and isinstance(btracks[0], np.ndarray)): atlbrs = atracks btlbrs = btracks else: atlbrs = [track.tlwh_to_tlbr(track.pred_bbox) for track in atracks] btlbrs = [track.tlwh_to_tlbr(track.pred_bbox) for track in btracks] _ious = ious(atlbrs, btlbrs) return 1 - _ious # cost matrix def embedding_distance(tracks, detections, metric='cosine'): """ :param tracks: list[STrack] :param detections: list[BaseTrack] :param metric: :return: cost_matrix np.ndarray """ cost_matrix = np.zeros((len(tracks), len(detections)), dtype=np.float32) if cost_matrix.size == 0: return cost_matrix det_features = np.asarray([track.curr_feat for track in detections], dtype=np.float32) # for i, track in enumerate(tracks): # cost_matrix[i, :] = np.maximum(0.0, cdist(track.smooth_feat.reshape(1,-1), det_features, metric)) track_features = np.asarray([track.smooth_feat for track in tracks], dtype=np.float32) cost_matrix = np.maximum(0.0, cdist(track_features, det_features, metric)) # Normalized features return cost_matrix def gate_cost_matrix(kf, cost_matrix, tracks, detections, only_position=False): """Apply gating to the cost matrix based on predicted tracks and detected objects.""" if cost_matrix.size == 0: return cost_matrix gating_dim = 2 if only_position else 4 gating_threshold = chi2inv95[gating_dim] measurements = np.asarray([det.to_xyah() for det in detections]) for row, track in enumerate(tracks): gating_distance = kf.gating_distance(track.mean, track.covariance, measurements, only_position) cost_matrix[row, gating_distance > gating_threshold] = np.inf return cost_matrix def fuse_motion(kf, cost_matrix, tracks, detections, only_position=False, lambda_=0.98): """Fuse motion between tracks and detections with gating and Kalman filtering.""" if cost_matrix.size == 0: return cost_matrix gating_dim = 2 if only_position else 4 gating_threshold = chi2inv95[gating_dim] measurements = np.asarray([det.to_xyah() for det in detections]) for row, track in enumerate(tracks): gating_distance = kf.gating_distance(track.mean, track.covariance, measurements, only_position, metric='maha') cost_matrix[row, gating_distance > gating_threshold] = np.inf cost_matrix[row] = lambda_ * cost_matrix[row] + (1 - lambda_) * gating_distance return cost_matrix def fuse_iou(cost_matrix, tracks, detections): """Fuses ReID and IoU similarity matrices to yield a cost matrix for object tracking.""" if cost_matrix.size == 0: return cost_matrix reid_sim = 1 - cost_matrix iou_dist = iou_distance(tracks, detections) iou_sim = 1 - iou_dist fuse_sim = reid_sim * (1 + iou_sim) / 2 # det_scores = np.array([det.score for det in detections]) # det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0) return 1 - fuse_sim # fuse cost def fuse_score(cost_matrix, detections): """Fuses cost matrix with detection scores to produce a single similarity matrix.""" if cost_matrix.size == 0: return cost_matrix iou_sim = 1 - cost_matrix det_scores = np.array([det.score for det in detections]) det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0) fuse_sim = iou_sim * det_scores return 1 - fuse_sim # fuse_cost def bbox_ious(box1, box2, eps=1e-7): """ Calculate the Intersection over Union (IoU) between pairs of bounding boxes. Args: box1 (np.array): A numpy array of shape (n, 4) representing 'n' bounding boxes. Each row is in the format (x1, y1, x2, y2). box2 (np.array): A numpy array of shape (m, 4) representing 'm' bounding boxes. Each row is in the format (x1, y1, x2, y2). eps (float, optional): A small constant to prevent division by zero. Defaults to 1e-7. Returns: (np.array): A numpy array of shape (n, m) representing the IoU scores for each pair of bounding boxes from box1 and box2. Note: The bounding box coordinates are expected to be in the format (x1, y1, x2, y2). """ # Get the coordinates of bounding boxes b1_x1, b1_y1, b1_x2, b1_y2 = box1.T b2_x1, b2_y1, b2_x2, b2_y2 = box2.T # Intersection area inter_area = (np.minimum(b1_x2[:, None], b2_x2) - np.maximum(b1_x1[:, None], b2_x1)).clip(0) * \ (np.minimum(b1_y2[:, None], b2_y2) - np.maximum(b1_y1[:, None], b2_y1)).clip(0) # box2 area box1_area = (b1_x2 - b1_x1) * (b1_y2 - b1_y1) box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) return inter_area / (box2_area + box1_area[:, None] - inter_area + eps) ================================================ FILE: ultralytics/vit/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from .rtdetr import RTDETR from .sam import SAM __all__ = 'RTDETR', 'SAM' # allow simpler import ================================================ FILE: ultralytics/vit/rtdetr/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from .model import RTDETR from .predict import RTDETRPredictor from .val import RTDETRValidator __all__ = 'RTDETRPredictor', 'RTDETRValidator', 'RTDETR' ================================================ FILE: ultralytics/vit/rtdetr/model.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license """ RT-DETR model interface """ from pathlib import Path import torch.nn as nn from ultralytics.nn.tasks import RTDETRDetectionModel, attempt_load_one_weight, yaml_model_load from ultralytics.yolo.cfg import get_cfg from ultralytics.yolo.engine.exporter import Exporter from ultralytics.yolo.utils import DEFAULT_CFG, DEFAULT_CFG_DICT, LOGGER, RANK, ROOT, is_git_dir from ultralytics.yolo.utils.checks import check_imgsz from ultralytics.yolo.utils.torch_utils import model_info, smart_inference_mode from .predict import RTDETRPredictor from .train import RTDETRTrainer from .val import RTDETRValidator class RTDETR: def __init__(self, model='rtdetr-l.pt') -> None: if model and not model.endswith('.pt') and not model.endswith('.yaml'): raise NotImplementedError('RT-DETR only supports creating from pt file or yaml file.') # Load or create new YOLO model self.predictor = None self.ckpt = None suffix = Path(model).suffix if suffix == '.yaml': self._new(model) else: self._load(model) def _new(self, cfg: str, verbose=True): cfg_dict = yaml_model_load(cfg) self.cfg = cfg self.task = 'detect' self.model = RTDETRDetectionModel(cfg_dict, verbose=verbose) # build model # Below added to allow export from YAMLs self.model.args = DEFAULT_CFG_DICT # attach args to model self.model.task = self.task @smart_inference_mode() def _load(self, weights: str): self.model, self.ckpt = attempt_load_one_weight(weights) self.model.args = DEFAULT_CFG_DICT # attach args to model self.task = self.model.args['task'] @smart_inference_mode() def load(self, weights='yolov8n.pt'): """ Transfers parameters with matching names and shapes from 'weights' to model. """ if isinstance(weights, (str, Path)): weights, self.ckpt = attempt_load_one_weight(weights) self.model.load(weights) return self @smart_inference_mode() def predict(self, source=None, stream=False, **kwargs): """ Perform prediction using the YOLO model. Args: source (str | int | PIL | np.ndarray): The source of the image to make predictions on. Accepts all source types accepted by the YOLO model. stream (bool): Whether to stream the predictions or not. Defaults to False. **kwargs : Additional keyword arguments passed to the predictor. Check the 'configuration' section in the documentation for all available options. Returns: (List[ultralytics.yolo.engine.results.Results]): The prediction results. """ if source is None: source = ROOT / 'assets' if is_git_dir() else 'https://ultralytics.com/images/bus.jpg' LOGGER.warning(f"WARNING ⚠️ 'source' is missing. Using 'source={source}'.") overrides = dict(conf=0.25, task='detect', mode='predict') overrides.update(kwargs) # prefer kwargs if not self.predictor: self.predictor = RTDETRPredictor(overrides=overrides) self.predictor.setup_model(model=self.model) else: # only update args if predictor is already setup self.predictor.args = get_cfg(self.predictor.args, overrides) return self.predictor(source, stream=stream) def train(self, **kwargs): """ Trains the model on a given dataset. Args: **kwargs (Any): Any number of arguments representing the training configuration. """ overrides = dict(task='detect', mode='train') overrides.update(kwargs) overrides['deterministic'] = False if not overrides.get('data'): raise AttributeError("Dataset required but missing, i.e. pass 'data=coco128.yaml'") if overrides.get('resume'): overrides['resume'] = self.ckpt_path self.task = overrides.get('task') or self.task self.trainer = RTDETRTrainer(overrides=overrides) if not overrides.get('resume'): # manually set model only if not resuming self.trainer.model = self.trainer.get_model(weights=self.model if self.ckpt else None, cfg=self.model.yaml) self.model = self.trainer.model self.trainer.train() # Update model and cfg after training if RANK in (-1, 0): self.model, _ = attempt_load_one_weight(str(self.trainer.best)) self.overrides = self.model.args self.metrics = getattr(self.trainer.validator, 'metrics', None) # TODO: no metrics returned by DDP def val(self, **kwargs): """Run validation given dataset.""" overrides = dict(task='detect', mode='val') overrides.update(kwargs) # prefer kwargs args = get_cfg(cfg=DEFAULT_CFG, overrides=overrides) args.imgsz = check_imgsz(args.imgsz, max_dim=1) validator = RTDETRValidator(args=args) validator(model=self.model) self.metrics = validator.metrics return validator.metrics def info(self, verbose=True): """Get model info""" return model_info(self.model, verbose=verbose) def _check_is_pytorch_model(self): """ Raises TypeError is model is not a PyTorch model """ pt_str = isinstance(self.model, (str, Path)) and Path(self.model).suffix == '.pt' pt_module = isinstance(self.model, nn.Module) if not (pt_module or pt_str): raise TypeError(f"model='{self.model}' must be a *.pt PyTorch model, but is a different type. " f'PyTorch models can be used to train, val, predict and export, i.e. ' f"'yolo export model=yolov8n.pt', but exported formats like ONNX, TensorRT etc. only " f"support 'predict' and 'val' modes, i.e. 'yolo predict model=yolov8n.onnx'.") def fuse(self): """Fuse PyTorch Conv2d and BatchNorm2d layers.""" self._check_is_pytorch_model() self.model.fuse() @smart_inference_mode() def export(self, **kwargs): """ Export model. Args: **kwargs : Any other args accepted by the predictors. To see all args check 'configuration' section in docs """ overrides = dict(task='detect') overrides.update(kwargs) overrides['mode'] = 'export' args = get_cfg(cfg=DEFAULT_CFG, overrides=overrides) args.task = self.task if args.imgsz == DEFAULT_CFG.imgsz: args.imgsz = self.model.args['imgsz'] # use trained imgsz unless custom value is passed if args.batch == DEFAULT_CFG.batch: args.batch = 1 # default to 1 if not modified return Exporter(overrides=args)(model=self.model) def __call__(self, source=None, stream=False, **kwargs): """Calls the 'predict' function with given arguments to perform object detection.""" return self.predict(source, stream, **kwargs) def __getattr__(self, attr): """Raises error if object has no requested attribute.""" name = self.__class__.__name__ raise AttributeError(f"'{name}' object has no attribute '{attr}'. See valid attributes below.\n{self.__doc__}") ================================================ FILE: ultralytics/vit/rtdetr/predict.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import torch from ultralytics.yolo.data.augment import LetterBox from ultralytics.yolo.engine.predictor import BasePredictor from ultralytics.yolo.engine.results import Results from ultralytics.yolo.utils import ops class RTDETRPredictor(BasePredictor): def postprocess(self, preds, img, orig_imgs): """Postprocess predictions and returns a list of Results objects.""" bboxes, scores = preds[:2] # (1, bs, 300, 4), (1, bs, 300, nc) bboxes, scores = bboxes.squeeze_(0), scores.squeeze_(0) results = [] for i, bbox in enumerate(bboxes): # (300, 4) bbox = ops.xywh2xyxy(bbox) score, cls = scores[i].max(-1, keepdim=True) # (300, 1) idx = score.squeeze(-1) > self.args.conf # (300, ) if self.args.classes is not None: idx = (cls == torch.tensor(self.args.classes, device=cls.device)).any(1) & idx pred = torch.cat([bbox, score, cls], dim=-1)[idx] # filter orig_img = orig_imgs[i] if isinstance(orig_imgs, list) else orig_imgs oh, ow = orig_img.shape[:2] if not isinstance(orig_imgs, torch.Tensor): pred[..., [0, 2]] *= ow pred[..., [1, 3]] *= oh path = self.batch[0] img_path = path[i] if isinstance(path, list) else path results.append(Results(orig_img=orig_img, path=img_path, names=self.model.names, boxes=pred)) return results def pre_transform(self, im): """Pre-transform input image before inference. Args: im (List(np.ndarray)): (N, 3, h, w) for tensor, [(h, w, 3) x N] for list. Return: A list of transformed imgs. """ # The size must be square(640) and scaleFilled. return [LetterBox(self.imgsz, auto=False, scaleFill=True)(image=x) for x in im] ================================================ FILE: ultralytics/vit/rtdetr/train.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from copy import copy import torch from ultralytics.nn.tasks import RTDETRDetectionModel from ultralytics.yolo.utils import DEFAULT_CFG, RANK, colorstr from ultralytics.yolo.v8.detect import DetectionTrainer from .val import RTDETRDataset, RTDETRValidator class RTDETRTrainer(DetectionTrainer): def get_model(self, cfg=None, weights=None, verbose=True): """Return a YOLO detection model.""" model = RTDETRDetectionModel(cfg, nc=self.data['nc'], verbose=verbose and RANK == -1) if weights: model.load(weights) return model def build_dataset(self, img_path, mode='val', batch=None): """Build RTDETR Dataset Args: img_path (str): Path to the folder containing images. mode (str): `train` mode or `val` mode, users are able to customize different augmentations for each mode. batch (int, optional): Size of batches, this is for `rect`. Defaults to None. """ return RTDETRDataset( img_path=img_path, imgsz=self.args.imgsz, batch_size=batch, augment=mode == 'train', # no augmentation hyp=self.args, rect=False, # no rect cache=self.args.cache or None, prefix=colorstr(f'{mode}: '), data=self.data) def get_validator(self): """Returns a DetectionValidator for RTDETR model validation.""" self.loss_names = 'giou_loss', 'cls_loss', 'l1_loss' return RTDETRValidator(self.test_loader, save_dir=self.save_dir, args=copy(self.args)) def preprocess_batch(self, batch): """Preprocesses a batch of images by scaling and converting to float.""" batch = super().preprocess_batch(batch) bs = len(batch['img']) batch_idx = batch['batch_idx'] gt_bbox, gt_class = [], [] for i in range(bs): gt_bbox.append(batch['bboxes'][batch_idx == i].to(batch_idx.device)) gt_class.append(batch['cls'][batch_idx == i].to(device=batch_idx.device, dtype=torch.long)) return batch def train(cfg=DEFAULT_CFG, use_python=False): """Train and optimize RTDETR model given training data and device.""" model = 'rtdetr-l.yaml' data = cfg.data or 'coco128.yaml' # or yolo.ClassificationDataset("mnist") device = cfg.device if cfg.device is not None else '' # NOTE: F.grid_sample which is in rt-detr does not support deterministic=True # NOTE: amp training causes nan outputs and end with error while doing bipartite graph matching args = dict(model=model, data=data, device=device, imgsz=640, exist_ok=True, batch=4, deterministic=False, amp=False) trainer = RTDETRTrainer(overrides=args) trainer.train() if __name__ == '__main__': train() ================================================ FILE: ultralytics/vit/rtdetr/val.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from pathlib import Path import cv2 import numpy as np import torch from ultralytics.yolo.data import YOLODataset from ultralytics.yolo.data.augment import Compose, Format, v8_transforms from ultralytics.yolo.utils import colorstr, ops from ultralytics.yolo.v8.detect import DetectionValidator __all__ = 'RTDETRValidator', # tuple or list # TODO: Temporarily, RT-DETR does not need padding. class RTDETRDataset(YOLODataset): def __init__(self, *args, data=None, **kwargs): super().__init__(*args, data=data, use_segments=False, use_keypoints=False, **kwargs) # NOTE: add stretch version load_image for rtdetr mosaic def load_image(self, i): """Loads 1 image from dataset index 'i', returns (im, resized hw).""" im, f, fn = self.ims[i], self.im_files[i], self.npy_files[i] if im is None: # not cached in RAM if fn.exists(): # load npy im = np.load(fn) else: # read image im = cv2.imread(f) # BGR if im is None: raise FileNotFoundError(f'Image Not Found {f}') h0, w0 = im.shape[:2] # orig hw im = cv2.resize(im, (self.imgsz, self.imgsz), interpolation=cv2.INTER_LINEAR) # Add to buffer if training with augmentations if self.augment: self.ims[i], self.im_hw0[i], self.im_hw[i] = im, (h0, w0), im.shape[:2] # im, hw_original, hw_resized self.buffer.append(i) if len(self.buffer) >= self.max_buffer_length: j = self.buffer.pop(0) self.ims[j], self.im_hw0[j], self.im_hw[j] = None, None, None return im, (h0, w0), im.shape[:2] return self.ims[i], self.im_hw0[i], self.im_hw[i] def build_transforms(self, hyp=None): """Temporarily, only for evaluation.""" if self.augment: hyp.mosaic = hyp.mosaic if self.augment and not self.rect else 0.0 hyp.mixup = hyp.mixup if self.augment and not self.rect else 0.0 transforms = v8_transforms(self, self.imgsz, hyp, stretch=True) else: # transforms = Compose([LetterBox(new_shape=(self.imgsz, self.imgsz), auto=False, scaleFill=True)]) transforms = Compose([]) transforms.append( Format(bbox_format='xywh', normalize=True, return_mask=self.use_segments, return_keypoint=self.use_keypoints, batch_idx=True, mask_ratio=hyp.mask_ratio, mask_overlap=hyp.overlap_mask)) return transforms class RTDETRValidator(DetectionValidator): def build_dataset(self, img_path, mode='val', batch=None): """Build YOLO Dataset Args: img_path (str): Path to the folder containing images. mode (str): `train` mode or `val` mode, users are able to customize different augmentations for each mode. batch (int, optional): Size of batches, this is for `rect`. Defaults to None. """ return RTDETRDataset( img_path=img_path, imgsz=self.args.imgsz, batch_size=batch, augment=False, # no augmentation hyp=self.args, rect=False, # no rect cache=self.args.cache or None, prefix=colorstr(f'{mode}: '), data=self.data) def postprocess(self, preds): """Apply Non-maximum suppression to prediction outputs.""" bboxes, scores = preds[:2] # (1, bs, 300, 4), (1, bs, 300, nc) bboxes, scores = bboxes.squeeze_(0), scores.squeeze_(0) # (bs, 300, 4) bs = len(bboxes) outputs = [torch.zeros((0, 6), device=bboxes.device)] * bs for i, bbox in enumerate(bboxes): # (300, 4) bbox = ops.xywh2xyxy(bbox) score, cls = scores[i].max(-1) # (300, ) # Do not need threshold for evaluation as only got 300 boxes here. # idx = score > self.args.conf pred = torch.cat([bbox, score[..., None], cls[..., None]], dim=-1) # filter # sort by confidence to correctly get internal metrics. pred = pred[score.argsort(descending=True)] outputs[i] = pred # [idx] return outputs def update_metrics(self, preds, batch): """Metrics.""" for si, pred in enumerate(preds): idx = batch['batch_idx'] == si cls = batch['cls'][idx] bbox = batch['bboxes'][idx] nl, npr = cls.shape[0], pred.shape[0] # number of labels, predictions shape = batch['ori_shape'][si] correct_bboxes = torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device) # init self.seen += 1 if npr == 0: if nl: self.stats.append((correct_bboxes, *torch.zeros((2, 0), device=self.device), cls.squeeze(-1))) if self.args.plots: self.confusion_matrix.process_batch(detections=None, labels=cls.squeeze(-1)) continue # Predictions if self.args.single_cls: pred[:, 5] = 0 predn = pred.clone() predn[..., [0, 2]] *= shape[1] # native-space pred predn[..., [1, 3]] *= shape[0] # native-space pred # Evaluate if nl: tbox = ops.xywh2xyxy(bbox) # target boxes tbox[..., [0, 2]] *= shape[1] # native-space pred tbox[..., [1, 3]] *= shape[0] # native-space pred labelsn = torch.cat((cls, tbox), 1) # native-space labels # NOTE: To get correct metrics, the inputs of `_process_batch` should always be float32 type. correct_bboxes = self._process_batch(predn.float(), labelsn) # TODO: maybe remove these `self.` arguments as they already are member variable if self.args.plots: self.confusion_matrix.process_batch(predn, labelsn) self.stats.append((correct_bboxes, pred[:, 4], pred[:, 5], cls.squeeze(-1))) # (conf, pcls, tcls) # Save if self.args.save_json: self.pred_to_json(predn, batch['im_file'][si]) if self.args.save_txt: file = self.save_dir / 'labels' / f'{Path(batch["im_file"][si]).stem}.txt' self.save_one_txt(predn, self.args.save_conf, shape, file) ================================================ FILE: ultralytics/vit/sam/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from .build import build_sam # noqa from .model import SAM # noqa from .modules.prompt_predictor import PromptPredictor # noqa ================================================ FILE: ultralytics/vit/sam/amg.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import math from copy import deepcopy from itertools import product from typing import Any, Dict, Generator, ItemsView, List, Tuple import numpy as np import torch class MaskData: """ A structure for storing masks and their related data in batched format. Implements basic filtering and concatenation. """ def __init__(self, **kwargs) -> None: """Initialize a MaskData object, ensuring all values are supported types.""" for v in kwargs.values(): assert isinstance( v, (list, np.ndarray, torch.Tensor)), 'MaskData only supports list, numpy arrays, and torch tensors.' self._stats = dict(**kwargs) def __setitem__(self, key: str, item: Any) -> None: """Set an item in the MaskData object, ensuring it is a supported type.""" assert isinstance( item, (list, np.ndarray, torch.Tensor)), 'MaskData only supports list, numpy arrays, and torch tensors.' self._stats[key] = item def __delitem__(self, key: str) -> None: """Delete an item from the MaskData object.""" del self._stats[key] def __getitem__(self, key: str) -> Any: """Get an item from the MaskData object.""" return self._stats[key] def items(self) -> ItemsView[str, Any]: """Return an ItemsView of the MaskData object.""" return self._stats.items() def filter(self, keep: torch.Tensor) -> None: """Filter the MaskData object based on the given boolean tensor.""" for k, v in self._stats.items(): if v is None: self._stats[k] = None elif isinstance(v, torch.Tensor): self._stats[k] = v[torch.as_tensor(keep, device=v.device)] elif isinstance(v, np.ndarray): self._stats[k] = v[keep.detach().cpu().numpy()] elif isinstance(v, list) and keep.dtype == torch.bool: self._stats[k] = [a for i, a in enumerate(v) if keep[i]] elif isinstance(v, list): self._stats[k] = [v[i] for i in keep] else: raise TypeError(f'MaskData key {k} has an unsupported type {type(v)}.') def cat(self, new_stats: 'MaskData') -> None: """Concatenate a new MaskData object to the current one.""" for k, v in new_stats.items(): if k not in self._stats or self._stats[k] is None: self._stats[k] = deepcopy(v) elif isinstance(v, torch.Tensor): self._stats[k] = torch.cat([self._stats[k], v], dim=0) elif isinstance(v, np.ndarray): self._stats[k] = np.concatenate([self._stats[k], v], axis=0) elif isinstance(v, list): self._stats[k] = self._stats[k] + deepcopy(v) else: raise TypeError(f'MaskData key {k} has an unsupported type {type(v)}.') def to_numpy(self) -> None: """Convert all torch tensors in the MaskData object to numpy arrays.""" for k, v in self._stats.items(): if isinstance(v, torch.Tensor): self._stats[k] = v.detach().cpu().numpy() def is_box_near_crop_edge(boxes: torch.Tensor, crop_box: List[int], orig_box: List[int], atol: float = 20.0) -> torch.Tensor: """Return a boolean tensor indicating if boxes are near the crop edge.""" crop_box_torch = torch.as_tensor(crop_box, dtype=torch.float, device=boxes.device) orig_box_torch = torch.as_tensor(orig_box, dtype=torch.float, device=boxes.device) boxes = uncrop_boxes_xyxy(boxes, crop_box).float() near_crop_edge = torch.isclose(boxes, crop_box_torch[None, :], atol=atol, rtol=0) near_image_edge = torch.isclose(boxes, orig_box_torch[None, :], atol=atol, rtol=0) near_crop_edge = torch.logical_and(near_crop_edge, ~near_image_edge) return torch.any(near_crop_edge, dim=1) def box_xyxy_to_xywh(box_xyxy: torch.Tensor) -> torch.Tensor: """Convert bounding boxes from XYXY format to XYWH format.""" box_xywh = deepcopy(box_xyxy) box_xywh[2] = box_xywh[2] - box_xywh[0] box_xywh[3] = box_xywh[3] - box_xywh[1] return box_xywh def batch_iterator(batch_size: int, *args) -> Generator[List[Any], None, None]: """Yield batches of data from the input arguments.""" assert args and all(len(a) == len(args[0]) for a in args), 'Batched iteration must have same-size inputs.' n_batches = len(args[0]) // batch_size + int(len(args[0]) % batch_size != 0) for b in range(n_batches): yield [arg[b * batch_size:(b + 1) * batch_size] for arg in args] def mask_to_rle_pytorch(tensor: torch.Tensor) -> List[Dict[str, Any]]: """Encode masks as uncompressed RLEs in the format expected by pycocotools.""" # Put in fortran order and flatten h,w b, h, w = tensor.shape tensor = tensor.permute(0, 2, 1).flatten(1) # Compute change indices diff = tensor[:, 1:] ^ tensor[:, :-1] change_indices = diff.nonzero() # Encode run length out = [] for i in range(b): cur_idxs = change_indices[change_indices[:, 0] == i, 1] cur_idxs = torch.cat([ torch.tensor([0], dtype=cur_idxs.dtype, device=cur_idxs.device), cur_idxs + 1, torch.tensor([h * w], dtype=cur_idxs.dtype, device=cur_idxs.device), ]) btw_idxs = cur_idxs[1:] - cur_idxs[:-1] counts = [] if tensor[i, 0] == 0 else [0] counts.extend(btw_idxs.detach().cpu().tolist()) out.append({'size': [h, w], 'counts': counts}) return out def rle_to_mask(rle: Dict[str, Any]) -> np.ndarray: """Compute a binary mask from an uncompressed RLE.""" h, w = rle['size'] mask = np.empty(h * w, dtype=bool) idx = 0 parity = False for count in rle['counts']: mask[idx:idx + count] = parity idx += count parity ^= True mask = mask.reshape(w, h) return mask.transpose() # Put in C order def area_from_rle(rle: Dict[str, Any]) -> int: """Calculate the area of a mask from its uncompressed RLE.""" return sum(rle['counts'][1::2]) def calculate_stability_score(masks: torch.Tensor, mask_threshold: float, threshold_offset: float) -> torch.Tensor: """ Computes the stability score for a batch of masks. The stability score is the IoU between the binary masks obtained by thresholding the predicted mask logits at high and low values. """ # One mask is always contained inside the other. # Save memory by preventing unnecessary cast to torch.int64 intersections = ((masks > (mask_threshold + threshold_offset)).sum(-1, dtype=torch.int16).sum(-1, dtype=torch.int32)) unions = ((masks > (mask_threshold - threshold_offset)).sum(-1, dtype=torch.int16).sum(-1, dtype=torch.int32)) return intersections / unions def build_point_grid(n_per_side: int) -> np.ndarray: """Generate a 2D grid of evenly spaced points in the range [0,1]x[0,1].""" offset = 1 / (2 * n_per_side) points_one_side = np.linspace(offset, 1 - offset, n_per_side) points_x = np.tile(points_one_side[None, :], (n_per_side, 1)) points_y = np.tile(points_one_side[:, None], (1, n_per_side)) return np.stack([points_x, points_y], axis=-1).reshape(-1, 2) def build_all_layer_point_grids(n_per_side: int, n_layers: int, scale_per_layer: int) -> List[np.ndarray]: """Generate point grids for all crop layers.""" return [build_point_grid(int(n_per_side / (scale_per_layer ** i))) for i in range(n_layers + 1)] def generate_crop_boxes(im_size: Tuple[int, ...], n_layers: int, overlap_ratio: float) -> Tuple[List[List[int]], List[int]]: """Generates a list of crop boxes of different sizes. Each layer has (2**i)**2 boxes for the ith layer.""" crop_boxes, layer_idxs = [], [] im_h, im_w = im_size short_side = min(im_h, im_w) # Original image crop_boxes.append([0, 0, im_w, im_h]) layer_idxs.append(0) def crop_len(orig_len, n_crops, overlap): """Crops bounding boxes to the size of the input image.""" return int(math.ceil((overlap * (n_crops - 1) + orig_len) / n_crops)) for i_layer in range(n_layers): n_crops_per_side = 2 ** (i_layer + 1) overlap = int(overlap_ratio * short_side * (2 / n_crops_per_side)) crop_w = crop_len(im_w, n_crops_per_side, overlap) crop_h = crop_len(im_h, n_crops_per_side, overlap) crop_box_x0 = [int((crop_w - overlap) * i) for i in range(n_crops_per_side)] crop_box_y0 = [int((crop_h - overlap) * i) for i in range(n_crops_per_side)] # Crops in XYWH format for x0, y0 in product(crop_box_x0, crop_box_y0): box = [x0, y0, min(x0 + crop_w, im_w), min(y0 + crop_h, im_h)] crop_boxes.append(box) layer_idxs.append(i_layer + 1) return crop_boxes, layer_idxs def uncrop_boxes_xyxy(boxes: torch.Tensor, crop_box: List[int]) -> torch.Tensor: """Uncrop bounding boxes by adding the crop box offset.""" x0, y0, _, _ = crop_box offset = torch.tensor([[x0, y0, x0, y0]], device=boxes.device) # Check if boxes has a channel dimension if len(boxes.shape) == 3: offset = offset.unsqueeze(1) return boxes + offset def uncrop_points(points: torch.Tensor, crop_box: List[int]) -> torch.Tensor: """Uncrop points by adding the crop box offset.""" x0, y0, _, _ = crop_box offset = torch.tensor([[x0, y0]], device=points.device) # Check if points has a channel dimension if len(points.shape) == 3: offset = offset.unsqueeze(1) return points + offset def uncrop_masks(masks: torch.Tensor, crop_box: List[int], orig_h: int, orig_w: int) -> torch.Tensor: """Uncrop masks by padding them to the original image size.""" x0, y0, x1, y1 = crop_box if x0 == 0 and y0 == 0 and x1 == orig_w and y1 == orig_h: return masks # Coordinate transform masks pad_x, pad_y = orig_w - (x1 - x0), orig_h - (y1 - y0) pad = (x0, pad_x - x0, y0, pad_y - y0) return torch.nn.functional.pad(masks, pad, value=0) def remove_small_regions(mask: np.ndarray, area_thresh: float, mode: str) -> Tuple[np.ndarray, bool]: """Remove small disconnected regions or holes in a mask, returning the mask and a modification indicator.""" import cv2 # type: ignore assert mode in {'holes', 'islands'} correct_holes = mode == 'holes' working_mask = (correct_holes ^ mask).astype(np.uint8) n_labels, regions, stats, _ = cv2.connectedComponentsWithStats(working_mask, 8) sizes = stats[:, -1][1:] # Row 0 is background label small_regions = [i + 1 for i, s in enumerate(sizes) if s < area_thresh] if not small_regions: return mask, False fill_labels = [0] + small_regions if not correct_holes: # If every region is below threshold, keep largest fill_labels = [i for i in range(n_labels) if i not in fill_labels] or [int(np.argmax(sizes)) + 1] mask = np.isin(regions, fill_labels) return mask, True def coco_encode_rle(uncompressed_rle: Dict[str, Any]) -> Dict[str, Any]: """Encode uncompressed RLE (run-length encoding) to COCO RLE format.""" from pycocotools import mask as mask_utils # type: ignore h, w = uncompressed_rle['size'] rle = mask_utils.frPyObjects(uncompressed_rle, h, w) rle['counts'] = rle['counts'].decode('utf-8') # Necessary to serialize with json return rle def batched_mask_to_box(masks: torch.Tensor) -> torch.Tensor: """ Calculates boxes in XYXY format around masks. Return [0,0,0,0] for an empty mask. For input shape C1xC2x...xHxW, the output shape is C1xC2x...x4. """ # torch.max below raises an error on empty inputs, just skip in this case if torch.numel(masks) == 0: return torch.zeros(*masks.shape[:-2], 4, device=masks.device) # Normalize shape to CxHxW shape = masks.shape h, w = shape[-2:] masks = masks.flatten(0, -3) if len(shape) > 2 else masks.unsqueeze(0) # Get top and bottom edges in_height, _ = torch.max(masks, dim=-1) in_height_coords = in_height * torch.arange(h, device=in_height.device)[None, :] bottom_edges, _ = torch.max(in_height_coords, dim=-1) in_height_coords = in_height_coords + h * (~in_height) top_edges, _ = torch.min(in_height_coords, dim=-1) # Get left and right edges in_width, _ = torch.max(masks, dim=-2) in_width_coords = in_width * torch.arange(w, device=in_width.device)[None, :] right_edges, _ = torch.max(in_width_coords, dim=-1) in_width_coords = in_width_coords + w * (~in_width) left_edges, _ = torch.min(in_width_coords, dim=-1) # If the mask is empty the right edge will be to the left of the left edge. # Replace these boxes with [0, 0, 0, 0] empty_filter = (right_edges < left_edges) | (bottom_edges < top_edges) out = torch.stack([left_edges, top_edges, right_edges, bottom_edges], dim=-1) out = out * (~empty_filter).unsqueeze(-1) # Return to original shape return out.reshape(*shape[:-2], 4) if len(shape) > 2 else out[0] ================================================ FILE: ultralytics/vit/sam/autosize.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. from copy import deepcopy from typing import Tuple import numpy as np import torch from torch.nn import functional as F from torchvision.transforms.functional import resize, to_pil_image # type: ignore class ResizeLongestSide: """ Resizes images to the longest side 'target_length', as well as provides methods for resizing coordinates and boxes. Provides methods for transforming both numpy array and batched torch tensors. """ def __init__(self, target_length: int) -> None: self.target_length = target_length def apply_image(self, image: np.ndarray) -> np.ndarray: """ Expects a numpy array with shape HxWxC in uint8 format. """ target_size = self.get_preprocess_shape(image.shape[0], image.shape[1], self.target_length) return np.array(resize(to_pil_image(image), target_size)) def apply_coords(self, coords: np.ndarray, original_size: Tuple[int, ...]) -> np.ndarray: """ Expects a numpy array of length 2 in the final dimension. Requires the original image size in (H, W) format. """ old_h, old_w = original_size new_h, new_w = self.get_preprocess_shape(original_size[0], original_size[1], self.target_length) coords = deepcopy(coords).astype(float) coords[..., 0] = coords[..., 0] * (new_w / old_w) coords[..., 1] = coords[..., 1] * (new_h / old_h) return coords def apply_boxes(self, boxes: np.ndarray, original_size: Tuple[int, ...]) -> np.ndarray: """ Expects a numpy array shape Bx4. Requires the original image size in (H, W) format. """ boxes = self.apply_coords(boxes.reshape(-1, 2, 2), original_size) return boxes.reshape(-1, 4) def apply_image_torch(self, image: torch.Tensor) -> torch.Tensor: """ Expects batched images with shape BxCxHxW and float format. This transformation may not exactly match apply_image. apply_image is the transformation expected by the model. """ # Expects an image in BCHW format. May not exactly match apply_image. target_size = self.get_preprocess_shape(image.shape[2], image.shape[3], self.target_length) return F.interpolate(image, target_size, mode='bilinear', align_corners=False, antialias=True) def apply_coords_torch(self, coords: torch.Tensor, original_size: Tuple[int, ...]) -> torch.Tensor: """ Expects a torch tensor with length 2 in the last dimension. Requires the original image size in (H, W) format. """ old_h, old_w = original_size new_h, new_w = self.get_preprocess_shape(original_size[0], original_size[1], self.target_length) coords = deepcopy(coords).to(torch.float) coords[..., 0] = coords[..., 0] * (new_w / old_w) coords[..., 1] = coords[..., 1] * (new_h / old_h) return coords def apply_boxes_torch(self, boxes: torch.Tensor, original_size: Tuple[int, ...]) -> torch.Tensor: """ Expects a torch tensor with shape Bx4. Requires the original image size in (H, W) format. """ boxes = self.apply_coords_torch(boxes.reshape(-1, 2, 2), original_size) return boxes.reshape(-1, 4) @staticmethod def get_preprocess_shape(oldh: int, oldw: int, long_side_length: int) -> Tuple[int, int]: """ Compute the output size given input size and target long side length. """ scale = long_side_length * 1.0 / max(oldh, oldw) newh, neww = oldh * scale, oldw * scale neww = int(neww + 0.5) newh = int(newh + 0.5) return (newh, neww) ================================================ FILE: ultralytics/vit/sam/build.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. from functools import partial import torch from ...yolo.utils.downloads import attempt_download_asset from .modules.decoders import MaskDecoder from .modules.encoders import ImageEncoderViT, PromptEncoder from .modules.sam import Sam from .modules.transformer import TwoWayTransformer def build_sam_vit_h(checkpoint=None): """Build and return a Segment Anything Model (SAM) h-size model.""" return _build_sam( encoder_embed_dim=1280, encoder_depth=32, encoder_num_heads=16, encoder_global_attn_indexes=[7, 15, 23, 31], checkpoint=checkpoint, ) def build_sam_vit_l(checkpoint=None): """Build and return a Segment Anything Model (SAM) l-size model.""" return _build_sam( encoder_embed_dim=1024, encoder_depth=24, encoder_num_heads=16, encoder_global_attn_indexes=[5, 11, 17, 23], checkpoint=checkpoint, ) def build_sam_vit_b(checkpoint=None): """Build and return a Segment Anything Model (SAM) b-size model.""" return _build_sam( encoder_embed_dim=768, encoder_depth=12, encoder_num_heads=12, encoder_global_attn_indexes=[2, 5, 8, 11], checkpoint=checkpoint, ) def _build_sam( encoder_embed_dim, encoder_depth, encoder_num_heads, encoder_global_attn_indexes, checkpoint=None, ): """Builds the selected SAM model architecture.""" prompt_embed_dim = 256 image_size = 1024 vit_patch_size = 16 image_embedding_size = image_size // vit_patch_size sam = Sam( image_encoder=ImageEncoderViT( depth=encoder_depth, embed_dim=encoder_embed_dim, img_size=image_size, mlp_ratio=4, norm_layer=partial(torch.nn.LayerNorm, eps=1e-6), num_heads=encoder_num_heads, patch_size=vit_patch_size, qkv_bias=True, use_rel_pos=True, global_attn_indexes=encoder_global_attn_indexes, window_size=14, out_chans=prompt_embed_dim, ), prompt_encoder=PromptEncoder( embed_dim=prompt_embed_dim, image_embedding_size=(image_embedding_size, image_embedding_size), input_image_size=(image_size, image_size), mask_in_chans=16, ), mask_decoder=MaskDecoder( num_multimask_outputs=3, transformer=TwoWayTransformer( depth=2, embedding_dim=prompt_embed_dim, mlp_dim=2048, num_heads=8, ), transformer_dim=prompt_embed_dim, iou_head_depth=3, iou_head_hidden_dim=256, ), pixel_mean=[123.675, 116.28, 103.53], pixel_std=[58.395, 57.12, 57.375], ) sam.eval() if checkpoint is not None: attempt_download_asset(checkpoint) with open(checkpoint, 'rb') as f: state_dict = torch.load(f) sam.load_state_dict(state_dict) return sam sam_model_map = { # "default": build_sam_vit_h, 'sam_h.pt': build_sam_vit_h, 'sam_l.pt': build_sam_vit_l, 'sam_b.pt': build_sam_vit_b, } def build_sam(ckpt='sam_b.pt'): """Build a SAM model specified by ckpt.""" model_builder = None for k in sam_model_map.keys(): if ckpt.endswith(k): model_builder = sam_model_map.get(k) if not model_builder: raise FileNotFoundError(f'{ckpt} is not a supported sam model. Available models are: \n {sam_model_map.keys()}') return model_builder(ckpt) ================================================ FILE: ultralytics/vit/sam/model.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license """ SAM model interface """ from ultralytics.yolo.cfg import get_cfg from ...yolo.utils.torch_utils import model_info from .build import build_sam from .predict import Predictor class SAM: def __init__(self, model='sam_b.pt') -> None: if model and not model.endswith('.pt') and not model.endswith('.pth'): # Should raise AssertionError instead? raise NotImplementedError('Segment anything prediction requires pre-trained checkpoint') self.model = build_sam(model) self.task = 'segment' # required self.predictor = None # reuse predictor def predict(self, source, stream=False, **kwargs): """Predicts and returns segmentation masks for given image or video source.""" overrides = dict(conf=0.25, task='segment', mode='predict') overrides.update(kwargs) # prefer kwargs if not self.predictor: self.predictor = Predictor(overrides=overrides) self.predictor.setup_model(model=self.model) else: # only update args if predictor is already setup self.predictor.args = get_cfg(self.predictor.args, overrides) return self.predictor(source, stream=stream) def train(self, **kwargs): """Function trains models but raises an error as SAM models do not support training.""" raise NotImplementedError("SAM models don't support training") def val(self, **kwargs): """Run validation given dataset.""" raise NotImplementedError("SAM models don't support validation") def __call__(self, source=None, stream=False, **kwargs): """Calls the 'predict' function with given arguments to perform object detection.""" return self.predict(source, stream, **kwargs) def __getattr__(self, attr): """Raises error if object has no requested attribute.""" name = self.__class__.__name__ raise AttributeError(f"'{name}' object has no attribute '{attr}'. See valid attributes below.\n{self.__doc__}") def info(self, detailed=False, verbose=True): """ Logs model info. Args: detailed (bool): Show detailed information about model. verbose (bool): Controls verbosity. """ return model_info(self.model, detailed=detailed, verbose=verbose) ================================================ FILE: ultralytics/vit/sam/modules/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license ================================================ FILE: ultralytics/vit/sam/modules/decoders.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from typing import List, Tuple, Type import torch from torch import nn from torch.nn import functional as F from ultralytics.nn.modules import LayerNorm2d class MaskDecoder(nn.Module): def __init__( self, *, transformer_dim: int, transformer: nn.Module, num_multimask_outputs: int = 3, activation: Type[nn.Module] = nn.GELU, iou_head_depth: int = 3, iou_head_hidden_dim: int = 256, ) -> None: """ Predicts masks given an image and prompt embeddings, using a transformer architecture. Arguments: transformer_dim (int): the channel dimension of the transformer module transformer (nn.Module): the transformer used to predict masks num_multimask_outputs (int): the number of masks to predict when disambiguating masks activation (nn.Module): the type of activation to use when upscaling masks iou_head_depth (int): the depth of the MLP used to predict mask quality iou_head_hidden_dim (int): the hidden dimension of the MLP used to predict mask quality """ super().__init__() self.transformer_dim = transformer_dim self.transformer = transformer self.num_multimask_outputs = num_multimask_outputs self.iou_token = nn.Embedding(1, transformer_dim) self.num_mask_tokens = num_multimask_outputs + 1 self.mask_tokens = nn.Embedding(self.num_mask_tokens, transformer_dim) self.output_upscaling = nn.Sequential( nn.ConvTranspose2d(transformer_dim, transformer_dim // 4, kernel_size=2, stride=2), LayerNorm2d(transformer_dim // 4), activation(), nn.ConvTranspose2d(transformer_dim // 4, transformer_dim // 8, kernel_size=2, stride=2), activation(), ) self.output_hypernetworks_mlps = nn.ModuleList([ MLP(transformer_dim, transformer_dim, transformer_dim // 8, 3) for _ in range(self.num_mask_tokens)]) self.iou_prediction_head = MLP(transformer_dim, iou_head_hidden_dim, self.num_mask_tokens, iou_head_depth) def forward( self, image_embeddings: torch.Tensor, image_pe: torch.Tensor, sparse_prompt_embeddings: torch.Tensor, dense_prompt_embeddings: torch.Tensor, multimask_output: bool, ) -> Tuple[torch.Tensor, torch.Tensor]: """ Predict masks given image and prompt embeddings. Arguments: image_embeddings (torch.Tensor): the embeddings from the image encoder image_pe (torch.Tensor): positional encoding with the shape of image_embeddings sparse_prompt_embeddings (torch.Tensor): the embeddings of the points and boxes dense_prompt_embeddings (torch.Tensor): the embeddings of the mask inputs multimask_output (bool): Whether to return multiple masks or a single mask. Returns: torch.Tensor: batched predicted masks torch.Tensor: batched predictions of mask quality """ masks, iou_pred = self.predict_masks( image_embeddings=image_embeddings, image_pe=image_pe, sparse_prompt_embeddings=sparse_prompt_embeddings, dense_prompt_embeddings=dense_prompt_embeddings, ) # Select the correct mask or masks for output mask_slice = slice(1, None) if multimask_output else slice(0, 1) masks = masks[:, mask_slice, :, :] iou_pred = iou_pred[:, mask_slice] # Prepare output return masks, iou_pred def predict_masks( self, image_embeddings: torch.Tensor, image_pe: torch.Tensor, sparse_prompt_embeddings: torch.Tensor, dense_prompt_embeddings: torch.Tensor, ) -> Tuple[torch.Tensor, torch.Tensor]: """Predicts masks. See 'forward' for more details.""" # Concatenate output tokens output_tokens = torch.cat([self.iou_token.weight, self.mask_tokens.weight], dim=0) output_tokens = output_tokens.unsqueeze(0).expand(sparse_prompt_embeddings.size(0), -1, -1) tokens = torch.cat((output_tokens, sparse_prompt_embeddings), dim=1) # Expand per-image data in batch direction to be per-mask src = torch.repeat_interleave(image_embeddings, tokens.shape[0], dim=0) src = src + dense_prompt_embeddings pos_src = torch.repeat_interleave(image_pe, tokens.shape[0], dim=0) b, c, h, w = src.shape # Run the transformer hs, src = self.transformer(src, pos_src, tokens) iou_token_out = hs[:, 0, :] mask_tokens_out = hs[:, 1:(1 + self.num_mask_tokens), :] # Upscale mask embeddings and predict masks using the mask tokens src = src.transpose(1, 2).view(b, c, h, w) upscaled_embedding = self.output_upscaling(src) hyper_in_list: List[torch.Tensor] = [ self.output_hypernetworks_mlps[i](mask_tokens_out[:, i, :]) for i in range(self.num_mask_tokens)] hyper_in = torch.stack(hyper_in_list, dim=1) b, c, h, w = upscaled_embedding.shape masks = (hyper_in @ upscaled_embedding.view(b, c, h * w)).view(b, -1, h, w) # Generate mask quality predictions iou_pred = self.iou_prediction_head(iou_token_out) return masks, iou_pred class MLP(nn.Module): """ Lightly adapted from https://github.com/facebookresearch/MaskFormer/blob/main/mask_former/modeling/transformer/transformer_predictor.py """ def __init__( self, input_dim: int, hidden_dim: int, output_dim: int, num_layers: int, sigmoid_output: bool = False, ) -> None: super().__init__() self.num_layers = num_layers h = [hidden_dim] * (num_layers - 1) self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) self.sigmoid_output = sigmoid_output def forward(self, x): """Executes feedforward within the neural network module and applies activation.""" for i, layer in enumerate(self.layers): x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) if self.sigmoid_output: x = torch.sigmoid(x) return x ================================================ FILE: ultralytics/vit/sam/modules/encoders.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from typing import Any, Optional, Tuple, Type import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from ultralytics.nn.modules import LayerNorm2d, MLPBlock # This class and its supporting functions below lightly adapted from the ViTDet backbone available at: https://github.com/facebookresearch/detectron2/blob/main/detectron2/modeling/backbone/vit.py # noqa class ImageEncoderViT(nn.Module): def __init__( self, img_size: int = 1024, patch_size: int = 16, in_chans: int = 3, embed_dim: int = 768, depth: int = 12, num_heads: int = 12, mlp_ratio: float = 4.0, out_chans: int = 256, qkv_bias: bool = True, norm_layer: Type[nn.Module] = nn.LayerNorm, act_layer: Type[nn.Module] = nn.GELU, use_abs_pos: bool = True, use_rel_pos: bool = False, rel_pos_zero_init: bool = True, window_size: int = 0, global_attn_indexes: Tuple[int, ...] = (), ) -> None: """ Args: img_size (int): Input image size. patch_size (int): Patch size. in_chans (int): Number of input image channels. embed_dim (int): Patch embedding dimension. depth (int): Depth of ViT. num_heads (int): Number of attention heads in each ViT block. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool): If True, add a learnable bias to query, key, value. norm_layer (nn.Module): Normalization layer. act_layer (nn.Module): Activation layer. use_abs_pos (bool): If True, use absolute positional embeddings. use_rel_pos (bool): If True, add relative positional embeddings to the attention map. rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. window_size (int): Window size for window attention blocks. global_attn_indexes (list): Indexes for blocks using global attention. """ super().__init__() self.img_size = img_size self.patch_embed = PatchEmbed( kernel_size=(patch_size, patch_size), stride=(patch_size, patch_size), in_chans=in_chans, embed_dim=embed_dim, ) self.pos_embed: Optional[nn.Parameter] = None if use_abs_pos: # Initialize absolute positional embedding with pretrain image size. self.pos_embed = nn.Parameter(torch.zeros(1, img_size // patch_size, img_size // patch_size, embed_dim)) self.blocks = nn.ModuleList() for i in range(depth): block = Block( dim=embed_dim, num_heads=num_heads, mlp_ratio=mlp_ratio, qkv_bias=qkv_bias, norm_layer=norm_layer, act_layer=act_layer, use_rel_pos=use_rel_pos, rel_pos_zero_init=rel_pos_zero_init, window_size=window_size if i not in global_attn_indexes else 0, input_size=(img_size // patch_size, img_size // patch_size), ) self.blocks.append(block) self.neck = nn.Sequential( nn.Conv2d( embed_dim, out_chans, kernel_size=1, bias=False, ), LayerNorm2d(out_chans), nn.Conv2d( out_chans, out_chans, kernel_size=3, padding=1, bias=False, ), LayerNorm2d(out_chans), ) def forward(self, x: torch.Tensor) -> torch.Tensor: x = self.patch_embed(x) if self.pos_embed is not None: x = x + self.pos_embed for blk in self.blocks: x = blk(x) x = self.neck(x.permute(0, 3, 1, 2)) return x class PromptEncoder(nn.Module): def __init__( self, embed_dim: int, image_embedding_size: Tuple[int, int], input_image_size: Tuple[int, int], mask_in_chans: int, activation: Type[nn.Module] = nn.GELU, ) -> None: """ Encodes prompts for input to SAM's mask decoder. Arguments: embed_dim (int): The prompts' embedding dimension image_embedding_size (tuple(int, int)): The spatial size of the image embedding, as (H, W). input_image_size (int): The padded size of the image as input to the image encoder, as (H, W). mask_in_chans (int): The number of hidden channels used for encoding input masks. activation (nn.Module): The activation to use when encoding input masks. """ super().__init__() self.embed_dim = embed_dim self.input_image_size = input_image_size self.image_embedding_size = image_embedding_size self.pe_layer = PositionEmbeddingRandom(embed_dim // 2) self.num_point_embeddings: int = 4 # pos/neg point + 2 box corners point_embeddings = [nn.Embedding(1, embed_dim) for _ in range(self.num_point_embeddings)] self.point_embeddings = nn.ModuleList(point_embeddings) self.not_a_point_embed = nn.Embedding(1, embed_dim) self.mask_input_size = (4 * image_embedding_size[0], 4 * image_embedding_size[1]) self.mask_downscaling = nn.Sequential( nn.Conv2d(1, mask_in_chans // 4, kernel_size=2, stride=2), LayerNorm2d(mask_in_chans // 4), activation(), nn.Conv2d(mask_in_chans // 4, mask_in_chans, kernel_size=2, stride=2), LayerNorm2d(mask_in_chans), activation(), nn.Conv2d(mask_in_chans, embed_dim, kernel_size=1), ) self.no_mask_embed = nn.Embedding(1, embed_dim) def get_dense_pe(self) -> torch.Tensor: """ Returns the positional encoding used to encode point prompts, applied to a dense set of points the shape of the image encoding. Returns: torch.Tensor: Positional encoding with shape 1x(embed_dim)x(embedding_h)x(embedding_w) """ return self.pe_layer(self.image_embedding_size).unsqueeze(0) def _embed_points( self, points: torch.Tensor, labels: torch.Tensor, pad: bool, ) -> torch.Tensor: """Embeds point prompts.""" points = points + 0.5 # Shift to center of pixel if pad: padding_point = torch.zeros((points.shape[0], 1, 2), device=points.device) padding_label = -torch.ones((labels.shape[0], 1), device=labels.device) points = torch.cat([points, padding_point], dim=1) labels = torch.cat([labels, padding_label], dim=1) point_embedding = self.pe_layer.forward_with_coords(points, self.input_image_size) point_embedding[labels == -1] = 0.0 point_embedding[labels == -1] += self.not_a_point_embed.weight point_embedding[labels == 0] += self.point_embeddings[0].weight point_embedding[labels == 1] += self.point_embeddings[1].weight return point_embedding def _embed_boxes(self, boxes: torch.Tensor) -> torch.Tensor: """Embeds box prompts.""" boxes = boxes + 0.5 # Shift to center of pixel coords = boxes.reshape(-1, 2, 2) corner_embedding = self.pe_layer.forward_with_coords(coords, self.input_image_size) corner_embedding[:, 0, :] += self.point_embeddings[2].weight corner_embedding[:, 1, :] += self.point_embeddings[3].weight return corner_embedding def _embed_masks(self, masks: torch.Tensor) -> torch.Tensor: """Embeds mask inputs.""" return self.mask_downscaling(masks) def _get_batch_size( self, points: Optional[Tuple[torch.Tensor, torch.Tensor]], boxes: Optional[torch.Tensor], masks: Optional[torch.Tensor], ) -> int: """ Gets the batch size of the output given the batch size of the input prompts. """ if points is not None: return points[0].shape[0] elif boxes is not None: return boxes.shape[0] elif masks is not None: return masks.shape[0] else: return 1 def _get_device(self) -> torch.device: return self.point_embeddings[0].weight.device def forward( self, points: Optional[Tuple[torch.Tensor, torch.Tensor]], boxes: Optional[torch.Tensor], masks: Optional[torch.Tensor], ) -> Tuple[torch.Tensor, torch.Tensor]: """ Embeds different types of prompts, returning both sparse and dense embeddings. Arguments: points (tuple(torch.Tensor, torch.Tensor), None): point coordinates and labels to embed. boxes (torch.Tensor, None): boxes to embed masks (torch.Tensor, None): masks to embed Returns: torch.Tensor: sparse embeddings for the points and boxes, with shape BxNx(embed_dim), where N is determined by the number of input points and boxes. torch.Tensor: dense embeddings for the masks, in the shape Bx(embed_dim)x(embed_H)x(embed_W) """ bs = self._get_batch_size(points, boxes, masks) sparse_embeddings = torch.empty((bs, 0, self.embed_dim), device=self._get_device()) if points is not None: coords, labels = points point_embeddings = self._embed_points(coords, labels, pad=(boxes is None)) sparse_embeddings = torch.cat([sparse_embeddings, point_embeddings], dim=1) if boxes is not None: box_embeddings = self._embed_boxes(boxes) sparse_embeddings = torch.cat([sparse_embeddings, box_embeddings], dim=1) if masks is not None: dense_embeddings = self._embed_masks(masks) else: dense_embeddings = self.no_mask_embed.weight.reshape(1, -1, 1, 1).expand(bs, -1, self.image_embedding_size[0], self.image_embedding_size[1]) return sparse_embeddings, dense_embeddings class PositionEmbeddingRandom(nn.Module): """ Positional encoding using random spatial frequencies. """ def __init__(self, num_pos_feats: int = 64, scale: Optional[float] = None) -> None: super().__init__() if scale is None or scale <= 0.0: scale = 1.0 self.register_buffer( 'positional_encoding_gaussian_matrix', scale * torch.randn((2, num_pos_feats)), ) def _pe_encoding(self, coords: torch.Tensor) -> torch.Tensor: """Positionally encode points that are normalized to [0,1].""" # assuming coords are in [0, 1]^2 square and have d_1 x ... x d_n x 2 shape coords = 2 * coords - 1 coords = coords @ self.positional_encoding_gaussian_matrix coords = 2 * np.pi * coords # outputs d_1 x ... x d_n x C shape return torch.cat([torch.sin(coords), torch.cos(coords)], dim=-1) def forward(self, size: Tuple[int, int]) -> torch.Tensor: """Generate positional encoding for a grid of the specified size.""" h, w = size device: Any = self.positional_encoding_gaussian_matrix.device grid = torch.ones((h, w), device=device, dtype=torch.float32) y_embed = grid.cumsum(dim=0) - 0.5 x_embed = grid.cumsum(dim=1) - 0.5 y_embed = y_embed / h x_embed = x_embed / w pe = self._pe_encoding(torch.stack([x_embed, y_embed], dim=-1)) return pe.permute(2, 0, 1) # C x H x W def forward_with_coords(self, coords_input: torch.Tensor, image_size: Tuple[int, int]) -> torch.Tensor: """Positionally encode points that are not normalized to [0,1].""" coords = coords_input.clone() coords[:, :, 0] = coords[:, :, 0] / image_size[1] coords[:, :, 1] = coords[:, :, 1] / image_size[0] return self._pe_encoding(coords.to(torch.float)) # B x N x C class Block(nn.Module): """Transformer blocks with support of window attention and residual propagation blocks""" def __init__( self, dim: int, num_heads: int, mlp_ratio: float = 4.0, qkv_bias: bool = True, norm_layer: Type[nn.Module] = nn.LayerNorm, act_layer: Type[nn.Module] = nn.GELU, use_rel_pos: bool = False, rel_pos_zero_init: bool = True, window_size: int = 0, input_size: Optional[Tuple[int, int]] = None, ) -> None: """ Args: dim (int): Number of input channels. num_heads (int): Number of attention heads in each ViT block. mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. qkv_bias (bool): If True, add a learnable bias to query, key, value. norm_layer (nn.Module): Normalization layer. act_layer (nn.Module): Activation layer. use_rel_pos (bool): If True, add relative positional embeddings to the attention map. rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. window_size (int): Window size for window attention blocks. If it equals 0, then use global attention. input_size (tuple(int, int), None): Input resolution for calculating the relative positional parameter size. """ super().__init__() self.norm1 = norm_layer(dim) self.attn = Attention( dim, num_heads=num_heads, qkv_bias=qkv_bias, use_rel_pos=use_rel_pos, rel_pos_zero_init=rel_pos_zero_init, input_size=input_size if window_size == 0 else (window_size, window_size), ) self.norm2 = norm_layer(dim) self.mlp = MLPBlock(embedding_dim=dim, mlp_dim=int(dim * mlp_ratio), act=act_layer) self.window_size = window_size def forward(self, x: torch.Tensor) -> torch.Tensor: shortcut = x x = self.norm1(x) # Window partition if self.window_size > 0: H, W = x.shape[1], x.shape[2] x, pad_hw = window_partition(x, self.window_size) x = self.attn(x) # Reverse window partition if self.window_size > 0: x = window_unpartition(x, self.window_size, pad_hw, (H, W)) x = shortcut + x x = x + self.mlp(self.norm2(x)) return x class Attention(nn.Module): """Multi-head Attention block with relative position embeddings.""" def __init__( self, dim: int, num_heads: int = 8, qkv_bias: bool = True, use_rel_pos: bool = False, rel_pos_zero_init: bool = True, input_size: Optional[Tuple[int, int]] = None, ) -> None: """ Args: dim (int): Number of input channels. num_heads (int): Number of attention heads. qkv_bias (bool): If True, add a learnable bias to query, key, value. rel_pos_zero_init (bool): If True, zero initialize relative positional parameters. input_size (tuple(int, int), None): Input resolution for calculating the relative positional parameter size. """ super().__init__() self.num_heads = num_heads head_dim = dim // num_heads self.scale = head_dim ** -0.5 self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) self.proj = nn.Linear(dim, dim) self.use_rel_pos = use_rel_pos if self.use_rel_pos: assert (input_size is not None), 'Input size must be provided if using relative positional encoding.' # initialize relative positional embeddings self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim)) self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim)) def forward(self, x: torch.Tensor) -> torch.Tensor: B, H, W, _ = x.shape # qkv with shape (3, B, nHead, H * W, C) qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) # q, k, v with shape (B * nHead, H * W, C) q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0) attn = (q * self.scale) @ k.transpose(-2, -1) if self.use_rel_pos: attn = add_decomposed_rel_pos(attn, q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W)) attn = attn.softmax(dim=-1) x = (attn @ v).view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1) x = self.proj(x) return x def window_partition(x: torch.Tensor, window_size: int) -> Tuple[torch.Tensor, Tuple[int, int]]: """ Partition into non-overlapping windows with padding if needed. Args: x (tensor): input tokens with [B, H, W, C]. window_size (int): window size. Returns: windows: windows after partition with [B * num_windows, window_size, window_size, C]. (Hp, Wp): padded height and width before partition """ B, H, W, C = x.shape pad_h = (window_size - H % window_size) % window_size pad_w = (window_size - W % window_size) % window_size if pad_h > 0 or pad_w > 0: x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h)) Hp, Wp = H + pad_h, W + pad_w x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C) windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) return windows, (Hp, Wp) def window_unpartition(windows: torch.Tensor, window_size: int, pad_hw: Tuple[int, int], hw: Tuple[int, int]) -> torch.Tensor: """ Window unpartition into original sequences and removing padding. Args: windows (tensor): input tokens with [B * num_windows, window_size, window_size, C]. window_size (int): window size. pad_hw (Tuple): padded height and width (Hp, Wp). hw (Tuple): original height and width (H, W) before padding. Returns: x: unpartitioned sequences with [B, H, W, C]. """ Hp, Wp = pad_hw H, W = hw B = windows.shape[0] // (Hp * Wp // window_size // window_size) x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1) x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1) if Hp > H or Wp > W: x = x[:, :H, :W, :].contiguous() return x def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor: """ Get relative positional embeddings according to the relative positions of query and key sizes. Args: q_size (int): size of query q. k_size (int): size of key k. rel_pos (Tensor): relative position embeddings (L, C). Returns: Extracted positional embeddings according to relative positions. """ max_rel_dist = int(2 * max(q_size, k_size) - 1) # Interpolate rel pos if needed. if rel_pos.shape[0] != max_rel_dist: # Interpolate rel pos. rel_pos_resized = F.interpolate( rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1), size=max_rel_dist, mode='linear', ) rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0) else: rel_pos_resized = rel_pos # Scale the coords with short length if shapes for q and k are different. q_coords = torch.arange(q_size)[:, None] * max(k_size / q_size, 1.0) k_coords = torch.arange(k_size)[None, :] * max(q_size / k_size, 1.0) relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0) return rel_pos_resized[relative_coords.long()] def add_decomposed_rel_pos( attn: torch.Tensor, q: torch.Tensor, rel_pos_h: torch.Tensor, rel_pos_w: torch.Tensor, q_size: Tuple[int, int], k_size: Tuple[int, int], ) -> torch.Tensor: """ Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`. https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py # noqa B950 Args: attn (Tensor): attention map. q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C). rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis. rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis. q_size (Tuple): spatial sequence size of query q with (q_h, q_w). k_size (Tuple): spatial sequence size of key k with (k_h, k_w). Returns: attn (Tensor): attention map with added relative positional embeddings. """ q_h, q_w = q_size k_h, k_w = k_size Rh = get_rel_pos(q_h, k_h, rel_pos_h) Rw = get_rel_pos(q_w, k_w, rel_pos_w) B, _, dim = q.shape r_q = q.reshape(B, q_h, q_w, dim) rel_h = torch.einsum('bhwc,hkc->bhwk', r_q, Rh) rel_w = torch.einsum('bhwc,wkc->bhwk', r_q, Rw) attn = (attn.view(B, q_h, q_w, k_h, k_w) + rel_h[:, :, :, :, None] + rel_w[:, :, :, None, :]).view( B, q_h * q_w, k_h * k_w) return attn class PatchEmbed(nn.Module): """ Image to Patch Embedding. """ def __init__( self, kernel_size: Tuple[int, int] = (16, 16), stride: Tuple[int, int] = (16, 16), padding: Tuple[int, int] = (0, 0), in_chans: int = 3, embed_dim: int = 768, ) -> None: """ Args: kernel_size (Tuple): kernel size of the projection layer. stride (Tuple): stride of the projection layer. padding (Tuple): padding size of the projection layer. in_chans (int): Number of input image channels. embed_dim (int): Patch embedding dimension. """ super().__init__() self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding) def forward(self, x: torch.Tensor) -> torch.Tensor: x = self.proj(x) # B C H W -> B H W C x = x.permute(0, 2, 3, 1) return x ================================================ FILE: ultralytics/vit/sam/modules/mask_generator.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. from typing import Any, Dict, List, Optional, Tuple import numpy as np import torch from torchvision.ops.boxes import batched_nms, box_area # type: ignore from ..amg import (MaskData, area_from_rle, batch_iterator, batched_mask_to_box, box_xyxy_to_xywh, build_all_layer_point_grids, calculate_stability_score, coco_encode_rle, generate_crop_boxes, is_box_near_crop_edge, mask_to_rle_pytorch, remove_small_regions, rle_to_mask, uncrop_boxes_xyxy, uncrop_masks, uncrop_points) from .prompt_predictor import PromptPredictor from .sam import Sam class SamAutomaticMaskGenerator: def __init__( self, model: Sam, points_per_side: Optional[int] = 32, points_per_batch: int = 64, pred_iou_thresh: float = 0.88, stability_score_thresh: float = 0.95, stability_score_offset: float = 1.0, box_nms_thresh: float = 0.7, crop_n_layers: int = 0, crop_nms_thresh: float = 0.7, crop_overlap_ratio: float = 512 / 1500, crop_n_points_downscale_factor: int = 1, point_grids: Optional[List[np.ndarray]] = None, min_mask_region_area: int = 0, output_mode: str = 'binary_mask', ) -> None: """ Using a SAM model, generates masks for the entire image. Generates a grid of point prompts over the image, then filters low quality and duplicate masks. The default settings are chosen for SAM with a ViT-H backbone. Arguments: model (Sam): The SAM model to use for mask prediction. points_per_side (int, None): The number of points to be sampled along one side of the image. The total number of points is points_per_side**2. If None, 'point_grids' must provide explicit point sampling. points_per_batch (int): Sets the number of points run simultaneously by the model. Higher numbers may be faster but use more GPU memory. pred_iou_thresh (float): A filtering threshold in [0,1], using the model's predicted mask quality. stability_score_thresh (float): A filtering threshold in [0,1], using the stability of the mask under changes to the cutoff used to binarize the model's mask predictions. stability_score_offset (float): The amount to shift the cutoff when calculated the stability score. box_nms_thresh (float): The box IoU cutoff used by non-maximal suppression to filter duplicate masks. crop_n_layers (int): If >0, mask prediction will be run again on crops of the image. Sets the number of layers to run, where each layer has 2**i_layer number of image crops. crop_nms_thresh (float): The box IoU cutoff used by non-maximal suppression to filter duplicate masks between different crops. crop_overlap_ratio (float): Sets the degree to which crops overlap. In the first crop layer, crops will overlap by this fraction of the image length. Later layers with more crops scale down this overlap. crop_n_points_downscale_factor (int): The number of points-per-side sampled in layer n is scaled down by crop_n_points_downscale_factor**n. point_grids (list(np.ndarray), None): A list over explicit grids of points used for sampling, normalized to [0,1]. The nth grid in the list is used in the nth crop layer. Exclusive with points_per_side. min_mask_region_area (int): If >0, postprocessing will be applied to remove disconnected regions and holes in masks with area smaller than min_mask_region_area. Requires opencv. output_mode (str): The form masks are returned in. Can be 'binary_mask', 'uncompressed_rle', or 'coco_rle'. 'coco_rle' requires pycocotools. For large resolutions, 'binary_mask' may consume large amounts of memory. """ assert (points_per_side is None) != (point_grids is None), \ 'Exactly one of points_per_side or point_grid must be provided.' if points_per_side is not None: self.point_grids = build_all_layer_point_grids( points_per_side, crop_n_layers, crop_n_points_downscale_factor, ) elif point_grids is not None: self.point_grids = point_grids else: raise ValueError("Can't have both points_per_side and point_grid be None.") assert output_mode in {'binary_mask', 'uncompressed_rle', 'coco_rle'}, f'Unknown output_mode {output_mode}.' if output_mode == 'coco_rle': from pycocotools import mask as mask_utils # type: ignore # noqa: F401 if min_mask_region_area > 0: import cv2 # type: ignore # noqa: F401 self.predictor = PromptPredictor(model) self.points_per_batch = points_per_batch self.pred_iou_thresh = pred_iou_thresh self.stability_score_thresh = stability_score_thresh self.stability_score_offset = stability_score_offset self.box_nms_thresh = box_nms_thresh self.crop_n_layers = crop_n_layers self.crop_nms_thresh = crop_nms_thresh self.crop_overlap_ratio = crop_overlap_ratio self.crop_n_points_downscale_factor = crop_n_points_downscale_factor self.min_mask_region_area = min_mask_region_area self.output_mode = output_mode # TODO: Temporary implementation for compatibility def __call__(self, image: np.ndarray, augment=False, visualize=False) -> List[Dict[str, Any]]: return self.generate(image) @torch.no_grad() def generate(self, image: np.ndarray) -> List[Dict[str, Any]]: """ Generates masks for the given image. Arguments: image (np.ndarray): The image to generate masks for, in HWC uint8 format. Returns: list(dict(str, any)): A list over records for masks. Each record is a dict containing the following keys: segmentation (dict(str, any), np.ndarray): The mask. If output_mode='binary_mask', is an array of shape HW. Otherwise, is a dictionary containing the RLE. bbox (list(float)): The box around the mask, in XYWH format. area (int): The area in pixels of the mask. predicted_iou (float): The model's own prediction of the mask's quality. This is filtered by the pred_iou_thresh parameter. point_coords (list(list(float))): The point coordinates input to the model to generate this mask. stability_score (float): A measure of the mask's quality. This is filtered on using the stability_score_thresh parameter. crop_box (list(float)): The crop of the image used to generate the mask, given in XYWH format. """ # Generate masks mask_data = self._generate_masks(image) # Filter small disconnected regions and holes in masks if self.min_mask_region_area > 0: mask_data = self.postprocess_small_regions( mask_data, self.min_mask_region_area, max(self.box_nms_thresh, self.crop_nms_thresh), ) # Encode masks if self.output_mode == 'coco_rle': mask_data['segmentations'] = [coco_encode_rle(rle) for rle in mask_data['rles']] elif self.output_mode == 'binary_mask': mask_data['segmentations'] = [rle_to_mask(rle) for rle in mask_data['rles']] else: mask_data['segmentations'] = mask_data['rles'] # Write mask records curr_anns = [] for idx in range(len(mask_data['segmentations'])): ann = { 'segmentation': mask_data['segmentations'][idx], 'area': area_from_rle(mask_data['rles'][idx]), 'bbox': box_xyxy_to_xywh(mask_data['boxes'][idx]).tolist(), 'predicted_iou': mask_data['iou_preds'][idx].item(), 'point_coords': [mask_data['points'][idx].tolist()], 'stability_score': mask_data['stability_score'][idx].item(), 'crop_box': box_xyxy_to_xywh(mask_data['crop_boxes'][idx]).tolist(), } curr_anns.append(ann) return curr_anns def _generate_masks(self, image: np.ndarray) -> MaskData: orig_size = image.shape[:2] crop_boxes, layer_idxs = generate_crop_boxes(orig_size, self.crop_n_layers, self.crop_overlap_ratio) # Iterate over image crops data = MaskData() for crop_box, layer_idx in zip(crop_boxes, layer_idxs): crop_data = self._process_crop(image, crop_box, layer_idx, orig_size) data.cat(crop_data) # Remove duplicate masks between crops if len(crop_boxes) > 1: # Prefer masks from smaller crops scores = 1 / box_area(data['crop_boxes']) scores = scores.to(data['boxes'].device) keep_by_nms = batched_nms( data['boxes'].float(), scores, torch.zeros_like(data['boxes'][:, 0]), # categories iou_threshold=self.crop_nms_thresh, ) data.filter(keep_by_nms) data.to_numpy() return data def _process_crop( self, image: np.ndarray, crop_box: List[int], crop_layer_idx: int, orig_size: Tuple[int, ...], ) -> MaskData: # Crop the image and calculate embeddings x0, y0, x1, y1 = crop_box cropped_im = image[y0:y1, x0:x1, :] cropped_im_size = cropped_im.shape[:2] self.predictor.set_image(cropped_im) # Get points for this crop points_scale = np.array(cropped_im_size)[None, ::-1] points_for_image = self.point_grids[crop_layer_idx] * points_scale # Generate masks for this crop in batches data = MaskData() for (points, ) in batch_iterator(self.points_per_batch, points_for_image): batch_data = self._process_batch(points, cropped_im_size, crop_box, orig_size) data.cat(batch_data) del batch_data self.predictor.reset_image() # Remove duplicates within this crop. keep_by_nms = batched_nms( data['boxes'].float(), data['iou_preds'], torch.zeros_like(data['boxes'][:, 0]), # categories iou_threshold=self.box_nms_thresh, ) data.filter(keep_by_nms) # Return to the original image frame data['boxes'] = uncrop_boxes_xyxy(data['boxes'], crop_box) data['points'] = uncrop_points(data['points'], crop_box) data['crop_boxes'] = torch.tensor([crop_box for _ in range(len(data['rles']))]) return data def _process_batch( self, points: np.ndarray, im_size: Tuple[int, ...], crop_box: List[int], orig_size: Tuple[int, ...], ) -> MaskData: orig_h, orig_w = orig_size # Run model on this batch transformed_points = self.predictor.transform.apply_coords(points, im_size) in_points = torch.as_tensor(transformed_points, device=self.predictor.device) in_labels = torch.ones(in_points.shape[0], dtype=torch.int, device=in_points.device) masks, iou_preds, _ = self.predictor.predict_torch( in_points[:, None, :], in_labels[:, None], multimask_output=True, return_logits=True, ) # Serialize predictions and store in MaskData data = MaskData( masks=masks.flatten(0, 1), iou_preds=iou_preds.flatten(0, 1), points=torch.as_tensor(points.repeat(masks.shape[1], axis=0)), ) del masks # Filter by predicted IoU if self.pred_iou_thresh > 0.0: keep_mask = data['iou_preds'] > self.pred_iou_thresh data.filter(keep_mask) # Calculate stability score data['stability_score'] = calculate_stability_score(data['masks'], self.predictor.model.mask_threshold, self.stability_score_offset) if self.stability_score_thresh > 0.0: keep_mask = data['stability_score'] >= self.stability_score_thresh data.filter(keep_mask) # Threshold masks and calculate boxes data['masks'] = data['masks'] > self.predictor.model.mask_threshold data['boxes'] = batched_mask_to_box(data['masks']) # Filter boxes that touch crop boundaries keep_mask = ~is_box_near_crop_edge(data['boxes'], crop_box, [0, 0, orig_w, orig_h]) if not torch.all(keep_mask): data.filter(keep_mask) # Compress to RLE data['masks'] = uncrop_masks(data['masks'], crop_box, orig_h, orig_w) data['rles'] = mask_to_rle_pytorch(data['masks']) del data['masks'] return data @staticmethod def postprocess_small_regions(mask_data: MaskData, min_area: int, nms_thresh: float) -> MaskData: """ Removes small disconnected regions and holes in masks, then reruns box NMS to remove any new duplicates. Edits mask_data in place. Requires open-cv as a dependency. """ if len(mask_data['rles']) == 0: return mask_data # Filter small disconnected regions and holes new_masks = [] scores = [] for rle in mask_data['rles']: mask = rle_to_mask(rle) mask, changed = remove_small_regions(mask, min_area, mode='holes') unchanged = not changed mask, changed = remove_small_regions(mask, min_area, mode='islands') unchanged = unchanged and not changed new_masks.append(torch.as_tensor(mask).unsqueeze(0)) # Give score=0 to changed masks and score=1 to unchanged masks # so NMS will prefer ones that didn't need postprocessing scores.append(float(unchanged)) # Recalculate boxes and remove any new duplicates masks = torch.cat(new_masks, dim=0) boxes = batched_mask_to_box(masks) keep_by_nms = batched_nms( boxes.float(), torch.as_tensor(scores), torch.zeros_like(boxes[:, 0]), # categories iou_threshold=nms_thresh, ) # Only recalculate RLEs for masks that have changed for i_mask in keep_by_nms: if scores[i_mask] == 0.0: mask_torch = masks[i_mask].unsqueeze(0) mask_data['rles'][i_mask] = mask_to_rle_pytorch(mask_torch)[0] mask_data['boxes'][i_mask] = boxes[i_mask] # update res directly mask_data.filter(keep_by_nms) return mask_data ================================================ FILE: ultralytics/vit/sam/modules/prompt_predictor.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from typing import Optional, Tuple import numpy as np import torch from ..autosize import ResizeLongestSide from .sam import Sam class PromptPredictor: def __init__(self, sam_model: Sam) -> None: """ Uses SAM to calculate the image embedding for an image, and then allow repeated, efficient mask prediction given prompts. Arguments: sam_model (Sam): The model to use for mask prediction. """ super().__init__() self.model = sam_model self.transform = ResizeLongestSide(sam_model.image_encoder.img_size) self.reset_image() def set_image(self, image: np.ndarray, image_format: str = 'RGB') -> None: """ Calculates the image embeddings for the provided image, allowing masks to be predicted with the 'predict' method. Arguments: image (np.ndarray): The image for calculating masks. Expects an image in HWC uint8 format, with pixel values in [0, 255]. image_format (str): The color format of the image, in ['RGB', 'BGR']. """ assert image_format in {'RGB', 'BGR'}, f"image_format must be in ['RGB', 'BGR'], is {image_format}." if image_format != self.model.image_format: image = image[..., ::-1] # Transform the image to the form expected by the model input_image = self.transform.apply_image(image) input_image_torch = torch.as_tensor(input_image, device=self.device) input_image_torch = input_image_torch.permute(2, 0, 1).contiguous()[None, :, :, :] self.set_torch_image(input_image_torch, image.shape[:2]) @torch.no_grad() def set_torch_image(self, transformed_image: torch.Tensor, original_image_size: Tuple[int, ...]) -> None: """ Calculates the image embeddings for the provided image, allowing masks to be predicted with the 'predict' method. Expects the input image to be already transformed to the format expected by the model. Arguments: transformed_image (torch.Tensor): The input image, with shape 1x3xHxW, which has been transformed with ResizeLongestSide. original_image_size (tuple(int, int)): The size of the image before transformation, in (H, W) format. """ if len(transformed_image.shape) != 4 \ or transformed_image.shape[1] != 3 \ or max(*transformed_image.shape[2:]) != self.model.image_encoder.img_size: raise ValueError('set_torch_image input must be BCHW with long side {self.model.image_encoder.img_size}.') self.reset_image() self.original_size = original_image_size self.input_size = tuple(transformed_image.shape[-2:]) input_image = self.model.preprocess(transformed_image) self.features = self.model.image_encoder(input_image) self.is_image_set = True def predict( self, point_coords: Optional[np.ndarray] = None, point_labels: Optional[np.ndarray] = None, box: Optional[np.ndarray] = None, mask_input: Optional[np.ndarray] = None, multimask_output: bool = True, return_logits: bool = False, ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: """ Predict masks for the given input prompts, using the currently set image. Arguments: point_coords (np.ndarray, None): A Nx2 array of point prompts to the model. Each point is in (X,Y) in pixels. point_labels (np.ndarray, None): A length N array of labels for the point prompts. 1 indicates a foreground point and 0 indicates a background point. box (np.ndarray, None): A length 4 array given a box prompt to the model, in XYXY format. mask_input (np.ndarray): A low resolution mask input to the model, typically coming from a previous prediction iteration. Has form 1xHxW, where for SAM, H=W=256. multimask_output (bool): If true, the model will return three masks. For ambiguous input prompts (such as a single click), this will often produce better masks than a single prediction. If only a single mask is needed, the model's predicted quality score can be used to select the best mask. For non-ambiguous prompts, such as multiple input prompts, multimask_output=False can give better results. return_logits (bool): If true, returns un-thresholded masks logits instead of a binary mask. Returns: (np.ndarray): The output masks in CxHxW format, where C is the number of masks, and (H, W) is the original image size. (np.ndarray): An array of length C containing the model's predictions for the quality of each mask. (np.ndarray): An array of shape CxHxW, where C is the number of masks and H=W=256. These low resolution logits can be passed to a subsequent iteration as mask input. """ if not self.is_image_set: raise RuntimeError('An image must be set with .set_image(...) before mask prediction.') # Transform input prompts coords_torch, labels_torch, box_torch, mask_input_torch = None, None, None, None if point_coords is not None: assert (point_labels is not None), 'point_labels must be supplied if point_coords is supplied.' point_coords = self.transform.apply_coords(point_coords, self.original_size) coords_torch = torch.as_tensor(point_coords, dtype=torch.float, device=self.device) labels_torch = torch.as_tensor(point_labels, dtype=torch.int, device=self.device) coords_torch, labels_torch = coords_torch[None, :, :], labels_torch[None, :] if box is not None: box = self.transform.apply_boxes(box, self.original_size) box_torch = torch.as_tensor(box, dtype=torch.float, device=self.device) box_torch = box_torch[None, :] if mask_input is not None: mask_input_torch = torch.as_tensor(mask_input, dtype=torch.float, device=self.device) mask_input_torch = mask_input_torch[None, :, :, :] masks, iou_predictions, low_res_masks = self.predict_torch( coords_torch, labels_torch, box_torch, mask_input_torch, multimask_output, return_logits=return_logits, ) masks_np = masks[0].detach().cpu().numpy() iou_predictions_np = iou_predictions[0].detach().cpu().numpy() low_res_masks_np = low_res_masks[0].detach().cpu().numpy() return masks_np, iou_predictions_np, low_res_masks_np @torch.no_grad() def predict_torch( self, point_coords: Optional[torch.Tensor], point_labels: Optional[torch.Tensor], boxes: Optional[torch.Tensor] = None, mask_input: Optional[torch.Tensor] = None, multimask_output: bool = True, return_logits: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: """ Predict masks for the given input prompts, using the currently set image. Input prompts are batched torch tensors and are expected to already be transformed to the input frame using ResizeLongestSide. Arguments: point_coords (torch.Tensor, None): A BxNx2 array of point prompts to the model. Each point is in (X,Y) in pixels. point_labels (torch.Tensor, None): A BxN array of labels for the point prompts. 1 indicates a foreground point and 0 indicates a background point. boxes (np.ndarray, None): A Bx4 array given a box prompt to the model, in XYXY format. mask_input (np.ndarray): A low resolution mask input to the model, typically coming from a previous prediction iteration. Has form Bx1xHxW, where for SAM, H=W=256. Masks returned by a previous iteration of the predict method do not need further transformation. multimask_output (bool): If true, the model will return three masks. For ambiguous input prompts (such as a single click), this will often produce better masks than a single prediction. If only a single mask is needed, the model's predicted quality score can be used to select the best mask. For non-ambiguous prompts, such as multiple input prompts, multimask_output=False can give better results. return_logits (bool): If true, returns un-thresholded masks logits instead of a binary mask. Returns: (torch.Tensor): The output masks in BxCxHxW format, where C is the number of masks, and (H, W) is the original image size. (torch.Tensor): An array of shape BxC containing the model's predictions for the quality of each mask. (torch.Tensor): An array of shape BxCxHxW, where C is the number of masks and H=W=256. These low res logits can be passed to a subsequent iteration as mask input. """ if not self.is_image_set: raise RuntimeError('An image must be set with .set_image(...) before mask prediction.') points = (point_coords, point_labels) if point_coords is not None else None # Embed prompts sparse_embeddings, dense_embeddings = self.model.prompt_encoder( points=points, boxes=boxes, masks=mask_input, ) # Predict masks low_res_masks, iou_predictions = self.model.mask_decoder( image_embeddings=self.features, image_pe=self.model.prompt_encoder.get_dense_pe(), sparse_prompt_embeddings=sparse_embeddings, dense_prompt_embeddings=dense_embeddings, multimask_output=multimask_output, ) # Upscale the masks to the original image resolution masks = self.model.postprocess_masks(low_res_masks, self.input_size, self.original_size) if not return_logits: masks = masks > self.model.mask_threshold return masks, iou_predictions, low_res_masks def get_image_embedding(self) -> torch.Tensor: """ Returns the image embeddings for the currently set image, with shape 1xCxHxW, where C is the embedding dimension and (H,W) are the embedding spatial dimension of SAM (typically C=256, H=W=64). """ if not self.is_image_set: raise RuntimeError('An image must be set with .set_image(...) to generate an embedding.') assert self.features is not None, 'Features must exist if an image has been set.' return self.features @property def device(self) -> torch.device: return self.model.device def reset_image(self) -> None: """Resets the currently set image.""" self.is_image_set = False self.features = None self.orig_h = None self.orig_w = None self.input_h = None self.input_w = None ================================================ FILE: ultralytics/vit/sam/modules/sam.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. from typing import Any, Dict, List, Tuple import torch from torch import nn from torch.nn import functional as F from .decoders import MaskDecoder from .encoders import ImageEncoderViT, PromptEncoder class Sam(nn.Module): mask_threshold: float = 0.0 image_format: str = 'RGB' def __init__(self, image_encoder: ImageEncoderViT, prompt_encoder: PromptEncoder, mask_decoder: MaskDecoder, pixel_mean: List[float] = None, pixel_std: List[float] = None) -> None: """ SAM predicts object masks from an image and input prompts. Arguments: image_encoder (ImageEncoderViT): The backbone used to encode the image into image embeddings that allow for efficient mask prediction. prompt_encoder (PromptEncoder): Encodes various types of input prompts. mask_decoder (MaskDecoder): Predicts masks from the image embeddings and encoded prompts. pixel_mean (list(float)): Mean values for normalizing pixels in the input image. pixel_std (list(float)): Std values for normalizing pixels in the input image. """ if pixel_mean is None: pixel_mean = [123.675, 116.28, 103.53] if pixel_std is None: pixel_std = [58.395, 57.12, 57.375] super().__init__() self.image_encoder = image_encoder self.prompt_encoder = prompt_encoder self.mask_decoder = mask_decoder self.register_buffer('pixel_mean', torch.Tensor(pixel_mean).view(-1, 1, 1), False) self.register_buffer('pixel_std', torch.Tensor(pixel_std).view(-1, 1, 1), False) @property def device(self) -> Any: return self.pixel_mean.device @torch.no_grad() def forward( self, batched_input: List[Dict[str, Any]], multimask_output: bool, ) -> List[Dict[str, torch.Tensor]]: """ Predicts masks end-to-end from provided images and prompts. If prompts are not known in advance, using SamPredictor is recommended over calling the model directly. Arguments: batched_input (list(dict)): A list over input images, each a dictionary with the following keys. A prompt key can be excluded if it is not present. 'image': The image as a torch tensor in 3xHxW format, already transformed for input to the model. 'original_size': (tuple(int, int)) The original size of the image before transformation, as (H, W). 'point_coords': (torch.Tensor) Batched point prompts for this image, with shape BxNx2. Already transformed to the input frame of the model. 'point_labels': (torch.Tensor) Batched labels for point prompts, with shape BxN. 'boxes': (torch.Tensor) Batched box inputs, with shape Bx4. Already transformed to the input frame of the model. 'mask_inputs': (torch.Tensor) Batched mask inputs to the model, in the form Bx1xHxW. multimask_output (bool): Whether the model should predict multiple disambiguating masks, or return a single mask. Returns: (list(dict)): A list over input images, where each element is as dictionary with the following keys. 'masks': (torch.Tensor) Batched binary mask predictions, with shape BxCxHxW, where B is the number of input prompts, C is determined by multimask_output, and (H, W) is the original size of the image. 'iou_predictions': (torch.Tensor) The model's predictions of mask quality, in shape BxC. 'low_res_logits': (torch.Tensor) Low resolution logits with shape BxCxHxW, where H=W=256. Can be passed as mask input to subsequent iterations of prediction. """ input_images = torch.stack([self.preprocess(x['image']) for x in batched_input], dim=0) image_embeddings = self.image_encoder(input_images) outputs = [] for image_record, curr_embedding in zip(batched_input, image_embeddings): if 'point_coords' in image_record: points = (image_record['point_coords'], image_record['point_labels']) else: points = None sparse_embeddings, dense_embeddings = self.prompt_encoder( points=points, boxes=image_record.get('boxes', None), masks=image_record.get('mask_inputs', None), ) low_res_masks, iou_predictions = self.mask_decoder( image_embeddings=curr_embedding.unsqueeze(0), image_pe=self.prompt_encoder.get_dense_pe(), sparse_prompt_embeddings=sparse_embeddings, dense_prompt_embeddings=dense_embeddings, multimask_output=multimask_output, ) masks = self.postprocess_masks( low_res_masks, input_size=image_record['image'].shape[-2:], original_size=image_record['original_size'], ) masks = masks > self.mask_threshold outputs.append({ 'masks': masks, 'iou_predictions': iou_predictions, 'low_res_logits': low_res_masks, }) return outputs def postprocess_masks( self, masks: torch.Tensor, input_size: Tuple[int, ...], original_size: Tuple[int, ...], ) -> torch.Tensor: """ Remove padding and upscale masks to the original image size. Arguments: masks (torch.Tensor): Batched masks from the mask_decoder, in BxCxHxW format. input_size (tuple(int, int)): The size of the image input to the model, in (H, W) format. Used to remove padding. original_size (tuple(int, int)): The original size of the image before resizing for input to the model, in (H, W) format. Returns: (torch.Tensor): Batched masks in BxCxHxW format, where (H, W) is given by original_size. """ masks = F.interpolate( masks, (self.image_encoder.img_size, self.image_encoder.img_size), mode='bilinear', align_corners=False, ) masks = masks[..., :input_size[0], :input_size[1]] masks = F.interpolate(masks, original_size, mode='bilinear', align_corners=False) return masks def preprocess(self, x: torch.Tensor) -> torch.Tensor: """Normalize pixel values and pad to a square input.""" # Normalize colors x = (x - self.pixel_mean) / self.pixel_std # Pad h, w = x.shape[-2:] padh = self.image_encoder.img_size - h padw = self.image_encoder.img_size - w return F.pad(x, (0, padw, 0, padh)) ================================================ FILE: ultralytics/vit/sam/modules/transformer.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import math from typing import Tuple, Type import torch from torch import Tensor, nn from ultralytics.nn.modules import MLPBlock class TwoWayTransformer(nn.Module): def __init__( self, depth: int, embedding_dim: int, num_heads: int, mlp_dim: int, activation: Type[nn.Module] = nn.ReLU, attention_downsample_rate: int = 2, ) -> None: """ A transformer decoder that attends to an input image using queries whose positional embedding is supplied. Args: depth (int): number of layers in the transformer embedding_dim (int): the channel dimension for the input embeddings num_heads (int): the number of heads for multihead attention. Must divide embedding_dim mlp_dim (int): the channel dimension internal to the MLP block activation (nn.Module): the activation to use in the MLP block """ super().__init__() self.depth = depth self.embedding_dim = embedding_dim self.num_heads = num_heads self.mlp_dim = mlp_dim self.layers = nn.ModuleList() for i in range(depth): self.layers.append( TwoWayAttentionBlock( embedding_dim=embedding_dim, num_heads=num_heads, mlp_dim=mlp_dim, activation=activation, attention_downsample_rate=attention_downsample_rate, skip_first_layer_pe=(i == 0), )) self.final_attn_token_to_image = Attention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate) self.norm_final_attn = nn.LayerNorm(embedding_dim) def forward( self, image_embedding: Tensor, image_pe: Tensor, point_embedding: Tensor, ) -> Tuple[Tensor, Tensor]: """ Args: image_embedding (torch.Tensor): image to attend to. Should be shape B x embedding_dim x h x w for any h and w. image_pe (torch.Tensor): the positional encoding to add to the image. Must have the same shape as image_embedding. point_embedding (torch.Tensor): the embedding to add to the query points. Must have shape B x N_points x embedding_dim for any N_points. Returns: torch.Tensor: the processed point_embedding torch.Tensor: the processed image_embedding """ # BxCxHxW -> BxHWxC == B x N_image_tokens x C bs, c, h, w = image_embedding.shape image_embedding = image_embedding.flatten(2).permute(0, 2, 1) image_pe = image_pe.flatten(2).permute(0, 2, 1) # Prepare queries queries = point_embedding keys = image_embedding # Apply transformer blocks and final layernorm for layer in self.layers: queries, keys = layer( queries=queries, keys=keys, query_pe=point_embedding, key_pe=image_pe, ) # Apply the final attention layer from the points to the image q = queries + point_embedding k = keys + image_pe attn_out = self.final_attn_token_to_image(q=q, k=k, v=keys) queries = queries + attn_out queries = self.norm_final_attn(queries) return queries, keys class TwoWayAttentionBlock(nn.Module): def __init__( self, embedding_dim: int, num_heads: int, mlp_dim: int = 2048, activation: Type[nn.Module] = nn.ReLU, attention_downsample_rate: int = 2, skip_first_layer_pe: bool = False, ) -> None: """ A transformer block with four layers: (1) self-attention of sparse inputs, (2) cross attention of sparse inputs to dense inputs, (3) mlp block on sparse inputs, and (4) cross attention of dense inputs to sparse inputs. Arguments: embedding_dim (int): the channel dimension of the embeddings num_heads (int): the number of heads in the attention layers mlp_dim (int): the hidden dimension of the mlp block activation (nn.Module): the activation of the mlp block skip_first_layer_pe (bool): skip the PE on the first layer """ super().__init__() self.self_attn = Attention(embedding_dim, num_heads) self.norm1 = nn.LayerNorm(embedding_dim) self.cross_attn_token_to_image = Attention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate) self.norm2 = nn.LayerNorm(embedding_dim) self.mlp = MLPBlock(embedding_dim, mlp_dim, activation) self.norm3 = nn.LayerNorm(embedding_dim) self.norm4 = nn.LayerNorm(embedding_dim) self.cross_attn_image_to_token = Attention(embedding_dim, num_heads, downsample_rate=attention_downsample_rate) self.skip_first_layer_pe = skip_first_layer_pe def forward(self, queries: Tensor, keys: Tensor, query_pe: Tensor, key_pe: Tensor) -> Tuple[Tensor, Tensor]: """Apply self-attention and cross-attention to queries and keys and return the processed embeddings.""" # Self attention block if self.skip_first_layer_pe: queries = self.self_attn(q=queries, k=queries, v=queries) else: q = queries + query_pe attn_out = self.self_attn(q=q, k=q, v=queries) queries = queries + attn_out queries = self.norm1(queries) # Cross attention block, tokens attending to image embedding q = queries + query_pe k = keys + key_pe attn_out = self.cross_attn_token_to_image(q=q, k=k, v=keys) queries = queries + attn_out queries = self.norm2(queries) # MLP block mlp_out = self.mlp(queries) queries = queries + mlp_out queries = self.norm3(queries) # Cross attention block, image embedding attending to tokens q = queries + query_pe k = keys + key_pe attn_out = self.cross_attn_image_to_token(q=k, k=q, v=queries) keys = keys + attn_out keys = self.norm4(keys) return queries, keys class Attention(nn.Module): """ An attention layer that allows for downscaling the size of the embedding after projection to queries, keys, and values. """ def __init__( self, embedding_dim: int, num_heads: int, downsample_rate: int = 1, ) -> None: super().__init__() self.embedding_dim = embedding_dim self.internal_dim = embedding_dim // downsample_rate self.num_heads = num_heads assert self.internal_dim % num_heads == 0, 'num_heads must divide embedding_dim.' self.q_proj = nn.Linear(embedding_dim, self.internal_dim) self.k_proj = nn.Linear(embedding_dim, self.internal_dim) self.v_proj = nn.Linear(embedding_dim, self.internal_dim) self.out_proj = nn.Linear(self.internal_dim, embedding_dim) def _separate_heads(self, x: Tensor, num_heads: int) -> Tensor: """Separate the input tensor into the specified number of attention heads.""" b, n, c = x.shape x = x.reshape(b, n, num_heads, c // num_heads) return x.transpose(1, 2) # B x N_heads x N_tokens x C_per_head def _recombine_heads(self, x: Tensor) -> Tensor: """Recombine the separated attention heads into a single tensor.""" b, n_heads, n_tokens, c_per_head = x.shape x = x.transpose(1, 2) return x.reshape(b, n_tokens, n_heads * c_per_head) # B x N_tokens x C def forward(self, q: Tensor, k: Tensor, v: Tensor) -> Tensor: """Compute the attention output given the input query, key, and value tensors.""" # Input projections q = self.q_proj(q) k = self.k_proj(k) v = self.v_proj(v) # Separate into heads q = self._separate_heads(q, self.num_heads) k = self._separate_heads(k, self.num_heads) v = self._separate_heads(v, self.num_heads) # Attention _, _, _, c_per_head = q.shape attn = q @ k.permute(0, 1, 3, 2) # B x N_heads x N_tokens x N_tokens attn = attn / math.sqrt(c_per_head) attn = torch.softmax(attn, dim=-1) # Get output out = attn @ v out = self._recombine_heads(out) out = self.out_proj(out) return out ================================================ FILE: ultralytics/vit/sam/predict.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import numpy as np import torch from ultralytics.yolo.engine.predictor import BasePredictor from ultralytics.yolo.engine.results import Results from ultralytics.yolo.utils.torch_utils import select_device from .modules.mask_generator import SamAutomaticMaskGenerator class Predictor(BasePredictor): def preprocess(self, im): """Prepares input image for inference.""" # TODO: Only support bs=1 for now # im = ResizeLongestSide(1024).apply_image(im[0]) # im = torch.as_tensor(im, device=self.device) # im = im.permute(2, 0, 1).contiguous()[None, :, :, :] return im[0] def setup_model(self, model): """Set up YOLO model with specified thresholds and device.""" device = select_device(self.args.device) model.eval() self.model = SamAutomaticMaskGenerator(model.to(device), pred_iou_thresh=self.args.conf, box_nms_thresh=self.args.iou) self.device = device # TODO: Temporary settings for compatibility self.model.pt = False self.model.triton = False self.model.stride = 32 self.model.fp16 = False self.done_warmup = True def postprocess(self, preds, path, orig_imgs): """Postprocesses inference output predictions to create detection masks for objects.""" names = dict(enumerate(list(range(len(preds))))) results = [] # TODO for i, pred in enumerate([preds]): masks = torch.from_numpy(np.stack([p['segmentation'] for p in pred], axis=0)) orig_img = orig_imgs[i] if isinstance(orig_imgs, list) else orig_imgs path = self.batch[0] img_path = path[i] if isinstance(path, list) else path results.append(Results(orig_img=orig_img, path=img_path, names=names, masks=masks)) return results # def __call__(self, source=None, model=None, stream=False): # frame = cv2.imread(source) # preds = self.model.generate(frame) # return self.postprocess(preds, source, frame) ================================================ FILE: ultralytics/vit/utils/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license ================================================ FILE: ultralytics/vit/utils/loss.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import torch import torch.nn as nn import torch.nn.functional as F from ultralytics.vit.utils.ops import HungarianMatcher from ultralytics.yolo.utils.loss import FocalLoss, VarifocalLoss from ultralytics.yolo.utils.metrics import bbox_iou class DETRLoss(nn.Module): def __init__(self, nc=80, loss_gain=None, aux_loss=True, use_fl=True, use_vfl=False, use_uni_match=False, uni_match_ind=0): """ DETR loss function. Args: nc (int): The number of classes. loss_gain (dict): The coefficient of loss. aux_loss (bool): If 'aux_loss = True', loss at each decoder layer are to be used. use_vfl (bool): Use VarifocalLoss or not. use_uni_match (bool): Whether to use a fixed layer to assign labels for auxiliary branch. uni_match_ind (int): The fixed indices of a layer. """ super().__init__() if loss_gain is None: loss_gain = {'class': 1, 'bbox': 5, 'giou': 2, 'no_object': 0.1, 'mask': 1, 'dice': 1} self.nc = nc self.matcher = HungarianMatcher(cost_gain={'class': 2, 'bbox': 5, 'giou': 2}) self.loss_gain = loss_gain self.aux_loss = aux_loss self.fl = FocalLoss() if use_fl else None self.vfl = VarifocalLoss() if use_vfl else None self.use_uni_match = use_uni_match self.uni_match_ind = uni_match_ind self.device = None def _get_loss_class(self, pred_scores, targets, gt_scores, num_gts, postfix=''): # logits: [b, query, num_classes], gt_class: list[[n, 1]] name_class = f'loss_class{postfix}' bs, nq = pred_scores.shape[:2] # one_hot = F.one_hot(targets, self.nc + 1)[..., :-1] # (bs, num_queries, num_classes) one_hot = torch.zeros((bs, nq, self.nc + 1), dtype=torch.int64, device=targets.device) one_hot.scatter_(2, targets.unsqueeze(-1), 1) one_hot = one_hot[..., :-1] gt_scores = gt_scores.view(bs, nq, 1) * one_hot if self.fl: if num_gts and self.vfl: loss_cls = self.vfl(pred_scores, gt_scores, one_hot) else: loss_cls = self.fl(pred_scores, one_hot.float()) loss_cls /= max(num_gts, 1) / nq else: loss_cls = nn.BCEWithLogitsLoss(reduction='none')(pred_scores, gt_scores).mean(1).sum() # YOLO CLS loss return {name_class: loss_cls.squeeze() * self.loss_gain['class']} def _get_loss_bbox(self, pred_bboxes, gt_bboxes, postfix=''): # boxes: [b, query, 4], gt_bbox: list[[n, 4]] name_bbox = f'loss_bbox{postfix}' name_giou = f'loss_giou{postfix}' loss = {} if len(gt_bboxes) == 0: loss[name_bbox] = torch.tensor(0., device=self.device) loss[name_giou] = torch.tensor(0., device=self.device) return loss loss[name_bbox] = self.loss_gain['bbox'] * F.l1_loss(pred_bboxes, gt_bboxes, reduction='sum') / len(gt_bboxes) loss[name_giou] = 1.0 - bbox_iou(pred_bboxes, gt_bboxes, xywh=True, GIoU=True) loss[name_giou] = loss[name_giou].sum() / len(gt_bboxes) loss[name_giou] = self.loss_gain['giou'] * loss[name_giou] loss = {k: v.squeeze() for k, v in loss.items()} return loss def _get_loss_mask(self, masks, gt_mask, match_indices, postfix=''): # masks: [b, query, h, w], gt_mask: list[[n, H, W]] name_mask = f'loss_mask{postfix}' name_dice = f'loss_dice{postfix}' loss = {} if sum(len(a) for a in gt_mask) == 0: loss[name_mask] = torch.tensor(0., device=self.device) loss[name_dice] = torch.tensor(0., device=self.device) return loss num_gts = len(gt_mask) src_masks, target_masks = self._get_assigned_bboxes(masks, gt_mask, match_indices) src_masks = F.interpolate(src_masks.unsqueeze(0), size=target_masks.shape[-2:], mode='bilinear')[0] # TODO: torch does not have `sigmoid_focal_loss`, but it's not urgent since we don't use mask branch for now. loss[name_mask] = self.loss_gain['mask'] * F.sigmoid_focal_loss(src_masks, target_masks, torch.tensor([num_gts], dtype=torch.float32)) loss[name_dice] = self.loss_gain['dice'] * self._dice_loss(src_masks, target_masks, num_gts) return loss def _dice_loss(self, inputs, targets, num_gts): inputs = F.sigmoid(inputs) inputs = inputs.flatten(1) targets = targets.flatten(1) numerator = 2 * (inputs * targets).sum(1) denominator = inputs.sum(-1) + targets.sum(-1) loss = 1 - (numerator + 1) / (denominator + 1) return loss.sum() / num_gts def _get_loss_aux(self, pred_bboxes, pred_scores, gt_bboxes, gt_cls, gt_groups, match_indices=None, postfix='', masks=None, gt_mask=None): """Get auxiliary losses""" # NOTE: loss class, bbox, giou, mask, dice loss = torch.zeros(5 if masks is not None else 3, device=pred_bboxes.device) if match_indices is None and self.use_uni_match: match_indices = self.matcher(pred_bboxes[self.uni_match_ind], pred_scores[self.uni_match_ind], gt_bboxes, gt_cls, gt_groups, masks=masks[self.uni_match_ind] if masks is not None else None, gt_mask=gt_mask) for i, (aux_bboxes, aux_scores) in enumerate(zip(pred_bboxes, pred_scores)): aux_masks = masks[i] if masks is not None else None loss_ = self._get_loss(aux_bboxes, aux_scores, gt_bboxes, gt_cls, gt_groups, masks=aux_masks, gt_mask=gt_mask, postfix=postfix, match_indices=match_indices) loss[0] += loss_[f'loss_class{postfix}'] loss[1] += loss_[f'loss_bbox{postfix}'] loss[2] += loss_[f'loss_giou{postfix}'] # if masks is not None and gt_mask is not None: # loss_ = self._get_loss_mask(aux_masks, gt_mask, match_indices, postfix) # loss[3] += loss_[f'loss_mask{postfix}'] # loss[4] += loss_[f'loss_dice{postfix}'] loss = { f'loss_class_aux{postfix}': loss[0], f'loss_bbox_aux{postfix}': loss[1], f'loss_giou_aux{postfix}': loss[2]} # if masks is not None and gt_mask is not None: # loss[f'loss_mask_aux{postfix}'] = loss[3] # loss[f'loss_dice_aux{postfix}'] = loss[4] return loss def _get_index(self, match_indices): batch_idx = torch.cat([torch.full_like(src, i) for i, (src, _) in enumerate(match_indices)]) src_idx = torch.cat([src for (src, _) in match_indices]) dst_idx = torch.cat([dst for (_, dst) in match_indices]) return (batch_idx, src_idx), dst_idx def _get_assigned_bboxes(self, pred_bboxes, gt_bboxes, match_indices): pred_assigned = torch.cat([ t[I] if len(I) > 0 else torch.zeros(0, t.shape[-1], device=self.device) for t, (I, _) in zip(pred_bboxes, match_indices)]) gt_assigned = torch.cat([ t[J] if len(J) > 0 else torch.zeros(0, t.shape[-1], device=self.device) for t, (_, J) in zip(gt_bboxes, match_indices)]) return pred_assigned, gt_assigned def _get_loss(self, pred_bboxes, pred_scores, gt_bboxes, gt_cls, gt_groups, masks=None, gt_mask=None, postfix='', match_indices=None): """Get losses""" if match_indices is None: match_indices = self.matcher(pred_bboxes, pred_scores, gt_bboxes, gt_cls, gt_groups, masks=masks, gt_mask=gt_mask) idx, gt_idx = self._get_index(match_indices) pred_bboxes, gt_bboxes = pred_bboxes[idx], gt_bboxes[gt_idx] bs, nq = pred_scores.shape[:2] targets = torch.full((bs, nq), self.nc, device=pred_scores.device, dtype=gt_cls.dtype) targets[idx] = gt_cls[gt_idx] gt_scores = torch.zeros([bs, nq], device=pred_scores.device) if len(gt_bboxes): gt_scores[idx] = bbox_iou(pred_bboxes.detach(), gt_bboxes, xywh=True).squeeze(-1) loss = {} loss.update(self._get_loss_class(pred_scores, targets, gt_scores, len(gt_bboxes), postfix)) loss.update(self._get_loss_bbox(pred_bboxes, gt_bboxes, postfix)) # if masks is not None and gt_mask is not None: # loss.update(self._get_loss_mask(masks, gt_mask, match_indices, postfix)) return loss def forward(self, pred_bboxes, pred_scores, batch, postfix='', **kwargs): """ Args: pred_bboxes (torch.Tensor): [l, b, query, 4] pred_scores (torch.Tensor): [l, b, query, num_classes] batch (dict): A dict includes: gt_cls (torch.Tensor) with shape [num_gts, ], gt_bboxes (torch.Tensor): [num_gts, 4], gt_groups (List(int)): a list of batch size length includes the number of gts of each image. postfix (str): postfix of loss name. """ self.device = pred_bboxes.device match_indices = kwargs.get('match_indices', None) gt_cls, gt_bboxes, gt_groups = batch['cls'], batch['bboxes'], batch['gt_groups'] total_loss = self._get_loss(pred_bboxes[-1], pred_scores[-1], gt_bboxes, gt_cls, gt_groups, postfix=postfix, match_indices=match_indices) if self.aux_loss: total_loss.update( self._get_loss_aux(pred_bboxes[:-1], pred_scores[:-1], gt_bboxes, gt_cls, gt_groups, match_indices, postfix)) return total_loss class RTDETRDetectionLoss(DETRLoss): def forward(self, preds, batch, dn_bboxes=None, dn_scores=None, dn_meta=None): pred_bboxes, pred_scores = preds total_loss = super().forward(pred_bboxes, pred_scores, batch) if dn_meta is not None: dn_pos_idx, dn_num_group = dn_meta['dn_pos_idx'], dn_meta['dn_num_group'] assert len(batch['gt_groups']) == len(dn_pos_idx) # denoising match indices match_indices = self.get_dn_match_indices(dn_pos_idx, dn_num_group, batch['gt_groups']) # compute denoising training loss dn_loss = super().forward(dn_bboxes, dn_scores, batch, postfix='_dn', match_indices=match_indices) total_loss.update(dn_loss) else: total_loss.update({f'{k}_dn': torch.tensor(0., device=self.device) for k in total_loss.keys()}) return total_loss @staticmethod def get_dn_match_indices(dn_pos_idx, dn_num_group, gt_groups): """Get the match indices for denoising. Args: dn_pos_idx (List[torch.Tensor]): A list includes positive indices of denoising. dn_num_group (int): The number of groups of denoising. gt_groups (List(int)): a list of batch size length includes the number of gts of each image. Returns: dn_match_indices (List(tuple)): Matched indices. """ dn_match_indices = [] idx_groups = torch.as_tensor([0, *gt_groups[:-1]]).cumsum_(0) for i, num_gt in enumerate(gt_groups): if num_gt > 0: gt_idx = torch.arange(end=num_gt, dtype=torch.int32) + idx_groups[i] gt_idx = gt_idx.repeat(dn_num_group) assert len(dn_pos_idx[i]) == len(gt_idx), 'Expected the same length, ' f'but got {len(dn_pos_idx[i])} and {len(gt_idx)} respectively.' dn_match_indices.append((dn_pos_idx[i], gt_idx)) else: dn_match_indices.append((torch.zeros([0], dtype=torch.int32), torch.zeros([0], dtype=torch.int32))) return dn_match_indices ================================================ FILE: ultralytics/vit/utils/ops.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import torch import torch.nn as nn import torch.nn.functional as F from scipy.optimize import linear_sum_assignment from ultralytics.yolo.utils.metrics import bbox_iou from ultralytics.yolo.utils.ops import xywh2xyxy, xyxy2xywh class HungarianMatcher(nn.Module): """ A module implementing the HungarianMatcher, which is a differentiable module to solve the assignment problem in an end-to-end fashion. HungarianMatcher performs optimal assignment over predicted and ground truth bounding boxes using a cost function that considers classification scores, bounding box coordinates, and optionally, mask predictions. Attributes: cost_gain (dict): Dictionary of cost coefficients for different components: 'class', 'bbox', 'giou', 'mask', and 'dice'. use_fl (bool): Indicates whether to use Focal Loss for the classification cost calculation. with_mask (bool): Indicates whether the model makes mask predictions. num_sample_points (int): The number of sample points used in mask cost calculation. alpha (float): The alpha factor in Focal Loss calculation. gamma (float): The gamma factor in Focal Loss calculation. Methods: forward(pred_bboxes, pred_scores, gt_bboxes, gt_cls, gt_groups, masks=None, gt_mask=None): Computes the assignment between predictions and ground truths for a batch. _cost_mask(bs, num_gts, masks=None, gt_mask=None): Computes the mask cost and dice cost if masks are predicted. """ def __init__(self, cost_gain=None, use_fl=True, with_mask=False, num_sample_points=12544, alpha=0.25, gamma=2.0): super().__init__() if cost_gain is None: cost_gain = {'class': 1, 'bbox': 5, 'giou': 2, 'mask': 1, 'dice': 1} self.cost_gain = cost_gain self.use_fl = use_fl self.with_mask = with_mask self.num_sample_points = num_sample_points self.alpha = alpha self.gamma = gamma def forward(self, pred_bboxes, pred_scores, gt_bboxes, gt_cls, gt_groups, masks=None, gt_mask=None): """ Forward pass for HungarianMatcher. This function computes costs based on prediction and ground truth (classification cost, L1 cost between boxes and GIoU cost between boxes) and finds the optimal matching between predictions and ground truth based on these costs. Args: pred_bboxes (Tensor): Predicted bounding boxes with shape [batch_size, num_queries, 4]. pred_scores (Tensor): Predicted scores with shape [batch_size, num_queries, num_classes]. gt_cls (torch.Tensor): Ground truth classes with shape [num_gts, ]. gt_bboxes (torch.Tensor): Ground truth bounding boxes with shape [num_gts, 4]. gt_groups (List[int]): List of length equal to batch size, containing the number of ground truths for each image. masks (Tensor, optional): Predicted masks with shape [batch_size, num_queries, height, width]. Defaults to None. gt_mask (List[Tensor], optional): List of ground truth masks, each with shape [num_masks, Height, Width]. Defaults to None. Returns: (List[Tuple[Tensor, Tensor]]): A list of size batch_size, each element is a tuple (index_i, index_j), where: - index_i is the tensor of indices of the selected predictions (in order) - index_j is the tensor of indices of the corresponding selected ground truth targets (in order) For each batch element, it holds: len(index_i) = len(index_j) = min(num_queries, num_target_boxes) """ bs, nq, nc = pred_scores.shape if sum(gt_groups) == 0: return [(torch.tensor([], dtype=torch.int32), torch.tensor([], dtype=torch.int32)) for _ in range(bs)] # We flatten to compute the cost matrices in a batch # [batch_size * num_queries, num_classes] pred_scores = pred_scores.detach().view(-1, nc) pred_scores = F.sigmoid(pred_scores) if self.use_fl else F.softmax(pred_scores, dim=-1) # [batch_size * num_queries, 4] pred_bboxes = pred_bboxes.detach().view(-1, 4) # Compute the classification cost pred_scores = pred_scores[:, gt_cls] if self.use_fl: neg_cost_class = (1 - self.alpha) * (pred_scores ** self.gamma) * (-(1 - pred_scores + 1e-8).log()) pos_cost_class = self.alpha * ((1 - pred_scores) ** self.gamma) * (-(pred_scores + 1e-8).log()) cost_class = pos_cost_class - neg_cost_class else: cost_class = -pred_scores # Compute the L1 cost between boxes cost_bbox = (pred_bboxes.unsqueeze(1) - gt_bboxes.unsqueeze(0)).abs().sum(-1) # (bs*num_queries, num_gt) # Compute the GIoU cost between boxes, (bs*num_queries, num_gt) cost_giou = 1.0 - bbox_iou(pred_bboxes.unsqueeze(1), gt_bboxes.unsqueeze(0), xywh=True, GIoU=True).squeeze(-1) # Final cost matrix C = self.cost_gain['class'] * cost_class + \ self.cost_gain['bbox'] * cost_bbox + \ self.cost_gain['giou'] * cost_giou # Compute the mask cost and dice cost if self.with_mask: C += self._cost_mask(bs, gt_groups, masks, gt_mask) C = C.view(bs, nq, -1).cpu() indices = [linear_sum_assignment(c[i]) for i, c in enumerate(C.split(gt_groups, -1))] gt_groups = torch.as_tensor([0, *gt_groups[:-1]]).cumsum_(0) # (idx for queries, idx for gt) return [(torch.tensor(i, dtype=torch.int32), torch.tensor(j, dtype=torch.int32) + gt_groups[k]) for k, (i, j) in enumerate(indices)] def _cost_mask(self, bs, num_gts, masks=None, gt_mask=None): assert masks is not None and gt_mask is not None, 'Make sure the input has `mask` and `gt_mask`' # all masks share the same set of points for efficient matching sample_points = torch.rand([bs, 1, self.num_sample_points, 2]) sample_points = 2.0 * sample_points - 1.0 out_mask = F.grid_sample(masks.detach(), sample_points, align_corners=False).squeeze(-2) out_mask = out_mask.flatten(0, 1) tgt_mask = torch.cat(gt_mask).unsqueeze(1) sample_points = torch.cat([a.repeat(b, 1, 1, 1) for a, b in zip(sample_points, num_gts) if b > 0]) tgt_mask = F.grid_sample(tgt_mask, sample_points, align_corners=False).squeeze([1, 2]) with torch.cuda.amp.autocast(False): # binary cross entropy cost pos_cost_mask = F.binary_cross_entropy_with_logits(out_mask, torch.ones_like(out_mask), reduction='none') neg_cost_mask = F.binary_cross_entropy_with_logits(out_mask, torch.zeros_like(out_mask), reduction='none') cost_mask = torch.matmul(pos_cost_mask, tgt_mask.T) + torch.matmul(neg_cost_mask, 1 - tgt_mask.T) cost_mask /= self.num_sample_points # dice cost out_mask = F.sigmoid(out_mask) numerator = 2 * torch.matmul(out_mask, tgt_mask.T) denominator = out_mask.sum(-1, keepdim=True) + tgt_mask.sum(-1).unsqueeze(0) cost_dice = 1 - (numerator + 1) / (denominator + 1) C = self.cost_gain['mask'] * cost_mask + self.cost_gain['dice'] * cost_dice return C def get_cdn_group(batch, num_classes, num_queries, class_embed, num_dn=100, cls_noise_ratio=0.5, box_noise_scale=1.0, training=False): """ Get contrastive denoising training group. This function creates a contrastive denoising training group with positive and negative samples from the ground truths (gt). It applies noise to the class labels and bounding box coordinates, and returns the modified labels, bounding boxes, attention mask and meta information. Args: batch (dict): A dict that includes 'gt_cls' (torch.Tensor with shape [num_gts, ]), 'gt_bboxes' (torch.Tensor with shape [num_gts, 4]), 'gt_groups' (List(int)) which is a list of batch size length indicating the number of gts of each image. num_classes (int): Number of classes. num_queries (int): Number of queries. class_embed (torch.Tensor): Embedding weights to map class labels to embedding space. num_dn (int, optional): Number of denoising. Defaults to 100. cls_noise_ratio (float, optional): Noise ratio for class labels. Defaults to 0.5. box_noise_scale (float, optional): Noise scale for bounding box coordinates. Defaults to 1.0. training (bool, optional): If it's in training mode. Defaults to False. Returns: (Tuple[Optional[Tensor], Optional[Tensor], Optional[Tensor], Optional[Dict]]): The modified class embeddings, bounding boxes, attention mask and meta information for denoising. If not in training mode or 'num_dn' is less than or equal to 0, the function returns None for all elements in the tuple. """ if (not training) or num_dn <= 0: return None, None, None, None gt_groups = batch['gt_groups'] total_num = sum(gt_groups) max_nums = max(gt_groups) if max_nums == 0: return None, None, None, None num_group = num_dn // max_nums num_group = 1 if num_group == 0 else num_group # pad gt to max_num of a batch bs = len(gt_groups) gt_cls = batch['cls'] # (bs*num, ) gt_bbox = batch['bboxes'] # bs*num, 4 b_idx = batch['batch_idx'] # each group has positive and negative queries. dn_cls = gt_cls.repeat(2 * num_group) # (2*num_group*bs*num, ) dn_bbox = gt_bbox.repeat(2 * num_group, 1) # 2*num_group*bs*num, 4 dn_b_idx = b_idx.repeat(2 * num_group).view(-1) # (2*num_group*bs*num, ) # positive and negative mask # (bs*num*num_group, ), the second total_num*num_group part as negative samples neg_idx = torch.arange(total_num * num_group, dtype=torch.long, device=gt_bbox.device) + num_group * total_num if cls_noise_ratio > 0: # half of bbox prob mask = torch.rand(dn_cls.shape) < (cls_noise_ratio * 0.5) idx = torch.nonzero(mask).squeeze(-1) # randomly put a new one here new_label = torch.randint_like(idx, 0, num_classes, dtype=dn_cls.dtype, device=dn_cls.device) dn_cls[idx] = new_label if box_noise_scale > 0: known_bbox = xywh2xyxy(dn_bbox) diff = (dn_bbox[..., 2:] * 0.5).repeat(1, 2) * box_noise_scale # 2*num_group*bs*num, 4 rand_sign = torch.randint_like(dn_bbox, 0, 2) * 2.0 - 1.0 rand_part = torch.rand_like(dn_bbox) rand_part[neg_idx] += 1.0 rand_part *= rand_sign known_bbox += rand_part * diff known_bbox.clip_(min=0.0, max=1.0) dn_bbox = xyxy2xywh(known_bbox) dn_bbox = inverse_sigmoid(dn_bbox) # total denoising queries num_dn = int(max_nums * 2 * num_group) # class_embed = torch.cat([class_embed, torch.zeros([1, class_embed.shape[-1]], device=class_embed.device)]) dn_cls_embed = class_embed[dn_cls] # bs*num * 2 * num_group, 256 padding_cls = torch.zeros(bs, num_dn, dn_cls_embed.shape[-1], device=gt_cls.device) padding_bbox = torch.zeros(bs, num_dn, 4, device=gt_bbox.device) map_indices = torch.cat([torch.tensor(range(num), dtype=torch.long) for num in gt_groups]) pos_idx = torch.stack([map_indices + max_nums * i for i in range(num_group)], dim=0) map_indices = torch.cat([map_indices + max_nums * i for i in range(2 * num_group)]) padding_cls[(dn_b_idx, map_indices)] = dn_cls_embed padding_bbox[(dn_b_idx, map_indices)] = dn_bbox tgt_size = num_dn + num_queries attn_mask = torch.zeros([tgt_size, tgt_size], dtype=torch.bool) # match query cannot see the reconstruct attn_mask[num_dn:, :num_dn] = True # reconstruct cannot see each other for i in range(num_group): if i == 0: attn_mask[max_nums * 2 * i:max_nums * 2 * (i + 1), max_nums * 2 * (i + 1):num_dn] = True if i == num_group - 1: attn_mask[max_nums * 2 * i:max_nums * 2 * (i + 1), :max_nums * i * 2] = True else: attn_mask[max_nums * 2 * i:max_nums * 2 * (i + 1), max_nums * 2 * (i + 1):num_dn] = True attn_mask[max_nums * 2 * i:max_nums * 2 * (i + 1), :max_nums * 2 * i] = True dn_meta = { 'dn_pos_idx': [p.reshape(-1) for p in pos_idx.cpu().split(list(gt_groups), dim=1)], 'dn_num_group': num_group, 'dn_num_split': [num_dn, num_queries]} return padding_cls.to(class_embed.device), padding_bbox.to(class_embed.device), attn_mask.to( class_embed.device), dn_meta def inverse_sigmoid(x, eps=1e-6): """Inverse sigmoid function.""" x = x.clip(min=0., max=1.) return torch.log(x / (1 - x + eps) + eps) ================================================ FILE: ultralytics/yolo/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from . import v8 __all__ = 'v8', # tuple or list ================================================ FILE: ultralytics/yolo/cfg/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import contextlib import re import shutil import sys from difflib import get_close_matches from pathlib import Path from types import SimpleNamespace from typing import Dict, List, Union from ultralytics.yolo.utils import (DEFAULT_CFG, DEFAULT_CFG_DICT, DEFAULT_CFG_PATH, LOGGER, ROOT, USER_CONFIG_DIR, IterableSimpleNamespace, __version__, checks, colorstr, deprecation_warn, get_settings, yaml_load, yaml_print) # Define valid tasks and modes MODES = 'train', 'val', 'predict', 'export', 'track', 'benchmark' TASKS = 'detect', 'segment', 'classify', 'pose' TASK2DATA = { 'detect': 'coco128.yaml', 'segment': 'coco128-seg.yaml', 'classify': 'imagenet100', 'pose': 'coco8-pose.yaml'} TASK2MODEL = { 'detect': 'yolov8n.pt', 'segment': 'yolov8n-seg.pt', 'classify': 'yolov8n-cls.pt', 'pose': 'yolov8n-pose.pt'} CLI_HELP_MSG = \ f""" Arguments received: {str(['yolo'] + sys.argv[1:])}. Ultralytics 'yolo' commands use the following syntax: yolo TASK MODE ARGS Where TASK (optional) is one of {TASKS} MODE (required) is one of {MODES} ARGS (optional) are any number of custom 'arg=value' pairs like 'imgsz=320' that override defaults. See all ARGS at https://docs.ultralytics.com/usage/cfg or with 'yolo cfg' 1. Train a detection model for 10 epochs with an initial learning_rate of 0.01 yolo train data=coco128.yaml model=yolov8n.pt epochs=10 lr0=0.01 2. Predict a YouTube video using a pretrained segmentation model at image size 320: yolo predict model=yolov8n-seg.pt source='https://youtu.be/Zgi9g1ksQHc' imgsz=320 3. Val a pretrained detection model at batch-size 1 and image size 640: yolo val model=yolov8n.pt data=coco128.yaml batch=1 imgsz=640 4. Export a YOLOv8n classification model to ONNX format at image size 224 by 128 (no TASK required) yolo export model=yolov8n-cls.pt format=onnx imgsz=224,128 5. Run special commands: yolo help yolo checks yolo version yolo settings yolo copy-cfg yolo cfg Docs: https://docs.ultralytics.com Community: https://community.ultralytics.com GitHub: https://github.com/ultralytics/ultralytics """ # Define keys for arg type checks CFG_FLOAT_KEYS = 'warmup_epochs', 'box', 'cls', 'dfl', 'degrees', 'shear' CFG_FRACTION_KEYS = ('dropout', 'iou', 'lr0', 'lrf', 'momentum', 'weight_decay', 'warmup_momentum', 'warmup_bias_lr', 'label_smoothing', 'hsv_h', 'hsv_s', 'hsv_v', 'translate', 'scale', 'perspective', 'flipud', 'fliplr', 'mosaic', 'mixup', 'copy_paste', 'conf', 'iou', 'fraction') # fraction floats 0.0 - 1.0 CFG_INT_KEYS = ('epochs', 'patience', 'batch', 'workers', 'seed', 'close_mosaic', 'mask_ratio', 'max_det', 'vid_stride', 'line_width', 'workspace', 'nbs', 'save_period') CFG_BOOL_KEYS = ('save', 'exist_ok', 'verbose', 'deterministic', 'single_cls', 'rect', 'cos_lr', 'overlap_mask', 'val', 'save_json', 'save_hybrid', 'half', 'dnn', 'plots', 'show', 'save_txt', 'save_conf', 'save_crop', 'show_labels', 'show_conf', 'visualize', 'augment', 'agnostic_nms', 'retina_masks', 'boxes', 'keras', 'optimize', 'int8', 'dynamic', 'simplify', 'nms', 'v5loader', 'profile') def cfg2dict(cfg): """ Convert a configuration object to a dictionary, whether it is a file path, a string, or a SimpleNamespace object. Args: cfg (str | Path | SimpleNamespace): Configuration object to be converted to a dictionary. Returns: cfg (dict): Configuration object in dictionary format. """ if isinstance(cfg, (str, Path)): cfg = yaml_load(cfg) # load dict elif isinstance(cfg, SimpleNamespace): cfg = vars(cfg) # convert to dict return cfg def get_cfg(cfg: Union[str, Path, Dict, SimpleNamespace] = DEFAULT_CFG_DICT, overrides: Dict = None): """ Load and merge configuration data from a file or dictionary. Args: cfg (str | Path | Dict | SimpleNamespace): Configuration data. overrides (str | Dict | optional): Overrides in the form of a file name or a dictionary. Default is None. Returns: (SimpleNamespace): Training arguments namespace. """ cfg = cfg2dict(cfg) # Merge overrides if overrides: overrides = cfg2dict(overrides) check_cfg_mismatch(cfg, overrides) cfg = {**cfg, **overrides} # merge cfg and overrides dicts (prefer overrides) # Special handling for numeric project/name for k in 'project', 'name': if k in cfg and isinstance(cfg[k], (int, float)): cfg[k] = str(cfg[k]) if cfg.get('name') == 'model': # assign model to 'name' arg cfg['name'] = cfg.get('model', '').split('.')[0] LOGGER.warning(f"WARNING ⚠️ 'name=model' automatically updated to 'name={cfg['name']}'.") # Type and Value checks for k, v in cfg.items(): if v is not None: # None values may be from optional args if k in CFG_FLOAT_KEYS and not isinstance(v, (int, float)): raise TypeError(f"'{k}={v}' is of invalid type {type(v).__name__}. " f"Valid '{k}' types are int (i.e. '{k}=0') or float (i.e. '{k}=0.5')") elif k in CFG_FRACTION_KEYS: if not isinstance(v, (int, float)): raise TypeError(f"'{k}={v}' is of invalid type {type(v).__name__}. " f"Valid '{k}' types are int (i.e. '{k}=0') or float (i.e. '{k}=0.5')") if not (0.0 <= v <= 1.0): raise ValueError(f"'{k}={v}' is an invalid value. " f"Valid '{k}' values are between 0.0 and 1.0.") elif k in CFG_INT_KEYS and not isinstance(v, int): raise TypeError(f"'{k}={v}' is of invalid type {type(v).__name__}. " f"'{k}' must be an int (i.e. '{k}=8')") elif k in CFG_BOOL_KEYS and not isinstance(v, bool): raise TypeError(f"'{k}={v}' is of invalid type {type(v).__name__}. " f"'{k}' must be a bool (i.e. '{k}=True' or '{k}=False')") # Return instance return IterableSimpleNamespace(**cfg) def _handle_deprecation(custom): """ Hardcoded function to handle deprecated config keys """ for key in custom.copy().keys(): if key == 'hide_labels': deprecation_warn(key, 'show_labels') custom['show_labels'] = custom.pop('hide_labels') == 'False' if key == 'hide_conf': deprecation_warn(key, 'show_conf') custom['show_conf'] = custom.pop('hide_conf') == 'False' if key == 'line_thickness': deprecation_warn(key, 'line_width') custom['line_width'] = custom.pop('line_thickness') return custom def check_cfg_mismatch(base: Dict, custom: Dict, e=None): """ This function checks for any mismatched keys between a custom configuration list and a base configuration list. If any mismatched keys are found, the function prints out similar keys from the base list and exits the program. Args: custom (Dict): a dictionary of custom configuration options base (Dict): a dictionary of base configuration options """ custom = _handle_deprecation(custom) base, custom = (set(x.keys()) for x in (base, custom)) mismatched = [x for x in custom if x not in base] if mismatched: string = '' for x in mismatched: matches = get_close_matches(x, base) # key list matches = [f'{k}={DEFAULT_CFG_DICT[k]}' if DEFAULT_CFG_DICT.get(k) is not None else k for k in matches] match_str = f'Similar arguments are i.e. {matches}.' if matches else '' string += f"'{colorstr('red', 'bold', x)}' is not a valid YOLO argument. {match_str}\n" raise SyntaxError(string + CLI_HELP_MSG) from e def merge_equals_args(args: List[str]) -> List[str]: """ Merges arguments around isolated '=' args in a list of strings. The function considers cases where the first argument ends with '=' or the second starts with '=', as well as when the middle one is an equals sign. Args: args (List[str]): A list of strings where each element is an argument. Returns: List[str]: A list of strings where the arguments around isolated '=' are merged. """ new_args = [] for i, arg in enumerate(args): if arg == '=' and 0 < i < len(args) - 1: # merge ['arg', '=', 'val'] new_args[-1] += f'={args[i + 1]}' del args[i + 1] elif arg.endswith('=') and i < len(args) - 1 and '=' not in args[i + 1]: # merge ['arg=', 'val'] new_args.append(f'{arg}{args[i + 1]}') del args[i + 1] elif arg.startswith('=') and i > 0: # merge ['arg', '=val'] new_args[-1] += arg else: new_args.append(arg) return new_args def handle_yolo_hub(args: List[str]) -> None: """ Handle Ultralytics HUB command-line interface (CLI) commands. This function processes Ultralytics HUB CLI commands such as login and logout. It should be called when executing a script with arguments related to HUB authentication. Args: args (List[str]): A list of command line arguments Example: python my_script.py hub login your_api_key """ from ultralytics import hub if args[0] == 'login': key = args[1] if len(args) > 1 else '' # Log in to Ultralytics HUB using the provided API key hub.login(key) elif args[0] == 'logout': # Log out from Ultralytics HUB hub.logout() def handle_yolo_settings(args: List[str]) -> None: """ Handle YOLO settings command-line interface (CLI) commands. This function processes YOLO settings CLI commands such as reset. It should be called when executing a script with arguments related to YOLO settings management. Args: args (List[str]): A list of command line arguments for YOLO settings management. Example: python my_script.py yolo settings reset """ path = USER_CONFIG_DIR / 'settings.yaml' # get SETTINGS YAML file path if any(args) and args[0] == 'reset': path.unlink() # delete the settings file get_settings() # create new settings LOGGER.info('Settings reset successfully') # inform the user that settings have been reset yaml_print(path) # print the current settings def entrypoint(debug=''): """ This function is the ultralytics package entrypoint, it's responsible for parsing the command line arguments passed to the package. This function allows for: - passing mandatory YOLO args as a list of strings - specifying the task to be performed, either 'detect', 'segment' or 'classify' - specifying the mode, either 'train', 'val', 'test', or 'predict' - running special modes like 'checks' - passing overrides to the package's configuration It uses the package's default cfg and initializes it using the passed overrides. Then it calls the CLI function with the composed cfg """ args = (debug.split(' ') if debug else sys.argv)[1:] if not args: # no arguments passed LOGGER.info(CLI_HELP_MSG) return special = { 'help': lambda: LOGGER.info(CLI_HELP_MSG), 'checks': checks.check_yolo, 'version': lambda: LOGGER.info(__version__), 'settings': lambda: handle_yolo_settings(args[1:]), 'cfg': lambda: yaml_print(DEFAULT_CFG_PATH), 'hub': lambda: handle_yolo_hub(args[1:]), 'login': lambda: handle_yolo_hub(args), 'copy-cfg': copy_default_cfg} full_args_dict = {**DEFAULT_CFG_DICT, **{k: None for k in TASKS}, **{k: None for k in MODES}, **special} # Define common mis-uses of special commands, i.e. -h, -help, --help special.update({k[0]: v for k, v in special.items()}) # singular special.update({k[:-1]: v for k, v in special.items() if len(k) > 1 and k.endswith('s')}) # singular special = {**special, **{f'-{k}': v for k, v in special.items()}, **{f'--{k}': v for k, v in special.items()}} overrides = {} # basic overrides, i.e. imgsz=320 for a in merge_equals_args(args): # merge spaces around '=' sign if a.startswith('--'): LOGGER.warning(f"WARNING ⚠️ '{a}' does not require leading dashes '--', updating to '{a[2:]}'.") a = a[2:] if a.endswith(','): LOGGER.warning(f"WARNING ⚠️ '{a}' does not require trailing comma ',', updating to '{a[:-1]}'.") a = a[:-1] if '=' in a: try: re.sub(r' *= *', '=', a) # remove spaces around equals sign k, v = a.split('=', 1) # split on first '=' sign assert v, f"missing '{k}' value" if k == 'cfg': # custom.yaml passed LOGGER.info(f'Overriding {DEFAULT_CFG_PATH} with {v}') overrides = {k: val for k, val in yaml_load(checks.check_yaml(v)).items() if k != 'cfg'} else: if v.lower() == 'none': v = None elif v.lower() == 'true': v = True elif v.lower() == 'false': v = False else: with contextlib.suppress(Exception): v = eval(v) overrides[k] = v except (NameError, SyntaxError, ValueError, AssertionError) as e: check_cfg_mismatch(full_args_dict, {a: ''}, e) elif a in TASKS: overrides['task'] = a elif a in MODES: overrides['mode'] = a elif a.lower() in special: special[a.lower()]() return elif a in DEFAULT_CFG_DICT and isinstance(DEFAULT_CFG_DICT[a], bool): overrides[a] = True # auto-True for default bool args, i.e. 'yolo show' sets show=True elif a in DEFAULT_CFG_DICT: raise SyntaxError(f"'{colorstr('red', 'bold', a)}' is a valid YOLO argument but is missing an '=' sign " f"to set its value, i.e. try '{a}={DEFAULT_CFG_DICT[a]}'\n{CLI_HELP_MSG}") else: check_cfg_mismatch(full_args_dict, {a: ''}) # Check keys check_cfg_mismatch(full_args_dict, overrides) # Mode mode = overrides.get('mode', None) if mode is None: mode = DEFAULT_CFG.mode or 'predict' LOGGER.warning(f"WARNING ⚠️ 'mode' is missing. Valid modes are {MODES}. Using default 'mode={mode}'.") elif mode not in MODES: if mode not in ('checks', checks): raise ValueError(f"Invalid 'mode={mode}'. Valid modes are {MODES}.\n{CLI_HELP_MSG}") LOGGER.warning("WARNING ⚠️ 'yolo mode=checks' is deprecated. Use 'yolo checks' instead.") checks.check_yolo() return # Task task = overrides.pop('task', None) if task: if task not in TASKS: raise ValueError(f"Invalid 'task={task}'. Valid tasks are {TASKS}.\n{CLI_HELP_MSG}") if 'model' not in overrides: overrides['model'] = TASK2MODEL[task] # Model model = overrides.pop('model', DEFAULT_CFG.model) if model is None: model = 'yolov8n.pt' LOGGER.warning(f"WARNING ⚠️ 'model' is missing. Using default 'model={model}'.") overrides['model'] = model if 'rtdetr' in model.lower(): # guess architecture from ultralytics import RTDETR model = RTDETR(model) # no task argument elif 'sam' in model.lower(): from ultralytics import SAM model = SAM(model) else: from ultralytics import YOLO model = YOLO(model, task=task) if isinstance(overrides.get('pretrained'), str): model.load(overrides['pretrained']) # Task Update if task != model.task: if task: LOGGER.warning(f"WARNING ⚠️ conflicting 'task={task}' passed with 'task={model.task}' model. " f"Ignoring 'task={task}' and updating to 'task={model.task}' to match model.") task = model.task # Mode if mode in ('predict', 'track') and 'source' not in overrides: overrides['source'] = DEFAULT_CFG.source or ROOT / 'assets' if (ROOT / 'assets').exists() \ else 'https://ultralytics.com/images/bus.jpg' LOGGER.warning(f"WARNING ⚠️ 'source' is missing. Using default 'source={overrides['source']}'.") elif mode in ('train', 'val'): if 'data' not in overrides: overrides['data'] = TASK2DATA.get(task or DEFAULT_CFG.task, DEFAULT_CFG.data) LOGGER.warning(f"WARNING ⚠️ 'data' is missing. Using default 'data={overrides['data']}'.") elif mode == 'export': if 'format' not in overrides: overrides['format'] = DEFAULT_CFG.format or 'torchscript' LOGGER.warning(f"WARNING ⚠️ 'format' is missing. Using default 'format={overrides['format']}'.") # Run command in python # getattr(model, mode)(**vars(get_cfg(overrides=overrides))) # default args using default.yaml getattr(model, mode)(**overrides) # default args from model # Special modes -------------------------------------------------------------------------------------------------------- def copy_default_cfg(): """Copy and create a new default configuration file with '_copy' appended to its name.""" new_file = Path.cwd() / DEFAULT_CFG_PATH.name.replace('.yaml', '_copy.yaml') shutil.copy2(DEFAULT_CFG_PATH, new_file) LOGGER.info(f'{DEFAULT_CFG_PATH} copied to {new_file}\n' f"Example YOLO command with this new custom cfg:\n yolo cfg='{new_file}' imgsz=320 batch=8") if __name__ == '__main__': # Example Usage: entrypoint(debug='yolo predict model=yolov8n.pt') entrypoint(debug='') ================================================ FILE: ultralytics/yolo/cfg/default.yaml ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license # Default training settings and hyperparameters for medium-augmentation COCO training task: detect # (str) YOLO task, i.e. detect, segment, classify, pose mode: train # (str) YOLO mode, i.e. train, val, predict, export, track, benchmark # Train settings ------------------------------------------------------------------------------------------------------- model: # (str, optional) path to model file, i.e. yolov8n.pt, yolov8n.yaml data: # (str, optional) path to data file, i.e. coco128.yaml epochs: 100 # (int) number of epochs to train for patience: 50 # (int) epochs to wait for no observable improvement for early stopping of training batch: 16 # (int) number of images per batch (-1 for AutoBatch) imgsz: 640 # (int) size of input images as integer or w,h save: True # (bool) save train checkpoints and predict results save_period: -1 # (int) Save checkpoint every x epochs (disabled if < 1) cache: False # (bool) True/ram, disk or False. Use cache for data loading device: # (int | str | list, optional) device to run on, i.e. cuda device=0 or device=0,1,2,3 or device=cpu workers: 8 # (int) number of worker threads for data loading (per RANK if DDP) project: # (str, optional) project name name: # (str, optional) experiment name, results saved to 'project/name' directory exist_ok: False # (bool) whether to overwrite existing experiment pretrained: True # (bool | str) whether to use a pretrained model (bool) or a model to load weights from (str) optimizer: auto # (str) optimizer to use, choices=[SGD, Adam, Adamax, AdamW, NAdam, RAdam, RMSProp, auto] verbose: True # (bool) whether to print verbose output seed: 0 # (int) random seed for reproducibility deterministic: True # (bool) whether to enable deterministic mode single_cls: False # (bool) train multi-class data as single-class rect: False # (bool) rectangular training if mode='train' or rectangular validation if mode='val' cos_lr: False # (bool) use cosine learning rate scheduler close_mosaic: 0 # (int) disable mosaic augmentation for final epochs resume: False # (bool) resume training from last checkpoint amp: True # (bool) Automatic Mixed Precision (AMP) training, choices=[True, False], True runs AMP check fraction: 1.0 # (float) dataset fraction to train on (default is 1.0, all images in train set) profile: False # (bool) profile ONNX and TensorRT speeds during training for loggers # Segmentation overlap_mask: True # (bool) masks should overlap during training (segment train only) mask_ratio: 4 # (int) mask downsample ratio (segment train only) # Classification dropout: 0.0 # (float) use dropout regularization (classify train only) # Val/Test settings ---------------------------------------------------------------------------------------------------- val: True # (bool) validate/test during training split: val # (str) dataset split to use for validation, i.e. 'val', 'test' or 'train' save_json: False # (bool) save results to JSON file save_hybrid: False # (bool) save hybrid version of labels (labels + additional predictions) conf: # (float, optional) object confidence threshold for detection (default 0.25 predict, 0.001 val) iou: 0.7 # (float) intersection over union (IoU) threshold for NMS max_det: 300 # (int) maximum number of detections per image half: False # (bool) use half precision (FP16) dnn: False # (bool) use OpenCV DNN for ONNX inference plots: True # (bool) save plots during train/val # Prediction settings -------------------------------------------------------------------------------------------------- source: # (str, optional) source directory for images or videos show: False # (bool) show results if possible save_txt: False # (bool) save results as .txt file save_conf: False # (bool) save results with confidence scores save_crop: False # (bool) save cropped images with results show_labels: True # (bool) show object labels in plots show_conf: True # (bool) show object confidence scores in plots vid_stride: 1 # (int) video frame-rate stride line_width: # (int, optional) line width of the bounding boxes, auto if missing visualize: False # (bool) visualize model features augment: False # (bool) apply image augmentation to prediction sources agnostic_nms: False # (bool) class-agnostic NMS classes: # (int | list[int], optional) filter results by class, i.e. class=0, or class=[0,2,3] retina_masks: False # (bool) use high-resolution segmentation masks boxes: True # (bool) Show boxes in segmentation predictions # Export settings ------------------------------------------------------------------------------------------------------ format: torchscript # (str) format to export to, choices at https://docs.ultralytics.com/modes/export/#export-formats keras: False # (bool) use Kera=s optimize: False # (bool) TorchScript: optimize for mobile int8: False # (bool) CoreML/TF INT8 quantization dynamic: False # (bool) ONNX/TF/TensorRT: dynamic axes simplify: False # (bool) ONNX: simplify model opset: # (int, optional) ONNX: opset version workspace: 4 # (int) TensorRT: workspace size (GB) nms: False # (bool) CoreML: add NMS # Hyperparameters ------------------------------------------------------------------------------------------------------ lr0: 0.01 # (float) initial learning rate (i.e. SGD=1E-2, Adam=1E-3) lrf: 0.01 # (float) final learning rate (lr0 * lrf) momentum: 0.937 # (float) SGD momentum/Adam beta1 weight_decay: 0.0005 # (float) optimizer weight decay 5e-4 warmup_epochs: 3.0 # (float) warmup epochs (fractions ok) warmup_momentum: 0.8 # (float) warmup initial momentum warmup_bias_lr: 0.1 # (float) warmup initial bias lr box: 7.5 # (float) box loss gain cls: 0.5 # (float) cls loss gain (scale with pixels) dfl: 1.5 # (float) dfl loss gain pose: 12.0 # (float) pose loss gain kobj: 1.0 # (float) keypoint obj loss gain label_smoothing: 0.0 # (float) label smoothing (fraction) nbs: 64 # (int) nominal batch size hsv_h: 0.015 # (float) image HSV-Hue augmentation (fraction) hsv_s: 0.7 # (float) image HSV-Saturation augmentation (fraction) hsv_v: 0.4 # (float) image HSV-Value augmentation (fraction) degrees: 0.0 # (float) image rotation (+/- deg) translate: 0.1 # (float) image translation (+/- fraction) scale: 0.5 # (float) image scale (+/- gain) shear: 0.0 # (float) image shear (+/- deg) perspective: 0.0 # (float) image perspective (+/- fraction), range 0-0.001 flipud: 0.0 # (float) image flip up-down (probability) fliplr: 0.5 # (float) image flip left-right (probability) mosaic: 1.0 # (float) image mosaic (probability) mixup: 0.0 # (float) image mixup (probability) copy_paste: 0.0 # (float) segment copy-paste (probability) # Custom config.yaml --------------------------------------------------------------------------------------------------- cfg: # (str, optional) for overriding defaults.yaml # Debug, do not modify ------------------------------------------------------------------------------------------------- v5loader: False # (bool) use legacy YOLOv5 dataloader (deprecated) # Tracker settings ------------------------------------------------------------------------------------------------------ tracker: botsort.yaml # (str) tracker type, choices=[botsort.yaml, bytetrack.yaml] ================================================ FILE: ultralytics/yolo/data/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from .base import BaseDataset from .build import build_dataloader, build_yolo_dataset, load_inference_source from .dataset import ClassificationDataset, SemanticDataset, YOLODataset from .dataset_wrappers import MixAndRectDataset __all__ = ('BaseDataset', 'ClassificationDataset', 'MixAndRectDataset', 'SemanticDataset', 'YOLODataset', 'build_yolo_dataset', 'build_dataloader', 'load_inference_source') ================================================ FILE: ultralytics/yolo/data/annotator.py ================================================ from pathlib import Path from ultralytics import YOLO from ultralytics.vit.sam import PromptPredictor, build_sam from ultralytics.yolo.utils.torch_utils import select_device def auto_annotate(data, det_model='yolov8x.pt', sam_model='sam_b.pt', device='', output_dir=None): """ Automatically annotates images using a YOLO object detection model and a SAM segmentation model. Args: data (str): Path to a folder containing images to be annotated. det_model (str, optional): Pre-trained YOLO detection model. Defaults to 'yolov8x.pt'. sam_model (str, optional): Pre-trained SAM segmentation model. Defaults to 'sam_b.pt'. device (str, optional): Device to run the models on. Defaults to an empty string (CPU or GPU, if available). output_dir (str | None | optional): Directory to save the annotated results. Defaults to a 'labels' folder in the same directory as 'data'. """ device = select_device(device) det_model = YOLO(det_model) sam_model = build_sam(sam_model) det_model.to(device) sam_model.to(device) if not output_dir: output_dir = Path(str(data)).parent / 'labels' Path(output_dir).mkdir(exist_ok=True, parents=True) prompt_predictor = PromptPredictor(sam_model) det_results = det_model(data, stream=True) for result in det_results: boxes = result.boxes.xyxy # Boxes object for bbox outputs class_ids = result.boxes.cls.int().tolist() # noqa if len(class_ids): prompt_predictor.set_image(result.orig_img) masks, _, _ = prompt_predictor.predict_torch( point_coords=None, point_labels=None, boxes=prompt_predictor.transform.apply_boxes_torch(boxes, result.orig_shape[:2]), multimask_output=False, ) result.update(masks=masks.squeeze(1)) segments = result.masks.xyn # noqa with open(str(Path(output_dir) / Path(result.path).stem) + '.txt', 'w') as f: for i in range(len(segments)): s = segments[i] if len(s) == 0: continue segment = map(str, segments[i].reshape(-1).tolist()) f.write(f'{class_ids[i]} ' + ' '.join(segment) + '\n') ================================================ FILE: ultralytics/yolo/data/augment.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import math import random from copy import deepcopy import cv2 import numpy as np import torch import torchvision.transforms as T from ..utils import LOGGER, colorstr from ..utils.checks import check_version from ..utils.instance import Instances from ..utils.metrics import bbox_ioa from ..utils.ops import segment2box from .utils import polygons2masks, polygons2masks_overlap POSE_FLIPLR_INDEX = [0, 2, 1, 4, 3, 6, 5, 8, 7, 10, 9, 12, 11, 14, 13, 16, 15] # TODO: we might need a BaseTransform to make all these augments be compatible with both classification and semantic class BaseTransform: def __init__(self) -> None: pass def apply_image(self, labels): """Applies image transformation to labels.""" pass def apply_instances(self, labels): """Applies transformations to input 'labels' and returns object instances.""" pass def apply_semantic(self, labels): """Applies semantic segmentation to an image.""" pass def __call__(self, labels): """Applies label transformations to an image, instances and semantic masks.""" self.apply_image(labels) self.apply_instances(labels) self.apply_semantic(labels) class Compose: def __init__(self, transforms): """Initializes the Compose object with a list of transforms.""" self.transforms = transforms def __call__(self, data): """Applies a series of transformations to input data.""" for t in self.transforms: data = t(data) return data def append(self, transform): """Appends a new transform to the existing list of transforms.""" self.transforms.append(transform) def tolist(self): """Converts list of transforms to a standard Python list.""" return self.transforms def __repr__(self): """Return string representation of object.""" format_string = f'{self.__class__.__name__}(' for t in self.transforms: format_string += '\n' format_string += f' {t}' format_string += '\n)' return format_string class BaseMixTransform: """This implementation is from mmyolo.""" def __init__(self, dataset, pre_transform=None, p=0.0) -> None: self.dataset = dataset self.pre_transform = pre_transform self.p = p def __call__(self, labels): """Applies pre-processing transforms and mixup/mosaic transforms to labels data.""" if random.uniform(0, 1) > self.p: return labels # Get index of one or three other images indexes = self.get_indexes() if isinstance(indexes, int): indexes = [indexes] # Get images information will be used for Mosaic or MixUp mix_labels = [self.dataset.get_image_and_label(i) for i in indexes] if self.pre_transform is not None: for i, data in enumerate(mix_labels): mix_labels[i] = self.pre_transform(data) labels['mix_labels'] = mix_labels # Mosaic or MixUp labels = self._mix_transform(labels) labels.pop('mix_labels', None) return labels def _mix_transform(self, labels): """Applies MixUp or Mosaic augmentation to the label dictionary.""" raise NotImplementedError def get_indexes(self): """Gets a list of shuffled indexes for mosaic augmentation.""" raise NotImplementedError class Mosaic(BaseMixTransform): """ Mosaic augmentation. This class performs mosaic augmentation by combining multiple (4 or 9) images into a single mosaic image. The augmentation is applied to a dataset with a given probability. Attributes: dataset: The dataset on which the mosaic augmentation is applied. imgsz (int, optional): Image size (height and width) after mosaic pipeline of a single image. Default to 640. p (float, optional): Probability of applying the mosaic augmentation. Must be in the range 0-1. Default to 1.0. n (int, optional): The grid size, either 4 (for 2x2) or 9 (for 3x3). """ def __init__(self, dataset, imgsz=640, p=1.0, n=4): """Initializes the object with a dataset, image size, probability, and border.""" assert 0 <= p <= 1.0, f'The probability should be in range [0, 1], but got {p}.' assert n in (4, 9), 'grid must be equal to 4 or 9.' super().__init__(dataset=dataset, p=p) self.dataset = dataset self.imgsz = imgsz self.border = (-imgsz // 2, -imgsz // 2) # width, height self.n = n def get_indexes(self, buffer=True): """Return a list of random indexes from the dataset.""" if buffer: # select images from buffer return random.choices(list(self.dataset.buffer), k=self.n - 1) else: # select any images return [random.randint(0, len(self.dataset) - 1) for _ in range(self.n - 1)] def _mix_transform(self, labels): """Apply mixup transformation to the input image and labels.""" assert labels.get('rect_shape', None) is None, 'rect and mosaic are mutually exclusive.' assert len(labels.get('mix_labels', [])), 'There are no other images for mosaic augment.' return self._mosaic4(labels) if self.n == 4 else self._mosaic9(labels) def _mosaic4(self, labels): """Create a 2x2 image mosaic.""" mosaic_labels = [] s = self.imgsz yc, xc = (int(random.uniform(-x, 2 * s + x)) for x in self.border) # mosaic center x, y for i in range(4): labels_patch = labels if i == 0 else labels['mix_labels'][i - 1] # Load image img = labels_patch['img'] h, w = labels_patch.pop('resized_shape') # Place img in img4 if i == 0: # top left img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image) x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image) elif i == 1: # top right x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h elif i == 2: # bottom left x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h) x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h) elif i == 3: # bottom right x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h) x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h) img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax] padw = x1a - x1b padh = y1a - y1b labels_patch = self._update_labels(labels_patch, padw, padh) mosaic_labels.append(labels_patch) final_labels = self._cat_labels(mosaic_labels) final_labels['img'] = img4 return final_labels def _mosaic9(self, labels): """Create a 3x3 image mosaic.""" mosaic_labels = [] s = self.imgsz hp, wp = -1, -1 # height, width previous for i in range(9): labels_patch = labels if i == 0 else labels['mix_labels'][i - 1] # Load image img = labels_patch['img'] h, w = labels_patch.pop('resized_shape') # Place img in img9 if i == 0: # center img9 = np.full((s * 3, s * 3, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles h0, w0 = h, w c = s, s, s + w, s + h # xmin, ymin, xmax, ymax (base) coordinates elif i == 1: # top c = s, s - h, s + w, s elif i == 2: # top right c = s + wp, s - h, s + wp + w, s elif i == 3: # right c = s + w0, s, s + w0 + w, s + h elif i == 4: # bottom right c = s + w0, s + hp, s + w0 + w, s + hp + h elif i == 5: # bottom c = s + w0 - w, s + h0, s + w0, s + h0 + h elif i == 6: # bottom left c = s + w0 - wp - w, s + h0, s + w0 - wp, s + h0 + h elif i == 7: # left c = s - w, s + h0 - h, s, s + h0 elif i == 8: # top left c = s - w, s + h0 - hp - h, s, s + h0 - hp padw, padh = c[:2] x1, y1, x2, y2 = (max(x, 0) for x in c) # allocate coords # Image img9[y1:y2, x1:x2] = img[y1 - padh:, x1 - padw:] # img9[ymin:ymax, xmin:xmax] hp, wp = h, w # height, width previous for next iteration # Labels assuming imgsz*2 mosaic size labels_patch = self._update_labels(labels_patch, padw + self.border[0], padh + self.border[1]) mosaic_labels.append(labels_patch) final_labels = self._cat_labels(mosaic_labels) final_labels['img'] = img9[-self.border[0]:self.border[0], -self.border[1]:self.border[1]] return final_labels @staticmethod def _update_labels(labels, padw, padh): """Update labels.""" nh, nw = labels['img'].shape[:2] labels['instances'].convert_bbox(format='xyxy') labels['instances'].denormalize(nw, nh) labels['instances'].add_padding(padw, padh) return labels def _cat_labels(self, mosaic_labels): """Return labels with mosaic border instances clipped.""" if len(mosaic_labels) == 0: return {} cls = [] instances = [] imgsz = self.imgsz * 2 # mosaic imgsz for labels in mosaic_labels: cls.append(labels['cls']) instances.append(labels['instances']) final_labels = { 'im_file': mosaic_labels[0]['im_file'], 'ori_shape': mosaic_labels[0]['ori_shape'], 'resized_shape': (imgsz, imgsz), 'cls': np.concatenate(cls, 0), 'instances': Instances.concatenate(instances, axis=0), 'mosaic_border': self.border} # final_labels final_labels['instances'].clip(imgsz, imgsz) good = final_labels['instances'].remove_zero_area_boxes() final_labels['cls'] = final_labels['cls'][good] return final_labels class MixUp(BaseMixTransform): def __init__(self, dataset, pre_transform=None, p=0.0) -> None: super().__init__(dataset=dataset, pre_transform=pre_transform, p=p) def get_indexes(self): """Get a random index from the dataset.""" return random.randint(0, len(self.dataset) - 1) def _mix_transform(self, labels): """Applies MixUp augmentation https://arxiv.org/pdf/1710.09412.pdf.""" r = np.random.beta(32.0, 32.0) # mixup ratio, alpha=beta=32.0 labels2 = labels['mix_labels'][0] labels['img'] = (labels['img'] * r + labels2['img'] * (1 - r)).astype(np.uint8) labels['instances'] = Instances.concatenate([labels['instances'], labels2['instances']], axis=0) labels['cls'] = np.concatenate([labels['cls'], labels2['cls']], 0) return labels class RandomPerspective: def __init__(self, degrees=0.0, translate=0.1, scale=0.5, shear=0.0, perspective=0.0, border=(0, 0), pre_transform=None): self.degrees = degrees self.translate = translate self.scale = scale self.shear = shear self.perspective = perspective # Mosaic border self.border = border self.pre_transform = pre_transform def affine_transform(self, img, border): """Center.""" C = np.eye(3, dtype=np.float32) C[0, 2] = -img.shape[1] / 2 # x translation (pixels) C[1, 2] = -img.shape[0] / 2 # y translation (pixels) # Perspective P = np.eye(3, dtype=np.float32) P[2, 0] = random.uniform(-self.perspective, self.perspective) # x perspective (about y) P[2, 1] = random.uniform(-self.perspective, self.perspective) # y perspective (about x) # Rotation and Scale R = np.eye(3, dtype=np.float32) a = random.uniform(-self.degrees, self.degrees) # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations s = random.uniform(1 - self.scale, 1 + self.scale) # s = 2 ** random.uniform(-scale, scale) R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s) # Shear S = np.eye(3, dtype=np.float32) S[0, 1] = math.tan(random.uniform(-self.shear, self.shear) * math.pi / 180) # x shear (deg) S[1, 0] = math.tan(random.uniform(-self.shear, self.shear) * math.pi / 180) # y shear (deg) # Translation T = np.eye(3, dtype=np.float32) T[0, 2] = random.uniform(0.5 - self.translate, 0.5 + self.translate) * self.size[0] # x translation (pixels) T[1, 2] = random.uniform(0.5 - self.translate, 0.5 + self.translate) * self.size[1] # y translation (pixels) # Combined rotation matrix M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT # Affine image if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed if self.perspective: img = cv2.warpPerspective(img, M, dsize=self.size, borderValue=(114, 114, 114)) else: # affine img = cv2.warpAffine(img, M[:2], dsize=self.size, borderValue=(114, 114, 114)) return img, M, s def apply_bboxes(self, bboxes, M): """ Apply affine to bboxes only. Args: bboxes (ndarray): list of bboxes, xyxy format, with shape (num_bboxes, 4). M (ndarray): affine matrix. Returns: new_bboxes (ndarray): bboxes after affine, [num_bboxes, 4]. """ n = len(bboxes) if n == 0: return bboxes xy = np.ones((n * 4, 3), dtype=bboxes.dtype) xy[:, :2] = bboxes[:, [0, 1, 2, 3, 0, 3, 2, 1]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1 xy = xy @ M.T # transform xy = (xy[:, :2] / xy[:, 2:3] if self.perspective else xy[:, :2]).reshape(n, 8) # perspective rescale or affine # Create new boxes x = xy[:, [0, 2, 4, 6]] y = xy[:, [1, 3, 5, 7]] return np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1)), dtype=bboxes.dtype).reshape(4, n).T def apply_segments(self, segments, M): """ Apply affine to segments and generate new bboxes from segments. Args: segments (ndarray): list of segments, [num_samples, 500, 2]. M (ndarray): affine matrix. Returns: new_segments (ndarray): list of segments after affine, [num_samples, 500, 2]. new_bboxes (ndarray): bboxes after affine, [N, 4]. """ n, num = segments.shape[:2] if n == 0: return [], segments xy = np.ones((n * num, 3), dtype=segments.dtype) segments = segments.reshape(-1, 2) xy[:, :2] = segments xy = xy @ M.T # transform xy = xy[:, :2] / xy[:, 2:3] segments = xy.reshape(n, -1, 2) bboxes = np.stack([segment2box(xy, self.size[0], self.size[1]) for xy in segments], 0) return bboxes, segments def apply_keypoints(self, keypoints, M): """ Apply affine to keypoints. Args: keypoints (ndarray): keypoints, [N, 17, 3]. M (ndarray): affine matrix. Return: new_keypoints (ndarray): keypoints after affine, [N, 17, 3]. """ n, nkpt = keypoints.shape[:2] if n == 0: return keypoints xy = np.ones((n * nkpt, 3), dtype=keypoints.dtype) visible = keypoints[..., 2].reshape(n * nkpt, 1) xy[:, :2] = keypoints[..., :2].reshape(n * nkpt, 2) xy = xy @ M.T # transform xy = xy[:, :2] / xy[:, 2:3] # perspective rescale or affine out_mask = (xy[:, 0] < 0) | (xy[:, 1] < 0) | (xy[:, 0] > self.size[0]) | (xy[:, 1] > self.size[1]) visible[out_mask] = 0 return np.concatenate([xy, visible], axis=-1).reshape(n, nkpt, 3) def __call__(self, labels): """ Affine images and targets. Args: labels (dict): a dict of `bboxes`, `segments`, `keypoints`. """ if self.pre_transform and 'mosaic_border' not in labels: labels = self.pre_transform(labels) labels.pop('ratio_pad') # do not need ratio pad img = labels['img'] cls = labels['cls'] instances = labels.pop('instances') # Make sure the coord formats are right instances.convert_bbox(format='xyxy') instances.denormalize(*img.shape[:2][::-1]) border = labels.pop('mosaic_border', self.border) self.size = img.shape[1] + border[1] * 2, img.shape[0] + border[0] * 2 # w, h # M is affine matrix # scale for func:`box_candidates` img, M, scale = self.affine_transform(img, border) bboxes = self.apply_bboxes(instances.bboxes, M) segments = instances.segments keypoints = instances.keypoints # Update bboxes if there are segments. if len(segments): bboxes, segments = self.apply_segments(segments, M) if keypoints is not None: keypoints = self.apply_keypoints(keypoints, M) new_instances = Instances(bboxes, segments, keypoints, bbox_format='xyxy', normalized=False) # Clip new_instances.clip(*self.size) # Filter instances instances.scale(scale_w=scale, scale_h=scale, bbox_only=True) # Make the bboxes have the same scale with new_bboxes i = self.box_candidates(box1=instances.bboxes.T, box2=new_instances.bboxes.T, area_thr=0.01 if len(segments) else 0.10) labels['instances'] = new_instances[i] labels['cls'] = cls[i] labels['img'] = img labels['resized_shape'] = img.shape[:2] return labels def box_candidates(self, box1, box2, wh_thr=2, ar_thr=100, area_thr=0.1, eps=1e-16): # box1(4,n), box2(4,n) # Compute box candidates: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio w1, h1 = box1[2] - box1[0], box1[3] - box1[1] w2, h2 = box2[2] - box2[0], box2[3] - box2[1] ar = np.maximum(w2 / (h2 + eps), h2 / (w2 + eps)) # aspect ratio return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + eps) > area_thr) & (ar < ar_thr) # candidates class RandomHSV: def __init__(self, hgain=0.5, sgain=0.5, vgain=0.5) -> None: self.hgain = hgain self.sgain = sgain self.vgain = vgain def __call__(self, labels): """Applies random horizontal or vertical flip to an image with a given probability.""" img = labels['img'] if self.hgain or self.sgain or self.vgain: r = np.random.uniform(-1, 1, 3) * [self.hgain, self.sgain, self.vgain] + 1 # random gains hue, sat, val = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV)) dtype = img.dtype # uint8 x = np.arange(0, 256, dtype=r.dtype) lut_hue = ((x * r[0]) % 180).astype(dtype) lut_sat = np.clip(x * r[1], 0, 255).astype(dtype) lut_val = np.clip(x * r[2], 0, 255).astype(dtype) im_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))) cv2.cvtColor(im_hsv, cv2.COLOR_HSV2BGR, dst=img) # no return needed return labels class RandomFlip: def __init__(self, p=0.5, direction='horizontal', flip_idx=None) -> None: assert direction in ['horizontal', 'vertical'], f'Support direction `horizontal` or `vertical`, got {direction}' assert 0 <= p <= 1.0 self.p = p self.direction = direction self.flip_idx = flip_idx def __call__(self, labels): """Resize image and padding for detection, instance segmentation, pose.""" img = labels['img'] instances = labels.pop('instances') instances.convert_bbox(format='xywh') h, w = img.shape[:2] h = 1 if instances.normalized else h w = 1 if instances.normalized else w # Flip up-down if self.direction == 'vertical' and random.random() < self.p: img = np.flipud(img) instances.flipud(h) if self.direction == 'horizontal' and random.random() < self.p: img = np.fliplr(img) instances.fliplr(w) # For keypoints if self.flip_idx is not None and instances.keypoints is not None: instances.keypoints = np.ascontiguousarray(instances.keypoints[:, self.flip_idx, :]) labels['img'] = np.ascontiguousarray(img) labels['instances'] = instances return labels class LetterBox: """Resize image and padding for detection, instance segmentation, pose.""" def __init__(self, new_shape=(640, 640), auto=False, scaleFill=False, scaleup=True, stride=32): """Initialize LetterBox object with specific parameters.""" self.new_shape = new_shape self.auto = auto self.scaleFill = scaleFill self.scaleup = scaleup self.stride = stride def __call__(self, labels=None, image=None): """Return updated labels and image with added border.""" if labels is None: labels = {} img = labels.get('img') if image is None else image shape = img.shape[:2] # current shape [height, width] new_shape = labels.pop('rect_shape', self.new_shape) if isinstance(new_shape, int): new_shape = (new_shape, new_shape) # Scale ratio (new / old) r = min(new_shape[0] / shape[0], new_shape[1] / shape[1]) if not self.scaleup: # only scale down, do not scale up (for better val mAP) r = min(r, 1.0) # Compute padding ratio = r, r # width, height ratios new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r)) dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding if self.auto: # minimum rectangle dw, dh = np.mod(dw, self.stride), np.mod(dh, self.stride) # wh padding elif self.scaleFill: # stretch dw, dh = 0.0, 0.0 new_unpad = (new_shape[1], new_shape[0]) ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios dw /= 2 # divide padding into 2 sides dh /= 2 if labels.get('ratio_pad'): labels['ratio_pad'] = (labels['ratio_pad'], (dw, dh)) # for evaluation if shape[::-1] != new_unpad: # resize img = cv2.resize(img, new_unpad, interpolation=cv2.INTER_LINEAR) top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1)) left, right = int(round(dw - 0.1)), int(round(dw + 0.1)) img = cv2.copyMakeBorder(img, top, bottom, left, right, cv2.BORDER_CONSTANT, value=(114, 114, 114)) # add border if len(labels): labels = self._update_labels(labels, ratio, dw, dh) labels['img'] = img labels['resized_shape'] = new_shape return labels else: return img def _update_labels(self, labels, ratio, padw, padh): """Update labels.""" labels['instances'].convert_bbox(format='xyxy') labels['instances'].denormalize(*labels['img'].shape[:2][::-1]) labels['instances'].scale(*ratio) labels['instances'].add_padding(padw, padh) return labels class CopyPaste: def __init__(self, p=0.5) -> None: self.p = p def __call__(self, labels): """Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy).""" im = labels['img'] cls = labels['cls'] h, w = im.shape[:2] instances = labels.pop('instances') instances.convert_bbox(format='xyxy') instances.denormalize(w, h) if self.p and len(instances.segments): n = len(instances) _, w, _ = im.shape # height, width, channels im_new = np.zeros(im.shape, np.uint8) # Calculate ioa first then select indexes randomly ins_flip = deepcopy(instances) ins_flip.fliplr(w) ioa = bbox_ioa(ins_flip.bboxes, instances.bboxes) # intersection over area, (N, M) indexes = np.nonzero((ioa < 0.30).all(1))[0] # (N, ) n = len(indexes) for j in random.sample(list(indexes), k=round(self.p * n)): cls = np.concatenate((cls, cls[[j]]), axis=0) instances = Instances.concatenate((instances, ins_flip[[j]]), axis=0) cv2.drawContours(im_new, instances.segments[[j]].astype(np.int32), -1, (1, 1, 1), cv2.FILLED) result = cv2.flip(im, 1) # augment segments (flip left-right) i = cv2.flip(im_new, 1).astype(bool) im[i] = result[i] # cv2.imwrite('debug.jpg', im) # debug labels['img'] = im labels['cls'] = cls labels['instances'] = instances return labels class Albumentations: # YOLOv8 Albumentations class (optional, only used if package is installed) def __init__(self, p=1.0): """Initialize the transform object for YOLO bbox formatted params.""" self.p = p self.transform = None prefix = colorstr('albumentations: ') try: import albumentations as A check_version(A.__version__, '1.0.3', hard=True) # version requirement T = [ A.Blur(p=0.01), A.MedianBlur(p=0.01), A.ToGray(p=0.01), A.CLAHE(p=0.01), A.RandomBrightnessContrast(p=0.0), A.RandomGamma(p=0.0), A.ImageCompression(quality_lower=75, p=0.0)] # transforms self.transform = A.Compose(T, bbox_params=A.BboxParams(format='yolo', label_fields=['class_labels'])) LOGGER.info(prefix + ', '.join(f'{x}'.replace('always_apply=False, ', '') for x in T if x.p)) except ImportError: # package not installed, skip pass except Exception as e: LOGGER.info(f'{prefix}{e}') def __call__(self, labels): """Generates object detections and returns a dictionary with detection results.""" im = labels['img'] cls = labels['cls'] if len(cls): labels['instances'].convert_bbox('xywh') labels['instances'].normalize(*im.shape[:2][::-1]) bboxes = labels['instances'].bboxes # TODO: add supports of segments and keypoints if self.transform and random.random() < self.p: new = self.transform(image=im, bboxes=bboxes, class_labels=cls) # transformed if len(new['class_labels']) > 0: # skip update if no bbox in new im labels['img'] = new['image'] labels['cls'] = np.array(new['class_labels']) bboxes = np.array(new['bboxes'], dtype=np.float32) labels['instances'].update(bboxes=bboxes) return labels # TODO: technically this is not an augmentation, maybe we should put this to another files class Format: def __init__(self, bbox_format='xywh', normalize=True, return_mask=False, return_keypoint=False, mask_ratio=4, mask_overlap=True, batch_idx=True): self.bbox_format = bbox_format self.normalize = normalize self.return_mask = return_mask # set False when training detection only self.return_keypoint = return_keypoint self.mask_ratio = mask_ratio self.mask_overlap = mask_overlap self.batch_idx = batch_idx # keep the batch indexes def __call__(self, labels): """Return formatted image, classes, bounding boxes & keypoints to be used by 'collate_fn'.""" img = labels.pop('img') h, w = img.shape[:2] cls = labels.pop('cls') instances = labels.pop('instances') instances.convert_bbox(format=self.bbox_format) instances.denormalize(w, h) nl = len(instances) if self.return_mask: if nl: masks, instances, cls = self._format_segments(instances, cls, w, h) masks = torch.from_numpy(masks) else: masks = torch.zeros(1 if self.mask_overlap else nl, img.shape[0] // self.mask_ratio, img.shape[1] // self.mask_ratio) labels['masks'] = masks if self.normalize: instances.normalize(w, h) labels['img'] = self._format_img(img) labels['cls'] = torch.from_numpy(cls) if nl else torch.zeros(nl) labels['bboxes'] = torch.from_numpy(instances.bboxes) if nl else torch.zeros((nl, 4)) if self.return_keypoint: labels['keypoints'] = torch.from_numpy(instances.keypoints) # Then we can use collate_fn if self.batch_idx: labels['batch_idx'] = torch.zeros(nl) return labels def _format_img(self, img): """Format the image for YOLOv5 from Numpy array to PyTorch tensor.""" if len(img.shape) < 3: img = np.expand_dims(img, -1) img = np.ascontiguousarray(img.transpose(2, 0, 1)[::-1]) img = torch.from_numpy(img) return img def _format_segments(self, instances, cls, w, h): """convert polygon points to bitmap.""" segments = instances.segments if self.mask_overlap: masks, sorted_idx = polygons2masks_overlap((h, w), segments, downsample_ratio=self.mask_ratio) masks = masks[None] # (640, 640) -> (1, 640, 640) instances = instances[sorted_idx] cls = cls[sorted_idx] else: masks = polygons2masks((h, w), segments, color=1, downsample_ratio=self.mask_ratio) return masks, instances, cls def v8_transforms(dataset, imgsz, hyp, stretch=False): """Convert images to a size suitable for YOLOv8 training.""" pre_transform = Compose([ Mosaic(dataset, imgsz=imgsz, p=hyp.mosaic), CopyPaste(p=hyp.copy_paste), RandomPerspective( degrees=hyp.degrees, translate=hyp.translate, scale=hyp.scale, shear=hyp.shear, perspective=hyp.perspective, pre_transform=None if stretch else LetterBox(new_shape=(imgsz, imgsz)), )]) flip_idx = dataset.data.get('flip_idx', None) # for keypoints augmentation if dataset.use_keypoints: kpt_shape = dataset.data.get('kpt_shape', None) if flip_idx is None and hyp.fliplr > 0.0: hyp.fliplr = 0.0 LOGGER.warning("WARNING ⚠️ No 'flip_idx' array defined in data.yaml, setting augmentation 'fliplr=0.0'") elif flip_idx and (len(flip_idx) != kpt_shape[0]): raise ValueError(f'data.yaml flip_idx={flip_idx} length must be equal to kpt_shape[0]={kpt_shape[0]}') return Compose([ pre_transform, MixUp(dataset, pre_transform=pre_transform, p=hyp.mixup), Albumentations(p=1.0), RandomHSV(hgain=hyp.hsv_h, sgain=hyp.hsv_s, vgain=hyp.hsv_v), RandomFlip(direction='vertical', p=hyp.flipud), RandomFlip(direction='horizontal', p=hyp.fliplr, flip_idx=flip_idx)]) # transforms # Classification augmentations ----------------------------------------------------------------------------------------- def classify_transforms(size=224, mean=(0.0, 0.0, 0.0), std=(1.0, 1.0, 1.0)): # IMAGENET_MEAN, IMAGENET_STD # Transforms to apply if albumentations not installed if not isinstance(size, int): raise TypeError(f'classify_transforms() size {size} must be integer, not (list, tuple)') if any(mean) or any(std): return T.Compose([CenterCrop(size), ToTensor(), T.Normalize(mean, std, inplace=True)]) else: return T.Compose([CenterCrop(size), ToTensor()]) def hsv2colorjitter(h, s, v): """Map HSV (hue, saturation, value) jitter into ColorJitter values (brightness, contrast, saturation, hue)""" return v, v, s, h def classify_albumentations( augment=True, size=224, scale=(0.08, 1.0), hflip=0.5, vflip=0.0, hsv_h=0.015, # image HSV-Hue augmentation (fraction) hsv_s=0.7, # image HSV-Saturation augmentation (fraction) hsv_v=0.4, # image HSV-Value augmentation (fraction) mean=(0.0, 0.0, 0.0), # IMAGENET_MEAN std=(1.0, 1.0, 1.0), # IMAGENET_STD auto_aug=False, ): # YOLOv8 classification Albumentations (optional, only used if package is installed) prefix = colorstr('albumentations: ') try: import albumentations as A from albumentations.pytorch import ToTensorV2 check_version(A.__version__, '1.0.3', hard=True) # version requirement if augment: # Resize and crop T = [A.RandomResizedCrop(height=size, width=size, scale=scale)] if auto_aug: # TODO: implement AugMix, AutoAug & RandAug in albumentations LOGGER.info(f'{prefix}auto augmentations are currently not supported') else: if hflip > 0: T += [A.HorizontalFlip(p=hflip)] if vflip > 0: T += [A.VerticalFlip(p=vflip)] if any((hsv_h, hsv_s, hsv_v)): T += [A.ColorJitter(*hsv2colorjitter(hsv_h, hsv_s, hsv_v))] # brightness, contrast, saturation, hue else: # Use fixed crop for eval set (reproducibility) T = [A.SmallestMaxSize(max_size=size), A.CenterCrop(height=size, width=size)] T += [A.Normalize(mean=mean, std=std), ToTensorV2()] # Normalize and convert to Tensor LOGGER.info(prefix + ', '.join(f'{x}'.replace('always_apply=False, ', '') for x in T if x.p)) return A.Compose(T) except ImportError: # package not installed, skip pass except Exception as e: LOGGER.info(f'{prefix}{e}') class ClassifyLetterBox: # YOLOv8 LetterBox class for image preprocessing, i.e. T.Compose([LetterBox(size), ToTensor()]) def __init__(self, size=(640, 640), auto=False, stride=32): """Resizes image and crops it to center with max dimensions 'h' and 'w'.""" super().__init__() self.h, self.w = (size, size) if isinstance(size, int) else size self.auto = auto # pass max size integer, automatically solve for short side using stride self.stride = stride # used with auto def __call__(self, im): # im = np.array HWC imh, imw = im.shape[:2] r = min(self.h / imh, self.w / imw) # ratio of new/old h, w = round(imh * r), round(imw * r) # resized image hs, ws = (math.ceil(x / self.stride) * self.stride for x in (h, w)) if self.auto else self.h, self.w top, left = round((hs - h) / 2 - 0.1), round((ws - w) / 2 - 0.1) im_out = np.full((self.h, self.w, 3), 114, dtype=im.dtype) im_out[top:top + h, left:left + w] = cv2.resize(im, (w, h), interpolation=cv2.INTER_LINEAR) return im_out class CenterCrop: # YOLOv8 CenterCrop class for image preprocessing, i.e. T.Compose([CenterCrop(size), ToTensor()]) def __init__(self, size=640): """Converts an image from numpy array to PyTorch tensor.""" super().__init__() self.h, self.w = (size, size) if isinstance(size, int) else size def __call__(self, im): # im = np.array HWC imh, imw = im.shape[:2] m = min(imh, imw) # min dimension top, left = (imh - m) // 2, (imw - m) // 2 return cv2.resize(im[top:top + m, left:left + m], (self.w, self.h), interpolation=cv2.INTER_LINEAR) class ToTensor: # YOLOv8 ToTensor class for image preprocessing, i.e. T.Compose([LetterBox(size), ToTensor()]) def __init__(self, half=False): """Initialize YOLOv8 ToTensor object with optional half-precision support.""" super().__init__() self.half = half def __call__(self, im): # im = np.array HWC in BGR order im = np.ascontiguousarray(im.transpose((2, 0, 1))[::-1]) # HWC to CHW -> BGR to RGB -> contiguous im = torch.from_numpy(im) # to torch im = im.half() if self.half else im.float() # uint8 to fp16/32 im /= 255.0 # 0-255 to 0.0-1.0 return im ================================================ FILE: ultralytics/yolo/data/base.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import glob import math import os import random from copy import deepcopy from multiprocessing.pool import ThreadPool from pathlib import Path from typing import Optional import cv2 import numpy as np import psutil from torch.utils.data import Dataset from tqdm import tqdm from ..utils import DEFAULT_CFG, LOCAL_RANK, LOGGER, NUM_THREADS, TQDM_BAR_FORMAT from .utils import HELP_URL, IMG_FORMATS class BaseDataset(Dataset): """ Base dataset class for loading and processing image data. Args: img_path (str): Path to the folder containing images. imgsz (int, optional): Image size. Defaults to 640. cache (bool, optional): Cache images to RAM or disk during training. Defaults to False. augment (bool, optional): If True, data augmentation is applied. Defaults to True. hyp (dict, optional): Hyperparameters to apply data augmentation. Defaults to None. prefix (str, optional): Prefix to print in log messages. Defaults to ''. rect (bool, optional): If True, rectangular training is used. Defaults to False. batch_size (int, optional): Size of batches. Defaults to None. stride (int, optional): Stride. Defaults to 32. pad (float, optional): Padding. Defaults to 0.0. single_cls (bool, optional): If True, single class training is used. Defaults to False. classes (list): List of included classes. Default is None. fraction (float): Fraction of dataset to utilize. Default is 1.0 (use all data). Attributes: im_files (list): List of image file paths. labels (list): List of label data dictionaries. ni (int): Number of images in the dataset. ims (list): List of loaded images. npy_files (list): List of numpy file paths. transforms (callable): Image transformation function. """ def __init__(self, img_path, imgsz=640, cache=False, augment=True, hyp=DEFAULT_CFG, prefix='', rect=False, batch_size=16, stride=32, pad=0.5, single_cls=False, classes=None, fraction=1.0): super().__init__() self.img_path = img_path self.imgsz = imgsz self.augment = augment self.single_cls = single_cls self.prefix = prefix self.fraction = fraction self.im_files = self.get_img_files(self.img_path) self.labels = self.get_labels() self.update_labels(include_class=classes) # single_cls and include_class self.ni = len(self.labels) # number of images self.rect = rect self.batch_size = batch_size self.stride = stride self.pad = pad if self.rect: assert self.batch_size is not None self.set_rectangle() # Buffer thread for mosaic images self.buffer = [] # buffer size = batch size self.max_buffer_length = min((self.ni, self.batch_size * 8, 1000)) if self.augment else 0 # Cache stuff if cache == 'ram' and not self.check_cache_ram(): cache = False self.ims, self.im_hw0, self.im_hw = [None] * self.ni, [None] * self.ni, [None] * self.ni self.npy_files = [Path(f).with_suffix('.npy') for f in self.im_files] if cache: self.cache_images(cache) # Transforms self.transforms = self.build_transforms(hyp=hyp) def get_img_files(self, img_path): """Read image files.""" try: f = [] # image files for p in img_path if isinstance(img_path, list) else [img_path]: p = Path(p) # os-agnostic if p.is_dir(): # dir f += glob.glob(str(p / '**' / '*.*'), recursive=True) # F = list(p.rglob('*.*')) # pathlib elif p.is_file(): # file with open(p) as t: t = t.read().strip().splitlines() parent = str(p.parent) + os.sep f += [x.replace('./', parent) if x.startswith('./') else x for x in t] # local to global path # F += [p.parent / x.lstrip(os.sep) for x in t] # local to global path (pathlib) else: raise FileNotFoundError(f'{self.prefix}{p} does not exist') im_files = sorted(x.replace('/', os.sep) for x in f if x.split('.')[-1].lower() in IMG_FORMATS) # self.img_files = sorted([x for x in f if x.suffix[1:].lower() in IMG_FORMATS]) # pathlib assert im_files, f'{self.prefix}No images found' except Exception as e: raise FileNotFoundError(f'{self.prefix}Error loading data from {img_path}\n{HELP_URL}') from e if self.fraction < 1: im_files = im_files[:round(len(im_files) * self.fraction)] return im_files def update_labels(self, include_class: Optional[list]): """include_class, filter labels to include only these classes (optional).""" include_class_array = np.array(include_class).reshape(1, -1) for i in range(len(self.labels)): if include_class is not None: cls = self.labels[i]['cls'] bboxes = self.labels[i]['bboxes'] segments = self.labels[i]['segments'] keypoints = self.labels[i]['keypoints'] j = (cls == include_class_array).any(1) self.labels[i]['cls'] = cls[j] self.labels[i]['bboxes'] = bboxes[j] if segments: self.labels[i]['segments'] = [segments[si] for si, idx in enumerate(j) if idx] if keypoints is not None: self.labels[i]['keypoints'] = keypoints[j] if self.single_cls: self.labels[i]['cls'][:, 0] = 0 def load_image(self, i): """Loads 1 image from dataset index 'i', returns (im, resized hw).""" im, f, fn = self.ims[i], self.im_files[i], self.npy_files[i] if im is None: # not cached in RAM if fn.exists(): # load npy im = np.load(fn) else: # read image im = cv2.imread(f) # BGR if im is None: raise FileNotFoundError(f'Image Not Found {f}') h0, w0 = im.shape[:2] # orig hw r = self.imgsz / max(h0, w0) # ratio if r != 1: # if sizes are not equal interp = cv2.INTER_LINEAR if (self.augment or r > 1) else cv2.INTER_AREA im = cv2.resize(im, (min(math.ceil(w0 * r), self.imgsz), min(math.ceil(h0 * r), self.imgsz)), interpolation=interp) # Add to buffer if training with augmentations if self.augment: self.ims[i], self.im_hw0[i], self.im_hw[i] = im, (h0, w0), im.shape[:2] # im, hw_original, hw_resized self.buffer.append(i) if len(self.buffer) >= self.max_buffer_length: j = self.buffer.pop(0) self.ims[j], self.im_hw0[j], self.im_hw[j] = None, None, None return im, (h0, w0), im.shape[:2] return self.ims[i], self.im_hw0[i], self.im_hw[i] def cache_images(self, cache): """Cache images to memory or disk.""" b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes fcn = self.cache_images_to_disk if cache == 'disk' else self.load_image with ThreadPool(NUM_THREADS) as pool: results = pool.imap(fcn, range(self.ni)) pbar = tqdm(enumerate(results), total=self.ni, bar_format=TQDM_BAR_FORMAT, disable=LOCAL_RANK > 0) for i, x in pbar: if cache == 'disk': b += self.npy_files[i].stat().st_size else: # 'ram' self.ims[i], self.im_hw0[i], self.im_hw[i] = x # im, hw_orig, hw_resized = load_image(self, i) b += self.ims[i].nbytes pbar.desc = f'{self.prefix}Caching images ({b / gb:.1f}GB {cache})' pbar.close() def cache_images_to_disk(self, i): """Saves an image as an *.npy file for faster loading.""" f = self.npy_files[i] if not f.exists(): np.save(f.as_posix(), cv2.imread(self.im_files[i])) def check_cache_ram(self, safety_margin=0.5): """Check image caching requirements vs available memory.""" b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes n = min(self.ni, 30) # extrapolate from 30 random images for _ in range(n): im = cv2.imread(random.choice(self.im_files)) # sample image ratio = self.imgsz / max(im.shape[0], im.shape[1]) # max(h, w) # ratio b += im.nbytes * ratio ** 2 mem_required = b * self.ni / n * (1 + safety_margin) # GB required to cache dataset into RAM mem = psutil.virtual_memory() cache = mem_required < mem.available # to cache or not to cache, that is the question if not cache: LOGGER.info(f'{self.prefix}{mem_required / gb:.1f}GB RAM required to cache images ' f'with {int(safety_margin * 100)}% safety margin but only ' f'{mem.available / gb:.1f}/{mem.total / gb:.1f}GB available, ' f"{'caching images ✅' if cache else 'not caching images ⚠️'}") return cache def set_rectangle(self): """Sets the shape of bounding boxes for YOLO detections as rectangles.""" bi = np.floor(np.arange(self.ni) / self.batch_size).astype(int) # batch index nb = bi[-1] + 1 # number of batches s = np.array([x.pop('shape') for x in self.labels]) # hw ar = s[:, 0] / s[:, 1] # aspect ratio irect = ar.argsort() self.im_files = [self.im_files[i] for i in irect] self.labels = [self.labels[i] for i in irect] ar = ar[irect] # Set training image shapes shapes = [[1, 1]] * nb for i in range(nb): ari = ar[bi == i] mini, maxi = ari.min(), ari.max() if maxi < 1: shapes[i] = [maxi, 1] elif mini > 1: shapes[i] = [1, 1 / mini] self.batch_shapes = np.ceil(np.array(shapes) * self.imgsz / self.stride + self.pad).astype(int) * self.stride self.batch = bi # batch index of image def __getitem__(self, index): """Returns transformed label information for given index.""" return self.transforms(self.get_image_and_label(index)) def get_image_and_label(self, index): """Get and return label information from the dataset.""" label = deepcopy(self.labels[index]) # requires deepcopy() https://github.com/ultralytics/ultralytics/pull/1948 label.pop('shape', None) # shape is for rect, remove it label['img'], label['ori_shape'], label['resized_shape'] = self.load_image(index) label['ratio_pad'] = (label['resized_shape'][0] / label['ori_shape'][0], label['resized_shape'][1] / label['ori_shape'][1]) # for evaluation if self.rect: label['rect_shape'] = self.batch_shapes[self.batch[index]] return self.update_labels_info(label) def __len__(self): """Returns the length of the labels list for the dataset.""" return len(self.labels) def update_labels_info(self, label): """custom your label format here.""" return label def build_transforms(self, hyp=None): """Users can custom augmentations here like: if self.augment: # Training transforms return Compose([]) else: # Val transforms return Compose([]) """ raise NotImplementedError def get_labels(self): """Users can custom their own format here. Make sure your output is a list with each element like below: dict( im_file=im_file, shape=shape, # format: (height, width) cls=cls, bboxes=bboxes, # xywh segments=segments, # xy keypoints=keypoints, # xy normalized=True, # or False bbox_format="xyxy", # or xywh, ltwh ) """ raise NotImplementedError ================================================ FILE: ultralytics/yolo/data/build.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import os import random from pathlib import Path import numpy as np import torch from PIL import Image from torch.utils.data import dataloader, distributed from ultralytics.yolo.data.dataloaders.stream_loaders import (LOADERS, LoadImages, LoadPilAndNumpy, LoadScreenshots, LoadStreams, LoadTensor, SourceTypes, autocast_list) from ultralytics.yolo.data.utils import IMG_FORMATS, VID_FORMATS from ultralytics.yolo.utils.checks import check_file from ..utils import RANK, colorstr from .dataset import YOLODataset from .utils import PIN_MEMORY class InfiniteDataLoader(dataloader.DataLoader): """Dataloader that reuses workers. Uses same syntax as vanilla DataLoader.""" def __init__(self, *args, **kwargs): """Dataloader that infinitely recycles workers, inherits from DataLoader.""" super().__init__(*args, **kwargs) object.__setattr__(self, 'batch_sampler', _RepeatSampler(self.batch_sampler)) self.iterator = super().__iter__() def __len__(self): """Returns the length of the batch sampler's sampler.""" return len(self.batch_sampler.sampler) def __iter__(self): """Creates a sampler that repeats indefinitely.""" for _ in range(len(self)): yield next(self.iterator) def reset(self): """Reset iterator. This is useful when we want to modify settings of dataset while training. """ self.iterator = self._get_iterator() class _RepeatSampler: """ Sampler that repeats forever. Args: sampler (Dataset.sampler): The sampler to repeat. """ def __init__(self, sampler): """Initializes an object that repeats a given sampler indefinitely.""" self.sampler = sampler def __iter__(self): """Iterates over the 'sampler' and yields its contents.""" while True: yield from iter(self.sampler) def seed_worker(worker_id): # noqa # Set dataloader worker seed https://pytorch.org/docs/stable/notes/randomness.html#dataloader worker_seed = torch.initial_seed() % 2 ** 32 np.random.seed(worker_seed) random.seed(worker_seed) def build_yolo_dataset(cfg, img_path, batch, data, mode='train', rect=False, stride=32): """Build YOLO Dataset""" return YOLODataset( img_path=img_path, imgsz=cfg.imgsz, batch_size=batch, augment=mode == 'train', # augmentation hyp=cfg, # TODO: probably add a get_hyps_from_cfg function rect=cfg.rect or rect, # rectangular batches cache=cfg.cache or None, single_cls=cfg.single_cls or False, stride=int(stride), pad=0.0 if mode == 'train' else 0.5, prefix=colorstr(f'{mode}: '), use_segments=cfg.task == 'segment', use_keypoints=cfg.task == 'pose', classes=cfg.classes, data=data, fraction=cfg.fraction if mode == 'train' else 1.0) def build_dataloader(dataset, batch, workers, shuffle=True, rank=-1): """Return an InfiniteDataLoader or DataLoader for training or validation set.""" batch = min(batch, len(dataset)) nd = torch.cuda.device_count() # number of CUDA devices nw = min([os.cpu_count() // max(nd, 1), batch if batch > 1 else 0, workers]) # number of workers sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle) generator = torch.Generator() generator.manual_seed(6148914691236517205 + RANK) return InfiniteDataLoader(dataset=dataset, batch_size=batch, shuffle=shuffle and sampler is None, num_workers=nw, sampler=sampler, pin_memory=PIN_MEMORY, collate_fn=getattr(dataset, 'collate_fn', None), worker_init_fn=seed_worker, generator=generator) def check_source(source): """Check source type and return corresponding flag values.""" webcam, screenshot, from_img, in_memory, tensor = False, False, False, False, False if isinstance(source, (str, int, Path)): # int for local usb camera source = str(source) is_file = Path(source).suffix[1:] in (IMG_FORMATS + VID_FORMATS) is_url = source.lower().startswith(('https://', 'http://', 'rtsp://', 'rtmp://')) webcam = source.isnumeric() or source.endswith('.streams') or (is_url and not is_file) screenshot = source.lower() == 'screen' if is_url and is_file: source = check_file(source) # download elif isinstance(source, tuple(LOADERS)): in_memory = True elif isinstance(source, (list, tuple)): source = autocast_list(source) # convert all list elements to PIL or np arrays from_img = True elif isinstance(source, (Image.Image, np.ndarray)): from_img = True elif isinstance(source, torch.Tensor): tensor = True else: raise TypeError('Unsupported image type. For supported types see https://docs.ultralytics.com/modes/predict') return source, webcam, screenshot, from_img, in_memory, tensor def load_inference_source(source=None, imgsz=640, vid_stride=1): """ Loads an inference source for object detection and applies necessary transformations. Args: source (str, Path, Tensor, PIL.Image, np.ndarray): The input source for inference. imgsz (int, optional): The size of the image for inference. Default is 640. vid_stride (int, optional): The frame interval for video sources. Default is 1. Returns: dataset (Dataset): A dataset object for the specified input source. """ source, webcam, screenshot, from_img, in_memory, tensor = check_source(source) source_type = source.source_type if in_memory else SourceTypes(webcam, screenshot, from_img, tensor) # Dataloader if tensor: dataset = LoadTensor(source) elif in_memory: dataset = source elif webcam: dataset = LoadStreams(source, imgsz=imgsz, vid_stride=vid_stride) elif screenshot: dataset = LoadScreenshots(source, imgsz=imgsz) elif from_img: dataset = LoadPilAndNumpy(source, imgsz=imgsz) else: dataset = LoadImages(source, imgsz=imgsz, vid_stride=vid_stride) # Attach source types to the dataset setattr(dataset, 'source_type', source_type) return dataset ================================================ FILE: ultralytics/yolo/data/converter.py ================================================ import json from collections import defaultdict from pathlib import Path import cv2 import numpy as np from tqdm import tqdm from ultralytics.yolo.utils.checks import check_requirements from ultralytics.yolo.utils.files import make_dirs def coco91_to_coco80_class(): """Converts 91-index COCO class IDs to 80-index COCO class IDs. Returns: (list): A list of 91 class IDs where the index represents the 80-index class ID and the value is the corresponding 91-index class ID. """ return [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, None, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, None, 24, 25, None, None, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, None, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, None, 60, None, None, 61, None, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, None, 73, 74, 75, 76, 77, 78, 79, None] def convert_coco(labels_dir='../coco/annotations/', use_segments=False, use_keypoints=False, cls91to80=True): """Converts COCO dataset annotations to a format suitable for training YOLOv5 models. Args: labels_dir (str, optional): Path to directory containing COCO dataset annotation files. use_segments (bool, optional): Whether to include segmentation masks in the output. use_keypoints (bool, optional): Whether to include keypoint annotations in the output. cls91to80 (bool, optional): Whether to map 91 COCO class IDs to the corresponding 80 COCO class IDs. Raises: FileNotFoundError: If the labels_dir path does not exist. Example Usage: convert_coco(labels_dir='../coco/annotations/', use_segments=True, use_keypoints=True, cls91to80=True) Output: Generates output files in the specified output directory. """ save_dir = make_dirs('yolo_labels') # output directory coco80 = coco91_to_coco80_class() # Import json for json_file in sorted(Path(labels_dir).resolve().glob('*.json')): fn = Path(save_dir) / 'labels' / json_file.stem.replace('instances_', '') # folder name fn.mkdir(parents=True, exist_ok=True) with open(json_file) as f: data = json.load(f) # Create image dict images = {'%g' % x['id']: x for x in data['images']} # Create image-annotations dict imgToAnns = defaultdict(list) for ann in data['annotations']: imgToAnns[ann['image_id']].append(ann) # Write labels file for img_id, anns in tqdm(imgToAnns.items(), desc=f'Annotations {json_file}'): img = images['%g' % img_id] h, w, f = img['height'], img['width'], img['file_name'] bboxes = [] segments = [] keypoints = [] for ann in anns: if ann['iscrowd']: continue # The COCO box format is [top left x, top left y, width, height] box = np.array(ann['bbox'], dtype=np.float64) box[:2] += box[2:] / 2 # xy top-left corner to center box[[0, 2]] /= w # normalize x box[[1, 3]] /= h # normalize y if box[2] <= 0 or box[3] <= 0: # if w <= 0 and h <= 0 continue cls = coco80[ann['category_id'] - 1] if cls91to80 else ann['category_id'] - 1 # class box = [cls] + box.tolist() if box not in bboxes: bboxes.append(box) if use_segments and ann.get('segmentation') is not None: if len(ann['segmentation']) == 0: segments.append([]) continue if isinstance(ann['segmentation'], dict): ann['segmentation'] = rle2polygon(ann['segmentation']) if len(ann['segmentation']) > 1: s = merge_multi_segment(ann['segmentation']) s = (np.concatenate(s, axis=0) / np.array([w, h])).reshape(-1).tolist() else: s = [j for i in ann['segmentation'] for j in i] # all segments concatenated s = (np.array(s).reshape(-1, 2) / np.array([w, h])).reshape(-1).tolist() s = [cls] + s if s not in segments: segments.append(s) if use_keypoints and ann.get('keypoints') is not None: k = (np.array(ann['keypoints']).reshape(-1, 3) / np.array([w, h, 1])).reshape(-1).tolist() k = box + k keypoints.append(k) # Write with open((fn / f).with_suffix('.txt'), 'a') as file: for i in range(len(bboxes)): if use_keypoints: line = *(keypoints[i]), # cls, box, keypoints else: line = *(segments[i] if use_segments and len(segments[i]) > 0 else bboxes[i]), # cls, box or segments file.write(('%g ' * len(line)).rstrip() % line + '\n') def rle2polygon(segmentation): """ Convert Run-Length Encoding (RLE) mask to polygon coordinates. Args: segmentation (dict, list): RLE mask representation of the object segmentation. Returns: (list): A list of lists representing the polygon coordinates for each contour. Note: Requires the 'pycocotools' package to be installed. """ check_requirements('pycocotools') from pycocotools import mask m = mask.decode(segmentation) m[m > 0] = 255 contours, _ = cv2.findContours(m, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_TC89_KCOS) polygons = [] for contour in contours: epsilon = 0.001 * cv2.arcLength(contour, True) contour_approx = cv2.approxPolyDP(contour, epsilon, True) polygon = contour_approx.flatten().tolist() polygons.append(polygon) return polygons def min_index(arr1, arr2): """ Find a pair of indexes with the shortest distance between two arrays of 2D points. Args: arr1 (np.array): A NumPy array of shape (N, 2) representing N 2D points. arr2 (np.array): A NumPy array of shape (M, 2) representing M 2D points. Returns: (tuple): A tuple containing the indexes of the points with the shortest distance in arr1 and arr2 respectively. """ dis = ((arr1[:, None, :] - arr2[None, :, :]) ** 2).sum(-1) return np.unravel_index(np.argmin(dis, axis=None), dis.shape) def merge_multi_segment(segments): """ Merge multiple segments into one list by connecting the coordinates with the minimum distance between each segment. This function connects these coordinates with a thin line to merge all segments into one. Args: segments (List[List]): Original segmentations in COCO's JSON file. Each element is a list of coordinates, like [segmentation1, segmentation2,...]. Returns: s (List[np.ndarray]): A list of connected segments represented as NumPy arrays. """ s = [] segments = [np.array(i).reshape(-1, 2) for i in segments] idx_list = [[] for _ in range(len(segments))] # record the indexes with min distance between each segment for i in range(1, len(segments)): idx1, idx2 = min_index(segments[i - 1], segments[i]) idx_list[i - 1].append(idx1) idx_list[i].append(idx2) # use two round to connect all the segments for k in range(2): # forward connection if k == 0: for i, idx in enumerate(idx_list): # middle segments have two indexes # reverse the index of middle segments if len(idx) == 2 and idx[0] > idx[1]: idx = idx[::-1] segments[i] = segments[i][::-1, :] segments[i] = np.roll(segments[i], -idx[0], axis=0) segments[i] = np.concatenate([segments[i], segments[i][:1]]) # deal with the first segment and the last one if i in [0, len(idx_list) - 1]: s.append(segments[i]) else: idx = [0, idx[1] - idx[0]] s.append(segments[i][idx[0]:idx[1] + 1]) else: for i in range(len(idx_list) - 1, -1, -1): if i not in [0, len(idx_list) - 1]: idx = idx_list[i] nidx = abs(idx[1] - idx[0]) s.append(segments[i][nidx:]) return s def delete_dsstore(path='../datasets'): """Delete Apple .DS_Store files in the specified directory and its subdirectories.""" from pathlib import Path files = list(Path(path).rglob('.DS_store')) print(files) for f in files: f.unlink() if __name__ == '__main__': source = 'COCO' if source == 'COCO': convert_coco( '../datasets/coco/annotations', # directory with *.json use_segments=False, use_keypoints=True, cls91to80=False) ================================================ FILE: ultralytics/yolo/data/dataloaders/__init__.py ================================================ ================================================ FILE: ultralytics/yolo/data/dataloaders/stream_loaders.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import glob import math import os import time from dataclasses import dataclass from pathlib import Path from threading import Thread from urllib.parse import urlparse import cv2 import numpy as np import requests import torch from PIL import Image from ultralytics.yolo.data.utils import IMG_FORMATS, VID_FORMATS from ultralytics.yolo.utils import LOGGER, ROOT, is_colab, is_kaggle, ops from ultralytics.yolo.utils.checks import check_requirements @dataclass class SourceTypes: webcam: bool = False screenshot: bool = False from_img: bool = False tensor: bool = False class LoadStreams: # YOLOv8 streamloader, i.e. `yolo predict source='rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP streams` def __init__(self, sources='file.streams', imgsz=640, vid_stride=1): """Initialize instance variables and check for consistent input stream shapes.""" torch.backends.cudnn.benchmark = True # faster for fixed-size inference self.mode = 'stream' self.imgsz = imgsz self.vid_stride = vid_stride # video frame-rate stride sources = Path(sources).read_text().rsplit() if os.path.isfile(sources) else [sources] n = len(sources) self.sources = [ops.clean_str(x) for x in sources] # clean source names for later self.imgs, self.fps, self.frames, self.threads = [None] * n, [0] * n, [0] * n, [None] * n for i, s in enumerate(sources): # index, source # Start thread to read frames from video stream st = f'{i + 1}/{n}: {s}... ' if urlparse(s).hostname in ('www.youtube.com', 'youtube.com', 'youtu.be'): # if source is YouTube video # YouTube format i.e. 'https://www.youtube.com/watch?v=Zgi9g1ksQHc' or 'https://youtu.be/Zgi9g1ksQHc' s = get_best_youtube_url(s) s = eval(s) if s.isnumeric() else s # i.e. s = '0' local webcam if s == 0 and (is_colab() or is_kaggle()): raise NotImplementedError("'source=0' webcam not supported in Colab and Kaggle notebooks. " "Try running 'source=0' in a local environment.") cap = cv2.VideoCapture(s) if not cap.isOpened(): raise ConnectionError(f'{st}Failed to open {s}') w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = cap.get(cv2.CAP_PROP_FPS) # warning: may return 0 or nan self.frames[i] = max(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)), 0) or float('inf') # infinite stream fallback self.fps[i] = max((fps if math.isfinite(fps) else 0) % 100, 0) or 30 # 30 FPS fallback success, self.imgs[i] = cap.read() # guarantee first frame if not success or self.imgs[i] is None: raise ConnectionError(f'{st}Failed to read images from {s}') self.threads[i] = Thread(target=self.update, args=([i, cap, s]), daemon=True) LOGGER.info(f'{st}Success ✅ ({self.frames[i]} frames of shape {w}x{h} at {self.fps[i]:.2f} FPS)') self.threads[i].start() LOGGER.info('') # newline # Check for common shapes self.bs = self.__len__() def update(self, i, cap, stream): """Read stream `i` frames in daemon thread.""" n, f = 0, self.frames[i] # frame number, frame array while cap.isOpened() and n < f: n += 1 cap.grab() # .read() = .grab() followed by .retrieve() if n % self.vid_stride == 0: success, im = cap.retrieve() if success: self.imgs[i] = im else: LOGGER.warning('WARNING ⚠️ Video stream unresponsive, please check your IP camera connection.') self.imgs[i] = np.zeros_like(self.imgs[i]) cap.open(stream) # re-open stream if signal was lost time.sleep(0.0) # wait time def __iter__(self): """Iterates through YOLO image feed and re-opens unresponsive streams.""" self.count = -1 return self def __next__(self): """Returns source paths, transformed and original images for processing YOLOv5.""" self.count += 1 if not all(x.is_alive() for x in self.threads) or cv2.waitKey(1) == ord('q'): # q to quit cv2.destroyAllWindows() raise StopIteration im0 = self.imgs.copy() return self.sources, im0, None, '' def __len__(self): """Return the length of the sources object.""" return len(self.sources) # 1E12 frames = 32 streams at 30 FPS for 30 years class LoadScreenshots: # YOLOv8 screenshot dataloader, i.e. `yolo predict source=screen` def __init__(self, source, imgsz=640): """source = [screen_number left top width height] (pixels).""" check_requirements('mss') import mss # noqa source, *params = source.split() self.screen, left, top, width, height = 0, None, None, None, None # default to full screen 0 if len(params) == 1: self.screen = int(params[0]) elif len(params) == 4: left, top, width, height = (int(x) for x in params) elif len(params) == 5: self.screen, left, top, width, height = (int(x) for x in params) self.imgsz = imgsz self.mode = 'stream' self.frame = 0 self.sct = mss.mss() self.bs = 1 # Parse monitor shape monitor = self.sct.monitors[self.screen] self.top = monitor['top'] if top is None else (monitor['top'] + top) self.left = monitor['left'] if left is None else (monitor['left'] + left) self.width = width or monitor['width'] self.height = height or monitor['height'] self.monitor = {'left': self.left, 'top': self.top, 'width': self.width, 'height': self.height} def __iter__(self): """Returns an iterator of the object.""" return self def __next__(self): """mss screen capture: get raw pixels from the screen as np array.""" im0 = np.array(self.sct.grab(self.monitor))[:, :, :3] # [:, :, :3] BGRA to BGR s = f'screen {self.screen} (LTWH): {self.left},{self.top},{self.width},{self.height}: ' self.frame += 1 return str(self.screen), im0, None, s # screen, img, original img, im0s, s class LoadImages: # YOLOv8 image/video dataloader, i.e. `yolo predict source=image.jpg/vid.mp4` def __init__(self, path, imgsz=640, vid_stride=1): """Initialize the Dataloader and raise FileNotFoundError if file not found.""" if isinstance(path, str) and Path(path).suffix == '.txt': # *.txt file with img/vid/dir on each line path = Path(path).read_text().rsplit() files = [] for p in sorted(path) if isinstance(path, (list, tuple)) else [path]: p = str(Path(p).absolute()) # do not use .resolve() https://github.com/ultralytics/ultralytics/issues/2912 if '*' in p: files.extend(sorted(glob.glob(p, recursive=True))) # glob elif os.path.isdir(p): files.extend(sorted(glob.glob(os.path.join(p, '*.*')))) # dir elif os.path.isfile(p): files.append(p) # files else: raise FileNotFoundError(f'{p} does not exist') images = [x for x in files if x.split('.')[-1].lower() in IMG_FORMATS] videos = [x for x in files if x.split('.')[-1].lower() in VID_FORMATS] ni, nv = len(images), len(videos) self.imgsz = imgsz self.files = images + videos self.nf = ni + nv # number of files self.video_flag = [False] * ni + [True] * nv self.mode = 'image' self.vid_stride = vid_stride # video frame-rate stride self.bs = 1 if any(videos): self.orientation = None # rotation degrees self._new_video(videos[0]) # new video else: self.cap = None if self.nf == 0: raise FileNotFoundError(f'No images or videos found in {p}. ' f'Supported formats are:\nimages: {IMG_FORMATS}\nvideos: {VID_FORMATS}') def __iter__(self): """Returns an iterator object for VideoStream or ImageFolder.""" self.count = 0 return self def __next__(self): """Return next image, path and metadata from dataset.""" if self.count == self.nf: raise StopIteration path = self.files[self.count] if self.video_flag[self.count]: # Read video self.mode = 'video' for _ in range(self.vid_stride): self.cap.grab() success, im0 = self.cap.retrieve() while not success: self.count += 1 self.cap.release() if self.count == self.nf: # last video raise StopIteration path = self.files[self.count] self._new_video(path) success, im0 = self.cap.read() self.frame += 1 # im0 = self._cv2_rotate(im0) # for use if cv2 autorotation is False s = f'video {self.count + 1}/{self.nf} ({self.frame}/{self.frames}) {path}: ' else: # Read image self.count += 1 im0 = cv2.imread(path) # BGR if im0 is None: raise FileNotFoundError(f'Image Not Found {path}') s = f'image {self.count}/{self.nf} {path}: ' return [path], [im0], self.cap, s def _new_video(self, path): """Create a new video capture object.""" self.frame = 0 self.cap = cv2.VideoCapture(path) self.frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT) / self.vid_stride) if hasattr(cv2, 'CAP_PROP_ORIENTATION_META'): # cv2<4.6.0 compatibility self.orientation = int(self.cap.get(cv2.CAP_PROP_ORIENTATION_META)) # rotation degrees # Disable auto-orientation due to known issues in https://github.com/ultralytics/yolov5/issues/8493 # self.cap.set(cv2.CAP_PROP_ORIENTATION_AUTO, 0) def _cv2_rotate(self, im): """Rotate a cv2 video manually.""" if self.orientation == 0: return cv2.rotate(im, cv2.ROTATE_90_CLOCKWISE) elif self.orientation == 180: return cv2.rotate(im, cv2.ROTATE_90_COUNTERCLOCKWISE) elif self.orientation == 90: return cv2.rotate(im, cv2.ROTATE_180) return im def __len__(self): """Returns the number of files in the object.""" return self.nf # number of files class LoadPilAndNumpy: def __init__(self, im0, imgsz=640): """Initialize PIL and Numpy Dataloader.""" if not isinstance(im0, list): im0 = [im0] self.paths = [getattr(im, 'filename', f'image{i}.jpg') for i, im in enumerate(im0)] self.im0 = [self._single_check(im) for im in im0] self.imgsz = imgsz self.mode = 'image' # Generate fake paths self.bs = len(self.im0) @staticmethod def _single_check(im): """Validate and format an image to numpy array.""" assert isinstance(im, (Image.Image, np.ndarray)), f'Expected PIL/np.ndarray image type, but got {type(im)}' if isinstance(im, Image.Image): if im.mode != 'RGB': im = im.convert('RGB') im = np.asarray(im)[:, :, ::-1] im = np.ascontiguousarray(im) # contiguous return im def __len__(self): """Returns the length of the 'im0' attribute.""" return len(self.im0) def __next__(self): """Returns batch paths, images, processed images, None, ''.""" if self.count == 1: # loop only once as it's batch inference raise StopIteration self.count += 1 return self.paths, self.im0, None, '' def __iter__(self): """Enables iteration for class LoadPilAndNumpy.""" self.count = 0 return self class LoadTensor: def __init__(self, imgs) -> None: self.im0 = imgs self.bs = imgs.shape[0] self.mode = 'image' def __iter__(self): """Returns an iterator object.""" self.count = 0 return self def __next__(self): """Return next item in the iterator.""" if self.count == 1: raise StopIteration self.count += 1 return None, self.im0, None, '' # self.paths, im, self.im0, None, '' def __len__(self): """Returns the batch size.""" return self.bs def autocast_list(source): """ Merges a list of source of different types into a list of numpy arrays or PIL images """ files = [] for im in source: if isinstance(im, (str, Path)): # filename or uri files.append(Image.open(requests.get(im, stream=True).raw if str(im).startswith('http') else im)) elif isinstance(im, (Image.Image, np.ndarray)): # PIL or np Image files.append(im) else: raise TypeError(f'type {type(im).__name__} is not a supported Ultralytics prediction source type. \n' f'See https://docs.ultralytics.com/modes/predict for supported source types.') return files LOADERS = [LoadStreams, LoadPilAndNumpy, LoadImages, LoadScreenshots] def get_best_youtube_url(url, use_pafy=True): """ Retrieves the URL of the best quality MP4 video stream from a given YouTube video. This function uses the pafy or yt_dlp library to extract the video info from YouTube. It then finds the highest quality MP4 format that has video codec but no audio codec, and returns the URL of this video stream. Args: url (str): The URL of the YouTube video. use_pafy (bool): Use the pafy package, default=True, otherwise use yt_dlp package. Returns: (str): The URL of the best quality MP4 video stream, or None if no suitable stream is found. """ if use_pafy: check_requirements(('pafy', 'youtube_dl==2020.12.2')) import pafy # noqa return pafy.new(url).getbest(preftype='mp4').url else: check_requirements('yt-dlp') import yt_dlp with yt_dlp.YoutubeDL({'quiet': True}) as ydl: info_dict = ydl.extract_info(url, download=False) # extract info for f in info_dict.get('formats', None): if f['vcodec'] != 'none' and f['acodec'] == 'none' and f['ext'] == 'mp4': return f.get('url', None) if __name__ == '__main__': img = cv2.imread(str(ROOT / 'assets/bus.jpg')) dataset = LoadPilAndNumpy(im0=img) for d in dataset: print(d[0]) ================================================ FILE: ultralytics/yolo/data/dataloaders/v5augmentations.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license """ Image augmentation functions """ import math import random import cv2 import numpy as np import torch import torchvision.transforms as T import torchvision.transforms.functional as TF from ultralytics.yolo.utils import LOGGER, colorstr from ultralytics.yolo.utils.checks import check_version from ultralytics.yolo.utils.metrics import bbox_ioa from ultralytics.yolo.utils.ops import resample_segments, segment2box, xywhn2xyxy IMAGENET_MEAN = 0.485, 0.456, 0.406 # RGB mean IMAGENET_STD = 0.229, 0.224, 0.225 # RGB standard deviation class Albumentations: # YOLOv5 Albumentations class (optional, only used if package is installed) def __init__(self, size=640): """Instantiate object with image augmentations for YOLOv5.""" self.transform = None prefix = colorstr('albumentations: ') try: import albumentations as A check_version(A.__version__, '1.0.3', hard=True) # version requirement T = [ A.RandomResizedCrop(height=size, width=size, scale=(0.8, 1.0), ratio=(0.9, 1.11), p=0.0), A.Blur(p=0.01), A.MedianBlur(p=0.01), A.ToGray(p=0.01), A.CLAHE(p=0.01), A.RandomBrightnessContrast(p=0.0), A.RandomGamma(p=0.0), A.ImageCompression(quality_lower=75, p=0.0)] # transforms self.transform = A.Compose(T, bbox_params=A.BboxParams(format='yolo', label_fields=['class_labels'])) LOGGER.info(prefix + ', '.join(f'{x}'.replace('always_apply=False, ', '') for x in T if x.p)) except ImportError: # package not installed, skip pass except Exception as e: LOGGER.info(f'{prefix}{e}') def __call__(self, im, labels, p=1.0): """Transforms input image and labels with probability 'p'.""" if self.transform and random.random() < p: new = self.transform(image=im, bboxes=labels[:, 1:], class_labels=labels[:, 0]) # transformed im, labels = new['image'], np.array([[c, *b] for c, b in zip(new['class_labels'], new['bboxes'])]) return im, labels def normalize(x, mean=IMAGENET_MEAN, std=IMAGENET_STD, inplace=False): """Denormalize RGB images x per ImageNet stats in BCHW format, i.e. = (x - mean) / std.""" return TF.normalize(x, mean, std, inplace=inplace) def denormalize(x, mean=IMAGENET_MEAN, std=IMAGENET_STD): """Denormalize RGB images x per ImageNet stats in BCHW format, i.e. = x * std + mean.""" for i in range(3): x[:, i] = x[:, i] * std[i] + mean[i] return x def augment_hsv(im, hgain=0.5, sgain=0.5, vgain=0.5): """HSV color-space augmentation.""" if hgain or sgain or vgain: r = np.random.uniform(-1, 1, 3) * [hgain, sgain, vgain] + 1 # random gains hue, sat, val = cv2.split(cv2.cvtColor(im, cv2.COLOR_BGR2HSV)) dtype = im.dtype # uint8 x = np.arange(0, 256, dtype=r.dtype) lut_hue = ((x * r[0]) % 180).astype(dtype) lut_sat = np.clip(x * r[1], 0, 255).astype(dtype) lut_val = np.clip(x * r[2], 0, 255).astype(dtype) im_hsv = cv2.merge((cv2.LUT(hue, lut_hue), cv2.LUT(sat, lut_sat), cv2.LUT(val, lut_val))) cv2.cvtColor(im_hsv, cv2.COLOR_HSV2BGR, dst=im) # no return needed def hist_equalize(im, clahe=True, bgr=False): """Equalize histogram on BGR image 'im' with im.shape(n,m,3) and range 0-255.""" yuv = cv2.cvtColor(im, cv2.COLOR_BGR2YUV if bgr else cv2.COLOR_RGB2YUV) if clahe: c = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8, 8)) yuv[:, :, 0] = c.apply(yuv[:, :, 0]) else: yuv[:, :, 0] = cv2.equalizeHist(yuv[:, :, 0]) # equalize Y channel histogram return cv2.cvtColor(yuv, cv2.COLOR_YUV2BGR if bgr else cv2.COLOR_YUV2RGB) # convert YUV image to RGB def replicate(im, labels): """Replicate labels.""" h, w = im.shape[:2] boxes = labels[:, 1:].astype(int) x1, y1, x2, y2 = boxes.T s = ((x2 - x1) + (y2 - y1)) / 2 # side length (pixels) for i in s.argsort()[:round(s.size * 0.5)]: # smallest indices x1b, y1b, x2b, y2b = boxes[i] bh, bw = y2b - y1b, x2b - x1b yc, xc = int(random.uniform(0, h - bh)), int(random.uniform(0, w - bw)) # offset x, y x1a, y1a, x2a, y2a = [xc, yc, xc + bw, yc + bh] im[y1a:y2a, x1a:x2a] = im[y1b:y2b, x1b:x2b] # im4[ymin:ymax, xmin:xmax] labels = np.append(labels, [[labels[i, 0], x1a, y1a, x2a, y2a]], axis=0) return im, labels def letterbox(im, new_shape=(640, 640), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True, stride=32): """Resize and pad image while meeting stride-multiple constraints.""" shape = im.shape[:2] # current shape [height, width] if isinstance(new_shape, int): new_shape = (new_shape, new_shape) # Scale ratio (new / old) r = min(new_shape[0] / shape[0], new_shape[1] / shape[1]) if not scaleup: # only scale down, do not scale up (for better val mAP) r = min(r, 1.0) # Compute padding ratio = r, r # width, height ratios new_unpad = int(round(shape[1] * r)), int(round(shape[0] * r)) dw, dh = new_shape[1] - new_unpad[0], new_shape[0] - new_unpad[1] # wh padding if auto: # minimum rectangle dw, dh = np.mod(dw, stride), np.mod(dh, stride) # wh padding elif scaleFill: # stretch dw, dh = 0.0, 0.0 new_unpad = (new_shape[1], new_shape[0]) ratio = new_shape[1] / shape[1], new_shape[0] / shape[0] # width, height ratios dw /= 2 # divide padding into 2 sides dh /= 2 if shape[::-1] != new_unpad: # resize im = cv2.resize(im, new_unpad, interpolation=cv2.INTER_LINEAR) top, bottom = int(round(dh - 0.1)), int(round(dh + 0.1)) left, right = int(round(dw - 0.1)), int(round(dw + 0.1)) im = cv2.copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT, value=color) # add border return im, ratio, (dw, dh) def random_perspective(im, targets=(), segments=(), degrees=10, translate=.1, scale=.1, shear=10, perspective=0.0, border=(0, 0)): # torchvision.transforms.RandomAffine(degrees=(-10, 10), translate=(0.1, 0.1), scale=(0.9, 1.1), shear=(-10, 10)) # targets = [cls, xyxy] height = im.shape[0] + border[0] * 2 # shape(h,w,c) width = im.shape[1] + border[1] * 2 # Center C = np.eye(3) C[0, 2] = -im.shape[1] / 2 # x translation (pixels) C[1, 2] = -im.shape[0] / 2 # y translation (pixels) # Perspective P = np.eye(3) P[2, 0] = random.uniform(-perspective, perspective) # x perspective (about y) P[2, 1] = random.uniform(-perspective, perspective) # y perspective (about x) # Rotation and Scale R = np.eye(3) a = random.uniform(-degrees, degrees) # a += random.choice([-180, -90, 0, 90]) # add 90deg rotations to small rotations s = random.uniform(1 - scale, 1 + scale) # s = 2 ** random.uniform(-scale, scale) R[:2] = cv2.getRotationMatrix2D(angle=a, center=(0, 0), scale=s) # Shear S = np.eye(3) S[0, 1] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # x shear (deg) S[1, 0] = math.tan(random.uniform(-shear, shear) * math.pi / 180) # y shear (deg) # Translation T = np.eye(3) T[0, 2] = random.uniform(0.5 - translate, 0.5 + translate) * width # x translation (pixels) T[1, 2] = random.uniform(0.5 - translate, 0.5 + translate) * height # y translation (pixels) # Combined rotation matrix M = T @ S @ R @ P @ C # order of operations (right to left) is IMPORTANT if (border[0] != 0) or (border[1] != 0) or (M != np.eye(3)).any(): # image changed if perspective: im = cv2.warpPerspective(im, M, dsize=(width, height), borderValue=(114, 114, 114)) else: # affine im = cv2.warpAffine(im, M[:2], dsize=(width, height), borderValue=(114, 114, 114)) # Visualize # import matplotlib.pyplot as plt # ax = plt.subplots(1, 2, figsize=(12, 6))[1].ravel() # ax[0].imshow(im[:, :, ::-1]) # base # ax[1].imshow(im2[:, :, ::-1]) # warped # Transform label coordinates n = len(targets) if n: use_segments = any(x.any() for x in segments) new = np.zeros((n, 4)) if use_segments: # warp segments segments = resample_segments(segments) # upsample for i, segment in enumerate(segments): xy = np.ones((len(segment), 3)) xy[:, :2] = segment xy = xy @ M.T # transform xy = xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2] # perspective rescale or affine # Clip new[i] = segment2box(xy, width, height) else: # warp boxes xy = np.ones((n * 4, 3)) xy[:, :2] = targets[:, [1, 2, 3, 4, 1, 4, 3, 2]].reshape(n * 4, 2) # x1y1, x2y2, x1y2, x2y1 xy = xy @ M.T # transform xy = (xy[:, :2] / xy[:, 2:3] if perspective else xy[:, :2]).reshape(n, 8) # perspective rescale or affine # Create new boxes x = xy[:, [0, 2, 4, 6]] y = xy[:, [1, 3, 5, 7]] new = np.concatenate((x.min(1), y.min(1), x.max(1), y.max(1))).reshape(4, n).T # Clip new[:, [0, 2]] = new[:, [0, 2]].clip(0, width) new[:, [1, 3]] = new[:, [1, 3]].clip(0, height) # Filter candidates i = box_candidates(box1=targets[:, 1:5].T * s, box2=new.T, area_thr=0.01 if use_segments else 0.10) targets = targets[i] targets[:, 1:5] = new[i] return im, targets def copy_paste(im, labels, segments, p=0.5): """Implement Copy-Paste augmentation https://arxiv.org/abs/2012.07177, labels as nx5 np.array(cls, xyxy).""" n = len(segments) if p and n: h, w, c = im.shape # height, width, channels im_new = np.zeros(im.shape, np.uint8) # Calculate ioa first then select indexes randomly boxes = np.stack([w - labels[:, 3], labels[:, 2], w - labels[:, 1], labels[:, 4]], axis=-1) # (n, 4) ioa = bbox_ioa(boxes, labels[:, 1:5]) # intersection over area indexes = np.nonzero((ioa < 0.30).all(1))[0] # (N, ) n = len(indexes) for j in random.sample(list(indexes), k=round(p * n)): l, box, s = labels[j], boxes[j], segments[j] labels = np.concatenate((labels, [[l[0], *box]]), 0) segments.append(np.concatenate((w - s[:, 0:1], s[:, 1:2]), 1)) cv2.drawContours(im_new, [segments[j].astype(np.int32)], -1, (1, 1, 1), cv2.FILLED) result = cv2.flip(im, 1) # augment segments (flip left-right) i = cv2.flip(im_new, 1).astype(bool) im[i] = result[i] # cv2.imwrite('debug.jpg', im) # debug return im, labels, segments def cutout(im, labels, p=0.5): """Applies image cutout augmentation https://arxiv.org/abs/1708.04552.""" if random.random() < p: h, w = im.shape[:2] scales = [0.5] * 1 + [0.25] * 2 + [0.125] * 4 + [0.0625] * 8 + [0.03125] * 16 # image size fraction for s in scales: mask_h = random.randint(1, int(h * s)) # create random masks mask_w = random.randint(1, int(w * s)) # Box xmin = max(0, random.randint(0, w) - mask_w // 2) ymin = max(0, random.randint(0, h) - mask_h // 2) xmax = min(w, xmin + mask_w) ymax = min(h, ymin + mask_h) # Apply random color mask im[ymin:ymax, xmin:xmax] = [random.randint(64, 191) for _ in range(3)] # Return unobscured labels if len(labels) and s > 0.03: box = np.array([[xmin, ymin, xmax, ymax]], dtype=np.float32) ioa = bbox_ioa(box, xywhn2xyxy(labels[:, 1:5], w, h))[0] # intersection over area labels = labels[ioa < 0.60] # remove >60% obscured labels return labels def mixup(im, labels, im2, labels2): """Applies MixUp augmentation https://arxiv.org/pdf/1710.09412.pdf.""" r = np.random.beta(32.0, 32.0) # mixup ratio, alpha=beta=32.0 im = (im * r + im2 * (1 - r)).astype(np.uint8) labels = np.concatenate((labels, labels2), 0) return im, labels def box_candidates(box1, box2, wh_thr=2, ar_thr=100, area_thr=0.1, eps=1e-16): # box1(4,n), box2(4,n) # Compute candidate boxes: box1 before augment, box2 after augment, wh_thr (pixels), aspect_ratio_thr, area_ratio w1, h1 = box1[2] - box1[0], box1[3] - box1[1] w2, h2 = box2[2] - box2[0], box2[3] - box2[1] ar = np.maximum(w2 / (h2 + eps), h2 / (w2 + eps)) # aspect ratio return (w2 > wh_thr) & (h2 > wh_thr) & (w2 * h2 / (w1 * h1 + eps) > area_thr) & (ar < ar_thr) # candidates def classify_albumentations( augment=True, size=224, scale=(0.08, 1.0), ratio=(0.75, 1.0 / 0.75), # 0.75, 1.33 hflip=0.5, vflip=0.0, jitter=0.4, mean=IMAGENET_MEAN, std=IMAGENET_STD, auto_aug=False): # YOLOv5 classification Albumentations (optional, only used if package is installed) prefix = colorstr('albumentations: ') try: import albumentations as A from albumentations.pytorch import ToTensorV2 check_version(A.__version__, '1.0.3', hard=True) # version requirement if augment: # Resize and crop T = [A.RandomResizedCrop(height=size, width=size, scale=scale, ratio=ratio)] if auto_aug: # TODO: implement AugMix, AutoAug & RandAug in albumentation LOGGER.info(f'{prefix}auto augmentations are currently not supported') else: if hflip > 0: T += [A.HorizontalFlip(p=hflip)] if vflip > 0: T += [A.VerticalFlip(p=vflip)] if jitter > 0: jitter = float(jitter) T += [A.ColorJitter(jitter, jitter, jitter, 0)] # brightness, contrast, satuaration, 0 hue else: # Use fixed crop for eval set (reproducibility) T = [A.SmallestMaxSize(max_size=size), A.CenterCrop(height=size, width=size)] T += [A.Normalize(mean=mean, std=std), ToTensorV2()] # Normalize and convert to Tensor LOGGER.info(prefix + ', '.join(f'{x}'.replace('always_apply=False, ', '') for x in T if x.p)) return A.Compose(T) except ImportError: # package not installed, skip LOGGER.warning(f'{prefix}⚠️ not found, install with `pip install albumentations` (recommended)') except Exception as e: LOGGER.info(f'{prefix}{e}') def classify_transforms(size=224): """Transforms to apply if albumentations not installed.""" assert isinstance(size, int), f'ERROR: classify_transforms size {size} must be integer, not (list, tuple)' # T.Compose([T.ToTensor(), T.Resize(size), T.CenterCrop(size), T.Normalize(IMAGENET_MEAN, IMAGENET_STD)]) return T.Compose([CenterCrop(size), ToTensor(), T.Normalize(IMAGENET_MEAN, IMAGENET_STD)]) class LetterBox: # YOLOv5 LetterBox class for image preprocessing, i.e. T.Compose([LetterBox(size), ToTensor()]) def __init__(self, size=(640, 640), auto=False, stride=32): """Resizes and crops an image to a specified size for YOLOv5 preprocessing.""" super().__init__() self.h, self.w = (size, size) if isinstance(size, int) else size self.auto = auto # pass max size integer, automatically solve for short side using stride self.stride = stride # used with auto def __call__(self, im): # im = np.array HWC imh, imw = im.shape[:2] r = min(self.h / imh, self.w / imw) # ratio of new/old h, w = round(imh * r), round(imw * r) # resized image hs, ws = (math.ceil(x / self.stride) * self.stride for x in (h, w)) if self.auto else self.h, self.w top, left = round((hs - h) / 2 - 0.1), round((ws - w) / 2 - 0.1) im_out = np.full((self.h, self.w, 3), 114, dtype=im.dtype) im_out[top:top + h, left:left + w] = cv2.resize(im, (w, h), interpolation=cv2.INTER_LINEAR) return im_out class CenterCrop: # YOLOv5 CenterCrop class for image preprocessing, i.e. T.Compose([CenterCrop(size), ToTensor()]) def __init__(self, size=640): """Converts input image into tensor for YOLOv5 processing.""" super().__init__() self.h, self.w = (size, size) if isinstance(size, int) else size def __call__(self, im): # im = np.array HWC imh, imw = im.shape[:2] m = min(imh, imw) # min dimension top, left = (imh - m) // 2, (imw - m) // 2 return cv2.resize(im[top:top + m, left:left + m], (self.w, self.h), interpolation=cv2.INTER_LINEAR) class ToTensor: # YOLOv5 ToTensor class for image preprocessing, i.e. T.Compose([LetterBox(size), ToTensor()]) def __init__(self, half=False): """Initialize ToTensor class for YOLOv5 image preprocessing.""" super().__init__() self.half = half def __call__(self, im): # im = np.array HWC in BGR order im = np.ascontiguousarray(im.transpose((2, 0, 1))[::-1]) # HWC to CHW -> BGR to RGB -> contiguous im = torch.from_numpy(im) # to torch im = im.half() if self.half else im.float() # uint8 to fp16/32 im /= 255.0 # 0-255 to 0.0-1.0 return im ================================================ FILE: ultralytics/yolo/data/dataloaders/v5loader.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license """ Dataloaders and dataset utils """ import contextlib import glob import hashlib import math import os import random import shutil import time from itertools import repeat from multiprocessing.pool import ThreadPool from pathlib import Path from threading import Thread from urllib.parse import urlparse import cv2 import numpy as np import psutil import torch import torchvision from PIL import ExifTags, Image, ImageOps from torch.utils.data import DataLoader, Dataset, dataloader, distributed from tqdm import tqdm from ultralytics.yolo.utils import (DATASETS_DIR, LOGGER, NUM_THREADS, TQDM_BAR_FORMAT, is_colab, is_dir_writeable, is_kaggle) from ultralytics.yolo.utils.checks import check_requirements from ultralytics.yolo.utils.ops import clean_str, segments2boxes, xyn2xy, xywh2xyxy, xywhn2xyxy, xyxy2xywhn from ultralytics.yolo.utils.torch_utils import torch_distributed_zero_first from .v5augmentations import (Albumentations, augment_hsv, classify_albumentations, classify_transforms, copy_paste, letterbox, mixup, random_perspective) # Parameters HELP_URL = 'See https://docs.ultralytics.com/yolov5/tutorials/train_custom_data' IMG_FORMATS = 'bmp', 'dng', 'jpeg', 'jpg', 'mpo', 'png', 'tif', 'tiff', 'webp', 'pfm' # include image suffixes VID_FORMATS = 'asf', 'avi', 'gif', 'm4v', 'mkv', 'mov', 'mp4', 'mpeg', 'mpg', 'ts', 'wmv' # include video suffixes LOCAL_RANK = int(os.getenv('LOCAL_RANK', -1)) # https://pytorch.org/docs/stable/elastic/run.html RANK = int(os.getenv('RANK', -1)) PIN_MEMORY = str(os.getenv('PIN_MEMORY', True)).lower() == 'true' # global pin_memory for dataloaders # Get orientation exif tag for orientation in ExifTags.TAGS.keys(): if ExifTags.TAGS[orientation] == 'Orientation': break def get_hash(paths): """Returns a single hash value of a list of paths (files or dirs).""" size = sum(os.path.getsize(p) for p in paths if os.path.exists(p)) # sizes h = hashlib.sha256(str(size).encode()) # hash sizes h.update(''.join(paths).encode()) # hash paths return h.hexdigest() # return hash def exif_size(img): """Returns exif-corrected PIL size.""" s = img.size # (width, height) with contextlib.suppress(Exception): rotation = dict(img._getexif().items())[orientation] if rotation in [6, 8]: # rotation 270 or 90 s = (s[1], s[0]) return s def exif_transpose(image): """ Transpose a PIL image accordingly if it has an EXIF Orientation tag. Inplace version of https://github.com/python-pillow/Pillow/blob/master/src/PIL/ImageOps.py exif_transpose() :param image: The image to transpose. :return: An image. """ exif = image.getexif() orientation = exif.get(0x0112, 1) # default 1 if orientation > 1: method = { 2: Image.FLIP_LEFT_RIGHT, 3: Image.ROTATE_180, 4: Image.FLIP_TOP_BOTTOM, 5: Image.TRANSPOSE, 6: Image.ROTATE_270, 7: Image.TRANSVERSE, 8: Image.ROTATE_90}.get(orientation) if method is not None: image = image.transpose(method) del exif[0x0112] image.info['exif'] = exif.tobytes() return image def seed_worker(worker_id): """Set dataloader worker seed https://pytorch.org/docs/stable/notes/randomness.html#dataloader.""" worker_seed = torch.initial_seed() % 2 ** 32 np.random.seed(worker_seed) random.seed(worker_seed) def create_dataloader(path, imgsz, batch_size, stride, single_cls=False, hyp=None, augment=False, cache=False, pad=0.0, rect=False, rank=-1, workers=8, image_weights=False, close_mosaic=False, min_items=0, prefix='', shuffle=False, seed=0): if rect and shuffle: LOGGER.warning('WARNING ⚠️ --rect is incompatible with DataLoader shuffle, setting shuffle=False') shuffle = False with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP dataset = LoadImagesAndLabels( path, imgsz, batch_size, augment=augment, # augmentation hyp=hyp, # hyperparameters rect=rect, # rectangular batches cache_images=cache, single_cls=single_cls, stride=int(stride), pad=pad, image_weights=image_weights, min_items=min_items, prefix=prefix) batch_size = min(batch_size, len(dataset)) nd = torch.cuda.device_count() # number of CUDA devices nw = min([os.cpu_count() // max(nd, 1), batch_size if batch_size > 1 else 0, workers]) # number of workers sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle) loader = DataLoader if image_weights or close_mosaic else InfiniteDataLoader # DataLoader allows attribute updates generator = torch.Generator() generator.manual_seed(6148914691236517205 + seed + RANK) return loader(dataset, batch_size=batch_size, shuffle=shuffle and sampler is None, num_workers=nw, sampler=sampler, pin_memory=PIN_MEMORY, collate_fn=LoadImagesAndLabels.collate_fn, worker_init_fn=seed_worker, generator=generator), dataset class InfiniteDataLoader(dataloader.DataLoader): """Dataloader that reuses workers Uses same syntax as vanilla DataLoader """ def __init__(self, *args, **kwargs): """Dataloader that reuses workers for same syntax as vanilla DataLoader.""" super().__init__(*args, **kwargs) object.__setattr__(self, 'batch_sampler', _RepeatSampler(self.batch_sampler)) self.iterator = super().__iter__() def __len__(self): """Returns the length of batch_sampler's sampler.""" return len(self.batch_sampler.sampler) def __iter__(self): """Creates a sampler that infinitely repeats.""" for _ in range(len(self)): yield next(self.iterator) class _RepeatSampler: """Sampler that repeats forever Args: sampler (Dataset.sampler): The sampler to repeat. """ def __init__(self, sampler): """Sampler that repeats dataset samples infinitely.""" self.sampler = sampler def __iter__(self): """Infinite loop iterating over a given sampler.""" while True: yield from iter(self.sampler) class LoadScreenshots: # YOLOv5 screenshot dataloader, i.e. `python detect.py --source "screen 0 100 100 512 256"` def __init__(self, source, img_size=640, stride=32, auto=True, transforms=None): """source = [screen_number left top width height] (pixels).""" check_requirements('mss') import mss source, *params = source.split() self.screen, left, top, width, height = 0, None, None, None, None # default to full screen 0 if len(params) == 1: self.screen = int(params[0]) elif len(params) == 4: left, top, width, height = (int(x) for x in params) elif len(params) == 5: self.screen, left, top, width, height = (int(x) for x in params) self.img_size = img_size self.stride = stride self.transforms = transforms self.auto = auto self.mode = 'stream' self.frame = 0 self.sct = mss.mss() # Parse monitor shape monitor = self.sct.monitors[self.screen] self.top = monitor['top'] if top is None else (monitor['top'] + top) self.left = monitor['left'] if left is None else (monitor['left'] + left) self.width = width or monitor['width'] self.height = height or monitor['height'] self.monitor = {'left': self.left, 'top': self.top, 'width': self.width, 'height': self.height} def __iter__(self): """Iterates over objects with the same structure as the monitor attribute.""" return self def __next__(self): """mss screen capture: get raw pixels from the screen as np array.""" im0 = np.array(self.sct.grab(self.monitor))[:, :, :3] # [:, :, :3] BGRA to BGR s = f'screen {self.screen} (LTWH): {self.left},{self.top},{self.width},{self.height}: ' if self.transforms: im = self.transforms(im0) # transforms else: im = letterbox(im0, self.img_size, stride=self.stride, auto=self.auto)[0] # padded resize im = im.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB im = np.ascontiguousarray(im) # contiguous self.frame += 1 return str(self.screen), im, im0, None, s # screen, img, original img, im0s, s class LoadImages: # YOLOv5 image/video dataloader, i.e. `python detect.py --source image.jpg/vid.mp4` def __init__(self, path, img_size=640, stride=32, auto=True, transforms=None, vid_stride=1): """Initialize instance variables and check for valid input.""" if isinstance(path, str) and Path(path).suffix == '.txt': # *.txt file with img/vid/dir on each line path = Path(path).read_text().rsplit() files = [] for p in sorted(path) if isinstance(path, (list, tuple)) else [path]: p = str(Path(p).resolve()) if '*' in p: files.extend(sorted(glob.glob(p, recursive=True))) # glob elif os.path.isdir(p): files.extend(sorted(glob.glob(os.path.join(p, '*.*')))) # dir elif os.path.isfile(p): files.append(p) # files else: raise FileNotFoundError(f'{p} does not exist') images = [x for x in files if x.split('.')[-1].lower() in IMG_FORMATS] videos = [x for x in files if x.split('.')[-1].lower() in VID_FORMATS] ni, nv = len(images), len(videos) self.img_size = img_size self.stride = stride self.files = images + videos self.nf = ni + nv # number of files self.video_flag = [False] * ni + [True] * nv self.mode = 'image' self.auto = auto self.transforms = transforms # optional self.vid_stride = vid_stride # video frame-rate stride if any(videos): self._new_video(videos[0]) # new video else: self.cap = None assert self.nf > 0, f'No images or videos found in {p}. ' \ f'Supported formats are:\nimages: {IMG_FORMATS}\nvideos: {VID_FORMATS}' def __iter__(self): """Returns an iterator object for iterating over images or videos found in a directory.""" self.count = 0 return self def __next__(self): """Iterator's next item, performs transformation on image and returns path, transformed image, original image, capture and size.""" if self.count == self.nf: raise StopIteration path = self.files[self.count] if self.video_flag[self.count]: # Read video self.mode = 'video' for _ in range(self.vid_stride): self.cap.grab() ret_val, im0 = self.cap.retrieve() while not ret_val: self.count += 1 self.cap.release() if self.count == self.nf: # last video raise StopIteration path = self.files[self.count] self._new_video(path) ret_val, im0 = self.cap.read() self.frame += 1 # im0 = self._cv2_rotate(im0) # for use if cv2 autorotation is False s = f'video {self.count + 1}/{self.nf} ({self.frame}/{self.frames}) {path}: ' else: # Read image self.count += 1 im0 = cv2.imread(path) # BGR assert im0 is not None, f'Image Not Found {path}' s = f'image {self.count}/{self.nf} {path}: ' if self.transforms: im = self.transforms(im0) # transforms else: im = letterbox(im0, self.img_size, stride=self.stride, auto=self.auto)[0] # padded resize im = im.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB im = np.ascontiguousarray(im) # contiguous return path, im, im0, self.cap, s def _new_video(self, path): """Create a new video capture object.""" self.frame = 0 self.cap = cv2.VideoCapture(path) self.frames = int(self.cap.get(cv2.CAP_PROP_FRAME_COUNT) / self.vid_stride) self.orientation = int(self.cap.get(cv2.CAP_PROP_ORIENTATION_META)) # rotation degrees # self.cap.set(cv2.CAP_PROP_ORIENTATION_AUTO, 0) # disable https://github.com/ultralytics/yolov5/issues/8493 def _cv2_rotate(self, im): """Rotate a cv2 video manually.""" if self.orientation == 0: return cv2.rotate(im, cv2.ROTATE_90_CLOCKWISE) elif self.orientation == 180: return cv2.rotate(im, cv2.ROTATE_90_COUNTERCLOCKWISE) elif self.orientation == 90: return cv2.rotate(im, cv2.ROTATE_180) return im def __len__(self): """Returns the number of files in the class instance.""" return self.nf # number of files class LoadStreams: # YOLOv5 streamloader, i.e. `python detect.py --source 'rtsp://example.com/media.mp4' # RTSP, RTMP, HTTP streams` def __init__(self, sources='file.streams', img_size=640, stride=32, auto=True, transforms=None, vid_stride=1): """Initialize YOLO detector with optional transforms and check input shapes.""" torch.backends.cudnn.benchmark = True # faster for fixed-size inference self.mode = 'stream' self.img_size = img_size self.stride = stride self.vid_stride = vid_stride # video frame-rate stride sources = Path(sources).read_text().rsplit() if os.path.isfile(sources) else [sources] n = len(sources) self.sources = [clean_str(x) for x in sources] # clean source names for later self.imgs, self.fps, self.frames, self.threads = [None] * n, [0] * n, [0] * n, [None] * n for i, s in enumerate(sources): # index, source # Start thread to read frames from video stream st = f'{i + 1}/{n}: {s}... ' if urlparse(s).hostname in ('www.youtube.com', 'youtube.com', 'youtu.be'): # if source is YouTube video # YouTube format i.e. 'https://www.youtube.com/watch?v=Zgi9g1ksQHc' or 'https://youtu.be/Zgi9g1ksQHc' check_requirements(('pafy', 'youtube_dl==2020.12.2')) import pafy s = pafy.new(s).getbest(preftype='mp4').url # YouTube URL s = eval(s) if s.isnumeric() else s # i.e. s = '0' local webcam if s == 0: assert not is_colab(), '--source 0 webcam unsupported on Colab. Rerun command in a local environment.' assert not is_kaggle(), '--source 0 webcam unsupported on Kaggle. Rerun command in a local environment.' cap = cv2.VideoCapture(s) assert cap.isOpened(), f'{st}Failed to open {s}' w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) fps = cap.get(cv2.CAP_PROP_FPS) # warning: may return 0 or nan self.frames[i] = max(int(cap.get(cv2.CAP_PROP_FRAME_COUNT)), 0) or float('inf') # infinite stream fallback self.fps[i] = max((fps if math.isfinite(fps) else 0) % 100, 0) or 30 # 30 FPS fallback _, self.imgs[i] = cap.read() # guarantee first frame self.threads[i] = Thread(target=self.update, args=([i, cap, s]), daemon=True) LOGGER.info(f'{st} Success ({self.frames[i]} frames {w}x{h} at {self.fps[i]:.2f} FPS)') self.threads[i].start() LOGGER.info('') # newline # Check for common shapes s = np.stack([letterbox(x, img_size, stride=stride, auto=auto)[0].shape for x in self.imgs]) self.rect = np.unique(s, axis=0).shape[0] == 1 # rect inference if all shapes equal self.auto = auto and self.rect self.transforms = transforms # optional if not self.rect: LOGGER.warning('WARNING ⚠️ Stream shapes differ. For optimal performance supply similarly-shaped streams.') def update(self, i, cap, stream): """Read stream `i` frames in daemon thread.""" n, f = 0, self.frames[i] # frame number, frame array while cap.isOpened() and n < f: n += 1 cap.grab() # .read() = .grab() followed by .retrieve() if n % self.vid_stride == 0: success, im = cap.retrieve() if success: self.imgs[i] = im else: LOGGER.warning('WARNING ⚠️ Video stream unresponsive, please check your IP camera connection.') self.imgs[i] = np.zeros_like(self.imgs[i]) cap.open(stream) # re-open stream if signal was lost time.sleep(0.0) # wait time def __iter__(self): """Iterator that returns the class instance.""" self.count = -1 return self def __next__(self): """Return a tuple containing transformed and resized image data.""" self.count += 1 if not all(x.is_alive() for x in self.threads) or cv2.waitKey(1) == ord('q'): # q to quit cv2.destroyAllWindows() raise StopIteration im0 = self.imgs.copy() if self.transforms: im = np.stack([self.transforms(x) for x in im0]) # transforms else: im = np.stack([letterbox(x, self.img_size, stride=self.stride, auto=self.auto)[0] for x in im0]) # resize im = im[..., ::-1].transpose((0, 3, 1, 2)) # BGR to RGB, BHWC to BCHW im = np.ascontiguousarray(im) # contiguous return self.sources, im, im0, None, '' def __len__(self): """Returns the number of sources as the length of the object.""" return len(self.sources) # 1E12 frames = 32 streams at 30 FPS for 30 years def img2label_paths(img_paths): """Define label paths as a function of image paths.""" sa, sb = f'{os.sep}images{os.sep}', f'{os.sep}labels{os.sep}' # /images/, /labels/ substrings return [sb.join(x.rsplit(sa, 1)).rsplit('.', 1)[0] + '.txt' for x in img_paths] class LoadImagesAndLabels(Dataset): """YOLOv5 train_loader/val_loader, loads images and labels for training and validation.""" cache_version = 0.6 # dataset labels *.cache version rand_interp_methods = [cv2.INTER_NEAREST, cv2.INTER_LINEAR, cv2.INTER_CUBIC, cv2.INTER_AREA, cv2.INTER_LANCZOS4] def __init__(self, path, img_size=640, batch_size=16, augment=False, hyp=None, rect=False, image_weights=False, cache_images=False, single_cls=False, stride=32, pad=0.0, min_items=0, prefix=''): self.img_size = img_size self.augment = augment self.hyp = hyp self.image_weights = image_weights self.rect = False if image_weights else rect self.mosaic = self.augment and not self.rect # load 4 images at a time into a mosaic (only during training) self.mosaic_border = [-img_size // 2, -img_size // 2] self.stride = stride self.path = path self.albumentations = Albumentations(size=img_size) if augment else None try: f = [] # image files for p in path if isinstance(path, list) else [path]: p = Path(p) # os-agnostic if p.is_dir(): # dir f += glob.glob(str(p / '**' / '*.*'), recursive=True) # f = list(p.rglob('*.*')) # pathlib elif p.is_file(): # file with open(p) as t: t = t.read().strip().splitlines() parent = str(p.parent) + os.sep f += [x.replace('./', parent, 1) if x.startswith('./') else x for x in t] # to global path # f += [p.parent / x.lstrip(os.sep) for x in t] # to global path (pathlib) else: raise FileNotFoundError(f'{prefix}{p} does not exist') self.im_files = sorted(x.replace('/', os.sep) for x in f if x.split('.')[-1].lower() in IMG_FORMATS) # self.img_files = sorted([x for x in f if x.suffix[1:].lower() in IMG_FORMATS]) # pathlib assert self.im_files, f'{prefix}No images found' except Exception as e: raise FileNotFoundError(f'{prefix}Error loading data from {path}: {e}\n{HELP_URL}') from e # Check cache self.label_files = img2label_paths(self.im_files) # labels cache_path = (p if p.is_file() else Path(self.label_files[0]).parent).with_suffix('.cache') try: cache, exists = np.load(cache_path, allow_pickle=True).item(), True # load dict assert cache['version'] == self.cache_version # matches current version assert cache['hash'] == get_hash(self.label_files + self.im_files) # identical hash except (FileNotFoundError, AssertionError, AttributeError): cache, exists = self.cache_labels(cache_path, prefix), False # run cache ops # Display cache nf, nm, ne, nc, n = cache.pop('results') # found, missing, empty, corrupt, total if exists and LOCAL_RANK in (-1, 0): d = f'Scanning {cache_path}... {nf} images, {nm + ne} backgrounds, {nc} corrupt' tqdm(None, desc=prefix + d, total=n, initial=n, bar_format=TQDM_BAR_FORMAT) # display cache results if cache['msgs']: LOGGER.info('\n'.join(cache['msgs'])) # display warnings assert nf > 0 or not augment, f'{prefix}No labels found in {cache_path}, can not start training. {HELP_URL}' # Read cache [cache.pop(k) for k in ('hash', 'version', 'msgs')] # remove items labels, shapes, self.segments = zip(*cache.values()) nl = len(np.concatenate(labels, 0)) # number of labels assert nl > 0 or not augment, f'{prefix}All labels empty in {cache_path}, can not start training. {HELP_URL}' self.labels = list(labels) self.shapes = np.array(shapes) self.im_files = list(cache.keys()) # update self.label_files = img2label_paths(cache.keys()) # update # Filter images if min_items: include = np.array([len(x) >= min_items for x in self.labels]).nonzero()[0].astype(int) LOGGER.info(f'{prefix}{n - len(include)}/{n} images filtered from dataset') self.im_files = [self.im_files[i] for i in include] self.label_files = [self.label_files[i] for i in include] self.labels = [self.labels[i] for i in include] self.segments = [self.segments[i] for i in include] self.shapes = self.shapes[include] # wh # Create indices n = len(self.shapes) # number of images bi = np.floor(np.arange(n) / batch_size).astype(int) # batch index nb = bi[-1] + 1 # number of batches self.batch = bi # batch index of image self.n = n self.indices = range(n) # Update labels include_class = [] # filter labels to include only these classes (optional) include_class_array = np.array(include_class).reshape(1, -1) for i, (label, segment) in enumerate(zip(self.labels, self.segments)): if include_class: j = (label[:, 0:1] == include_class_array).any(1) self.labels[i] = label[j] if segment: self.segments[i] = [segment[si] for si, idx in enumerate(j) if idx] if single_cls: # single-class training, merge all classes into 0 self.labels[i][:, 0] = 0 # Rectangular Training if self.rect: # Sort by aspect ratio s = self.shapes # wh ar = s[:, 1] / s[:, 0] # aspect ratio irect = ar.argsort() self.im_files = [self.im_files[i] for i in irect] self.label_files = [self.label_files[i] for i in irect] self.labels = [self.labels[i] for i in irect] self.segments = [self.segments[i] for i in irect] self.shapes = s[irect] # wh ar = ar[irect] # Set training image shapes shapes = [[1, 1]] * nb for i in range(nb): ari = ar[bi == i] mini, maxi = ari.min(), ari.max() if maxi < 1: shapes[i] = [maxi, 1] elif mini > 1: shapes[i] = [1, 1 / mini] self.batch_shapes = np.ceil(np.array(shapes) * img_size / stride + pad).astype(int) * stride # Cache images into RAM/disk for faster training if cache_images == 'ram' and not self.check_cache_ram(prefix=prefix): cache_images = False self.ims = [None] * n self.npy_files = [Path(f).with_suffix('.npy') for f in self.im_files] if cache_images: b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes self.im_hw0, self.im_hw = [None] * n, [None] * n fcn = self.cache_images_to_disk if cache_images == 'disk' else self.load_image with ThreadPool(NUM_THREADS) as pool: results = pool.imap(fcn, range(n)) pbar = tqdm(enumerate(results), total=n, bar_format=TQDM_BAR_FORMAT, disable=LOCAL_RANK > 0) for i, x in pbar: if cache_images == 'disk': b += self.npy_files[i].stat().st_size else: # 'ram' self.ims[i], self.im_hw0[i], self.im_hw[i] = x # im, hw_orig, hw_resized = load_image(self, i) b += self.ims[i].nbytes pbar.desc = f'{prefix}Caching images ({b / gb:.1f}GB {cache_images})' pbar.close() def check_cache_ram(self, safety_margin=0.1, prefix=''): """Check image caching requirements vs available memory.""" b, gb = 0, 1 << 30 # bytes of cached images, bytes per gigabytes n = min(self.n, 30) # extrapolate from 30 random images for _ in range(n): im = cv2.imread(random.choice(self.im_files)) # sample image ratio = self.img_size / max(im.shape[0], im.shape[1]) # max(h, w) # ratio b += im.nbytes * ratio ** 2 mem_required = b * self.n / n # GB required to cache dataset into RAM mem = psutil.virtual_memory() cache = mem_required * (1 + safety_margin) < mem.available # to cache or not to cache, that is the question if not cache: LOGGER.info(f'{prefix}{mem_required / gb:.1f}GB RAM required, ' f'{mem.available / gb:.1f}/{mem.total / gb:.1f}GB available, ' f"{'caching images ✅' if cache else 'not caching images ⚠️'}") return cache def cache_labels(self, path=Path('./labels.cache'), prefix=''): """Cache labels and save as numpy file for next time.""" # Cache dataset labels, check images and read shapes if path.exists(): path.unlink() # remove *.cache file if exists x = {} # dict nm, nf, ne, nc, msgs = 0, 0, 0, 0, [] # number missing, found, empty, corrupt, messages desc = f'{prefix}Scanning {path.parent / path.stem}...' total = len(self.im_files) with ThreadPool(NUM_THREADS) as pool: results = pool.imap(verify_image_label, zip(self.im_files, self.label_files, repeat(prefix))) pbar = tqdm(results, desc=desc, total=total, bar_format=TQDM_BAR_FORMAT) for im_file, lb, shape, segments, nm_f, nf_f, ne_f, nc_f, msg in pbar: nm += nm_f nf += nf_f ne += ne_f nc += nc_f if im_file: x[im_file] = [lb, shape, segments] if msg: msgs.append(msg) pbar.desc = f'{desc} {nf} images, {nm + ne} backgrounds, {nc} corrupt' pbar.close() if msgs: LOGGER.info('\n'.join(msgs)) if nf == 0: LOGGER.warning(f'{prefix}WARNING ⚠️ No labels found in {path}. {HELP_URL}') x['hash'] = get_hash(self.label_files + self.im_files) x['results'] = nf, nm, ne, nc, len(self.im_files) x['msgs'] = msgs # warnings x['version'] = self.cache_version # cache version if is_dir_writeable(path.parent): np.save(str(path), x) # save cache for next time path.with_suffix('.cache.npy').rename(path) # remove .npy suffix LOGGER.info(f'{prefix}New cache created: {path}') else: LOGGER.warning(f'{prefix}WARNING ⚠️ Cache directory {path.parent} is not writeable') # not writeable return x def __len__(self): """Returns the length of 'im_files' attribute.""" return len(self.im_files) def __getitem__(self, index): """Get a sample and its corresponding label, filename and shape from the dataset.""" index = self.indices[index] # linear, shuffled, or image_weights hyp = self.hyp mosaic = self.mosaic and random.random() < hyp['mosaic'] if mosaic: # Load mosaic img, labels = self.load_mosaic(index) shapes = None # MixUp augmentation if random.random() < hyp['mixup']: img, labels = mixup(img, labels, *self.load_mosaic(random.randint(0, self.n - 1))) else: # Load image img, (h0, w0), (h, w) = self.load_image(index) # Letterbox shape = self.batch_shapes[self.batch[index]] if self.rect else self.img_size # final letterboxed shape img, ratio, pad = letterbox(img, shape, auto=False, scaleup=self.augment) shapes = (h0, w0), ((h / h0, w / w0), pad) # for COCO mAP rescaling labels = self.labels[index].copy() if labels.size: # normalized xywh to pixel xyxy format labels[:, 1:] = xywhn2xyxy(labels[:, 1:], ratio[0] * w, ratio[1] * h, padw=pad[0], padh=pad[1]) if self.augment: img, labels = random_perspective(img, labels, degrees=hyp['degrees'], translate=hyp['translate'], scale=hyp['scale'], shear=hyp['shear'], perspective=hyp['perspective']) nl = len(labels) # number of labels if nl: labels[:, 1:5] = xyxy2xywhn(labels[:, 1:5], w=img.shape[1], h=img.shape[0], clip=True, eps=1E-3) if self.augment: # Albumentations img, labels = self.albumentations(img, labels) nl = len(labels) # update after albumentations # HSV color-space augment_hsv(img, hgain=hyp['hsv_h'], sgain=hyp['hsv_s'], vgain=hyp['hsv_v']) # Flip up-down if random.random() < hyp['flipud']: img = np.flipud(img) if nl: labels[:, 2] = 1 - labels[:, 2] # Flip left-right if random.random() < hyp['fliplr']: img = np.fliplr(img) if nl: labels[:, 1] = 1 - labels[:, 1] # Cutouts # labels = cutout(img, labels, p=0.5) # nl = len(labels) # update after cutout labels_out = torch.zeros((nl, 6)) if nl: labels_out[:, 1:] = torch.from_numpy(labels) # Convert img = img.transpose((2, 0, 1))[::-1] # HWC to CHW, BGR to RGB img = np.ascontiguousarray(img) return torch.from_numpy(img), labels_out, self.im_files[index], shapes def load_image(self, i): """Loads 1 image from dataset index 'i', returns (im, original hw, resized hw).""" im, f, fn = self.ims[i], self.im_files[i], self.npy_files[i], if im is None: # not cached in RAM if fn.exists(): # load npy im = np.load(fn) else: # read image im = cv2.imread(f) # BGR assert im is not None, f'Image Not Found {f}' h0, w0 = im.shape[:2] # orig hw r = self.img_size / max(h0, w0) # ratio if r != 1: # if sizes are not equal interp = cv2.INTER_LINEAR if (self.augment or r > 1) else cv2.INTER_AREA im = cv2.resize(im, (math.ceil(w0 * r), math.ceil(h0 * r)), interpolation=interp) return im, (h0, w0), im.shape[:2] # im, hw_original, hw_resized return self.ims[i], self.im_hw0[i], self.im_hw[i] # im, hw_original, hw_resized def cache_images_to_disk(self, i): """Saves an image as an *.npy file for faster loading.""" f = self.npy_files[i] if not f.exists(): np.save(f.as_posix(), cv2.imread(self.im_files[i])) def load_mosaic(self, index): """YOLOv5 4-mosaic loader. Loads 1 image + 3 random images into a 4-image mosaic.""" labels4, segments4 = [], [] s = self.img_size yc, xc = (int(random.uniform(-x, 2 * s + x)) for x in self.mosaic_border) # mosaic center x, y indices = [index] + random.choices(self.indices, k=3) # 3 additional image indices random.shuffle(indices) for i, index in enumerate(indices): # Load image img, _, (h, w) = self.load_image(index) # Place img in img4 if i == 0: # top left img4 = np.full((s * 2, s * 2, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles x1a, y1a, x2a, y2a = max(xc - w, 0), max(yc - h, 0), xc, yc # xmin, ymin, xmax, ymax (large image) x1b, y1b, x2b, y2b = w - (x2a - x1a), h - (y2a - y1a), w, h # xmin, ymin, xmax, ymax (small image) elif i == 1: # top right x1a, y1a, x2a, y2a = xc, max(yc - h, 0), min(xc + w, s * 2), yc x1b, y1b, x2b, y2b = 0, h - (y2a - y1a), min(w, x2a - x1a), h elif i == 2: # bottom left x1a, y1a, x2a, y2a = max(xc - w, 0), yc, xc, min(s * 2, yc + h) x1b, y1b, x2b, y2b = w - (x2a - x1a), 0, w, min(y2a - y1a, h) elif i == 3: # bottom right x1a, y1a, x2a, y2a = xc, yc, min(xc + w, s * 2), min(s * 2, yc + h) x1b, y1b, x2b, y2b = 0, 0, min(w, x2a - x1a), min(y2a - y1a, h) img4[y1a:y2a, x1a:x2a] = img[y1b:y2b, x1b:x2b] # img4[ymin:ymax, xmin:xmax] padw = x1a - x1b padh = y1a - y1b # Labels labels, segments = self.labels[index].copy(), self.segments[index].copy() if labels.size: labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padw, padh) # normalized xywh to pixel xyxy format segments = [xyn2xy(x, w, h, padw, padh) for x in segments] labels4.append(labels) segments4.extend(segments) # Concat/clip labels labels4 = np.concatenate(labels4, 0) for x in (labels4[:, 1:], *segments4): np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective() # img4, labels4 = replicate(img4, labels4) # replicate # Augment img4, labels4, segments4 = copy_paste(img4, labels4, segments4, p=self.hyp['copy_paste']) img4, labels4 = random_perspective(img4, labels4, segments4, degrees=self.hyp['degrees'], translate=self.hyp['translate'], scale=self.hyp['scale'], shear=self.hyp['shear'], perspective=self.hyp['perspective'], border=self.mosaic_border) # border to remove return img4, labels4 def load_mosaic9(self, index): """YOLOv5 9-mosaic loader. Loads 1 image + 8 random images into a 9-image mosaic.""" labels9, segments9 = [], [] s = self.img_size indices = [index] + random.choices(self.indices, k=8) # 8 additional image indices random.shuffle(indices) hp, wp = -1, -1 # height, width previous for i, index in enumerate(indices): # Load image img, _, (h, w) = self.load_image(index) # Place img in img9 if i == 0: # center img9 = np.full((s * 3, s * 3, img.shape[2]), 114, dtype=np.uint8) # base image with 4 tiles h0, w0 = h, w c = s, s, s + w, s + h # xmin, ymin, xmax, ymax (base) coordinates elif i == 1: # top c = s, s - h, s + w, s elif i == 2: # top right c = s + wp, s - h, s + wp + w, s elif i == 3: # right c = s + w0, s, s + w0 + w, s + h elif i == 4: # bottom right c = s + w0, s + hp, s + w0 + w, s + hp + h elif i == 5: # bottom c = s + w0 - w, s + h0, s + w0, s + h0 + h elif i == 6: # bottom left c = s + w0 - wp - w, s + h0, s + w0 - wp, s + h0 + h elif i == 7: # left c = s - w, s + h0 - h, s, s + h0 elif i == 8: # top left c = s - w, s + h0 - hp - h, s, s + h0 - hp padx, pady = c[:2] x1, y1, x2, y2 = (max(x, 0) for x in c) # allocate coords # Labels labels, segments = self.labels[index].copy(), self.segments[index].copy() if labels.size: labels[:, 1:] = xywhn2xyxy(labels[:, 1:], w, h, padx, pady) # normalized xywh to pixel xyxy format segments = [xyn2xy(x, w, h, padx, pady) for x in segments] labels9.append(labels) segments9.extend(segments) # Image img9[y1:y2, x1:x2] = img[y1 - pady:, x1 - padx:] # img9[ymin:ymax, xmin:xmax] hp, wp = h, w # height, width previous # Offset yc, xc = (int(random.uniform(0, s)) for _ in self.mosaic_border) # mosaic center x, y img9 = img9[yc:yc + 2 * s, xc:xc + 2 * s] # Concat/clip labels labels9 = np.concatenate(labels9, 0) labels9[:, [1, 3]] -= xc labels9[:, [2, 4]] -= yc c = np.array([xc, yc]) # centers segments9 = [x - c for x in segments9] for x in (labels9[:, 1:], *segments9): np.clip(x, 0, 2 * s, out=x) # clip when using random_perspective() # img9, labels9 = replicate(img9, labels9) # replicate # Augment img9, labels9, segments9 = copy_paste(img9, labels9, segments9, p=self.hyp['copy_paste']) img9, labels9 = random_perspective(img9, labels9, segments9, degrees=self.hyp['degrees'], translate=self.hyp['translate'], scale=self.hyp['scale'], shear=self.hyp['shear'], perspective=self.hyp['perspective'], border=self.mosaic_border) # border to remove return img9, labels9 @staticmethod def collate_fn(batch): """YOLOv8 collate function, outputs dict.""" im, label, path, shapes = zip(*batch) # transposed for i, lb in enumerate(label): lb[:, 0] = i # add target image index for build_targets() batch_idx, cls, bboxes = torch.cat(label, 0).split((1, 1, 4), dim=1) return { 'ori_shape': tuple((x[0] if x else None) for x in shapes), 'ratio_pad': tuple((x[1] if x else None) for x in shapes), 'im_file': path, 'img': torch.stack(im, 0), 'cls': cls, 'bboxes': bboxes, 'batch_idx': batch_idx.view(-1)} @staticmethod def collate_fn_old(batch): """YOLOv5 original collate function.""" im, label, path, shapes = zip(*batch) # transposed for i, lb in enumerate(label): lb[:, 0] = i # add target image index for build_targets() return torch.stack(im, 0), torch.cat(label, 0), path, shapes # Ancillary functions -------------------------------------------------------------------------------------------------- def flatten_recursive(path=DATASETS_DIR / 'coco128'): """Flatten a recursive directory by bringing all files to top level.""" new_path = Path(f'{str(path)}_flat') if os.path.exists(new_path): shutil.rmtree(new_path) # delete output folder os.makedirs(new_path) # make new output folder for file in tqdm(glob.glob(f'{str(Path(path))}/**/*.*', recursive=True)): shutil.copyfile(file, new_path / Path(file).name) def extract_boxes(path=DATASETS_DIR / 'coco128'): # from utils.dataloaders import *; extract_boxes() # Convert detection dataset into classification dataset, with one directory per class path = Path(path) # images dir shutil.rmtree(path / 'classification') if (path / 'classification').is_dir() else None # remove existing files = list(path.rglob('*.*')) n = len(files) # number of files for im_file in tqdm(files, total=n): if im_file.suffix[1:] in IMG_FORMATS: # Image im = cv2.imread(str(im_file))[..., ::-1] # BGR to RGB h, w = im.shape[:2] # Labels lb_file = Path(img2label_paths([str(im_file)])[0]) if Path(lb_file).exists(): with open(lb_file) as f: lb = np.array([x.split() for x in f.read().strip().splitlines()], dtype=np.float32) # labels for j, x in enumerate(lb): c = int(x[0]) # class f = (path / 'classifier') / f'{c}' / f'{path.stem}_{im_file.stem}_{j}.jpg' # new filename if not f.parent.is_dir(): f.parent.mkdir(parents=True) b = x[1:] * [w, h, w, h] # box # B[2:] = b[2:].max() # rectangle to square b[2:] = b[2:] * 1.2 + 3 # pad b = xywh2xyxy(b.reshape(-1, 4)).ravel().astype(int) b[[0, 2]] = np.clip(b[[0, 2]], 0, w) # clip boxes outside of image b[[1, 3]] = np.clip(b[[1, 3]], 0, h) assert cv2.imwrite(str(f), im[b[1]:b[3], b[0]:b[2]]), f'box failure in {f}' def autosplit(path=DATASETS_DIR / 'coco128/images', weights=(0.9, 0.1, 0.0), annotated_only=False): """Autosplit a dataset into train/val/test splits and save path/autosplit_*.txt files Usage: from utils.dataloaders import *; autosplit() Arguments path: Path to images directory weights: Train, val, test weights (list, tuple) annotated_only: Only use images with an annotated txt file """ path = Path(path) # images dir files = sorted(x for x in path.rglob('*.*') if x.suffix[1:].lower() in IMG_FORMATS) # image files only n = len(files) # number of files random.seed(0) # for reproducibility indices = random.choices([0, 1, 2], weights=weights, k=n) # assign each image to a split txt = ['autosplit_train.txt', 'autosplit_val.txt', 'autosplit_test.txt'] # 3 txt files for x in txt: if (path.parent / x).exists(): (path.parent / x).unlink() # remove existing print(f'Autosplitting images from {path}' + ', using *.txt labeled images only' * annotated_only) for i, img in tqdm(zip(indices, files), total=n): if not annotated_only or Path(img2label_paths([str(img)])[0]).exists(): # check label with open(path.parent / txt[i], 'a') as f: f.write(f'./{img.relative_to(path.parent).as_posix()}' + '\n') # add image to txt file def verify_image_label(args): """Verify one image-label pair.""" im_file, lb_file, prefix = args nm, nf, ne, nc, msg, segments = 0, 0, 0, 0, '', [] # number (missing, found, empty, corrupt), message, segments try: # Verify images im = Image.open(im_file) im.verify() # PIL verify shape = exif_size(im) # image size assert (shape[0] > 9) & (shape[1] > 9), f'image size {shape} <10 pixels' assert im.format.lower() in IMG_FORMATS, f'invalid image format {im.format}' if im.format.lower() in ('jpg', 'jpeg'): with open(im_file, 'rb') as f: f.seek(-2, 2) if f.read() != b'\xff\xd9': # corrupt JPEG ImageOps.exif_transpose(Image.open(im_file)).save(im_file, 'JPEG', subsampling=0, quality=100) msg = f'{prefix}WARNING ⚠️ {im_file}: corrupt JPEG restored and saved' # Verify labels if os.path.isfile(lb_file): nf = 1 # label found with open(lb_file) as f: lb = [x.split() for x in f.read().strip().splitlines() if len(x)] if any(len(x) > 6 for x in lb): # is segment classes = np.array([x[0] for x in lb], dtype=np.float32) segments = [np.array(x[1:], dtype=np.float32).reshape(-1, 2) for x in lb] # (cls, xy1...) lb = np.concatenate((classes.reshape(-1, 1), segments2boxes(segments)), 1) # (cls, xywh) lb = np.array(lb, dtype=np.float32) nl = len(lb) if nl: assert lb.shape[1] == 5, f'labels require 5 columns, {lb.shape[1]} columns detected' assert (lb >= 0).all(), f'negative label values {lb[lb < 0]}' assert (lb[:, 1:] <= 1).all(), f'non-normalized or out of bounds coordinates {lb[:, 1:][lb[:, 1:] > 1]}' _, i = np.unique(lb, axis=0, return_index=True) if len(i) < nl: # duplicate row check lb = lb[i] # remove duplicates if segments: segments = [segments[x] for x in i] msg = f'{prefix}WARNING ⚠️ {im_file}: {nl - len(i)} duplicate labels removed' else: ne = 1 # label empty lb = np.zeros((0, 5), dtype=np.float32) else: nm = 1 # label missing lb = np.zeros((0, 5), dtype=np.float32) return im_file, lb, shape, segments, nm, nf, ne, nc, msg except Exception as e: nc = 1 msg = f'{prefix}WARNING ⚠️ {im_file}: ignoring corrupt image/label: {e}' return [None, None, None, None, nm, nf, ne, nc, msg] # Classification dataloaders ------------------------------------------------------------------------------------------- class ClassificationDataset(torchvision.datasets.ImageFolder): """ YOLOv5 Classification Dataset. Arguments root: Dataset path transform: torchvision transforms, used by default album_transform: Albumentations transforms, used if installed """ def __init__(self, root, augment, imgsz, cache=False): """Initialize YOLO dataset with root, augmentation, image size, and cache parameters.""" super().__init__(root=root) self.torch_transforms = classify_transforms(imgsz) self.album_transforms = classify_albumentations(augment, imgsz) if augment else None self.cache_ram = cache is True or cache == 'ram' self.cache_disk = cache == 'disk' self.samples = [list(x) + [Path(x[0]).with_suffix('.npy'), None] for x in self.samples] # file, index, npy, im def __getitem__(self, i): """Retrieves data items of 'dataset' via indices & creates InfiniteDataLoader.""" f, j, fn, im = self.samples[i] # filename, index, filename.with_suffix('.npy'), image if self.cache_ram and im is None: im = self.samples[i][3] = cv2.imread(f) elif self.cache_disk: if not fn.exists(): # load npy np.save(fn.as_posix(), cv2.imread(f)) im = np.load(fn) else: # read image im = cv2.imread(f) # BGR if self.album_transforms: sample = self.album_transforms(image=cv2.cvtColor(im, cv2.COLOR_BGR2RGB))['image'] else: sample = self.torch_transforms(im) return sample, j def create_classification_dataloader(path, imgsz=224, batch_size=16, augment=True, cache=False, rank=-1, workers=8, shuffle=True): """Returns Dataloader object to be used with YOLOv5 Classifier.""" with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP dataset = ClassificationDataset(root=path, imgsz=imgsz, augment=augment, cache=cache) batch_size = min(batch_size, len(dataset)) nd = torch.cuda.device_count() nw = min([os.cpu_count() // max(nd, 1), batch_size if batch_size > 1 else 0, workers]) sampler = None if rank == -1 else distributed.DistributedSampler(dataset, shuffle=shuffle) generator = torch.Generator() generator.manual_seed(6148914691236517205 + RANK) return InfiniteDataLoader(dataset, batch_size=batch_size, shuffle=shuffle and sampler is None, num_workers=nw, sampler=sampler, pin_memory=PIN_MEMORY, worker_init_fn=seed_worker, generator=generator) # or DataLoader(persistent_workers=True) ================================================ FILE: ultralytics/yolo/data/dataset.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from itertools import repeat from multiprocessing.pool import ThreadPool from pathlib import Path import cv2 import numpy as np import torch import torchvision from tqdm import tqdm from ..utils import LOCAL_RANK, NUM_THREADS, TQDM_BAR_FORMAT, is_dir_writeable from .augment import Compose, Format, Instances, LetterBox, classify_albumentations, classify_transforms, v8_transforms from .base import BaseDataset from .utils import HELP_URL, LOGGER, get_hash, img2label_paths, verify_image_label class YOLODataset(BaseDataset): """ Dataset class for loading object detection and/or segmentation labels in YOLO format. Args: data (dict, optional): A dataset YAML dictionary. Defaults to None. use_segments (bool, optional): If True, segmentation masks are used as labels. Defaults to False. use_keypoints (bool, optional): If True, keypoints are used as labels. Defaults to False. Returns: (torch.utils.data.Dataset): A PyTorch dataset object that can be used for training an object detection model. """ cache_version = '1.0.2' # dataset labels *.cache version, >= 1.0.0 for YOLOv8 rand_interp_methods = [cv2.INTER_NEAREST, cv2.INTER_LINEAR, cv2.INTER_CUBIC, cv2.INTER_AREA, cv2.INTER_LANCZOS4] def __init__(self, *args, data=None, use_segments=False, use_keypoints=False, **kwargs): self.use_segments = use_segments self.use_keypoints = use_keypoints self.data = data assert not (self.use_segments and self.use_keypoints), 'Can not use both segments and keypoints.' super().__init__(*args, **kwargs) def cache_labels(self, path=Path('./labels.cache')): """Cache dataset labels, check images and read shapes. Args: path (Path): path where to save the cache file (default: Path('./labels.cache')). Returns: (dict): labels. """ x = {'labels': []} nm, nf, ne, nc, msgs = 0, 0, 0, 0, [] # number missing, found, empty, corrupt, messages desc = f'{self.prefix}Scanning {path.parent / path.stem}...' total = len(self.im_files) nkpt, ndim = self.data.get('kpt_shape', (0, 0)) if self.use_keypoints and (nkpt <= 0 or ndim not in (2, 3)): raise ValueError("'kpt_shape' in data.yaml missing or incorrect. Should be a list with [number of " "keypoints, number of dims (2 for x,y or 3 for x,y,visible)], i.e. 'kpt_shape: [17, 3]'") with ThreadPool(NUM_THREADS) as pool: results = pool.imap(func=verify_image_label, iterable=zip(self.im_files, self.label_files, repeat(self.prefix), repeat(self.use_keypoints), repeat(len(self.data['names'])), repeat(nkpt), repeat(ndim))) pbar = tqdm(results, desc=desc, total=total, bar_format=TQDM_BAR_FORMAT) for im_file, lb, shape, segments, keypoint, nm_f, nf_f, ne_f, nc_f, msg in pbar: nm += nm_f nf += nf_f ne += ne_f nc += nc_f if im_file: x['labels'].append( dict( im_file=im_file, shape=shape, cls=lb[:, 0:1], # n, 1 bboxes=lb[:, 1:], # n, 4 segments=segments, keypoints=keypoint, normalized=True, bbox_format='xywh')) if msg: msgs.append(msg) pbar.desc = f'{desc} {nf} images, {nm + ne} backgrounds, {nc} corrupt' pbar.close() if msgs: LOGGER.info('\n'.join(msgs)) if nf == 0: LOGGER.warning(f'{self.prefix}WARNING ⚠️ No labels found in {path}. {HELP_URL}') x['hash'] = get_hash(self.label_files + self.im_files) x['results'] = nf, nm, ne, nc, len(self.im_files) x['msgs'] = msgs # warnings x['version'] = self.cache_version # cache version if is_dir_writeable(path.parent): if path.exists(): path.unlink() # remove *.cache file if exists np.save(str(path), x) # save cache for next time path.with_suffix('.cache.npy').rename(path) # remove .npy suffix LOGGER.info(f'{self.prefix}New cache created: {path}') else: LOGGER.warning(f'{self.prefix}WARNING ⚠️ Cache directory {path.parent} is not writeable, cache not saved.') return x def get_labels(self): """Returns dictionary of labels for YOLO training.""" self.label_files = img2label_paths(self.im_files) cache_path = Path(self.label_files[0]).parent.with_suffix('.cache') try: import gc gc.disable() # reduce pickle load time https://github.com/ultralytics/ultralytics/pull/1585 cache, exists = np.load(str(cache_path), allow_pickle=True).item(), True # load dict gc.enable() assert cache['version'] == self.cache_version # matches current version assert cache['hash'] == get_hash(self.label_files + self.im_files) # identical hash except (FileNotFoundError, AssertionError, AttributeError): cache, exists = self.cache_labels(cache_path), False # run cache ops # Display cache nf, nm, ne, nc, n = cache.pop('results') # found, missing, empty, corrupt, total if exists and LOCAL_RANK in (-1, 0): d = f'Scanning {cache_path}... {nf} images, {nm + ne} backgrounds, {nc} corrupt' tqdm(None, desc=self.prefix + d, total=n, initial=n, bar_format=TQDM_BAR_FORMAT) # display cache results if cache['msgs']: LOGGER.info('\n'.join(cache['msgs'])) # display warnings if nf == 0: # number of labels found raise FileNotFoundError(f'{self.prefix}No labels found in {cache_path}, can not start training. {HELP_URL}') # Read cache [cache.pop(k) for k in ('hash', 'version', 'msgs')] # remove items labels = cache['labels'] self.im_files = [lb['im_file'] for lb in labels] # update im_files # Check if the dataset is all boxes or all segments lengths = ((len(lb['cls']), len(lb['bboxes']), len(lb['segments'])) for lb in labels) len_cls, len_boxes, len_segments = (sum(x) for x in zip(*lengths)) if len_segments and len_boxes != len_segments: LOGGER.warning( f'WARNING ⚠️ Box and segment counts should be equal, but got len(segments) = {len_segments}, ' f'len(boxes) = {len_boxes}. To resolve this only boxes will be used and all segments will be removed. ' 'To avoid this please supply either a detect or segment dataset, not a detect-segment mixed dataset.') for lb in labels: lb['segments'] = [] if len_cls == 0: raise ValueError(f'All labels empty in {cache_path}, can not start training without labels. {HELP_URL}') return labels # TODO: use hyp config to set all these augmentations def build_transforms(self, hyp=None): """Builds and appends transforms to the list.""" if self.augment: hyp.mosaic = hyp.mosaic if self.augment and not self.rect else 0.0 hyp.mixup = hyp.mixup if self.augment and not self.rect else 0.0 transforms = v8_transforms(self, self.imgsz, hyp) else: transforms = Compose([LetterBox(new_shape=(self.imgsz, self.imgsz), scaleup=False)]) transforms.append( Format(bbox_format='xywh', normalize=True, return_mask=self.use_segments, return_keypoint=self.use_keypoints, batch_idx=True, mask_ratio=hyp.mask_ratio, mask_overlap=hyp.overlap_mask)) return transforms def close_mosaic(self, hyp): """Sets mosaic, copy_paste and mixup options to 0.0 and builds transformations.""" hyp.mosaic = 0.0 # set mosaic ratio=0.0 hyp.copy_paste = 0.0 # keep the same behavior as previous v8 close-mosaic hyp.mixup = 0.0 # keep the same behavior as previous v8 close-mosaic self.transforms = self.build_transforms(hyp) def update_labels_info(self, label): """custom your label format here.""" # NOTE: cls is not with bboxes now, classification and semantic segmentation need an independent cls label # we can make it also support classification and semantic segmentation by add or remove some dict keys there. bboxes = label.pop('bboxes') segments = label.pop('segments') keypoints = label.pop('keypoints', None) bbox_format = label.pop('bbox_format') normalized = label.pop('normalized') label['instances'] = Instances(bboxes, segments, keypoints, bbox_format=bbox_format, normalized=normalized) return label @staticmethod def collate_fn(batch): """Collates data samples into batches.""" new_batch = {} keys = batch[0].keys() values = list(zip(*[list(b.values()) for b in batch])) for i, k in enumerate(keys): value = values[i] if k == 'img': value = torch.stack(value, 0) if k in ['masks', 'keypoints', 'bboxes', 'cls']: value = torch.cat(value, 0) new_batch[k] = value new_batch['batch_idx'] = list(new_batch['batch_idx']) for i in range(len(new_batch['batch_idx'])): new_batch['batch_idx'][i] += i # add target image index for build_targets() new_batch['batch_idx'] = torch.cat(new_batch['batch_idx'], 0) return new_batch # Classification dataloaders ------------------------------------------------------------------------------------------- class ClassificationDataset(torchvision.datasets.ImageFolder): """ YOLO Classification Dataset. Args: root (str): Dataset path. Attributes: cache_ram (bool): True if images should be cached in RAM, False otherwise. cache_disk (bool): True if images should be cached on disk, False otherwise. samples (list): List of samples containing file, index, npy, and im. torch_transforms (callable): torchvision transforms applied to the dataset. album_transforms (callable, optional): Albumentations transforms applied to the dataset if augment is True. """ def __init__(self, root, args, augment=False, cache=False): """ Initialize YOLO object with root, image size, augmentations, and cache settings. Args: root (str): Dataset path. args (Namespace): Argument parser containing dataset related settings. augment (bool, optional): True if dataset should be augmented, False otherwise. Defaults to False. cache (bool | str | optional): Cache setting, can be True, False, 'ram' or 'disk'. Defaults to False. """ super().__init__(root=root) if augment and args.fraction < 1.0: # reduce training fraction self.samples = self.samples[:round(len(self.samples) * args.fraction)] self.cache_ram = cache is True or cache == 'ram' self.cache_disk = cache == 'disk' self.samples = [list(x) + [Path(x[0]).with_suffix('.npy'), None] for x in self.samples] # file, index, npy, im self.torch_transforms = classify_transforms(args.imgsz) self.album_transforms = classify_albumentations( augment=augment, size=args.imgsz, scale=(1.0 - args.scale, 1.0), # (0.08, 1.0) hflip=args.fliplr, vflip=args.flipud, hsv_h=args.hsv_h, # HSV-Hue augmentation (fraction) hsv_s=args.hsv_s, # HSV-Saturation augmentation (fraction) hsv_v=args.hsv_v, # HSV-Value augmentation (fraction) mean=(0.0, 0.0, 0.0), # IMAGENET_MEAN std=(1.0, 1.0, 1.0), # IMAGENET_STD auto_aug=False) if augment else None def __getitem__(self, i): """Returns subset of data and targets corresponding to given indices.""" f, j, fn, im = self.samples[i] # filename, index, filename.with_suffix('.npy'), image if self.cache_ram and im is None: im = self.samples[i][3] = cv2.imread(f) elif self.cache_disk: if not fn.exists(): # load npy np.save(fn.as_posix(), cv2.imread(f)) im = np.load(fn) else: # read image im = cv2.imread(f) # BGR if self.album_transforms: sample = self.album_transforms(image=cv2.cvtColor(im, cv2.COLOR_BGR2RGB))['image'] else: sample = self.torch_transforms(im) return {'img': sample, 'cls': j} def __len__(self) -> int: return len(self.samples) # TODO: support semantic segmentation class SemanticDataset(BaseDataset): def __init__(self): """Initialize a SemanticDataset object.""" super().__init__() ================================================ FILE: ultralytics/yolo/data/dataset_wrappers.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import collections from copy import deepcopy from .augment import LetterBox class MixAndRectDataset: """ A dataset class that applies mosaic and mixup transformations as well as rectangular training. Attributes: dataset: The base dataset. imgsz: The size of the images in the dataset. """ def __init__(self, dataset): """ Args: dataset (BaseDataset): The base dataset to apply transformations to. """ self.dataset = dataset self.imgsz = dataset.imgsz def __len__(self): """Returns the number of items in the dataset.""" return len(self.dataset) def __getitem__(self, index): """ Applies mosaic, mixup and rectangular training transformations to an item in the dataset. Args: index (int): Index of the item in the dataset. Returns: (dict): A dictionary containing the transformed item data. """ labels = deepcopy(self.dataset[index]) for transform in self.dataset.transforms.tolist(): # Mosaic and mixup if hasattr(transform, 'get_indexes'): indexes = transform.get_indexes(self.dataset) if not isinstance(indexes, collections.abc.Sequence): indexes = [indexes] labels['mix_labels'] = [deepcopy(self.dataset[index]) for index in indexes] if self.dataset.rect and isinstance(transform, LetterBox): transform.new_shape = self.dataset.batch_shapes[self.dataset.batch[index]] labels = transform(labels) if 'mix_labels' in labels: labels.pop('mix_labels') return labels ================================================ FILE: ultralytics/yolo/data/scripts/download_weights.sh ================================================ #!/bin/bash # Ultralytics YOLO 🚀, AGPL-3.0 license # Download latest models from https://github.com/ultralytics/assets/releases # Example usage: bash ultralytics/yolo/data/scripts/download_weights.sh # parent # └── weights # ├── yolov8n.pt ← downloads here # ├── yolov8s.pt # └── ... python - <
(pixels) | mAPval
50-95 | Speed
CPU ONNX
(ms) | Speed
{gpu} TensorRT
(ms) | params
(M) | FLOPs
(B) |' separator = '|-------------|---------------------|--------------------|------------------------------|-----------------------------------|------------------|-----------------|' print(f'\n\n{header}') print(separator) for row in table_rows: print(row) if __name__ == '__main__': # Benchmark all export formats benchmark() # Profiling models on ONNX and TensorRT ProfileModels(['yolov8n.yaml', 'yolov8s.yaml']) ================================================ FILE: ultralytics/yolo/utils/callbacks/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from .base import add_integration_callbacks, default_callbacks, get_default_callbacks __all__ = 'add_integration_callbacks', 'default_callbacks', 'get_default_callbacks' ================================================ FILE: ultralytics/yolo/utils/callbacks/base.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license """ Base callbacks """ from collections import defaultdict from copy import deepcopy # Trainer callbacks ---------------------------------------------------------------------------------------------------- def on_pretrain_routine_start(trainer): """Called before the pretraining routine starts.""" pass def on_pretrain_routine_end(trainer): """Called after the pretraining routine ends.""" pass def on_train_start(trainer): """Called when the training starts.""" pass def on_train_epoch_start(trainer): """Called at the start of each training epoch.""" pass def on_train_batch_start(trainer): """Called at the start of each training batch.""" pass def optimizer_step(trainer): """Called when the optimizer takes a step.""" pass def on_before_zero_grad(trainer): """Called before the gradients are set to zero.""" pass def on_train_batch_end(trainer): """Called at the end of each training batch.""" pass def on_train_epoch_end(trainer): """Called at the end of each training epoch.""" pass def on_fit_epoch_end(trainer): """Called at the end of each fit epoch (train + val).""" pass def on_model_save(trainer): """Called when the model is saved.""" pass def on_train_end(trainer): """Called when the training ends.""" pass def on_params_update(trainer): """Called when the model parameters are updated.""" pass def teardown(trainer): """Called during the teardown of the training process.""" pass # Validator callbacks -------------------------------------------------------------------------------------------------- def on_val_start(validator): """Called when the validation starts.""" pass def on_val_batch_start(validator): """Called at the start of each validation batch.""" pass def on_val_batch_end(validator): """Called at the end of each validation batch.""" pass def on_val_end(validator): """Called when the validation ends.""" pass # Predictor callbacks -------------------------------------------------------------------------------------------------- def on_predict_start(predictor): """Called when the prediction starts.""" pass def on_predict_batch_start(predictor): """Called at the start of each prediction batch.""" pass def on_predict_batch_end(predictor): """Called at the end of each prediction batch.""" pass def on_predict_postprocess_end(predictor): """Called after the post-processing of the prediction ends.""" pass def on_predict_end(predictor): """Called when the prediction ends.""" pass # Exporter callbacks --------------------------------------------------------------------------------------------------- def on_export_start(exporter): """Called when the model export starts.""" pass def on_export_end(exporter): """Called when the model export ends.""" pass default_callbacks = { # Run in trainer 'on_pretrain_routine_start': [on_pretrain_routine_start], 'on_pretrain_routine_end': [on_pretrain_routine_end], 'on_train_start': [on_train_start], 'on_train_epoch_start': [on_train_epoch_start], 'on_train_batch_start': [on_train_batch_start], 'optimizer_step': [optimizer_step], 'on_before_zero_grad': [on_before_zero_grad], 'on_train_batch_end': [on_train_batch_end], 'on_train_epoch_end': [on_train_epoch_end], 'on_fit_epoch_end': [on_fit_epoch_end], # fit = train + val 'on_model_save': [on_model_save], 'on_train_end': [on_train_end], 'on_params_update': [on_params_update], 'teardown': [teardown], # Run in validator 'on_val_start': [on_val_start], 'on_val_batch_start': [on_val_batch_start], 'on_val_batch_end': [on_val_batch_end], 'on_val_end': [on_val_end], # Run in predictor 'on_predict_start': [on_predict_start], 'on_predict_batch_start': [on_predict_batch_start], 'on_predict_postprocess_end': [on_predict_postprocess_end], 'on_predict_batch_end': [on_predict_batch_end], 'on_predict_end': [on_predict_end], # Run in exporter 'on_export_start': [on_export_start], 'on_export_end': [on_export_end]} def get_default_callbacks(): """ Return a copy of the default_callbacks dictionary with lists as default values. Returns: (defaultdict): A defaultdict with keys from default_callbacks and empty lists as default values. """ return defaultdict(list, deepcopy(default_callbacks)) def add_integration_callbacks(instance): """ Add integration callbacks from various sources to the instance's callbacks. Args: instance (Trainer, Predictor, Validator, Exporter): An object with a 'callbacks' attribute that is a dictionary of callback lists. """ from .clearml import callbacks as clearml_cb from .comet import callbacks as comet_cb from .dvc import callbacks as dvc_cb from .hub import callbacks as hub_cb from .mlflow import callbacks as mlflow_cb from .neptune import callbacks as neptune_cb from .raytune import callbacks as tune_cb from .tensorboard import callbacks as tensorboard_cb from .wb import callbacks as wb_cb for x in clearml_cb, comet_cb, hub_cb, mlflow_cb, neptune_cb, tune_cb, tensorboard_cb, wb_cb, dvc_cb: for k, v in x.items(): if v not in instance.callbacks[k]: # prevent duplicate callbacks addition instance.callbacks[k].append(v) # callback[name].append(func) ================================================ FILE: ultralytics/yolo/utils/callbacks/clearml.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import re import matplotlib.image as mpimg import matplotlib.pyplot as plt from ultralytics.yolo.utils import LOGGER, TESTS_RUNNING from ultralytics.yolo.utils.torch_utils import model_info_for_loggers try: import clearml from clearml import Task from clearml.binding.frameworks.pytorch_bind import PatchPyTorchModelIO from clearml.binding.matplotlib_bind import PatchedMatplotlib assert hasattr(clearml, '__version__') # verify package is not directory assert not TESTS_RUNNING # do not log pytest except (ImportError, AssertionError): clearml = None def _log_debug_samples(files, title='Debug Samples') -> None: """ Log files (images) as debug samples in the ClearML task. Args: files (list): A list of file paths in PosixPath format. title (str): A title that groups together images with the same values. """ task = Task.current_task() if task: for f in files: if f.exists(): it = re.search(r'_batch(\d+)', f.name) iteration = int(it.groups()[0]) if it else 0 task.get_logger().report_image(title=title, series=f.name.replace(it.group(), ''), local_path=str(f), iteration=iteration) def _log_plot(title, plot_path) -> None: """ Log an image as a plot in the plot section of ClearML. Args: title (str): The title of the plot. plot_path (str): The path to the saved image file. """ img = mpimg.imread(plot_path) fig = plt.figure() ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect='auto', xticks=[], yticks=[]) # no ticks ax.imshow(img) Task.current_task().get_logger().report_matplotlib_figure(title=title, series='', figure=fig, report_interactive=False) def on_pretrain_routine_start(trainer): """Runs at start of pretraining routine; initializes and connects/ logs task to ClearML.""" try: task = Task.current_task() if task: # Make sure the automatic pytorch and matplotlib bindings are disabled! # We are logging these plots and model files manually in the integration PatchPyTorchModelIO.update_current_task(None) PatchedMatplotlib.update_current_task(None) else: task = Task.init(project_name=trainer.args.project or 'YOLOv8', task_name=trainer.args.name, tags=['YOLOv8'], output_uri=True, reuse_last_task_id=False, auto_connect_frameworks={ 'pytorch': False, 'matplotlib': False}) LOGGER.warning('ClearML Initialized a new task. If you want to run remotely, ' 'please add clearml-init and connect your arguments before initializing YOLO.') task.connect(vars(trainer.args), name='General') except Exception as e: LOGGER.warning(f'WARNING ⚠️ ClearML installed but not initialized correctly, not logging this run. {e}') def on_train_epoch_end(trainer): task = Task.current_task() if task: """Logs debug samples for the first epoch of YOLO training.""" if trainer.epoch == 1: _log_debug_samples(sorted(trainer.save_dir.glob('train_batch*.jpg')), 'Mosaic') """Report the current training progress.""" for k, v in trainer.validator.metrics.results_dict.items(): task.get_logger().report_scalar('train', k, v, iteration=trainer.epoch) def on_fit_epoch_end(trainer): """Reports model information to logger at the end of an epoch.""" task = Task.current_task() if task: # You should have access to the validation bboxes under jdict task.get_logger().report_scalar(title='Epoch Time', series='Epoch Time', value=trainer.epoch_time, iteration=trainer.epoch) if trainer.epoch == 0: for k, v in model_info_for_loggers(trainer).items(): task.get_logger().report_single_value(k, v) def on_val_end(validator): """Logs validation results including labels and predictions.""" if Task.current_task(): # Log val_labels and val_pred _log_debug_samples(sorted(validator.save_dir.glob('val*.jpg')), 'Validation') def on_train_end(trainer): """Logs final model and its name on training completion.""" task = Task.current_task() if task: # Log final results, CM matrix + PR plots files = [ 'results.png', 'confusion_matrix.png', 'confusion_matrix_normalized.png', *(f'{x}_curve.png' for x in ('F1', 'PR', 'P', 'R'))] files = [(trainer.save_dir / f) for f in files if (trainer.save_dir / f).exists()] # filter for f in files: _log_plot(title=f.stem, plot_path=f) # Report final metrics for k, v in trainer.validator.metrics.results_dict.items(): task.get_logger().report_single_value(k, v) # Log the final model task.update_output_model(model_path=str(trainer.best), model_name=trainer.args.name, auto_delete_file=False) callbacks = { 'on_pretrain_routine_start': on_pretrain_routine_start, 'on_train_epoch_end': on_train_epoch_end, 'on_fit_epoch_end': on_fit_epoch_end, 'on_val_end': on_val_end, 'on_train_end': on_train_end} if clearml else {} ================================================ FILE: ultralytics/yolo/utils/callbacks/comet.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import os from pathlib import Path from ultralytics.yolo.utils import LOGGER, RANK, TESTS_RUNNING, ops from ultralytics.yolo.utils.torch_utils import model_info_for_loggers try: import comet_ml assert not TESTS_RUNNING # do not log pytest assert hasattr(comet_ml, '__version__') # verify package is not directory except (ImportError, AssertionError): comet_ml = None # Ensures certain logging functions only run for supported tasks COMET_SUPPORTED_TASKS = ['detect'] # Names of plots created by YOLOv8 that are logged to Comet EVALUATION_PLOT_NAMES = 'F1_curve', 'P_curve', 'R_curve', 'PR_curve', 'confusion_matrix' LABEL_PLOT_NAMES = 'labels', 'labels_correlogram' _comet_image_prediction_count = 0 def _get_comet_mode(): return os.getenv('COMET_MODE', 'online') def _get_comet_model_name(): return os.getenv('COMET_MODEL_NAME', 'YOLOv8') def _get_eval_batch_logging_interval(): return int(os.getenv('COMET_EVAL_BATCH_LOGGING_INTERVAL', 1)) def _get_max_image_predictions_to_log(): return int(os.getenv('COMET_MAX_IMAGE_PREDICTIONS', 100)) def _scale_confidence_score(score): scale = float(os.getenv('COMET_MAX_CONFIDENCE_SCORE', 100.0)) return score * scale def _should_log_confusion_matrix(): return os.getenv('COMET_EVAL_LOG_CONFUSION_MATRIX', 'false').lower() == 'true' def _should_log_image_predictions(): return os.getenv('COMET_EVAL_LOG_IMAGE_PREDICTIONS', 'true').lower() == 'true' def _get_experiment_type(mode, project_name): """Return an experiment based on mode and project name.""" if mode == 'offline': return comet_ml.OfflineExperiment(project_name=project_name) return comet_ml.Experiment(project_name=project_name) def _create_experiment(args): """Ensures that the experiment object is only created in a single process during distributed training.""" if RANK not in (-1, 0): return try: comet_mode = _get_comet_mode() _project_name = os.getenv('COMET_PROJECT_NAME', args.project) experiment = _get_experiment_type(comet_mode, _project_name) experiment.log_parameters(vars(args)) experiment.log_others({ 'eval_batch_logging_interval': _get_eval_batch_logging_interval(), 'log_confusion_matrix_on_eval': _should_log_confusion_matrix(), 'log_image_predictions': _should_log_image_predictions(), 'max_image_predictions': _get_max_image_predictions_to_log(), }) experiment.log_other('Created from', 'yolov8') except Exception as e: LOGGER.warning(f'WARNING ⚠️ Comet installed but not initialized correctly, not logging this run. {e}') def _fetch_trainer_metadata(trainer): """Returns metadata for YOLO training including epoch and asset saving status.""" curr_epoch = trainer.epoch + 1 train_num_steps_per_epoch = len(trainer.train_loader.dataset) // trainer.batch_size curr_step = curr_epoch * train_num_steps_per_epoch final_epoch = curr_epoch == trainer.epochs save = trainer.args.save save_period = trainer.args.save_period save_interval = curr_epoch % save_period == 0 save_assets = save and save_period > 0 and save_interval and not final_epoch return dict( curr_epoch=curr_epoch, curr_step=curr_step, save_assets=save_assets, final_epoch=final_epoch, ) def _scale_bounding_box_to_original_image_shape(box, resized_image_shape, original_image_shape, ratio_pad): """YOLOv8 resizes images during training and the label values are normalized based on this resized shape. This function rescales the bounding box labels to the original image shape. """ resized_image_height, resized_image_width = resized_image_shape # Convert normalized xywh format predictions to xyxy in resized scale format box = ops.xywhn2xyxy(box, h=resized_image_height, w=resized_image_width) # Scale box predictions from resized image scale back to original image scale box = ops.scale_boxes(resized_image_shape, box, original_image_shape, ratio_pad) # Convert bounding box format from xyxy to xywh for Comet logging box = ops.xyxy2xywh(box) # Adjust xy center to correspond top-left corner box[:2] -= box[2:] / 2 box = box.tolist() return box def _format_ground_truth_annotations_for_detection(img_idx, image_path, batch, class_name_map=None): """Format ground truth annotations for detection.""" indices = batch['batch_idx'] == img_idx bboxes = batch['bboxes'][indices] if len(bboxes) == 0: LOGGER.debug(f'COMET WARNING: Image: {image_path} has no bounding boxes labels') return None cls_labels = batch['cls'][indices].squeeze(1).tolist() if class_name_map: cls_labels = [str(class_name_map[label]) for label in cls_labels] original_image_shape = batch['ori_shape'][img_idx] resized_image_shape = batch['resized_shape'][img_idx] ratio_pad = batch['ratio_pad'][img_idx] data = [] for box, label in zip(bboxes, cls_labels): box = _scale_bounding_box_to_original_image_shape(box, resized_image_shape, original_image_shape, ratio_pad) data.append({ 'boxes': [box], 'label': f'gt_{label}', 'score': _scale_confidence_score(1.0), }) return {'name': 'ground_truth', 'data': data} def _format_prediction_annotations_for_detection(image_path, metadata, class_label_map=None): """Format YOLO predictions for object detection visualization.""" stem = image_path.stem image_id = int(stem) if stem.isnumeric() else stem predictions = metadata.get(image_id) if not predictions: LOGGER.debug(f'COMET WARNING: Image: {image_path} has no bounding boxes predictions') return None data = [] for prediction in predictions: boxes = prediction['bbox'] score = _scale_confidence_score(prediction['score']) cls_label = prediction['category_id'] if class_label_map: cls_label = str(class_label_map[cls_label]) data.append({'boxes': [boxes], 'label': cls_label, 'score': score}) return {'name': 'prediction', 'data': data} def _fetch_annotations(img_idx, image_path, batch, prediction_metadata_map, class_label_map): """Join the ground truth and prediction annotations if they exist.""" ground_truth_annotations = _format_ground_truth_annotations_for_detection(img_idx, image_path, batch, class_label_map) prediction_annotations = _format_prediction_annotations_for_detection(image_path, prediction_metadata_map, class_label_map) annotations = [ annotation for annotation in [ground_truth_annotations, prediction_annotations] if annotation is not None] return [annotations] if annotations else None def _create_prediction_metadata_map(model_predictions): """Create metadata map for model predictions by groupings them based on image ID.""" pred_metadata_map = {} for prediction in model_predictions: pred_metadata_map.setdefault(prediction['image_id'], []) pred_metadata_map[prediction['image_id']].append(prediction) return pred_metadata_map def _log_confusion_matrix(experiment, trainer, curr_step, curr_epoch): """Log the confusion matrix to Comet experiment.""" conf_mat = trainer.validator.confusion_matrix.matrix names = list(trainer.data['names'].values()) + ['background'] experiment.log_confusion_matrix( matrix=conf_mat, labels=names, max_categories=len(names), epoch=curr_epoch, step=curr_step, ) def _log_images(experiment, image_paths, curr_step, annotations=None): """Logs images to the experiment with optional annotations.""" if annotations: for image_path, annotation in zip(image_paths, annotations): experiment.log_image(image_path, name=image_path.stem, step=curr_step, annotations=annotation) else: for image_path in image_paths: experiment.log_image(image_path, name=image_path.stem, step=curr_step) def _log_image_predictions(experiment, validator, curr_step): """Logs predicted boxes for a single image during training.""" global _comet_image_prediction_count task = validator.args.task if task not in COMET_SUPPORTED_TASKS: return jdict = validator.jdict if not jdict: return predictions_metadata_map = _create_prediction_metadata_map(jdict) dataloader = validator.dataloader class_label_map = validator.names batch_logging_interval = _get_eval_batch_logging_interval() max_image_predictions = _get_max_image_predictions_to_log() for batch_idx, batch in enumerate(dataloader): if (batch_idx + 1) % batch_logging_interval != 0: continue image_paths = batch['im_file'] for img_idx, image_path in enumerate(image_paths): if _comet_image_prediction_count >= max_image_predictions: return image_path = Path(image_path) annotations = _fetch_annotations( img_idx, image_path, batch, predictions_metadata_map, class_label_map, ) _log_images( experiment, [image_path], curr_step, annotations=annotations, ) _comet_image_prediction_count += 1 def _log_plots(experiment, trainer): """Logs evaluation plots and label plots for the experiment.""" plot_filenames = [trainer.save_dir / f'{plots}.png' for plots in EVALUATION_PLOT_NAMES] _log_images(experiment, plot_filenames, None) label_plot_filenames = [trainer.save_dir / f'{labels}.jpg' for labels in LABEL_PLOT_NAMES] _log_images(experiment, label_plot_filenames, None) def _log_model(experiment, trainer): """Log the best-trained model to Comet.ml.""" model_name = _get_comet_model_name() experiment.log_model( model_name, file_or_folder=str(trainer.best), file_name='best.pt', overwrite=True, ) def on_pretrain_routine_start(trainer): """Creates or resumes a CometML experiment at the start of a YOLO pre-training routine.""" experiment = comet_ml.get_global_experiment() is_alive = getattr(experiment, 'alive', False) if not experiment or not is_alive: _create_experiment(trainer.args) def on_train_epoch_end(trainer): """Log metrics and save batch images at the end of training epochs.""" experiment = comet_ml.get_global_experiment() if not experiment: return metadata = _fetch_trainer_metadata(trainer) curr_epoch = metadata['curr_epoch'] curr_step = metadata['curr_step'] experiment.log_metrics( trainer.label_loss_items(trainer.tloss, prefix='train'), step=curr_step, epoch=curr_epoch, ) if curr_epoch == 1: _log_images(experiment, trainer.save_dir.glob('train_batch*.jpg'), curr_step) def on_fit_epoch_end(trainer): """Logs model assets at the end of each epoch.""" experiment = comet_ml.get_global_experiment() if not experiment: return metadata = _fetch_trainer_metadata(trainer) curr_epoch = metadata['curr_epoch'] curr_step = metadata['curr_step'] save_assets = metadata['save_assets'] experiment.log_metrics(trainer.metrics, step=curr_step, epoch=curr_epoch) experiment.log_metrics(trainer.lr, step=curr_step, epoch=curr_epoch) if curr_epoch == 1: experiment.log_metrics(model_info_for_loggers(trainer), step=curr_step, epoch=curr_epoch) if not save_assets: return _log_model(experiment, trainer) if _should_log_confusion_matrix(): _log_confusion_matrix(experiment, trainer, curr_step, curr_epoch) if _should_log_image_predictions(): _log_image_predictions(experiment, trainer.validator, curr_step) def on_train_end(trainer): """Perform operations at the end of training.""" experiment = comet_ml.get_global_experiment() if not experiment: return metadata = _fetch_trainer_metadata(trainer) curr_epoch = metadata['curr_epoch'] curr_step = metadata['curr_step'] plots = trainer.args.plots _log_model(experiment, trainer) if plots: _log_plots(experiment, trainer) _log_confusion_matrix(experiment, trainer, curr_step, curr_epoch) _log_image_predictions(experiment, trainer.validator, curr_step) experiment.end() global _comet_image_prediction_count _comet_image_prediction_count = 0 callbacks = { 'on_pretrain_routine_start': on_pretrain_routine_start, 'on_train_epoch_end': on_train_epoch_end, 'on_fit_epoch_end': on_fit_epoch_end, 'on_train_end': on_train_end} if comet_ml else {} ================================================ FILE: ultralytics/yolo/utils/callbacks/dvc.py ================================================ # Ultralytics YOLO 🚀, GPL-3.0 license import os import pkg_resources as pkg from ultralytics.yolo.utils import LOGGER, TESTS_RUNNING from ultralytics.yolo.utils.torch_utils import model_info_for_loggers try: from importlib.metadata import version import dvclive assert not TESTS_RUNNING # do not log pytest ver = version('dvclive') if pkg.parse_version(ver) < pkg.parse_version('2.11.0'): LOGGER.debug(f'DVCLive is detected but version {ver} is incompatible (>=2.11 required).') dvclive = None # noqa: F811 except (ImportError, AssertionError, TypeError): dvclive = None # DVCLive logger instance live = None _processed_plots = {} # `on_fit_epoch_end` is called on final validation (probably need to be fixed) # for now this is the way we distinguish final evaluation of the best model vs # last epoch validation _training_epoch = False def _logger_disabled(): return os.getenv('ULTRALYTICS_DVC_DISABLED', 'false').lower() == 'true' def _log_images(image_path, prefix=''): if live: live.log_image(os.path.join(prefix, image_path.name), image_path) def _log_plots(plots, prefix=''): for name, params in plots.items(): timestamp = params['timestamp'] if _processed_plots.get(name) != timestamp: _log_images(name, prefix) _processed_plots[name] = timestamp def _log_confusion_matrix(validator): targets = [] preds = [] matrix = validator.confusion_matrix.matrix names = list(validator.names.values()) if validator.confusion_matrix.task == 'detect': names += ['background'] for ti, pred in enumerate(matrix.T.astype(int)): for pi, num in enumerate(pred): targets.extend([names[ti]] * num) preds.extend([names[pi]] * num) live.log_sklearn_plot('confusion_matrix', targets, preds, name='cf.json', normalized=True) def on_pretrain_routine_start(trainer): try: global live if not _logger_disabled(): live = dvclive.Live(save_dvc_exp=True) LOGGER.info( 'DVCLive is detected and auto logging is enabled (can be disabled with `ULTRALYTICS_DVC_DISABLED=true`).' ) else: LOGGER.debug('DVCLive is detected and auto logging is disabled via `ULTRALYTICS_DVC_DISABLED`.') live = None except Exception as e: LOGGER.warning(f'WARNING ⚠️ DVCLive installed but not initialized correctly, not logging this run. {e}') def on_pretrain_routine_end(trainer): _log_plots(trainer.plots, 'train') def on_train_start(trainer): if live: live.log_params(trainer.args) def on_train_epoch_start(trainer): global _training_epoch _training_epoch = True def on_fit_epoch_end(trainer): global _training_epoch if live and _training_epoch: all_metrics = {**trainer.label_loss_items(trainer.tloss, prefix='train'), **trainer.metrics, **trainer.lr} for metric, value in all_metrics.items(): live.log_metric(metric, value) if trainer.epoch == 0: for metric, value in model_info_for_loggers(trainer).items(): live.log_metric(metric, value, plot=False) _log_plots(trainer.plots, 'train') _log_plots(trainer.validator.plots, 'val') live.next_step() _training_epoch = False def on_train_end(trainer): if live: # At the end log the best metrics. It runs validator on the best model internally. all_metrics = {**trainer.label_loss_items(trainer.tloss, prefix='train'), **trainer.metrics, **trainer.lr} for metric, value in all_metrics.items(): live.log_metric(metric, value, plot=False) _log_plots(trainer.plots, 'eval') _log_plots(trainer.validator.plots, 'eval') _log_confusion_matrix(trainer.validator) if trainer.best.exists(): live.log_artifact(trainer.best, copy=True) live.end() callbacks = { 'on_pretrain_routine_start': on_pretrain_routine_start, 'on_pretrain_routine_end': on_pretrain_routine_end, 'on_train_start': on_train_start, 'on_train_epoch_start': on_train_epoch_start, 'on_fit_epoch_end': on_fit_epoch_end, 'on_train_end': on_train_end} if dvclive else {} ================================================ FILE: ultralytics/yolo/utils/callbacks/hub.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import json from time import time from ultralytics.hub.utils import PREFIX, events from ultralytics.yolo.utils import LOGGER from ultralytics.yolo.utils.torch_utils import model_info_for_loggers def on_pretrain_routine_end(trainer): """Logs info before starting timer for upload rate limit.""" session = getattr(trainer, 'hub_session', None) if session: # Start timer for upload rate limit LOGGER.info(f'{PREFIX}View model at https://hub.ultralytics.com/models/{session.model_id} 🚀') session.timers = {'metrics': time(), 'ckpt': time()} # start timer on session.rate_limit def on_fit_epoch_end(trainer): """Uploads training progress metrics at the end of each epoch.""" session = getattr(trainer, 'hub_session', None) if session: # Upload metrics after val end all_plots = {**trainer.label_loss_items(trainer.tloss, prefix='train'), **trainer.metrics} if trainer.epoch == 0: all_plots = {**all_plots, **model_info_for_loggers(trainer)} session.metrics_queue[trainer.epoch] = json.dumps(all_plots) if time() - session.timers['metrics'] > session.rate_limits['metrics']: session.upload_metrics() session.timers['metrics'] = time() # reset timer session.metrics_queue = {} # reset queue def on_model_save(trainer): """Saves checkpoints to Ultralytics HUB with rate limiting.""" session = getattr(trainer, 'hub_session', None) if session: # Upload checkpoints with rate limiting is_best = trainer.best_fitness == trainer.fitness if time() - session.timers['ckpt'] > session.rate_limits['ckpt']: LOGGER.info(f'{PREFIX}Uploading checkpoint https://hub.ultralytics.com/models/{session.model_id}') session.upload_model(trainer.epoch, trainer.last, is_best) session.timers['ckpt'] = time() # reset timer def on_train_end(trainer): """Upload final model and metrics to Ultralytics HUB at the end of training.""" session = getattr(trainer, 'hub_session', None) if session: # Upload final model and metrics with exponential standoff LOGGER.info(f'{PREFIX}Syncing final model...') session.upload_model(trainer.epoch, trainer.best, map=trainer.metrics.get('metrics/mAP50-95(B)', 0), final=True) session.alive = False # stop heartbeats LOGGER.info(f'{PREFIX}Done ✅\n' f'{PREFIX}View model at https://hub.ultralytics.com/models/{session.model_id} 🚀') def on_train_start(trainer): """Run events on train start.""" events(trainer.args) def on_val_start(validator): """Runs events on validation start.""" events(validator.args) def on_predict_start(predictor): """Run events on predict start.""" events(predictor.args) def on_export_start(exporter): """Run events on export start.""" events(exporter.args) callbacks = { 'on_pretrain_routine_end': on_pretrain_routine_end, 'on_fit_epoch_end': on_fit_epoch_end, 'on_model_save': on_model_save, 'on_train_end': on_train_end, 'on_train_start': on_train_start, 'on_val_start': on_val_start, 'on_predict_start': on_predict_start, 'on_export_start': on_export_start} ================================================ FILE: ultralytics/yolo/utils/callbacks/mlflow.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import os import re from pathlib import Path from ultralytics.yolo.utils import LOGGER, TESTS_RUNNING, colorstr try: import mlflow assert not TESTS_RUNNING # do not log pytest assert hasattr(mlflow, '__version__') # verify package is not directory except (ImportError, AssertionError): mlflow = None def on_pretrain_routine_end(trainer): """Logs training parameters to MLflow.""" global mlflow, run, run_id, experiment_name if os.environ.get('MLFLOW_TRACKING_URI') is None: mlflow = None if mlflow: mlflow_location = os.environ['MLFLOW_TRACKING_URI'] # "http://192.168.xxx.xxx:5000" mlflow.set_tracking_uri(mlflow_location) experiment_name = os.environ.get('MLFLOW_EXPERIMENT') or trainer.args.project or '/Shared/YOLOv8' experiment = mlflow.get_experiment_by_name(experiment_name) if experiment is None: mlflow.create_experiment(experiment_name) mlflow.set_experiment(experiment_name) prefix = colorstr('MLFlow: ') try: run, active_run = mlflow, mlflow.active_run() if not active_run: active_run = mlflow.start_run(experiment_id=experiment.experiment_id) run_id = active_run.info.run_id LOGGER.info(f'{prefix}Using run_id({run_id}) at {mlflow_location}') run.log_params(vars(trainer.model.args)) except Exception as err: LOGGER.error(f'{prefix}Failing init - {repr(err)}') LOGGER.warning(f'{prefix}Continuing without Mlflow') def on_fit_epoch_end(trainer): """Logs training metrics to Mlflow.""" if mlflow: metrics_dict = {f"{re.sub('[()]', '', k)}": float(v) for k, v in trainer.metrics.items()} run.log_metrics(metrics=metrics_dict, step=trainer.epoch) def on_train_end(trainer): """Called at end of train loop to log model artifact info.""" if mlflow: root_dir = Path(__file__).resolve().parents[3] run.log_artifact(trainer.last) run.log_artifact(trainer.best) run.pyfunc.log_model(artifact_path=experiment_name, code_path=[str(root_dir)], artifacts={'model_path': str(trainer.save_dir)}, python_model=run.pyfunc.PythonModel()) callbacks = { 'on_pretrain_routine_end': on_pretrain_routine_end, 'on_fit_epoch_end': on_fit_epoch_end, 'on_train_end': on_train_end} if mlflow else {} ================================================ FILE: ultralytics/yolo/utils/callbacks/neptune.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import matplotlib.image as mpimg import matplotlib.pyplot as plt from ultralytics.yolo.utils import LOGGER, TESTS_RUNNING from ultralytics.yolo.utils.torch_utils import model_info_for_loggers try: import neptune from neptune.types import File assert not TESTS_RUNNING # do not log pytest assert hasattr(neptune, '__version__') except (ImportError, AssertionError): neptune = None run = None # NeptuneAI experiment logger instance def _log_scalars(scalars, step=0): """Log scalars to the NeptuneAI experiment logger.""" if run: for k, v in scalars.items(): run[k].append(value=v, step=step) def _log_images(imgs_dict, group=''): """Log scalars to the NeptuneAI experiment logger.""" if run: for k, v in imgs_dict.items(): run[f'{group}/{k}'].upload(File(v)) def _log_plot(title, plot_path): """Log plots to the NeptuneAI experiment logger.""" """ Log image as plot in the plot section of NeptuneAI arguments: title (str) Title of the plot plot_path (PosixPath or str) Path to the saved image file """ img = mpimg.imread(plot_path) fig = plt.figure() ax = fig.add_axes([0, 0, 1, 1], frameon=False, aspect='auto', xticks=[], yticks=[]) # no ticks ax.imshow(img) run[f'Plots/{title}'].upload(fig) def on_pretrain_routine_start(trainer): """Callback function called before the training routine starts.""" try: global run run = neptune.init_run(project=trainer.args.project or 'YOLOv8', name=trainer.args.name, tags=['YOLOv8']) run['Configuration/Hyperparameters'] = {k: '' if v is None else v for k, v in vars(trainer.args).items()} except Exception as e: LOGGER.warning(f'WARNING ⚠️ NeptuneAI installed but not initialized correctly, not logging this run. {e}') def on_train_epoch_end(trainer): """Callback function called at end of each training epoch.""" _log_scalars(trainer.label_loss_items(trainer.tloss, prefix='train'), trainer.epoch + 1) _log_scalars(trainer.lr, trainer.epoch + 1) if trainer.epoch == 1: _log_images({f.stem: str(f) for f in trainer.save_dir.glob('train_batch*.jpg')}, 'Mosaic') def on_fit_epoch_end(trainer): """Callback function called at end of each fit (train+val) epoch.""" if run and trainer.epoch == 0: run['Configuration/Model'] = model_info_for_loggers(trainer) _log_scalars(trainer.metrics, trainer.epoch + 1) def on_val_end(validator): """Callback function called at end of each validation.""" if run: # Log val_labels and val_pred _log_images({f.stem: str(f) for f in validator.save_dir.glob('val*.jpg')}, 'Validation') def on_train_end(trainer): """Callback function called at end of training.""" if run: # Log final results, CM matrix + PR plots files = [ 'results.png', 'confusion_matrix.png', 'confusion_matrix_normalized.png', *(f'{x}_curve.png' for x in ('F1', 'PR', 'P', 'R'))] files = [(trainer.save_dir / f) for f in files if (trainer.save_dir / f).exists()] # filter for f in files: _log_plot(title=f.stem, plot_path=f) # Log the final model run[f'weights/{trainer.args.name or trainer.args.task}/{str(trainer.best.name)}'].upload(File(str( trainer.best))) callbacks = { 'on_pretrain_routine_start': on_pretrain_routine_start, 'on_train_epoch_end': on_train_epoch_end, 'on_fit_epoch_end': on_fit_epoch_end, 'on_val_end': on_val_end, 'on_train_end': on_train_end} if neptune else {} ================================================ FILE: ultralytics/yolo/utils/callbacks/raytune.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license try: import ray from ray import tune from ray.air import session except (ImportError, AssertionError): tune = None def on_fit_epoch_end(trainer): """Sends training metrics to Ray Tune at end of each epoch.""" if ray.tune.is_session_enabled(): metrics = trainer.metrics metrics['epoch'] = trainer.epoch session.report(metrics) callbacks = { 'on_fit_epoch_end': on_fit_epoch_end, } if tune else {} ================================================ FILE: ultralytics/yolo/utils/callbacks/tensorboard.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from ultralytics.yolo.utils import LOGGER, TESTS_RUNNING, colorstr try: from torch.utils.tensorboard import SummaryWriter assert not TESTS_RUNNING # do not log pytest except (ImportError, AssertionError): SummaryWriter = None writer = None # TensorBoard SummaryWriter instance def _log_scalars(scalars, step=0): """Logs scalar values to TensorBoard.""" if writer: for k, v in scalars.items(): writer.add_scalar(k, v, step) def on_pretrain_routine_start(trainer): """Initialize TensorBoard logging with SummaryWriter.""" if SummaryWriter: try: global writer writer = SummaryWriter(str(trainer.save_dir)) prefix = colorstr('TensorBoard: ') LOGGER.info(f"{prefix}Start with 'tensorboard --logdir {trainer.save_dir}', view at http://localhost:6006/") except Exception as e: LOGGER.warning(f'WARNING ⚠️ TensorBoard not initialized correctly, not logging this run. {e}') def on_batch_end(trainer): """Logs scalar statistics at the end of a training batch.""" _log_scalars(trainer.label_loss_items(trainer.tloss, prefix='train'), trainer.epoch + 1) def on_fit_epoch_end(trainer): """Logs epoch metrics at end of training epoch.""" _log_scalars(trainer.metrics, trainer.epoch + 1) callbacks = { 'on_pretrain_routine_start': on_pretrain_routine_start, 'on_fit_epoch_end': on_fit_epoch_end, 'on_batch_end': on_batch_end} ================================================ FILE: ultralytics/yolo/utils/callbacks/wb.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from ultralytics.yolo.utils import TESTS_RUNNING from ultralytics.yolo.utils.torch_utils import model_info_for_loggers try: import wandb as wb assert hasattr(wb, '__version__') assert not TESTS_RUNNING # do not log pytest except (ImportError, AssertionError): wb = None _processed_plots = {} def _log_plots(plots, step): for name, params in plots.items(): timestamp = params['timestamp'] if _processed_plots.get(name, None) != timestamp: wb.run.log({name.stem: wb.Image(str(name))}, step=step) _processed_plots[name] = timestamp def on_pretrain_routine_start(trainer): """Initiate and start project if module is present.""" wb.run or wb.init(project=trainer.args.project or 'YOLOv8', name=trainer.args.name, config=vars(trainer.args)) def on_fit_epoch_end(trainer): """Logs training metrics and model information at the end of an epoch.""" wb.run.log(trainer.metrics, step=trainer.epoch + 1) _log_plots(trainer.plots, step=trainer.epoch + 1) _log_plots(trainer.validator.plots, step=trainer.epoch + 1) if trainer.epoch == 0: wb.run.log(model_info_for_loggers(trainer), step=trainer.epoch + 1) def on_train_epoch_end(trainer): """Log metrics and save images at the end of each training epoch.""" wb.run.log(trainer.label_loss_items(trainer.tloss, prefix='train'), step=trainer.epoch + 1) wb.run.log(trainer.lr, step=trainer.epoch + 1) if trainer.epoch == 1: _log_plots(trainer.plots, step=trainer.epoch + 1) def on_train_end(trainer): """Save the best model as an artifact at end of training.""" _log_plots(trainer.validator.plots, step=trainer.epoch + 1) _log_plots(trainer.plots, step=trainer.epoch + 1) art = wb.Artifact(type='model', name=f'run_{wb.run.id}_model') if trainer.best.exists(): art.add_file(trainer.best) wb.run.log_artifact(art) callbacks = { 'on_pretrain_routine_start': on_pretrain_routine_start, 'on_train_epoch_end': on_train_epoch_end, 'on_fit_epoch_end': on_fit_epoch_end, 'on_train_end': on_train_end} if wb else {} ================================================ FILE: ultralytics/yolo/utils/checks.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import contextlib import glob import inspect import math import os import platform import re import shutil import subprocess from pathlib import Path from typing import Optional import cv2 import numpy as np import pkg_resources as pkg import psutil import requests import torch from matplotlib import font_manager from ultralytics.yolo.utils import (AUTOINSTALL, LOGGER, ONLINE, ROOT, USER_CONFIG_DIR, TryExcept, clean_url, colorstr, downloads, emojis, is_colab, is_docker, is_jupyter, is_kaggle, is_online, is_pip_package, url2file) def is_ascii(s) -> bool: """ Check if a string is composed of only ASCII characters. Args: s (str): String to be checked. Returns: bool: True if the string is composed only of ASCII characters, False otherwise. """ # Convert list, tuple, None, etc. to string s = str(s) # Check if the string is composed of only ASCII characters return all(ord(c) < 128 for c in s) def check_imgsz(imgsz, stride=32, min_dim=1, max_dim=2, floor=0): """ Verify image size is a multiple of the given stride in each dimension. If the image size is not a multiple of the stride, update it to the nearest multiple of the stride that is greater than or equal to the given floor value. Args: imgsz (int | cList[int]): Image size. stride (int): Stride value. min_dim (int): Minimum number of dimensions. floor (int): Minimum allowed value for image size. Returns: (List[int]): Updated image size. """ # Convert stride to integer if it is a tensor stride = int(stride.max() if isinstance(stride, torch.Tensor) else stride) # Convert image size to list if it is an integer if isinstance(imgsz, int): imgsz = [imgsz] elif isinstance(imgsz, (list, tuple)): imgsz = list(imgsz) else: raise TypeError(f"'imgsz={imgsz}' is of invalid type {type(imgsz).__name__}. " f"Valid imgsz types are int i.e. 'imgsz=640' or list i.e. 'imgsz=[640,640]'") # Apply max_dim if len(imgsz) > max_dim: msg = "'train' and 'val' imgsz must be an integer, while 'predict' and 'export' imgsz may be a [h, w] list " \ "or an integer, i.e. 'yolo export imgsz=640,480' or 'yolo export imgsz=640'" if max_dim != 1: raise ValueError(f'imgsz={imgsz} is not a valid image size. {msg}') LOGGER.warning(f"WARNING ⚠️ updating to 'imgsz={max(imgsz)}'. {msg}") imgsz = [max(imgsz)] # Make image size a multiple of the stride sz = [max(math.ceil(x / stride) * stride, floor) for x in imgsz] # Print warning message if image size was updated if sz != imgsz: LOGGER.warning(f'WARNING ⚠️ imgsz={imgsz} must be multiple of max stride {stride}, updating to {sz}') # Add missing dimensions if necessary sz = [sz[0], sz[0]] if min_dim == 2 and len(sz) == 1 else sz[0] if min_dim == 1 and len(sz) == 1 else sz return sz def check_version(current: str = '0.0.0', minimum: str = '0.0.0', name: str = 'version ', pinned: bool = False, hard: bool = False, verbose: bool = False) -> bool: """ Check current version against the required minimum version. Args: current (str): Current version. minimum (str): Required minimum version. name (str): Name to be used in warning message. pinned (bool): If True, versions must match exactly. If False, minimum version must be satisfied. hard (bool): If True, raise an AssertionError if the minimum version is not met. verbose (bool): If True, print warning message if minimum version is not met. Returns: (bool): True if minimum version is met, False otherwise. """ current, minimum = (pkg.parse_version(x) for x in (current, minimum)) result = (current == minimum) if pinned else (current >= minimum) # bool warning_message = f'WARNING ⚠️ {name}{minimum} is required by YOLOv8, but {name}{current} is currently installed' if hard: assert result, emojis(warning_message) # assert min requirements met if verbose and not result: LOGGER.warning(warning_message) return result def check_latest_pypi_version(package_name='ultralytics'): """ Returns the latest version of a PyPI package without downloading or installing it. Parameters: package_name (str): The name of the package to find the latest version for. Returns: (str): The latest version of the package. """ with contextlib.suppress(Exception): requests.packages.urllib3.disable_warnings() # Disable the InsecureRequestWarning response = requests.get(f'https://pypi.org/pypi/{package_name}/json', timeout=3) if response.status_code == 200: return response.json()['info']['version'] return None def check_pip_update_available(): """ Checks if a new version of the ultralytics package is available on PyPI. Returns: (bool): True if an update is available, False otherwise. """ if ONLINE and is_pip_package(): with contextlib.suppress(Exception): from ultralytics import __version__ latest = check_latest_pypi_version() if pkg.parse_version(__version__) < pkg.parse_version(latest): # update is available LOGGER.info(f'New https://pypi.org/project/ultralytics/{latest} available 😃 ' f"Update with 'pip install -U ultralytics'") return True return False def check_font(font='Arial.ttf'): """ Find font locally or download to user's configuration directory if it does not already exist. Args: font (str): Path or name of font. Returns: file (Path): Resolved font file path. """ name = Path(font).name # Check USER_CONFIG_DIR file = USER_CONFIG_DIR / name if file.exists(): return file # Check system fonts matches = [s for s in font_manager.findSystemFonts() if font in s] if any(matches): return matches[0] # Download to USER_CONFIG_DIR if missing url = f'https://ultralytics.com/assets/{name}' if downloads.is_url(url): downloads.safe_download(url=url, file=file) return file def check_python(minimum: str = '3.7.0') -> bool: """ Check current python version against the required minimum version. Args: minimum (str): Required minimum version of python. Returns: None """ return check_version(platform.python_version(), minimum, name='Python ', hard=True) @TryExcept() def check_requirements(requirements=ROOT.parent / 'requirements.txt', exclude=(), install=True, cmds=''): """ Check if installed dependencies meet YOLOv8 requirements and attempt to auto-update if needed. Args: requirements (Union[Path, str, List[str]]): Path to a requirements.txt file, a single package requirement as a string, or a list of package requirements as strings. exclude (Tuple[str]): Tuple of package names to exclude from checking. install (bool): If True, attempt to auto-update packages that don't meet requirements. cmds (str): Additional commands to pass to the pip install command when auto-updating. """ prefix = colorstr('red', 'bold', 'requirements:') check_python() # check python version file = None if isinstance(requirements, Path): # requirements.txt file file = requirements.resolve() assert file.exists(), f'{prefix} {file} not found, check failed.' with file.open() as f: requirements = [f'{x.name}{x.specifier}' for x in pkg.parse_requirements(f) if x.name not in exclude] elif isinstance(requirements, str): requirements = [requirements] s = '' # console string n = 0 # number of packages updates for r in requirements: try: pkg.require(r) except (pkg.VersionConflict, pkg.DistributionNotFound): # exception if requirements not met try: # attempt to import (slower but more accurate) import importlib importlib.import_module(next(pkg.parse_requirements(r)).name) except ImportError: s += f'"{r}" ' n += 1 if s: if install and AUTOINSTALL: # check environment variable LOGGER.info(f"{prefix} Ultralytics requirement{'s' * (n > 1)} {s}not found, attempting AutoUpdate...") try: assert is_online(), 'AutoUpdate skipped (offline)' LOGGER.info(subprocess.check_output(f'pip install --no-cache {s} {cmds}', shell=True).decode()) s = f"{prefix} {n} package{'s' * (n > 1)} updated per {file or requirements}\n" \ f"{prefix} ⚠️ {colorstr('bold', 'Restart runtime or rerun command for updates to take effect')}\n" LOGGER.info(s) except Exception as e: LOGGER.warning(f'{prefix} ❌ {e}') return False else: return False return True def check_suffix(file='yolov8n.pt', suffix='.pt', msg=''): """Check file(s) for acceptable suffix.""" if file and suffix: if isinstance(suffix, str): suffix = (suffix, ) for f in file if isinstance(file, (list, tuple)) else [file]: s = Path(f).suffix.lower().strip() # file suffix if len(s): assert s in suffix, f'{msg}{f} acceptable suffix is {suffix}, not {s}' def check_yolov5u_filename(file: str, verbose: bool = True): """Replace legacy YOLOv5 filenames with updated YOLOv5u filenames.""" if ('yolov3' in file or 'yolov5' in file) and 'u' not in file: original_file = file file = re.sub(r'(.*yolov5([nsmlx]))\.pt', '\\1u.pt', file) # i.e. yolov5n.pt -> yolov5nu.pt file = re.sub(r'(.*yolov5([nsmlx])6)\.pt', '\\1u.pt', file) # i.e. yolov5n6.pt -> yolov5n6u.pt file = re.sub(r'(.*yolov3(|-tiny|-spp))\.pt', '\\1u.pt', file) # i.e. yolov3-spp.pt -> yolov3-sppu.pt if file != original_file and verbose: LOGGER.info(f"PRO TIP 💡 Replace 'model={original_file}' with new 'model={file}'.\nYOLOv5 'u' models are " f'trained with https://github.com/ultralytics/ultralytics and feature improved performance vs ' f'standard YOLOv5 models trained with https://github.com/ultralytics/yolov5.\n') return file def check_file(file, suffix='', download=True, hard=True): """Search/download file (if necessary) and return path.""" check_suffix(file, suffix) # optional file = str(file).strip() # convert to string and strip spaces file = check_yolov5u_filename(file) # yolov5n -> yolov5nu if not file or ('://' not in file and Path(file).exists()): # exists ('://' check required in Windows Python<3.10) return file elif download and file.lower().startswith(('https://', 'http://', 'rtsp://', 'rtmp://')): # download url = file # warning: Pathlib turns :// -> :/ file = url2file(file) # '%2F' to '/', split https://url.com/file.txt?auth if Path(file).exists(): LOGGER.info(f'Found {clean_url(url)} locally at {file}') # file already exists else: downloads.safe_download(url=url, file=file, unzip=False) return file else: # search files = [] for d in 'models', 'datasets', 'tracker/cfg', 'yolo/cfg': # search directories files.extend(glob.glob(str(ROOT / d / '**' / file), recursive=True)) # find file if not files and hard: raise FileNotFoundError(f"'{file}' does not exist") elif len(files) > 1 and hard: raise FileNotFoundError(f"Multiple files match '{file}', specify exact path: {files}") return files[0] if len(files) else [] # return file def check_yaml(file, suffix=('.yaml', '.yml'), hard=True): """Search/download YAML file (if necessary) and return path, checking suffix.""" return check_file(file, suffix, hard=hard) def check_imshow(warn=False): """Check if environment supports image displays.""" try: assert not any((is_colab(), is_kaggle(), is_docker())) cv2.imshow('test', np.zeros((1, 1, 3))) cv2.waitKey(1) cv2.destroyAllWindows() cv2.waitKey(1) return True except Exception as e: if warn: LOGGER.warning(f'WARNING ⚠️ Environment does not support cv2.imshow() or PIL Image.show()\n{e}') return False def check_yolo(verbose=True, device=''): """Return a human-readable YOLO software and hardware summary.""" from ultralytics.yolo.utils.torch_utils import select_device if is_jupyter(): if check_requirements('wandb', install=False): os.system('pip uninstall -y wandb') # uninstall wandb: unwanted account creation prompt with infinite hang if is_colab(): shutil.rmtree('sample_data', ignore_errors=True) # remove colab /sample_data directory if verbose: # System info gib = 1 << 30 # bytes per GiB ram = psutil.virtual_memory().total total, used, free = shutil.disk_usage('/') s = f'({os.cpu_count()} CPUs, {ram / gib:.1f} GB RAM, {(total - free) / gib:.1f}/{total / gib:.1f} GB disk)' with contextlib.suppress(Exception): # clear display if ipython is installed from IPython import display display.clear_output() else: s = '' select_device(device=device, newline=False) LOGGER.info(f'Setup complete ✅ {s}') def check_amp(model): """ This function checks the PyTorch Automatic Mixed Precision (AMP) functionality of a YOLOv8 model. If the checks fail, it means there are anomalies with AMP on the system that may cause NaN losses or zero-mAP results, so AMP will be disabled during training. Args: model (nn.Module): A YOLOv8 model instance. Returns: (bool): Returns True if the AMP functionality works correctly with YOLOv8 model, else False. Raises: AssertionError: If the AMP checks fail, indicating anomalies with the AMP functionality on the system. """ device = next(model.parameters()).device # get model device if device.type in ('cpu', 'mps'): return False # AMP only used on CUDA devices def amp_allclose(m, im): """All close FP32 vs AMP results.""" a = m(im, device=device, verbose=False)[0].boxes.data # FP32 inference with torch.cuda.amp.autocast(True): b = m(im, device=device, verbose=False)[0].boxes.data # AMP inference del m return a.shape == b.shape and torch.allclose(a, b.float(), atol=0.5) # close to 0.5 absolute tolerance f = ROOT / 'assets/bus.jpg' # image to check im = f if f.exists() else 'https://ultralytics.com/images/bus.jpg' if ONLINE else np.ones((640, 640, 3)) prefix = colorstr('AMP: ') LOGGER.info(f'{prefix}running Automatic Mixed Precision (AMP) checks with YOLOv8n...') warning_msg = "Setting 'amp=True'. If you experience zero-mAP or NaN losses you can disable AMP with amp=False." try: from ultralytics import YOLO assert amp_allclose(YOLO('yolov8n.pt'), im) LOGGER.info(f'{prefix}checks passed ✅') except ConnectionError: LOGGER.warning(f'{prefix}checks skipped ⚠️, offline and unable to download YOLOv8n. {warning_msg}') except (AttributeError, ModuleNotFoundError): LOGGER.warning( f'{prefix}checks skipped ⚠️. Unable to load YOLOv8n due to possible Ultralytics package modifications. {warning_msg}' ) except AssertionError: LOGGER.warning(f'{prefix}checks failed ❌. Anomalies were detected with AMP on your system that may lead to ' f'NaN losses or zero-mAP results, so AMP will be disabled during training.') return False return True def git_describe(path=ROOT): # path must be a directory # Return human-readable git description, i.e. v5.0-5-g3e25f1e https://git-scm.com/docs/git-describe try: assert (Path(path) / '.git').is_dir() return subprocess.check_output(f'git -C {path} describe --tags --long --always', shell=True).decode()[:-1] except AssertionError: return '' def print_args(args: Optional[dict] = None, show_file=True, show_func=False): """Print function arguments (optional args dict).""" def strip_auth(v): """Clean longer Ultralytics HUB URLs by stripping potential authentication information.""" return clean_url(v) if (isinstance(v, str) and v.startswith('http') and len(v) > 100) else v x = inspect.currentframe().f_back # previous frame file, _, func, _, _ = inspect.getframeinfo(x) if args is None: # get args automatically args, _, _, frm = inspect.getargvalues(x) args = {k: v for k, v in frm.items() if k in args} try: file = Path(file).resolve().relative_to(ROOT).with_suffix('') except ValueError: file = Path(file).stem s = (f'{file}: ' if show_file else '') + (f'{func}: ' if show_func else '') LOGGER.info(colorstr(s) + ', '.join(f'{k}={strip_auth(v)}' for k, v in args.items())) ================================================ FILE: ultralytics/yolo/utils/dist.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import os import re import shutil import socket import sys import tempfile from pathlib import Path from . import USER_CONFIG_DIR from .torch_utils import TORCH_1_9 def find_free_network_port() -> int: """Finds a free port on localhost. It is useful in single-node training when we don't want to connect to a real main node but have to set the `MASTER_PORT` environment variable. """ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(('127.0.0.1', 0)) return s.getsockname()[1] # port def generate_ddp_file(trainer): """Generates a DDP file and returns its file name.""" module, name = f'{trainer.__class__.__module__}.{trainer.__class__.__name__}'.rsplit('.', 1) content = f'''overrides = {vars(trainer.args)} \nif __name__ == "__main__": from {module} import {name} from ultralytics.yolo.utils import DEFAULT_CFG_DICT cfg = DEFAULT_CFG_DICT.copy() cfg.update(save_dir='') # handle the extra key 'save_dir' trainer = {name}(cfg=cfg, overrides=overrides) trainer.train()''' (USER_CONFIG_DIR / 'DDP').mkdir(exist_ok=True) with tempfile.NamedTemporaryFile(prefix='_temp_', suffix=f'{id(trainer)}.py', mode='w+', encoding='utf-8', dir=USER_CONFIG_DIR / 'DDP', delete=False) as file: file.write(content) return file.name def generate_ddp_command(world_size, trainer): """Generates and returns command for distributed training.""" import __main__ # noqa local import to avoid https://github.com/Lightning-AI/lightning/issues/15218 if not trainer.resume: shutil.rmtree(trainer.save_dir) # remove the save_dir file = str(Path(sys.argv[0]).resolve()) safe_pattern = re.compile(r'^[a-zA-Z0-9_. /\\-]{1,128}$') # allowed characters and maximum of 100 characters if not (safe_pattern.match(file) and Path(file).exists() and file.endswith('.py')): # using CLI file = generate_ddp_file(trainer) dist_cmd = 'torch.distributed.run' if TORCH_1_9 else 'torch.distributed.launch' port = find_free_network_port() cmd = [sys.executable, '-m', dist_cmd, '--nproc_per_node', f'{world_size}', '--master_port', f'{port}', file] return cmd, file def ddp_cleanup(trainer, file): """Delete temp file if created.""" if f'{id(trainer)}.py' in file: # if temp_file suffix in file os.remove(file) ================================================ FILE: ultralytics/yolo/utils/downloads.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import contextlib import shutil import subprocess from itertools import repeat from multiprocessing.pool import ThreadPool from pathlib import Path from urllib import parse, request from zipfile import BadZipFile, ZipFile, is_zipfile import requests import torch from tqdm import tqdm from ultralytics.yolo.utils import LOGGER, checks, clean_url, emojis, is_online, url2file GITHUB_ASSET_NAMES = [f'yolov8{k}{suffix}.pt' for k in 'nsmlx' for suffix in ('', '6', '-cls', '-seg', '-pose')] + \ [f'yolov5{k}u.pt' for k in 'nsmlx'] + \ [f'yolov3{k}u.pt' for k in ('', '-spp', '-tiny')] + \ [f'sam_{k}.pt' for k in 'bl'] + \ [f'rtdetr-{k}.pt' for k in 'lx'] GITHUB_ASSET_STEMS = [Path(k).stem for k in GITHUB_ASSET_NAMES] def is_url(url, check=True): """Check if string is URL and check if URL exists.""" with contextlib.suppress(Exception): url = str(url) result = parse.urlparse(url) assert all([result.scheme, result.netloc]) # check if is url if check: with request.urlopen(url) as response: return response.getcode() == 200 # check if exists online return True return False def unzip_file(file, path=None, exclude=('.DS_Store', '__MACOSX')): """ Unzips a *.zip file to the specified path, excluding files containing strings in the exclude list. If the zipfile does not contain a single top-level directory, the function will create a new directory with the same name as the zipfile (without the extension) to extract its contents. If a path is not provided, the function will use the parent directory of the zipfile as the default path. Args: file (str): The path to the zipfile to be extracted. path (str, optional): The path to extract the zipfile to. Defaults to None. exclude (tuple, optional): A tuple of filename strings to be excluded. Defaults to ('.DS_Store', '__MACOSX'). Raises: BadZipFile: If the provided file does not exist or is not a valid zipfile. Returns: (Path): The path to the directory where the zipfile was extracted. """ if not (Path(file).exists() and is_zipfile(file)): raise BadZipFile(f"File '{file}' does not exist or is a bad zip file.") if path is None: path = Path(file).parent # default path with ZipFile(file) as zipObj: file_list = [f for f in zipObj.namelist() if all(x not in f for x in exclude)] top_level_dirs = {Path(f).parts[0] for f in file_list} if len(top_level_dirs) > 1 or not file_list[0].endswith('/'): path = Path(path) / Path(file).stem # define new unzip directory for f in file_list: zipObj.extract(f, path=path) return path # return unzip dir def check_disk_space(url='https://ultralytics.com/assets/coco128.zip', sf=1.5, hard=True): """ Check if there is sufficient disk space to download and store a file. Args: url (str, optional): The URL to the file. Defaults to 'https://ultralytics.com/assets/coco128.zip'. sf (float, optional): Safety factor, the multiplier for the required free space. Defaults to 2.0. hard (bool, optional): Whether to throw an error or not on insufficient disk space. Defaults to True. Returns: (bool): True if there is sufficient disk space, False otherwise. """ with contextlib.suppress(Exception): gib = 1 << 30 # bytes per GiB data = int(requests.head(url).headers['Content-Length']) / gib # file size (GB) total, used, free = (x / gib for x in shutil.disk_usage('/')) # bytes if data * sf < free: return True # sufficient space # Insufficient space text = (f'WARNING ⚠️ Insufficient free disk space {free:.1f} GB < {data * sf:.3f} GB required, ' f'Please free {data * sf - free:.1f} GB additional disk space and try again.') if hard: raise MemoryError(text) else: LOGGER.warning(text) return False # Pass if error return True def safe_download(url, file=None, dir=None, unzip=True, delete=False, curl=False, retry=3, min_bytes=1E0, progress=True): """ Downloads files from a URL, with options for retrying, unzipping, and deleting the downloaded file. Args: url (str): The URL of the file to be downloaded. file (str, optional): The filename of the downloaded file. If not provided, the file will be saved with the same name as the URL. dir (str, optional): The directory to save the downloaded file. If not provided, the file will be saved in the current working directory. unzip (bool, optional): Whether to unzip the downloaded file. Default: True. delete (bool, optional): Whether to delete the downloaded file after unzipping. Default: False. curl (bool, optional): Whether to use curl command line tool for downloading. Default: False. retry (int, optional): The number of times to retry the download in case of failure. Default: 3. min_bytes (float, optional): The minimum number of bytes that the downloaded file should have, to be considered a successful download. Default: 1E0. progress (bool, optional): Whether to display a progress bar during the download. Default: True. """ f = dir / url2file(url) if dir else Path(file) # URL converted to filename if '://' not in str(url) and Path(url).is_file(): # URL exists ('://' check required in Windows Python<3.10) f = Path(url) # filename elif not f.is_file(): # URL and file do not exist assert dir or file, 'dir or file required for download' f = dir / url2file(url) if dir else Path(file) desc = f'Downloading {clean_url(url)} to {f}' LOGGER.info(f'{desc}...') f.parent.mkdir(parents=True, exist_ok=True) # make directory if missing check_disk_space(url) for i in range(retry + 1): try: if curl or i > 0: # curl download with retry, continue s = 'sS' * (not progress) # silent r = subprocess.run(['curl', '-#', f'-{s}L', url, '-o', f, '--retry', '3', '-C', '-']).returncode assert r == 0, f'Curl return value {r}' else: # urllib download method = 'torch' if method == 'torch': torch.hub.download_url_to_file(url, f, progress=progress) else: from ultralytics.yolo.utils import TQDM_BAR_FORMAT with request.urlopen(url) as response, tqdm(total=int(response.getheader('Content-Length', 0)), desc=desc, disable=not progress, unit='B', unit_scale=True, unit_divisor=1024, bar_format=TQDM_BAR_FORMAT) as pbar: with open(f, 'wb') as f_opened: for data in response: f_opened.write(data) pbar.update(len(data)) if f.exists(): if f.stat().st_size > min_bytes: break # success f.unlink() # remove partial downloads except Exception as e: if i == 0 and not is_online(): raise ConnectionError(emojis(f'❌ Download failure for {url}. Environment is not online.')) from e elif i >= retry: raise ConnectionError(emojis(f'❌ Download failure for {url}. Retry limit reached.')) from e LOGGER.warning(f'⚠️ Download failure, retrying {i + 1}/{retry} {url}...') if unzip and f.exists() and f.suffix in ('', '.zip', '.tar', '.gz'): unzip_dir = dir or f.parent # unzip to dir if provided else unzip in place LOGGER.info(f'Unzipping {f} to {unzip_dir}...') if is_zipfile(f): unzip_dir = unzip_file(file=f, path=unzip_dir) # unzip elif f.suffix == '.tar': subprocess.run(['tar', 'xf', f, '--directory', unzip_dir], check=True) # unzip elif f.suffix == '.gz': subprocess.run(['tar', 'xfz', f, '--directory', unzip_dir], check=True) # unzip if delete: f.unlink() # remove zip return unzip_dir def attempt_download_asset(file, repo='ultralytics/assets', release='v0.0.0'): """Attempt file download from GitHub release assets if not found locally. release = 'latest', 'v6.2', etc.""" from ultralytics.yolo.utils import SETTINGS # scoped for circular import def github_assets(repository, version='latest'): """Return GitHub repo tag and assets (i.e. ['yolov8n.pt', 'yolov8s.pt', ...]).""" if version != 'latest': version = f'tags/{version}' # i.e. tags/v6.2 response = requests.get(f'https://api.github.com/repos/{repository}/releases/{version}').json() # github api return response['tag_name'], [x['name'] for x in response['assets']] # tag, assets # YOLOv3/5u updates file = str(file) file = checks.check_yolov5u_filename(file) file = Path(file.strip().replace("'", '')) if file.exists(): return str(file) elif (SETTINGS['weights_dir'] / file).exists(): return str(SETTINGS['weights_dir'] / file) else: # URL specified name = Path(parse.unquote(str(file))).name # decode '%2F' to '/' etc. if str(file).startswith(('http:/', 'https:/')): # download url = str(file).replace(':/', '://') # Pathlib turns :// -> :/ file = url2file(name) # parse authentication https://url.com/file.txt?auth... if Path(file).is_file(): LOGGER.info(f'Found {clean_url(url)} locally at {file}') # file already exists else: safe_download(url=url, file=file, min_bytes=1E5) return file # GitHub assets assets = GITHUB_ASSET_NAMES try: tag, assets = github_assets(repo, release) except Exception: try: tag, assets = github_assets(repo) # latest release except Exception: try: tag = subprocess.check_output(['git', 'tag']).decode().split()[-1] except Exception: tag = release file.parent.mkdir(parents=True, exist_ok=True) # make parent dir (if required) if name in assets: safe_download(url=f'https://github.com/{repo}/releases/download/{tag}/{name}', file=file, min_bytes=1E5) return str(file) def download(url, dir=Path.cwd(), unzip=True, delete=False, curl=False, threads=1, retry=3): """Downloads and unzips files concurrently if threads > 1, else sequentially.""" dir = Path(dir) dir.mkdir(parents=True, exist_ok=True) # make directory if threads > 1: with ThreadPool(threads) as pool: pool.map( lambda x: safe_download( url=x[0], dir=x[1], unzip=unzip, delete=delete, curl=curl, retry=retry, progress=threads <= 1), zip(url, repeat(dir))) pool.close() pool.join() else: for u in [url] if isinstance(url, (str, Path)) else url: safe_download(url=u, dir=dir, unzip=unzip, delete=delete, curl=curl, retry=retry) ================================================ FILE: ultralytics/yolo/utils/errors.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from ultralytics.yolo.utils import emojis class HUBModelError(Exception): def __init__(self, message='Model not found. Please check model URL and try again.'): """Create an exception for when a model is not found.""" super().__init__(emojis(message)) ================================================ FILE: ultralytics/yolo/utils/files.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import contextlib import glob import os import shutil from datetime import datetime from pathlib import Path class WorkingDirectory(contextlib.ContextDecorator): """Usage: @WorkingDirectory(dir) decorator or 'with WorkingDirectory(dir):' context manager.""" def __init__(self, new_dir): """Sets the working directory to 'new_dir' upon instantiation.""" self.dir = new_dir # new dir self.cwd = Path.cwd().resolve() # current dir def __enter__(self): """Changes the current directory to the specified directory.""" os.chdir(self.dir) def __exit__(self, exc_type, exc_val, exc_tb): """Restore the current working directory on context exit.""" os.chdir(self.cwd) def increment_path(path, exist_ok=False, sep='', mkdir=False): """ Increments a file or directory path, i.e. runs/exp --> runs/exp{sep}2, runs/exp{sep}3, ... etc. If the path exists and exist_ok is not set to True, the path will be incremented by appending a number and sep to the end of the path. If the path is a file, the file extension will be preserved. If the path is a directory, the number will be appended directly to the end of the path. If mkdir is set to True, the path will be created as a directory if it does not already exist. Args: path (str, pathlib.Path): Path to increment. exist_ok (bool, optional): If True, the path will not be incremented and returned as-is. Defaults to False. sep (str, optional): Separator to use between the path and the incrementation number. Defaults to ''. mkdir (bool, optional): Create a directory if it does not exist. Defaults to False. Returns: (pathlib.Path): Incremented path. """ path = Path(path) # os-agnostic if path.exists() and not exist_ok: path, suffix = (path.with_suffix(''), path.suffix) if path.is_file() else (path, '') # Method 1 for n in range(2, 9999): p = f'{path}{sep}{n}{suffix}' # increment path if not os.path.exists(p): # break path = Path(p) if mkdir: path.mkdir(parents=True, exist_ok=True) # make directory return path def file_age(path=__file__): """Return days since last file update.""" dt = (datetime.now() - datetime.fromtimestamp(Path(path).stat().st_mtime)) # delta return dt.days # + dt.seconds / 86400 # fractional days def file_date(path=__file__): """Return human-readable file modification date, i.e. '2021-3-26'.""" t = datetime.fromtimestamp(Path(path).stat().st_mtime) return f'{t.year}-{t.month}-{t.day}' def file_size(path): """Return file/dir size (MB).""" if isinstance(path, (str, Path)): mb = 1 << 20 # bytes to MiB (1024 ** 2) path = Path(path) if path.is_file(): return path.stat().st_size / mb elif path.is_dir(): return sum(f.stat().st_size for f in path.glob('**/*') if f.is_file()) / mb return 0.0 def get_latest_run(search_dir='.'): """Return path to most recent 'last.pt' in /runs (i.e. to --resume from).""" last_list = glob.glob(f'{search_dir}/**/last*.pt', recursive=True) return max(last_list, key=os.path.getctime) if last_list else '' def make_dirs(dir='new_dir/'): # Create folders dir = Path(dir) if dir.exists(): shutil.rmtree(dir) # delete dir for p in dir, dir / 'labels', dir / 'images': p.mkdir(parents=True, exist_ok=True) # make dir return dir ================================================ FILE: ultralytics/yolo/utils/instance.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from collections import abc from itertools import repeat from numbers import Number from typing import List import numpy as np from .ops import ltwh2xywh, ltwh2xyxy, resample_segments, xywh2ltwh, xywh2xyxy, xyxy2ltwh, xyxy2xywh def _ntuple(n): """From PyTorch internals.""" def parse(x): """Parse bounding boxes format between XYWH and LTWH.""" return x if isinstance(x, abc.Iterable) else tuple(repeat(x, n)) return parse to_4tuple = _ntuple(4) # `xyxy` means left top and right bottom # `xywh` means center x, center y and width, height(yolo format) # `ltwh` means left top and width, height(coco format) _formats = ['xyxy', 'xywh', 'ltwh'] __all__ = 'Bboxes', # tuple or list class Bboxes: """Now only numpy is supported.""" def __init__(self, bboxes, format='xyxy') -> None: assert format in _formats, f'Invalid bounding box format: {format}, format must be one of {_formats}' bboxes = bboxes[None, :] if bboxes.ndim == 1 else bboxes assert bboxes.ndim == 2 assert bboxes.shape[1] == 4 self.bboxes = bboxes self.format = format # self.normalized = normalized # def convert(self, format): # assert format in _formats # if self.format == format: # bboxes = self.bboxes # elif self.format == "xyxy": # if format == "xywh": # bboxes = xyxy2xywh(self.bboxes) # else: # bboxes = xyxy2ltwh(self.bboxes) # elif self.format == "xywh": # if format == "xyxy": # bboxes = xywh2xyxy(self.bboxes) # else: # bboxes = xywh2ltwh(self.bboxes) # else: # if format == "xyxy": # bboxes = ltwh2xyxy(self.bboxes) # else: # bboxes = ltwh2xywh(self.bboxes) # # return Bboxes(bboxes, format) def convert(self, format): """Converts bounding box format from one type to another.""" assert format in _formats, f'Invalid bounding box format: {format}, format must be one of {_formats}' if self.format == format: return elif self.format == 'xyxy': bboxes = xyxy2xywh(self.bboxes) if format == 'xywh' else xyxy2ltwh(self.bboxes) elif self.format == 'xywh': bboxes = xywh2xyxy(self.bboxes) if format == 'xyxy' else xywh2ltwh(self.bboxes) else: bboxes = ltwh2xyxy(self.bboxes) if format == 'xyxy' else ltwh2xywh(self.bboxes) self.bboxes = bboxes self.format = format def areas(self): """Return box areas.""" self.convert('xyxy') return (self.bboxes[:, 2] - self.bboxes[:, 0]) * (self.bboxes[:, 3] - self.bboxes[:, 1]) # def denormalize(self, w, h): # if not self.normalized: # return # assert (self.bboxes <= 1.0).all() # self.bboxes[:, 0::2] *= w # self.bboxes[:, 1::2] *= h # self.normalized = False # # def normalize(self, w, h): # if self.normalized: # return # assert (self.bboxes > 1.0).any() # self.bboxes[:, 0::2] /= w # self.bboxes[:, 1::2] /= h # self.normalized = True def mul(self, scale): """ Args: scale (tuple | list | int): the scale for four coords. """ if isinstance(scale, Number): scale = to_4tuple(scale) assert isinstance(scale, (tuple, list)) assert len(scale) == 4 self.bboxes[:, 0] *= scale[0] self.bboxes[:, 1] *= scale[1] self.bboxes[:, 2] *= scale[2] self.bboxes[:, 3] *= scale[3] def add(self, offset): """ Args: offset (tuple | list | int): the offset for four coords. """ if isinstance(offset, Number): offset = to_4tuple(offset) assert isinstance(offset, (tuple, list)) assert len(offset) == 4 self.bboxes[:, 0] += offset[0] self.bboxes[:, 1] += offset[1] self.bboxes[:, 2] += offset[2] self.bboxes[:, 3] += offset[3] def __len__(self): """Return the number of boxes.""" return len(self.bboxes) @classmethod def concatenate(cls, boxes_list: List['Bboxes'], axis=0) -> 'Bboxes': """ Concatenate a list of Bboxes objects into a single Bboxes object. Args: boxes_list (List[Bboxes]): A list of Bboxes objects to concatenate. axis (int, optional): The axis along which to concatenate the bounding boxes. Defaults to 0. Returns: Bboxes: A new Bboxes object containing the concatenated bounding boxes. Note: The input should be a list or tuple of Bboxes objects. """ assert isinstance(boxes_list, (list, tuple)) if not boxes_list: return cls(np.empty(0)) assert all(isinstance(box, Bboxes) for box in boxes_list) if len(boxes_list) == 1: return boxes_list[0] return cls(np.concatenate([b.bboxes for b in boxes_list], axis=axis)) def __getitem__(self, index) -> 'Bboxes': """ Retrieve a specific bounding box or a set of bounding boxes using indexing. Args: index (int, slice, or np.ndarray): The index, slice, or boolean array to select the desired bounding boxes. Returns: Bboxes: A new Bboxes object containing the selected bounding boxes. Raises: AssertionError: If the indexed bounding boxes do not form a 2-dimensional matrix. Note: When using boolean indexing, make sure to provide a boolean array with the same length as the number of bounding boxes. """ if isinstance(index, int): return Bboxes(self.bboxes[index].view(1, -1)) b = self.bboxes[index] assert b.ndim == 2, f'Indexing on Bboxes with {index} failed to return a matrix!' return Bboxes(b) class Instances: def __init__(self, bboxes, segments=None, keypoints=None, bbox_format='xywh', normalized=True) -> None: """ Args: bboxes (ndarray): bboxes with shape [N, 4]. segments (list | ndarray): segments. keypoints (ndarray): keypoints(x, y, visible) with shape [N, 17, 3]. """ if segments is None: segments = [] self._bboxes = Bboxes(bboxes=bboxes, format=bbox_format) self.keypoints = keypoints self.normalized = normalized if len(segments) > 0: # list[np.array(1000, 2)] * num_samples segments = resample_segments(segments) # (N, 1000, 2) segments = np.stack(segments, axis=0) else: segments = np.zeros((0, 1000, 2), dtype=np.float32) self.segments = segments def convert_bbox(self, format): """Convert bounding box format.""" self._bboxes.convert(format=format) @property def bbox_areas(self): """Calculate the area of bounding boxes.""" return self._bboxes.areas() def scale(self, scale_w, scale_h, bbox_only=False): """this might be similar with denormalize func but without normalized sign.""" self._bboxes.mul(scale=(scale_w, scale_h, scale_w, scale_h)) if bbox_only: return self.segments[..., 0] *= scale_w self.segments[..., 1] *= scale_h if self.keypoints is not None: self.keypoints[..., 0] *= scale_w self.keypoints[..., 1] *= scale_h def denormalize(self, w, h): """Denormalizes boxes, segments, and keypoints from normalized coordinates.""" if not self.normalized: return self._bboxes.mul(scale=(w, h, w, h)) self.segments[..., 0] *= w self.segments[..., 1] *= h if self.keypoints is not None: self.keypoints[..., 0] *= w self.keypoints[..., 1] *= h self.normalized = False def normalize(self, w, h): """Normalize bounding boxes, segments, and keypoints to image dimensions.""" if self.normalized: return self._bboxes.mul(scale=(1 / w, 1 / h, 1 / w, 1 / h)) self.segments[..., 0] /= w self.segments[..., 1] /= h if self.keypoints is not None: self.keypoints[..., 0] /= w self.keypoints[..., 1] /= h self.normalized = True def add_padding(self, padw, padh): """Handle rect and mosaic situation.""" assert not self.normalized, 'you should add padding with absolute coordinates.' self._bboxes.add(offset=(padw, padh, padw, padh)) self.segments[..., 0] += padw self.segments[..., 1] += padh if self.keypoints is not None: self.keypoints[..., 0] += padw self.keypoints[..., 1] += padh def __getitem__(self, index) -> 'Instances': """ Retrieve a specific instance or a set of instances using indexing. Args: index (int, slice, or np.ndarray): The index, slice, or boolean array to select the desired instances. Returns: Instances: A new Instances object containing the selected bounding boxes, segments, and keypoints if present. Note: When using boolean indexing, make sure to provide a boolean array with the same length as the number of instances. """ segments = self.segments[index] if len(self.segments) else self.segments keypoints = self.keypoints[index] if self.keypoints is not None else None bboxes = self.bboxes[index] bbox_format = self._bboxes.format return Instances( bboxes=bboxes, segments=segments, keypoints=keypoints, bbox_format=bbox_format, normalized=self.normalized, ) def flipud(self, h): """Flips the coordinates of bounding boxes, segments, and keypoints vertically.""" if self._bboxes.format == 'xyxy': y1 = self.bboxes[:, 1].copy() y2 = self.bboxes[:, 3].copy() self.bboxes[:, 1] = h - y2 self.bboxes[:, 3] = h - y1 else: self.bboxes[:, 1] = h - self.bboxes[:, 1] self.segments[..., 1] = h - self.segments[..., 1] if self.keypoints is not None: self.keypoints[..., 1] = h - self.keypoints[..., 1] def fliplr(self, w): """Reverses the order of the bounding boxes and segments horizontally.""" if self._bboxes.format == 'xyxy': x1 = self.bboxes[:, 0].copy() x2 = self.bboxes[:, 2].copy() self.bboxes[:, 0] = w - x2 self.bboxes[:, 2] = w - x1 else: self.bboxes[:, 0] = w - self.bboxes[:, 0] self.segments[..., 0] = w - self.segments[..., 0] if self.keypoints is not None: self.keypoints[..., 0] = w - self.keypoints[..., 0] def clip(self, w, h): """Clips bounding boxes, segments, and keypoints values to stay within image boundaries.""" ori_format = self._bboxes.format self.convert_bbox(format='xyxy') self.bboxes[:, [0, 2]] = self.bboxes[:, [0, 2]].clip(0, w) self.bboxes[:, [1, 3]] = self.bboxes[:, [1, 3]].clip(0, h) if ori_format != 'xyxy': self.convert_bbox(format=ori_format) self.segments[..., 0] = self.segments[..., 0].clip(0, w) self.segments[..., 1] = self.segments[..., 1].clip(0, h) if self.keypoints is not None: self.keypoints[..., 0] = self.keypoints[..., 0].clip(0, w) self.keypoints[..., 1] = self.keypoints[..., 1].clip(0, h) def remove_zero_area_boxes(self): """Remove zero-area boxes, i.e. after clipping some boxes may have zero width or height. This removes them.""" good = self.bbox_areas > 0 if not all(good): self._bboxes = self._bboxes[good] if len(self.segments): self.segments = self.segments[good] if self.keypoints is not None: self.keypoints = self.keypoints[good] return good def update(self, bboxes, segments=None, keypoints=None): """Updates instance variables.""" self._bboxes = Bboxes(bboxes, format=self._bboxes.format) if segments is not None: self.segments = segments if keypoints is not None: self.keypoints = keypoints def __len__(self): """Return the length of the instance list.""" return len(self.bboxes) @classmethod def concatenate(cls, instances_list: List['Instances'], axis=0) -> 'Instances': """ Concatenates a list of Instances objects into a single Instances object. Args: instances_list (List[Instances]): A list of Instances objects to concatenate. axis (int, optional): The axis along which the arrays will be concatenated. Defaults to 0. Returns: Instances: A new Instances object containing the concatenated bounding boxes, segments, and keypoints if present. Note: The `Instances` objects in the list should have the same properties, such as the format of the bounding boxes, whether keypoints are present, and if the coordinates are normalized. """ assert isinstance(instances_list, (list, tuple)) if not instances_list: return cls(np.empty(0)) assert all(isinstance(instance, Instances) for instance in instances_list) if len(instances_list) == 1: return instances_list[0] use_keypoint = instances_list[0].keypoints is not None bbox_format = instances_list[0]._bboxes.format normalized = instances_list[0].normalized cat_boxes = np.concatenate([ins.bboxes for ins in instances_list], axis=axis) cat_segments = np.concatenate([b.segments for b in instances_list], axis=axis) cat_keypoints = np.concatenate([b.keypoints for b in instances_list], axis=axis) if use_keypoint else None return cls(cat_boxes, cat_segments, cat_keypoints, bbox_format, normalized) @property def bboxes(self): """Return bounding boxes.""" return self._bboxes.bboxes ================================================ FILE: ultralytics/yolo/utils/loss.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import torch import torch.nn as nn import torch.nn.functional as F from ultralytics.yolo.utils.metrics import OKS_SIGMA from ultralytics.yolo.utils.ops import crop_mask, xywh2xyxy, xyxy2xywh from ultralytics.yolo.utils.tal import TaskAlignedAssigner, dist2bbox, make_anchors from .metrics import bbox_iou from .tal import bbox2dist class VarifocalLoss(nn.Module): """Varifocal loss by Zhang et al. https://arxiv.org/abs/2008.13367.""" def __init__(self): """Initialize the VarifocalLoss class.""" super().__init__() def forward(self, pred_score, gt_score, label, alpha=0.75, gamma=2.0): """Computes varfocal loss.""" weight = alpha * pred_score.sigmoid().pow(gamma) * (1 - label) + gt_score * label with torch.cuda.amp.autocast(enabled=False): loss = (F.binary_cross_entropy_with_logits(pred_score.float(), gt_score.float(), reduction='none') * weight).mean(1).sum() return loss # Losses class FocalLoss(nn.Module): """Wraps focal loss around existing loss_fcn(), i.e. criteria = FocalLoss(nn.BCEWithLogitsLoss(), gamma=1.5).""" def __init__(self, ): super().__init__() def forward(self, pred, label, gamma=1.5, alpha=0.25): """Calculates and updates confusion matrix for object detection/classification tasks.""" loss = F.binary_cross_entropy_with_logits(pred, label, reduction='none') # p_t = torch.exp(-loss) # loss *= self.alpha * (1.000001 - p_t) ** self.gamma # non-zero power for gradient stability # TF implementation https://github.com/tensorflow/addons/blob/v0.7.1/tensorflow_addons/losses/focal_loss.py pred_prob = pred.sigmoid() # prob from logits p_t = label * pred_prob + (1 - label) * (1 - pred_prob) modulating_factor = (1.0 - p_t) ** gamma loss *= modulating_factor if alpha > 0: alpha_factor = label * alpha + (1 - label) * (1 - alpha) loss *= alpha_factor return loss.mean(1).sum() class BboxLoss(nn.Module): def __init__(self, reg_max, use_dfl=False): """Initialize the BboxLoss module with regularization maximum and DFL settings.""" super().__init__() self.reg_max = reg_max self.use_dfl = use_dfl def forward(self, pred_dist, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask): """IoU loss.""" weight = target_scores.sum(-1)[fg_mask].unsqueeze(-1) iou = bbox_iou(pred_bboxes[fg_mask], target_bboxes[fg_mask], xywh=False, CIoU=True) loss_iou = ((1.0 - iou) * weight).sum() / target_scores_sum # DFL loss if self.use_dfl: target_ltrb = bbox2dist(anchor_points, target_bboxes, self.reg_max) loss_dfl = self._df_loss(pred_dist[fg_mask].view(-1, self.reg_max + 1), target_ltrb[fg_mask]) * weight loss_dfl = loss_dfl.sum() / target_scores_sum else: loss_dfl = torch.tensor(0.0).to(pred_dist.device) return loss_iou, loss_dfl @staticmethod def _df_loss(pred_dist, target): """Return sum of left and right DFL losses.""" # Distribution Focal Loss (DFL) proposed in Generalized Focal Loss https://ieeexplore.ieee.org/document/9792391 tl = target.long() # target left tr = tl + 1 # target right wl = tr - target # weight left wr = 1 - wl # weight right return (F.cross_entropy(pred_dist, tl.view(-1), reduction='none').view(tl.shape) * wl + F.cross_entropy(pred_dist, tr.view(-1), reduction='none').view(tl.shape) * wr).mean(-1, keepdim=True) class KeypointLoss(nn.Module): def __init__(self, sigmas) -> None: super().__init__() self.sigmas = sigmas def forward(self, pred_kpts, gt_kpts, kpt_mask, area): """Calculates keypoint loss factor and Euclidean distance loss for predicted and actual keypoints.""" d = (pred_kpts[..., 0] - gt_kpts[..., 0]) ** 2 + (pred_kpts[..., 1] - gt_kpts[..., 1]) ** 2 kpt_loss_factor = (torch.sum(kpt_mask != 0) + torch.sum(kpt_mask == 0)) / (torch.sum(kpt_mask != 0) + 1e-9) # e = d / (2 * (area * self.sigmas) ** 2 + 1e-9) # from formula e = d / (2 * self.sigmas) ** 2 / (area + 1e-9) / 2 # from cocoeval return kpt_loss_factor * ((1 - torch.exp(-e)) * kpt_mask).mean() # Criterion class for computing Detection training losses class v8DetectionLoss: def __init__(self, model): # model must be de-paralleled device = next(model.parameters()).device # get model device h = model.args # hyperparameters m = model.model[-1] # Detect() module self.bce = nn.BCEWithLogitsLoss(reduction='none') self.hyp = h self.stride = m.stride # model strides self.nc = m.nc # number of classes self.no = m.no self.reg_max = m.reg_max self.device = device self.use_dfl = m.reg_max > 1 self.assigner = TaskAlignedAssigner(topk=10, num_classes=self.nc, alpha=0.5, beta=6.0) self.bbox_loss = BboxLoss(m.reg_max - 1, use_dfl=self.use_dfl).to(device) self.proj = torch.arange(m.reg_max, dtype=torch.float, device=device) def preprocess(self, targets, batch_size, scale_tensor): """Preprocesses the target counts and matches with the input batch size to output a tensor.""" if targets.shape[0] == 0: out = torch.zeros(batch_size, 0, 5, device=self.device) else: i = targets[:, 0] # image index _, counts = i.unique(return_counts=True) counts = counts.to(dtype=torch.int32) out = torch.zeros(batch_size, counts.max(), 5, device=self.device) for j in range(batch_size): matches = i == j n = matches.sum() if n: out[j, :n] = targets[matches, 1:] out[..., 1:5] = xywh2xyxy(out[..., 1:5].mul_(scale_tensor)) return out def bbox_decode(self, anchor_points, pred_dist): """Decode predicted object bounding box coordinates from anchor points and distribution.""" if self.use_dfl: b, a, c = pred_dist.shape # batch, anchors, channels pred_dist = pred_dist.view(b, a, 4, c // 4).softmax(3).matmul(self.proj.type(pred_dist.dtype)) # pred_dist = pred_dist.view(b, a, c // 4, 4).transpose(2,3).softmax(3).matmul(self.proj.type(pred_dist.dtype)) # pred_dist = (pred_dist.view(b, a, c // 4, 4).softmax(2) * self.proj.type(pred_dist.dtype).view(1, 1, -1, 1)).sum(2) return dist2bbox(pred_dist, anchor_points, xywh=False) def __call__(self, preds, batch): """Calculate the sum of the loss for box, cls and dfl multiplied by batch size.""" loss = torch.zeros(3, device=self.device) # box, cls, dfl feats = preds[1] if isinstance(preds, tuple) else preds pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split( (self.reg_max * 4, self.nc), 1) pred_scores = pred_scores.permute(0, 2, 1).contiguous() pred_distri = pred_distri.permute(0, 2, 1).contiguous() dtype = pred_scores.dtype batch_size = pred_scores.shape[0] imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w) anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5) # targets targets = torch.cat((batch['batch_idx'].view(-1, 1), batch['cls'].view(-1, 1), batch['bboxes']), 1) targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]]) gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0) # pboxes pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4) _, target_bboxes, target_scores, fg_mask, _ = self.assigner( pred_scores.detach().sigmoid(), (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype), anchor_points * stride_tensor, gt_labels, gt_bboxes, mask_gt) target_scores_sum = max(target_scores.sum(), 1) # cls loss # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way loss[1] = self.bce(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE # bbox loss if fg_mask.sum(): target_bboxes /= stride_tensor loss[0], loss[2] = self.bbox_loss(pred_distri, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask) loss[0] *= self.hyp.box # box gain loss[1] *= self.hyp.cls # cls gain loss[2] *= self.hyp.dfl # dfl gain return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl) # Criterion class for computing training losses class v8SegmentationLoss(v8DetectionLoss): def __init__(self, model): # model must be de-paralleled super().__init__(model) self.nm = model.model[-1].nm # number of masks self.overlap = model.args.overlap_mask def __call__(self, preds, batch): """Calculate and return the loss for the YOLO model.""" loss = torch.zeros(4, device=self.device) # box, cls, dfl feats, pred_masks, proto = preds if len(preds) == 3 else preds[1] batch_size, _, mask_h, mask_w = proto.shape # batch size, number of masks, mask height, mask width pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split( (self.reg_max * 4, self.nc), 1) # b, grids, .. pred_scores = pred_scores.permute(0, 2, 1).contiguous() pred_distri = pred_distri.permute(0, 2, 1).contiguous() pred_masks = pred_masks.permute(0, 2, 1).contiguous() dtype = pred_scores.dtype imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w) anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5) # targets try: batch_idx = batch['batch_idx'].view(-1, 1) targets = torch.cat((batch_idx, batch['cls'].view(-1, 1), batch['bboxes']), 1) targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]]) gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0) except RuntimeError as e: raise TypeError('ERROR ❌ segment dataset incorrectly formatted or not a segment dataset.\n' "This error can occur when incorrectly training a 'segment' model on a 'detect' dataset, " "i.e. 'yolo train model=yolov8n-seg.pt data=coco128.yaml'.\nVerify your dataset is a " "correctly formatted 'segment' dataset using 'data=coco128-seg.yaml' " 'as an example.\nSee https://docs.ultralytics.com/tasks/segment/ for help.') from e # pboxes pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4) _, target_bboxes, target_scores, fg_mask, target_gt_idx = self.assigner( pred_scores.detach().sigmoid(), (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype), anchor_points * stride_tensor, gt_labels, gt_bboxes, mask_gt) target_scores_sum = max(target_scores.sum(), 1) # cls loss # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way loss[2] = self.bce(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE if fg_mask.sum(): # bbox loss loss[0], loss[3] = self.bbox_loss(pred_distri, pred_bboxes, anchor_points, target_bboxes / stride_tensor, target_scores, target_scores_sum, fg_mask) # masks loss masks = batch['masks'].to(self.device).float() if tuple(masks.shape[-2:]) != (mask_h, mask_w): # downsample masks = F.interpolate(masks[None], (mask_h, mask_w), mode='nearest')[0] for i in range(batch_size): if fg_mask[i].sum(): mask_idx = target_gt_idx[i][fg_mask[i]] if self.overlap: gt_mask = torch.where(masks[[i]] == (mask_idx + 1).view(-1, 1, 1), 1.0, 0.0) else: gt_mask = masks[batch_idx.view(-1) == i][mask_idx] xyxyn = target_bboxes[i][fg_mask[i]] / imgsz[[1, 0, 1, 0]] marea = xyxy2xywh(xyxyn)[:, 2:].prod(1) mxyxy = xyxyn * torch.tensor([mask_w, mask_h, mask_w, mask_h], device=self.device) loss[1] += self.single_mask_loss(gt_mask, pred_masks[i][fg_mask[i]], proto[i], mxyxy, marea) # seg # WARNING: lines below prevents Multi-GPU DDP 'unused gradient' PyTorch errors, do not remove else: loss[1] += (proto * 0).sum() + (pred_masks * 0).sum() # inf sums may lead to nan loss # WARNING: lines below prevent Multi-GPU DDP 'unused gradient' PyTorch errors, do not remove else: loss[1] += (proto * 0).sum() + (pred_masks * 0).sum() # inf sums may lead to nan loss loss[0] *= self.hyp.box # box gain loss[1] *= self.hyp.box / batch_size # seg gain loss[2] *= self.hyp.cls # cls gain loss[3] *= self.hyp.dfl # dfl gain return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl) def single_mask_loss(self, gt_mask, pred, proto, xyxy, area): """Mask loss for one image.""" pred_mask = (pred @ proto.view(self.nm, -1)).view(-1, *proto.shape[1:]) # (n, 32) @ (32,80,80) -> (n,80,80) loss = F.binary_cross_entropy_with_logits(pred_mask, gt_mask, reduction='none') return (crop_mask(loss, xyxy).mean(dim=(1, 2)) / area).mean() # Criterion class for computing training losses class v8PoseLoss(v8DetectionLoss): def __init__(self, model): # model must be de-paralleled super().__init__(model) self.kpt_shape = model.model[-1].kpt_shape self.bce_pose = nn.BCEWithLogitsLoss() is_pose = self.kpt_shape == [17, 3] nkpt = self.kpt_shape[0] # number of keypoints sigmas = torch.from_numpy(OKS_SIGMA).to(self.device) if is_pose else torch.ones(nkpt, device=self.device) / nkpt self.keypoint_loss = KeypointLoss(sigmas=sigmas) def __call__(self, preds, batch): """Calculate the total loss and detach it.""" loss = torch.zeros(5, device=self.device) # box, cls, dfl, kpt_location, kpt_visibility feats, pred_kpts = preds if isinstance(preds[0], list) else preds[1] pred_distri, pred_scores = torch.cat([xi.view(feats[0].shape[0], self.no, -1) for xi in feats], 2).split( (self.reg_max * 4, self.nc), 1) # b, grids, .. pred_scores = pred_scores.permute(0, 2, 1).contiguous() pred_distri = pred_distri.permute(0, 2, 1).contiguous() pred_kpts = pred_kpts.permute(0, 2, 1).contiguous() dtype = pred_scores.dtype imgsz = torch.tensor(feats[0].shape[2:], device=self.device, dtype=dtype) * self.stride[0] # image size (h,w) anchor_points, stride_tensor = make_anchors(feats, self.stride, 0.5) # targets batch_size = pred_scores.shape[0] batch_idx = batch['batch_idx'].view(-1, 1) targets = torch.cat((batch_idx, batch['cls'].view(-1, 1), batch['bboxes']), 1) targets = self.preprocess(targets.to(self.device), batch_size, scale_tensor=imgsz[[1, 0, 1, 0]]) gt_labels, gt_bboxes = targets.split((1, 4), 2) # cls, xyxy mask_gt = gt_bboxes.sum(2, keepdim=True).gt_(0) # pboxes pred_bboxes = self.bbox_decode(anchor_points, pred_distri) # xyxy, (b, h*w, 4) pred_kpts = self.kpts_decode(anchor_points, pred_kpts.view(batch_size, -1, *self.kpt_shape)) # (b, h*w, 17, 3) _, target_bboxes, target_scores, fg_mask, target_gt_idx = self.assigner( pred_scores.detach().sigmoid(), (pred_bboxes.detach() * stride_tensor).type(gt_bboxes.dtype), anchor_points * stride_tensor, gt_labels, gt_bboxes, mask_gt) target_scores_sum = max(target_scores.sum(), 1) # cls loss # loss[1] = self.varifocal_loss(pred_scores, target_scores, target_labels) / target_scores_sum # VFL way loss[3] = self.bce(pred_scores, target_scores.to(dtype)).sum() / target_scores_sum # BCE # bbox loss if fg_mask.sum(): target_bboxes /= stride_tensor loss[0], loss[4] = self.bbox_loss(pred_distri, pred_bboxes, anchor_points, target_bboxes, target_scores, target_scores_sum, fg_mask) keypoints = batch['keypoints'].to(self.device).float().clone() keypoints[..., 0] *= imgsz[1] keypoints[..., 1] *= imgsz[0] for i in range(batch_size): if fg_mask[i].sum(): idx = target_gt_idx[i][fg_mask[i]] gt_kpt = keypoints[batch_idx.view(-1) == i][idx] # (n, 51) gt_kpt[..., 0] /= stride_tensor[fg_mask[i]] gt_kpt[..., 1] /= stride_tensor[fg_mask[i]] area = xyxy2xywh(target_bboxes[i][fg_mask[i]])[:, 2:].prod(1, keepdim=True) pred_kpt = pred_kpts[i][fg_mask[i]] kpt_mask = gt_kpt[..., 2] != 0 loss[1] += self.keypoint_loss(pred_kpt, gt_kpt, kpt_mask, area) # pose loss # kpt_score loss if pred_kpt.shape[-1] == 3: loss[2] += self.bce_pose(pred_kpt[..., 2], kpt_mask.float()) # keypoint obj loss loss[0] *= self.hyp.box # box gain loss[1] *= self.hyp.pose / batch_size # pose gain loss[2] *= self.hyp.kobj / batch_size # kobj gain loss[3] *= self.hyp.cls # cls gain loss[4] *= self.hyp.dfl # dfl gain return loss.sum() * batch_size, loss.detach() # loss(box, cls, dfl) def kpts_decode(self, anchor_points, pred_kpts): """Decodes predicted keypoints to image coordinates.""" y = pred_kpts.clone() y[..., :2] *= 2.0 y[..., 0] += anchor_points[:, [0]] - 0.5 y[..., 1] += anchor_points[:, [1]] - 0.5 return y class v8ClassificationLoss: def __call__(self, preds, batch): """Compute the classification loss between predictions and true labels.""" loss = torch.nn.functional.cross_entropy(preds, batch['cls'], reduction='sum') / 64 loss_items = loss.detach() return loss, loss_items ================================================ FILE: ultralytics/yolo/utils/metrics.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license """ Model validation metrics """ import math import warnings from pathlib import Path import matplotlib.pyplot as plt import numpy as np import torch from ultralytics.yolo.utils import LOGGER, SimpleClass, TryExcept, plt_settings OKS_SIGMA = np.array([.26, .25, .25, .35, .35, .79, .79, .72, .72, .62, .62, 1.07, 1.07, .87, .87, .89, .89]) / 10.0 # Boxes def box_area(box): """Return box area, where box shape is xyxy(4,n).""" return (box[2] - box[0]) * (box[3] - box[1]) def bbox_ioa(box1, box2, eps=1e-7): """ Calculate the intersection over box2 area given box1 and box2. Boxes are in x1y1x2y2 format. Args: box1 (np.array): A numpy array of shape (n, 4) representing n bounding boxes. box2 (np.array): A numpy array of shape (m, 4) representing m bounding boxes. eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. Returns: (np.array): A numpy array of shape (n, m) representing the intersection over box2 area. """ # Get the coordinates of bounding boxes b1_x1, b1_y1, b1_x2, b1_y2 = box1.T b2_x1, b2_y1, b2_x2, b2_y2 = box2.T # Intersection area inter_area = (np.minimum(b1_x2[:, None], b2_x2) - np.maximum(b1_x1[:, None], b2_x1)).clip(0) * \ (np.minimum(b1_y2[:, None], b2_y2) - np.maximum(b1_y1[:, None], b2_y1)).clip(0) # box2 area box2_area = (b2_x2 - b2_x1) * (b2_y2 - b2_y1) + eps # Intersection over box2 area return inter_area / box2_area def box_iou(box1, box2, eps=1e-7): """ Calculate intersection-over-union (IoU) of boxes. Both sets of boxes are expected to be in (x1, y1, x2, y2) format. Based on https://github.com/pytorch/vision/blob/master/torchvision/ops/boxes.py Args: box1 (torch.Tensor): A tensor of shape (N, 4) representing N bounding boxes. box2 (torch.Tensor): A tensor of shape (M, 4) representing M bounding boxes. eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. Returns: (torch.Tensor): An NxM tensor containing the pairwise IoU values for every element in box1 and box2. """ # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2) (a1, a2), (b1, b2) = box1.unsqueeze(1).chunk(2, 2), box2.unsqueeze(0).chunk(2, 2) inter = (torch.min(a2, b2) - torch.max(a1, b1)).clamp_(0).prod(2) # IoU = inter / (area1 + area2 - inter) return inter / ((a2 - a1).prod(2) + (b2 - b1).prod(2) - inter + eps) def bbox_iou(box1, box2, xywh=True, GIoU=False, DIoU=False, CIoU=False, eps=1e-7): """ Calculate Intersection over Union (IoU) of box1(1, 4) to box2(n, 4). Args: box1 (torch.Tensor): A tensor representing a single bounding box with shape (1, 4). box2 (torch.Tensor): A tensor representing n bounding boxes with shape (n, 4). xywh (bool, optional): If True, input boxes are in (x, y, w, h) format. If False, input boxes are in (x1, y1, x2, y2) format. Defaults to True. GIoU (bool, optional): If True, calculate Generalized IoU. Defaults to False. DIoU (bool, optional): If True, calculate Distance IoU. Defaults to False. CIoU (bool, optional): If True, calculate Complete IoU. Defaults to False. eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. Returns: (torch.Tensor): IoU, GIoU, DIoU, or CIoU values depending on the specified flags. """ # Get the coordinates of bounding boxes if xywh: # transform from xywh to xyxy (x1, y1, w1, h1), (x2, y2, w2, h2) = box1.chunk(4, -1), box2.chunk(4, -1) w1_, h1_, w2_, h2_ = w1 / 2, h1 / 2, w2 / 2, h2 / 2 b1_x1, b1_x2, b1_y1, b1_y2 = x1 - w1_, x1 + w1_, y1 - h1_, y1 + h1_ b2_x1, b2_x2, b2_y1, b2_y2 = x2 - w2_, x2 + w2_, y2 - h2_, y2 + h2_ else: # x1, y1, x2, y2 = box1 b1_x1, b1_y1, b1_x2, b1_y2 = box1.chunk(4, -1) b2_x1, b2_y1, b2_x2, b2_y2 = box2.chunk(4, -1) w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps # Intersection area inter = (b1_x2.minimum(b2_x2) - b1_x1.maximum(b2_x1)).clamp_(0) * \ (b1_y2.minimum(b2_y2) - b1_y1.maximum(b2_y1)).clamp_(0) # Union Area union = w1 * h1 + w2 * h2 - inter + eps # IoU iou = inter / union if CIoU or DIoU or GIoU: cw = b1_x2.maximum(b2_x2) - b1_x1.minimum(b2_x1) # convex (smallest enclosing box) width ch = b1_y2.maximum(b2_y2) - b1_y1.minimum(b2_y1) # convex height if CIoU or DIoU: # Distance or Complete IoU https://arxiv.org/abs/1911.08287v1 c2 = cw ** 2 + ch ** 2 + eps # convex diagonal squared rho2 = ((b2_x1 + b2_x2 - b1_x1 - b1_x2) ** 2 + (b2_y1 + b2_y2 - b1_y1 - b1_y2) ** 2) / 4 # center dist ** 2 if CIoU: # https://github.com/Zzh-tju/DIoU-SSD-pytorch/blob/master/utils/box/box_utils.py#L47 v = (4 / math.pi ** 2) * (torch.atan(w2 / h2) - torch.atan(w1 / h1)).pow(2) with torch.no_grad(): alpha = v / (v - iou + (1 + eps)) return iou - (rho2 / c2 + v * alpha) # CIoU return iou - rho2 / c2 # DIoU c_area = cw * ch + eps # convex area return iou - (c_area - union) / c_area # GIoU https://arxiv.org/pdf/1902.09630.pdf return iou # IoU def mask_iou(mask1, mask2, eps=1e-7): """ Calculate masks IoU. Args: mask1 (torch.Tensor): A tensor of shape (N, n) where N is the number of ground truth objects and n is the product of image width and height. mask2 (torch.Tensor): A tensor of shape (M, n) where M is the number of predicted objects and n is the product of image width and height. eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. Returns: (torch.Tensor): A tensor of shape (N, M) representing masks IoU. """ intersection = torch.matmul(mask1, mask2.T).clamp_(0) union = (mask1.sum(1)[:, None] + mask2.sum(1)[None]) - intersection # (area1 + area2) - intersection return intersection / (union + eps) def kpt_iou(kpt1, kpt2, area, sigma, eps=1e-7): """ Calculate Object Keypoint Similarity (OKS). Args: kpt1 (torch.Tensor): A tensor of shape (N, 17, 3) representing ground truth keypoints. kpt2 (torch.Tensor): A tensor of shape (M, 17, 3) representing predicted keypoints. area (torch.Tensor): A tensor of shape (N,) representing areas from ground truth. sigma (list): A list containing 17 values representing keypoint scales. eps (float, optional): A small value to avoid division by zero. Defaults to 1e-7. Returns: (torch.Tensor): A tensor of shape (N, M) representing keypoint similarities. """ d = (kpt1[:, None, :, 0] - kpt2[..., 0]) ** 2 + (kpt1[:, None, :, 1] - kpt2[..., 1]) ** 2 # (N, M, 17) sigma = torch.tensor(sigma, device=kpt1.device, dtype=kpt1.dtype) # (17, ) kpt_mask = kpt1[..., 2] != 0 # (N, 17) e = d / (2 * sigma) ** 2 / (area[:, None, None] + eps) / 2 # from cocoeval # e = d / ((area[None, :, None] + eps) * sigma) ** 2 / 2 # from formula return (torch.exp(-e) * kpt_mask[:, None]).sum(-1) / (kpt_mask.sum(-1)[:, None] + eps) def smooth_BCE(eps=0.1): # https://github.com/ultralytics/yolov3/issues/238#issuecomment-598028441 # return positive, negative label smoothing BCE targets return 1.0 - 0.5 * eps, 0.5 * eps class ConfusionMatrix: """ A class for calculating and updating a confusion matrix for object detection and classification tasks. Attributes: task (str): The type of task, either 'detect' or 'classify'. matrix (np.array): The confusion matrix, with dimensions depending on the task. nc (int): The number of classes. conf (float): The confidence threshold for detections. iou_thres (float): The Intersection over Union threshold. """ def __init__(self, nc, conf=0.25, iou_thres=0.45, task='detect'): """Initialize attributes for the YOLO model.""" self.task = task self.matrix = np.zeros((nc + 1, nc + 1)) if self.task == 'detect' else np.zeros((nc, nc)) self.nc = nc # number of classes self.conf = conf self.iou_thres = iou_thres def process_cls_preds(self, preds, targets): """ Update confusion matrix for classification task Args: preds (Array[N, min(nc,5)]): Predicted class labels. targets (Array[N, 1]): Ground truth class labels. """ preds, targets = torch.cat(preds)[:, 0], torch.cat(targets) for p, t in zip(preds.cpu().numpy(), targets.cpu().numpy()): self.matrix[p][t] += 1 def process_batch(self, detections, labels): """ Update confusion matrix for object detection task. Args: detections (Array[N, 6]): Detected bounding boxes and their associated information. Each row should contain (x1, y1, x2, y2, conf, class). labels (Array[M, 5]): Ground truth bounding boxes and their associated class labels. Each row should contain (class, x1, y1, x2, y2). """ if detections is None: gt_classes = labels.int() for gc in gt_classes: self.matrix[self.nc, gc] += 1 # background FN return detections = detections[detections[:, 4] > self.conf] gt_classes = labels[:, 0].int() detection_classes = detections[:, 5].int() iou = box_iou(labels[:, 1:], detections[:, :4]) x = torch.where(iou > self.iou_thres) if x[0].shape[0]: matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() if x[0].shape[0] > 1: matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 1], return_index=True)[1]] matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 0], return_index=True)[1]] else: matches = np.zeros((0, 3)) n = matches.shape[0] > 0 m0, m1, _ = matches.transpose().astype(int) for i, gc in enumerate(gt_classes): j = m0 == i if n and sum(j) == 1: self.matrix[detection_classes[m1[j]], gc] += 1 # correct else: self.matrix[self.nc, gc] += 1 # true background if n: for i, dc in enumerate(detection_classes): if not any(m1 == i): self.matrix[dc, self.nc] += 1 # predicted background def matrix(self): """Returns the confusion matrix.""" return self.matrix def tp_fp(self): """Returns true positives and false positives.""" tp = self.matrix.diagonal() # true positives fp = self.matrix.sum(1) - tp # false positives # fn = self.matrix.sum(0) - tp # false negatives (missed detections) return (tp[:-1], fp[:-1]) if self.task == 'detect' else (tp, fp) # remove background class if task=detect @TryExcept('WARNING ⚠️ ConfusionMatrix plot failure') @plt_settings() def plot(self, normalize=True, save_dir='', names=(), on_plot=None): """ Plot the confusion matrix using seaborn and save it to a file. Args: normalize (bool): Whether to normalize the confusion matrix. save_dir (str): Directory where the plot will be saved. names (tuple): Names of classes, used as labels on the plot. on_plot (func): An optional callback to pass plots path and data when they are rendered. """ import seaborn as sn array = self.matrix / ((self.matrix.sum(0).reshape(1, -1) + 1E-9) if normalize else 1) # normalize columns array[array < 0.005] = np.nan # don't annotate (would appear as 0.00) fig, ax = plt.subplots(1, 1, figsize=(12, 9), tight_layout=True) nc, nn = self.nc, len(names) # number of classes, names sn.set(font_scale=1.0 if nc < 50 else 0.8) # for label size labels = (0 < nn < 99) and (nn == nc) # apply names to ticklabels ticklabels = (list(names) + ['background']) if labels else 'auto' with warnings.catch_warnings(): warnings.simplefilter('ignore') # suppress empty matrix RuntimeWarning: All-NaN slice encountered sn.heatmap(array, ax=ax, annot=nc < 30, annot_kws={ 'size': 8}, cmap='Blues', fmt='.2f' if normalize else '.0f', square=True, vmin=0.0, xticklabels=ticklabels, yticklabels=ticklabels).set_facecolor((1, 1, 1)) title = 'Confusion Matrix' + ' Normalized' * normalize ax.set_xlabel('True') ax.set_ylabel('Predicted') ax.set_title(title) plot_fname = Path(save_dir) / f'{title.lower().replace(" ", "_")}.png' fig.savefig(plot_fname, dpi=250) plt.close(fig) if on_plot: on_plot(plot_fname) def print(self): """ Print the confusion matrix to the console. """ for i in range(self.nc + 1): LOGGER.info(' '.join(map(str, self.matrix[i]))) def smooth(y, f=0.05): """Box filter of fraction f.""" nf = round(len(y) * f * 2) // 2 + 1 # number of filter elements (must be odd) p = np.ones(nf // 2) # ones padding yp = np.concatenate((p * y[0], y, p * y[-1]), 0) # y padded return np.convolve(yp, np.ones(nf) / nf, mode='valid') # y-smoothed @plt_settings() def plot_pr_curve(px, py, ap, save_dir=Path('pr_curve.png'), names=(), on_plot=None): """Plots a precision-recall curve.""" fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True) py = np.stack(py, axis=1) if 0 < len(names) < 21: # display per-class legend if < 21 classes for i, y in enumerate(py.T): ax.plot(px, y, linewidth=1, label=f'{names[i]} {ap[i, 0]:.3f}') # plot(recall, precision) else: ax.plot(px, py, linewidth=1, color='grey') # plot(recall, precision) ax.plot(px, py.mean(1), linewidth=3, color='blue', label='all classes %.3f mAP@0.5' % ap[:, 0].mean()) ax.set_xlabel('Recall') ax.set_ylabel('Precision') ax.set_xlim(0, 1) ax.set_ylim(0, 1) ax.legend(bbox_to_anchor=(1.04, 1), loc='upper left') ax.set_title('Precision-Recall Curve') fig.savefig(save_dir, dpi=250) plt.close(fig) if on_plot: on_plot(save_dir) @plt_settings() def plot_mc_curve(px, py, save_dir=Path('mc_curve.png'), names=(), xlabel='Confidence', ylabel='Metric', on_plot=None): """Plots a metric-confidence curve.""" fig, ax = plt.subplots(1, 1, figsize=(9, 6), tight_layout=True) if 0 < len(names) < 21: # display per-class legend if < 21 classes for i, y in enumerate(py): ax.plot(px, y, linewidth=1, label=f'{names[i]}') # plot(confidence, metric) else: ax.plot(px, py.T, linewidth=1, color='grey') # plot(confidence, metric) y = smooth(py.mean(0), 0.05) ax.plot(px, y, linewidth=3, color='blue', label=f'all classes {y.max():.2f} at {px[y.argmax()]:.3f}') ax.set_xlabel(xlabel) ax.set_ylabel(ylabel) ax.set_xlim(0, 1) ax.set_ylim(0, 1) ax.legend(bbox_to_anchor=(1.04, 1), loc='upper left') ax.set_title(f'{ylabel}-Confidence Curve') fig.savefig(save_dir, dpi=250) plt.close(fig) if on_plot: on_plot(save_dir) def compute_ap(recall, precision): """ Compute the average precision (AP) given the recall and precision curves. Arguments: recall (list): The recall curve. precision (list): The precision curve. Returns: (float): Average precision. (np.ndarray): Precision envelope curve. (np.ndarray): Modified recall curve with sentinel values added at the beginning and end. """ # Append sentinel values to beginning and end mrec = np.concatenate(([0.0], recall, [1.0])) mpre = np.concatenate(([1.0], precision, [0.0])) # Compute the precision envelope mpre = np.flip(np.maximum.accumulate(np.flip(mpre))) # Integrate area under curve method = 'interp' # methods: 'continuous', 'interp' if method == 'interp': x = np.linspace(0, 1, 101) # 101-point interp (COCO) ap = np.trapz(np.interp(x, mrec, mpre), x) # integrate else: # 'continuous' i = np.where(mrec[1:] != mrec[:-1])[0] # points where x-axis (recall) changes ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) # area under curve return ap, mpre, mrec def ap_per_class(tp, conf, pred_cls, target_cls, plot=False, on_plot=None, save_dir=Path(), names=(), eps=1e-16, prefix=''): """ Computes the average precision per class for object detection evaluation. Args: tp (np.ndarray): Binary array indicating whether the detection is correct (True) or not (False). conf (np.ndarray): Array of confidence scores of the detections. pred_cls (np.ndarray): Array of predicted classes of the detections. target_cls (np.ndarray): Array of true classes of the detections. plot (bool, optional): Whether to plot PR curves or not. Defaults to False. on_plot (func, optional): A callback to pass plots path and data when they are rendered. Defaults to None. save_dir (Path, optional): Directory to save the PR curves. Defaults to an empty path. names (tuple, optional): Tuple of class names to plot PR curves. Defaults to an empty tuple. eps (float, optional): A small value to avoid division by zero. Defaults to 1e-16. prefix (str, optional): A prefix string for saving the plot files. Defaults to an empty string. Returns: (tuple): A tuple of six arrays and one array of unique classes, where: tp (np.ndarray): True positive counts for each class. fp (np.ndarray): False positive counts for each class. p (np.ndarray): Precision values at each confidence threshold. r (np.ndarray): Recall values at each confidence threshold. f1 (np.ndarray): F1-score values at each confidence threshold. ap (np.ndarray): Average precision for each class at different IoU thresholds. unique_classes (np.ndarray): An array of unique classes that have data. """ # Sort by objectness i = np.argsort(-conf) tp, conf, pred_cls = tp[i], conf[i], pred_cls[i] # Find unique classes unique_classes, nt = np.unique(target_cls, return_counts=True) nc = unique_classes.shape[0] # number of classes, number of detections # Create Precision-Recall curve and compute AP for each class px, py = np.linspace(0, 1, 1000), [] # for plotting ap, p, r = np.zeros((nc, tp.shape[1])), np.zeros((nc, 1000)), np.zeros((nc, 1000)) for ci, c in enumerate(unique_classes): i = pred_cls == c n_l = nt[ci] # number of labels n_p = i.sum() # number of predictions if n_p == 0 or n_l == 0: continue # Accumulate FPs and TPs fpc = (1 - tp[i]).cumsum(0) tpc = tp[i].cumsum(0) # Recall recall = tpc / (n_l + eps) # recall curve r[ci] = np.interp(-px, -conf[i], recall[:, 0], left=0) # negative x, xp because xp decreases # Precision precision = tpc / (tpc + fpc) # precision curve p[ci] = np.interp(-px, -conf[i], precision[:, 0], left=1) # p at pr_score # AP from recall-precision curve for j in range(tp.shape[1]): ap[ci, j], mpre, mrec = compute_ap(recall[:, j], precision[:, j]) if plot and j == 0: py.append(np.interp(px, mrec, mpre)) # precision at mAP@0.5 # Compute F1 (harmonic mean of precision and recall) f1 = 2 * p * r / (p + r + eps) names = [v for k, v in names.items() if k in unique_classes] # list: only classes that have data names = dict(enumerate(names)) # to dict if plot: plot_pr_curve(px, py, ap, save_dir / f'{prefix}PR_curve.png', names, on_plot=on_plot) plot_mc_curve(px, f1, save_dir / f'{prefix}F1_curve.png', names, ylabel='F1', on_plot=on_plot) plot_mc_curve(px, p, save_dir / f'{prefix}P_curve.png', names, ylabel='Precision', on_plot=on_plot) plot_mc_curve(px, r, save_dir / f'{prefix}R_curve.png', names, ylabel='Recall', on_plot=on_plot) i = smooth(f1.mean(0), 0.1).argmax() # max F1 index p, r, f1 = p[:, i], r[:, i], f1[:, i] tp = (r * nt).round() # true positives fp = (tp / (p + eps) - tp).round() # false positives return tp, fp, p, r, f1, ap, unique_classes.astype(int) class Metric(SimpleClass): """ Class for computing evaluation metrics for YOLOv8 model. Attributes: p (list): Precision for each class. Shape: (nc,). r (list): Recall for each class. Shape: (nc,). f1 (list): F1 score for each class. Shape: (nc,). all_ap (list): AP scores for all classes and all IoU thresholds. Shape: (nc, 10). ap_class_index (list): Index of class for each AP score. Shape: (nc,). nc (int): Number of classes. Methods: ap50(): AP at IoU threshold of 0.5 for all classes. Returns: List of AP scores. Shape: (nc,) or []. ap(): AP at IoU thresholds from 0.5 to 0.95 for all classes. Returns: List of AP scores. Shape: (nc,) or []. mp(): Mean precision of all classes. Returns: Float. mr(): Mean recall of all classes. Returns: Float. map50(): Mean AP at IoU threshold of 0.5 for all classes. Returns: Float. map75(): Mean AP at IoU threshold of 0.75 for all classes. Returns: Float. map(): Mean AP at IoU thresholds from 0.5 to 0.95 for all classes. Returns: Float. mean_results(): Mean of results, returns mp, mr, map50, map. class_result(i): Class-aware result, returns p[i], r[i], ap50[i], ap[i]. maps(): mAP of each class. Returns: Array of mAP scores, shape: (nc,). fitness(): Model fitness as a weighted combination of metrics. Returns: Float. update(results): Update metric attributes with new evaluation results. """ def __init__(self) -> None: self.p = [] # (nc, ) self.r = [] # (nc, ) self.f1 = [] # (nc, ) self.all_ap = [] # (nc, 10) self.ap_class_index = [] # (nc, ) self.nc = 0 @property def ap50(self): """ Returns the Average Precision (AP) at an IoU threshold of 0.5 for all classes. Returns: (np.ndarray, list): Array of shape (nc,) with AP50 values per class, or an empty list if not available. """ return self.all_ap[:, 0] if len(self.all_ap) else [] @property def ap(self): """ Returns the Average Precision (AP) at an IoU threshold of 0.5-0.95 for all classes. Returns: (np.ndarray, list): Array of shape (nc,) with AP50-95 values per class, or an empty list if not available. """ return self.all_ap.mean(1) if len(self.all_ap) else [] @property def mp(self): """ Returns the Mean Precision of all classes. Returns: (float): The mean precision of all classes. """ return self.p.mean() if len(self.p) else 0.0 @property def mr(self): """ Returns the Mean Recall of all classes. Returns: (float): The mean recall of all classes. """ return self.r.mean() if len(self.r) else 0.0 @property def map50(self): """ Returns the mean Average Precision (mAP) at an IoU threshold of 0.5. Returns: (float): The mAP50 at an IoU threshold of 0.5. """ return self.all_ap[:, 0].mean() if len(self.all_ap) else 0.0 @property def map75(self): """ Returns the mean Average Precision (mAP) at an IoU threshold of 0.75. Returns: (float): The mAP50 at an IoU threshold of 0.75. """ return self.all_ap[:, 5].mean() if len(self.all_ap) else 0.0 @property def map(self): """ Returns the mean Average Precision (mAP) over IoU thresholds of 0.5 - 0.95 in steps of 0.05. Returns: (float): The mAP over IoU thresholds of 0.5 - 0.95 in steps of 0.05. """ return self.all_ap.mean() if len(self.all_ap) else 0.0 def mean_results(self): """Mean of results, return mp, mr, map50, map.""" return [self.mp, self.mr, self.map50, self.map] def class_result(self, i): """class-aware result, return p[i], r[i], ap50[i], ap[i].""" return self.p[i], self.r[i], self.ap50[i], self.ap[i] @property def maps(self): """mAP of each class.""" maps = np.zeros(self.nc) + self.map for i, c in enumerate(self.ap_class_index): maps[c] = self.ap[i] return maps def fitness(self): """Model fitness as a weighted combination of metrics.""" w = [0.0, 0.0, 0.1, 0.9] # weights for [P, R, mAP@0.5, mAP@0.5:0.95] return (np.array(self.mean_results()) * w).sum() def update(self, results): """ Args: results (tuple): A tuple of (p, r, ap, f1, ap_class) """ self.p, self.r, self.f1, self.all_ap, self.ap_class_index = results class DetMetrics(SimpleClass): """ This class is a utility class for computing detection metrics such as precision, recall, and mean average precision (mAP) of an object detection model. Args: save_dir (Path): A path to the directory where the output plots will be saved. Defaults to current directory. plot (bool): A flag that indicates whether to plot precision-recall curves for each class. Defaults to False. on_plot (func): An optional callback to pass plots path and data when they are rendered. Defaults to None. names (tuple of str): A tuple of strings that represents the names of the classes. Defaults to an empty tuple. Attributes: save_dir (Path): A path to the directory where the output plots will be saved. plot (bool): A flag that indicates whether to plot the precision-recall curves for each class. on_plot (func): An optional callback to pass plots path and data when they are rendered. names (tuple of str): A tuple of strings that represents the names of the classes. box (Metric): An instance of the Metric class for storing the results of the detection metrics. speed (dict): A dictionary for storing the execution time of different parts of the detection process. Methods: process(tp, conf, pred_cls, target_cls): Updates the metric results with the latest batch of predictions. keys: Returns a list of keys for accessing the computed detection metrics. mean_results: Returns a list of mean values for the computed detection metrics. class_result(i): Returns a list of values for the computed detection metrics for a specific class. maps: Returns a dictionary of mean average precision (mAP) values for different IoU thresholds. fitness: Computes the fitness score based on the computed detection metrics. ap_class_index: Returns a list of class indices sorted by their average precision (AP) values. results_dict: Returns a dictionary that maps detection metric keys to their computed values. """ def __init__(self, save_dir=Path('.'), plot=False, on_plot=None, names=()) -> None: self.save_dir = save_dir self.plot = plot self.on_plot = on_plot self.names = names self.box = Metric() self.speed = {'preprocess': 0.0, 'inference': 0.0, 'loss': 0.0, 'postprocess': 0.0} def process(self, tp, conf, pred_cls, target_cls): """Process predicted results for object detection and update metrics.""" results = ap_per_class(tp, conf, pred_cls, target_cls, plot=self.plot, save_dir=self.save_dir, names=self.names, on_plot=self.on_plot)[2:] self.box.nc = len(self.names) self.box.update(results) @property def keys(self): """Returns a list of keys for accessing specific metrics.""" return ['metrics/precision(B)', 'metrics/recall(B)', 'metrics/mAP50(B)', 'metrics/mAP50-95(B)'] def mean_results(self): """Calculate mean of detected objects & return precision, recall, mAP50, and mAP50-95.""" return self.box.mean_results() def class_result(self, i): """Return the result of evaluating the performance of an object detection model on a specific class.""" return self.box.class_result(i) @property def maps(self): """Returns mean Average Precision (mAP) scores per class.""" return self.box.maps @property def fitness(self): """Returns the fitness of box object.""" return self.box.fitness() @property def ap_class_index(self): """Returns the average precision index per class.""" return self.box.ap_class_index @property def results_dict(self): """Returns dictionary of computed performance metrics and statistics.""" return dict(zip(self.keys + ['fitness'], self.mean_results() + [self.fitness])) class SegmentMetrics(SimpleClass): """ Calculates and aggregates detection and segmentation metrics over a given set of classes. Args: save_dir (Path): Path to the directory where the output plots should be saved. Default is the current directory. plot (bool): Whether to save the detection and segmentation plots. Default is False. on_plot (func): An optional callback to pass plots path and data when they are rendered. Defaults to None. names (list): List of class names. Default is an empty list. Attributes: save_dir (Path): Path to the directory where the output plots should be saved. plot (bool): Whether to save the detection and segmentation plots. on_plot (func): An optional callback to pass plots path and data when they are rendered. names (list): List of class names. box (Metric): An instance of the Metric class to calculate box detection metrics. seg (Metric): An instance of the Metric class to calculate mask segmentation metrics. speed (dict): Dictionary to store the time taken in different phases of inference. Methods: process(tp_m, tp_b, conf, pred_cls, target_cls): Processes metrics over the given set of predictions. mean_results(): Returns the mean of the detection and segmentation metrics over all the classes. class_result(i): Returns the detection and segmentation metrics of class `i`. maps: Returns the mean Average Precision (mAP) scores for IoU thresholds ranging from 0.50 to 0.95. fitness: Returns the fitness scores, which are a single weighted combination of metrics. ap_class_index: Returns the list of indices of classes used to compute Average Precision (AP). results_dict: Returns the dictionary containing all the detection and segmentation metrics and fitness score. """ def __init__(self, save_dir=Path('.'), plot=False, on_plot=None, names=()) -> None: self.save_dir = save_dir self.plot = plot self.on_plot = on_plot self.names = names self.box = Metric() self.seg = Metric() self.speed = {'preprocess': 0.0, 'inference': 0.0, 'loss': 0.0, 'postprocess': 0.0} def process(self, tp_b, tp_m, conf, pred_cls, target_cls): """ Processes the detection and segmentation metrics over the given set of predictions. Args: tp_b (list): List of True Positive boxes. tp_m (list): List of True Positive masks. conf (list): List of confidence scores. pred_cls (list): List of predicted classes. target_cls (list): List of target classes. """ results_mask = ap_per_class(tp_m, conf, pred_cls, target_cls, plot=self.plot, on_plot=self.on_plot, save_dir=self.save_dir, names=self.names, prefix='Mask')[2:] self.seg.nc = len(self.names) self.seg.update(results_mask) results_box = ap_per_class(tp_b, conf, pred_cls, target_cls, plot=self.plot, on_plot=self.on_plot, save_dir=self.save_dir, names=self.names, prefix='Box')[2:] self.box.nc = len(self.names) self.box.update(results_box) @property def keys(self): """Returns a list of keys for accessing metrics.""" return [ 'metrics/precision(B)', 'metrics/recall(B)', 'metrics/mAP50(B)', 'metrics/mAP50-95(B)', 'metrics/precision(M)', 'metrics/recall(M)', 'metrics/mAP50(M)', 'metrics/mAP50-95(M)'] def mean_results(self): """Return the mean metrics for bounding box and segmentation results.""" return self.box.mean_results() + self.seg.mean_results() def class_result(self, i): """Returns classification results for a specified class index.""" return self.box.class_result(i) + self.seg.class_result(i) @property def maps(self): """Returns mAP scores for object detection and semantic segmentation models.""" return self.box.maps + self.seg.maps @property def fitness(self): """Get the fitness score for both segmentation and bounding box models.""" return self.seg.fitness() + self.box.fitness() @property def ap_class_index(self): """Boxes and masks have the same ap_class_index.""" return self.box.ap_class_index @property def results_dict(self): """Returns results of object detection model for evaluation.""" return dict(zip(self.keys + ['fitness'], self.mean_results() + [self.fitness])) class PoseMetrics(SegmentMetrics): """ Calculates and aggregates detection and pose metrics over a given set of classes. Args: save_dir (Path): Path to the directory where the output plots should be saved. Default is the current directory. plot (bool): Whether to save the detection and segmentation plots. Default is False. on_plot (func): An optional callback to pass plots path and data when they are rendered. Defaults to None. names (list): List of class names. Default is an empty list. Attributes: save_dir (Path): Path to the directory where the output plots should be saved. plot (bool): Whether to save the detection and segmentation plots. on_plot (func): An optional callback to pass plots path and data when they are rendered. names (list): List of class names. box (Metric): An instance of the Metric class to calculate box detection metrics. pose (Metric): An instance of the Metric class to calculate mask segmentation metrics. speed (dict): Dictionary to store the time taken in different phases of inference. Methods: process(tp_m, tp_b, conf, pred_cls, target_cls): Processes metrics over the given set of predictions. mean_results(): Returns the mean of the detection and segmentation metrics over all the classes. class_result(i): Returns the detection and segmentation metrics of class `i`. maps: Returns the mean Average Precision (mAP) scores for IoU thresholds ranging from 0.50 to 0.95. fitness: Returns the fitness scores, which are a single weighted combination of metrics. ap_class_index: Returns the list of indices of classes used to compute Average Precision (AP). results_dict: Returns the dictionary containing all the detection and segmentation metrics and fitness score. """ def __init__(self, save_dir=Path('.'), plot=False, on_plot=None, names=()) -> None: super().__init__(save_dir, plot, names) self.save_dir = save_dir self.plot = plot self.on_plot = on_plot self.names = names self.box = Metric() self.pose = Metric() self.speed = {'preprocess': 0.0, 'inference': 0.0, 'loss': 0.0, 'postprocess': 0.0} def __getattr__(self, attr): """Raises an AttributeError if an invalid attribute is accessed.""" name = self.__class__.__name__ raise AttributeError(f"'{name}' object has no attribute '{attr}'. See valid attributes below.\n{self.__doc__}") def process(self, tp_b, tp_p, conf, pred_cls, target_cls): """ Processes the detection and pose metrics over the given set of predictions. Args: tp_b (list): List of True Positive boxes. tp_p (list): List of True Positive keypoints. conf (list): List of confidence scores. pred_cls (list): List of predicted classes. target_cls (list): List of target classes. """ results_pose = ap_per_class(tp_p, conf, pred_cls, target_cls, plot=self.plot, on_plot=self.on_plot, save_dir=self.save_dir, names=self.names, prefix='Pose')[2:] self.pose.nc = len(self.names) self.pose.update(results_pose) results_box = ap_per_class(tp_b, conf, pred_cls, target_cls, plot=self.plot, on_plot=self.on_plot, save_dir=self.save_dir, names=self.names, prefix='Box')[2:] self.box.nc = len(self.names) self.box.update(results_box) @property def keys(self): """Returns list of evaluation metric keys.""" return [ 'metrics/precision(B)', 'metrics/recall(B)', 'metrics/mAP50(B)', 'metrics/mAP50-95(B)', 'metrics/precision(P)', 'metrics/recall(P)', 'metrics/mAP50(P)', 'metrics/mAP50-95(P)'] def mean_results(self): """Return the mean results of box and pose.""" return self.box.mean_results() + self.pose.mean_results() def class_result(self, i): """Return the class-wise detection results for a specific class i.""" return self.box.class_result(i) + self.pose.class_result(i) @property def maps(self): """Returns the mean average precision (mAP) per class for both box and pose detections.""" return self.box.maps + self.pose.maps @property def fitness(self): """Computes classification metrics and speed using the `targets` and `pred` inputs.""" return self.pose.fitness() + self.box.fitness() class ClassifyMetrics(SimpleClass): """ Class for computing classification metrics including top-1 and top-5 accuracy. Attributes: top1 (float): The top-1 accuracy. top5 (float): The top-5 accuracy. speed (Dict[str, float]): A dictionary containing the time taken for each step in the pipeline. Properties: fitness (float): The fitness of the model, which is equal to top-5 accuracy. results_dict (Dict[str, Union[float, str]]): A dictionary containing the classification metrics and fitness. keys (List[str]): A list of keys for the results_dict. Methods: process(targets, pred): Processes the targets and predictions to compute classification metrics. """ def __init__(self) -> None: self.top1 = 0 self.top5 = 0 self.speed = {'preprocess': 0.0, 'inference': 0.0, 'loss': 0.0, 'postprocess': 0.0} def process(self, targets, pred): """Target classes and predicted classes.""" pred, targets = torch.cat(pred), torch.cat(targets) correct = (targets[:, None] == pred).float() acc = torch.stack((correct[:, 0], correct.max(1).values), dim=1) # (top1, top5) accuracy self.top1, self.top5 = acc.mean(0).tolist() @property def fitness(self): """Returns top-5 accuracy as fitness score.""" return self.top5 @property def results_dict(self): """Returns a dictionary with model's performance metrics and fitness score.""" return dict(zip(self.keys + ['fitness'], [self.top1, self.top5, self.fitness])) @property def keys(self): """Returns a list of keys for the results_dict property.""" return ['metrics/accuracy_top1', 'metrics/accuracy_top5'] ================================================ FILE: ultralytics/yolo/utils/ops.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import contextlib import math import re import time import cv2 import numpy as np import torch import torch.nn.functional as F import torchvision from ultralytics.yolo.utils import LOGGER from .metrics import box_iou class Profile(contextlib.ContextDecorator): """ YOLOv8 Profile class. Usage: as a decorator with @Profile() or as a context manager with 'with Profile():' """ def __init__(self, t=0.0): """ Initialize the Profile class. Args: t (float): Initial time. Defaults to 0.0. """ self.t = t self.cuda = torch.cuda.is_available() def __enter__(self): """ Start timing. """ self.start = self.time() return self def __exit__(self, type, value, traceback): """ Stop timing. """ self.dt = self.time() - self.start # delta-time self.t += self.dt # accumulate dt def time(self): """ Get current time. """ if self.cuda: torch.cuda.synchronize() return time.time() def coco80_to_coco91_class(): # converts 80-index (val2014) to 91-index (paper) # https://tech.amikelive.com/node-718/what-object-categories-labels-are-in-coco-dataset/ # a = np.loadtxt('data/coco.names', dtype='str', delimiter='\n') # b = np.loadtxt('data/coco_paper.names', dtype='str', delimiter='\n') # x1 = [list(a[i] == b).index(True) + 1 for i in range(80)] # darknet to coco # x2 = [list(b[i] == a).index(True) if any(b[i] == a) else None for i in range(91)] # coco to darknet return [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 67, 70, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 84, 85, 86, 87, 88, 89, 90] def segment2box(segment, width=640, height=640): """ Convert 1 segment label to 1 box label, applying inside-image constraint, i.e. (xy1, xy2, ...) to (xyxy) Args: segment (torch.Tensor): the segment label width (int): the width of the image. Defaults to 640 height (int): The height of the image. Defaults to 640 Returns: (np.ndarray): the minimum and maximum x and y values of the segment. """ # Convert 1 segment label to 1 box label, applying inside-image constraint, i.e. (xy1, xy2, ...) to (xyxy) x, y = segment.T # segment xy inside = (x >= 0) & (y >= 0) & (x <= width) & (y <= height) x, y, = x[inside], y[inside] return np.array([x.min(), y.min(), x.max(), y.max()], dtype=segment.dtype) if any(x) else np.zeros( 4, dtype=segment.dtype) # xyxy def scale_boxes(img1_shape, boxes, img0_shape, ratio_pad=None): """ Rescales bounding boxes (in the format of xyxy) from the shape of the image they were originally specified in (img1_shape) to the shape of a different image (img0_shape). Args: img1_shape (tuple): The shape of the image that the bounding boxes are for, in the format of (height, width). boxes (torch.Tensor): the bounding boxes of the objects in the image, in the format of (x1, y1, x2, y2) img0_shape (tuple): the shape of the target image, in the format of (height, width). ratio_pad (tuple): a tuple of (ratio, pad) for scaling the boxes. If not provided, the ratio and pad will be calculated based on the size difference between the two images. Returns: boxes (torch.Tensor): The scaled bounding boxes, in the format of (x1, y1, x2, y2) """ if ratio_pad is None: # calculate from img0_shape gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new pad = round((img1_shape[1] - img0_shape[1] * gain) / 2 - 0.1), round( (img1_shape[0] - img0_shape[0] * gain) / 2 - 0.1) # wh padding else: gain = ratio_pad[0][0] pad = ratio_pad[1] boxes[..., [0, 2]] -= pad[0] # x padding boxes[..., [1, 3]] -= pad[1] # y padding boxes[..., :4] /= gain clip_boxes(boxes, img0_shape) return boxes def make_divisible(x, divisor): """ Returns the nearest number that is divisible by the given divisor. Args: x (int): The number to make divisible. divisor (int | torch.Tensor): The divisor. Returns: (int): The nearest number divisible by the divisor. """ if isinstance(divisor, torch.Tensor): divisor = int(divisor.max()) # to int return math.ceil(x / divisor) * divisor def non_max_suppression( prediction, conf_thres=0.25, iou_thres=0.45, classes=None, agnostic=False, multi_label=False, labels=(), max_det=300, nc=0, # number of classes (optional) max_time_img=0.05, max_nms=30000, max_wh=7680, ): """ Perform non-maximum suppression (NMS) on a set of boxes, with support for masks and multiple labels per box. Arguments: prediction (torch.Tensor): A tensor of shape (batch_size, num_classes + 4 + num_masks, num_boxes) containing the predicted boxes, classes, and masks. The tensor should be in the format output by a model, such as YOLO. conf_thres (float): The confidence threshold below which boxes will be filtered out. Valid values are between 0.0 and 1.0. iou_thres (float): The IoU threshold below which boxes will be filtered out during NMS. Valid values are between 0.0 and 1.0. classes (List[int]): A list of class indices to consider. If None, all classes will be considered. agnostic (bool): If True, the model is agnostic to the number of classes, and all classes will be considered as one. multi_label (bool): If True, each box may have multiple labels. labels (List[List[Union[int, float, torch.Tensor]]]): A list of lists, where each inner list contains the apriori labels for a given image. The list should be in the format output by a dataloader, with each label being a tuple of (class_index, x1, y1, x2, y2). max_det (int): The maximum number of boxes to keep after NMS. nc (int, optional): The number of classes output by the model. Any indices after this will be considered masks. max_time_img (float): The maximum time (seconds) for processing one image. max_nms (int): The maximum number of boxes into torchvision.ops.nms(). max_wh (int): The maximum box width and height in pixels Returns: (List[torch.Tensor]): A list of length batch_size, where each element is a tensor of shape (num_boxes, 6 + num_masks) containing the kept boxes, with columns (x1, y1, x2, y2, confidence, class, mask1, mask2, ...). """ # Checks assert 0 <= conf_thres <= 1, f'Invalid Confidence threshold {conf_thres}, valid values are between 0.0 and 1.0' assert 0 <= iou_thres <= 1, f'Invalid IoU {iou_thres}, valid values are between 0.0 and 1.0' if isinstance(prediction, (list, tuple)): # YOLOv8 model in validation model, output = (inference_out, loss_out) prediction = prediction[0] # select only inference output device = prediction.device mps = 'mps' in device.type # Apple MPS if mps: # MPS not fully supported yet, convert tensors to CPU before NMS prediction = prediction.cpu() bs = prediction.shape[0] # batch size nc = nc or (prediction.shape[1] - 4) # number of classes nm = prediction.shape[1] - nc - 4 mi = 4 + nc # mask start index xc = prediction[:, 4:mi].amax(1) > conf_thres # candidates # Settings # min_wh = 2 # (pixels) minimum box width and height time_limit = 0.5 + max_time_img * bs # seconds to quit after redundant = True # require redundant detections multi_label &= nc > 1 # multiple labels per box (adds 0.5ms/img) merge = False # use merge-NMS t = time.time() output = [torch.zeros((0, 6 + nm), device=prediction.device)] * bs for xi, x in enumerate(prediction): # image index, image inference # Apply constraints # x[((x[:, 2:4] < min_wh) | (x[:, 2:4] > max_wh)).any(1), 4] = 0 # width-height x = x.transpose(0, -1)[xc[xi]] # confidence # Cat apriori labels if autolabelling if labels and len(labels[xi]): lb = labels[xi] v = torch.zeros((len(lb), nc + nm + 5), device=x.device) v[:, :4] = lb[:, 1:5] # box v[range(len(lb)), lb[:, 0].long() + 4] = 1.0 # cls x = torch.cat((x, v), 0) # If none remain process next image if not x.shape[0]: continue # Detections matrix nx6 (xyxy, conf, cls) box, cls, mask = x.split((4, nc, nm), 1) box = xywh2xyxy(box) # center_x, center_y, width, height) to (x1, y1, x2, y2) if multi_label: i, j = (cls > conf_thres).nonzero(as_tuple=False).T x = torch.cat((box[i], x[i, 4 + j, None], j[:, None].float(), mask[i]), 1) else: # best class only conf, j = cls.max(1, keepdim=True) x = torch.cat((box, conf, j.float(), mask), 1)[conf.view(-1) > conf_thres] # Filter by class if classes is not None: x = x[(x[:, 5:6] == torch.tensor(classes, device=x.device)).any(1)] # Apply finite constraint # if not torch.isfinite(x).all(): # x = x[torch.isfinite(x).all(1)] # Check shape n = x.shape[0] # number of boxes if not n: # no boxes continue x = x[x[:, 4].argsort(descending=True)[:max_nms]] # sort by confidence and remove excess boxes # Batched NMS c = x[:, 5:6] * (0 if agnostic else max_wh) # classes boxes, scores = x[:, :4] + c, x[:, 4] # boxes (offset by class), scores i = torchvision.ops.nms(boxes, scores, iou_thres) # NMS i = i[:max_det] # limit detections if merge and (1 < n < 3E3): # Merge NMS (boxes merged using weighted mean) # Update boxes as boxes(i,4) = weights(i,n) * boxes(n,4) iou = box_iou(boxes[i], boxes) > iou_thres # iou matrix weights = iou * scores[None] # box weights x[i, :4] = torch.mm(weights, x[:, :4]).float() / weights.sum(1, keepdim=True) # merged boxes if redundant: i = i[iou.sum(1) > 1] # require redundancy output[xi] = x[i] if mps: output[xi] = output[xi].to(device) if (time.time() - t) > time_limit: LOGGER.warning(f'WARNING ⚠️ NMS time limit {time_limit:.3f}s exceeded') break # time limit exceeded return output def clip_boxes(boxes, shape): """ It takes a list of bounding boxes and a shape (height, width) and clips the bounding boxes to the shape Args: boxes (torch.Tensor): the bounding boxes to clip shape (tuple): the shape of the image """ if isinstance(boxes, torch.Tensor): # faster individually boxes[..., 0].clamp_(0, shape[1]) # x1 boxes[..., 1].clamp_(0, shape[0]) # y1 boxes[..., 2].clamp_(0, shape[1]) # x2 boxes[..., 3].clamp_(0, shape[0]) # y2 else: # np.array (faster grouped) boxes[..., [0, 2]] = boxes[..., [0, 2]].clip(0, shape[1]) # x1, x2 boxes[..., [1, 3]] = boxes[..., [1, 3]].clip(0, shape[0]) # y1, y2 def clip_coords(coords, shape): """ Clip line coordinates to the image boundaries. Args: coords (torch.Tensor | numpy.ndarray): A list of line coordinates. shape (tuple): A tuple of integers representing the size of the image in the format (height, width). Returns: (None): The function modifies the input `coordinates` in place, by clipping each coordinate to the image boundaries. """ if isinstance(coords, torch.Tensor): # faster individually coords[..., 0].clamp_(0, shape[1]) # x coords[..., 1].clamp_(0, shape[0]) # y else: # np.array (faster grouped) coords[..., 0] = coords[..., 0].clip(0, shape[1]) # x coords[..., 1] = coords[..., 1].clip(0, shape[0]) # y def scale_image(masks, im0_shape, ratio_pad=None): """ Takes a mask, and resizes it to the original image size Args: masks (torch.Tensor): resized and padded masks/images, [h, w, num]/[h, w, 3]. im0_shape (tuple): the original image shape ratio_pad (tuple): the ratio of the padding to the original image. Returns: masks (torch.Tensor): The masks that are being returned. """ # Rescale coordinates (xyxy) from im1_shape to im0_shape im1_shape = masks.shape if im1_shape[:2] == im0_shape[:2]: return masks if ratio_pad is None: # calculate from im0_shape gain = min(im1_shape[0] / im0_shape[0], im1_shape[1] / im0_shape[1]) # gain = old / new pad = (im1_shape[1] - im0_shape[1] * gain) / 2, (im1_shape[0] - im0_shape[0] * gain) / 2 # wh padding else: gain = ratio_pad[0][0] pad = ratio_pad[1] top, left = int(pad[1]), int(pad[0]) # y, x bottom, right = int(im1_shape[0] - pad[1]), int(im1_shape[1] - pad[0]) if len(masks.shape) < 2: raise ValueError(f'"len of masks shape" should be 2 or 3, but got {len(masks.shape)}') masks = masks[top:bottom, left:right] # masks = masks.permute(2, 0, 1).contiguous() # masks = F.interpolate(masks[None], im0_shape[:2], mode='bilinear', align_corners=False)[0] # masks = masks.permute(1, 2, 0).contiguous() masks = cv2.resize(masks, (im0_shape[1], im0_shape[0])) if len(masks.shape) == 2: masks = masks[:, :, None] return masks def xyxy2xywh(x): """ Convert bounding box coordinates from (x1, y1, x2, y2) format to (x, y, width, height) format. Args: x (np.ndarray | torch.Tensor): The input bounding box coordinates in (x1, y1, x2, y2) format. Returns: y (np.ndarray | torch.Tensor): The bounding box coordinates in (x, y, width, height) format. """ y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) y[..., 0] = (x[..., 0] + x[..., 2]) / 2 # x center y[..., 1] = (x[..., 1] + x[..., 3]) / 2 # y center y[..., 2] = x[..., 2] - x[..., 0] # width y[..., 3] = x[..., 3] - x[..., 1] # height return y def xywh2xyxy(x): """ Convert bounding box coordinates from (x, y, width, height) format to (x1, y1, x2, y2) format where (x1, y1) is the top-left corner and (x2, y2) is the bottom-right corner. Args: x (np.ndarray | torch.Tensor): The input bounding box coordinates in (x, y, width, height) format. Returns: y (np.ndarray | torch.Tensor): The bounding box coordinates in (x1, y1, x2, y2) format. """ y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) y[..., 0] = x[..., 0] - x[..., 2] / 2 # top left x y[..., 1] = x[..., 1] - x[..., 3] / 2 # top left y y[..., 2] = x[..., 0] + x[..., 2] / 2 # bottom right x y[..., 3] = x[..., 1] + x[..., 3] / 2 # bottom right y return y def xywhn2xyxy(x, w=640, h=640, padw=0, padh=0): """ Convert normalized bounding box coordinates to pixel coordinates. Args: x (np.ndarray | torch.Tensor): The bounding box coordinates. w (int): Width of the image. Defaults to 640 h (int): Height of the image. Defaults to 640 padw (int): Padding width. Defaults to 0 padh (int): Padding height. Defaults to 0 Returns: y (np.ndarray | torch.Tensor): The coordinates of the bounding box in the format [x1, y1, x2, y2] where x1,y1 is the top-left corner, x2,y2 is the bottom-right corner of the bounding box. """ y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) y[..., 0] = w * (x[..., 0] - x[..., 2] / 2) + padw # top left x y[..., 1] = h * (x[..., 1] - x[..., 3] / 2) + padh # top left y y[..., 2] = w * (x[..., 0] + x[..., 2] / 2) + padw # bottom right x y[..., 3] = h * (x[..., 1] + x[..., 3] / 2) + padh # bottom right y return y def xyxy2xywhn(x, w=640, h=640, clip=False, eps=0.0): """ Convert bounding box coordinates from (x1, y1, x2, y2) format to (x, y, width, height, normalized) format. x, y, width and height are normalized to image dimensions Args: x (np.ndarray | torch.Tensor): The input bounding box coordinates in (x1, y1, x2, y2) format. w (int): The width of the image. Defaults to 640 h (int): The height of the image. Defaults to 640 clip (bool): If True, the boxes will be clipped to the image boundaries. Defaults to False eps (float): The minimum value of the box's width and height. Defaults to 0.0 Returns: y (np.ndarray | torch.Tensor): The bounding box coordinates in (x, y, width, height, normalized) format """ if clip: clip_boxes(x, (h - eps, w - eps)) # warning: inplace clip y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) y[..., 0] = ((x[..., 0] + x[..., 2]) / 2) / w # x center y[..., 1] = ((x[..., 1] + x[..., 3]) / 2) / h # y center y[..., 2] = (x[..., 2] - x[..., 0]) / w # width y[..., 3] = (x[..., 3] - x[..., 1]) / h # height return y def xyn2xy(x, w=640, h=640, padw=0, padh=0): """ Convert normalized coordinates to pixel coordinates of shape (n,2) Args: x (np.ndarray | torch.Tensor): The input tensor of normalized bounding box coordinates w (int): The width of the image. Defaults to 640 h (int): The height of the image. Defaults to 640 padw (int): The width of the padding. Defaults to 0 padh (int): The height of the padding. Defaults to 0 Returns: y (np.ndarray | torch.Tensor): The x and y coordinates of the top left corner of the bounding box """ y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) y[..., 0] = w * x[..., 0] + padw # top left x y[..., 1] = h * x[..., 1] + padh # top left y return y def xywh2ltwh(x): """ Convert the bounding box format from [x, y, w, h] to [x1, y1, w, h], where x1, y1 are the top-left coordinates. Args: x (np.ndarray | torch.Tensor): The input tensor with the bounding box coordinates in the xywh format Returns: y (np.ndarray | torch.Tensor): The bounding box coordinates in the xyltwh format """ y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) y[:, 0] = x[:, 0] - x[:, 2] / 2 # top left x y[:, 1] = x[:, 1] - x[:, 3] / 2 # top left y return y def xyxy2ltwh(x): """ Convert nx4 bounding boxes from [x1, y1, x2, y2] to [x1, y1, w, h], where xy1=top-left, xy2=bottom-right Args: x (np.ndarray | torch.Tensor): The input tensor with the bounding boxes coordinates in the xyxy format Returns: y (np.ndarray | torch.Tensor): The bounding box coordinates in the xyltwh format. """ y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) y[:, 2] = x[:, 2] - x[:, 0] # width y[:, 3] = x[:, 3] - x[:, 1] # height return y def ltwh2xywh(x): """ Convert nx4 boxes from [x1, y1, w, h] to [x, y, w, h] where xy1=top-left, xy=center Args: x (torch.Tensor): the input tensor """ y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) y[:, 0] = x[:, 0] + x[:, 2] / 2 # center x y[:, 1] = x[:, 1] + x[:, 3] / 2 # center y return y def ltwh2xyxy(x): """ It converts the bounding box from [x1, y1, w, h] to [x1, y1, x2, y2] where xy1=top-left, xy2=bottom-right Args: x (np.ndarray | torch.Tensor): the input image Returns: y (np.ndarray | torch.Tensor): the xyxy coordinates of the bounding boxes. """ y = x.clone() if isinstance(x, torch.Tensor) else np.copy(x) y[:, 2] = x[:, 2] + x[:, 0] # width y[:, 3] = x[:, 3] + x[:, 1] # height return y def segments2boxes(segments): """ It converts segment labels to box labels, i.e. (cls, xy1, xy2, ...) to (cls, xywh) Args: segments (list): list of segments, each segment is a list of points, each point is a list of x, y coordinates Returns: (np.ndarray): the xywh coordinates of the bounding boxes. """ boxes = [] for s in segments: x, y = s.T # segment xy boxes.append([x.min(), y.min(), x.max(), y.max()]) # cls, xyxy return xyxy2xywh(np.array(boxes)) # cls, xywh def resample_segments(segments, n=1000): """ Inputs a list of segments (n,2) and returns a list of segments (n,2) up-sampled to n points each. Args: segments (list): a list of (n,2) arrays, where n is the number of points in the segment. n (int): number of points to resample the segment to. Defaults to 1000 Returns: segments (list): the resampled segments. """ for i, s in enumerate(segments): s = np.concatenate((s, s[0:1, :]), axis=0) x = np.linspace(0, len(s) - 1, n) xp = np.arange(len(s)) segments[i] = np.concatenate([np.interp(x, xp, s[:, i]) for i in range(2)], dtype=np.float32).reshape(2, -1).T # segment xy return segments def crop_mask(masks, boxes): """ It takes a mask and a bounding box, and returns a mask that is cropped to the bounding box Args: masks (torch.Tensor): [h, w, n] tensor of masks boxes (torch.Tensor): [n, 4] tensor of bbox coordinates in relative point form Returns: (torch.Tensor): The masks are being cropped to the bounding box. """ n, h, w = masks.shape x1, y1, x2, y2 = torch.chunk(boxes[:, :, None], 4, 1) # x1 shape(n,1,1) r = torch.arange(w, device=masks.device, dtype=x1.dtype)[None, None, :] # rows shape(1,1,w) c = torch.arange(h, device=masks.device, dtype=x1.dtype)[None, :, None] # cols shape(1,h,1) return masks * ((r >= x1) * (r < x2) * (c >= y1) * (c < y2)) def process_mask_upsample(protos, masks_in, bboxes, shape): """ It takes the output of the mask head, and applies the mask to the bounding boxes. This produces masks of higher quality but is slower. Args: protos (torch.Tensor): [mask_dim, mask_h, mask_w] masks_in (torch.Tensor): [n, mask_dim], n is number of masks after nms bboxes (torch.Tensor): [n, 4], n is number of masks after nms shape (tuple): the size of the input image (h,w) Returns: (torch.Tensor): The upsampled masks. """ c, mh, mw = protos.shape # CHW masks = (masks_in @ protos.float().view(c, -1)).view(-1, mh, mw) masks = F.interpolate(masks[None], shape, mode='bilinear', align_corners=False)[0] # CHW masks = crop_mask(masks, bboxes) # CHW return masks.gt_(0) def process_mask(protos, masks_in, bboxes, shape, upsample=False): """ Apply masks to bounding boxes using the output of the mask head. Args: protos (torch.Tensor): A tensor of shape [mask_dim, mask_h, mask_w]. masks_in (torch.Tensor): A tensor of shape [n, mask_dim], where n is the number of masks after NMS. bboxes (torch.Tensor): A tensor of shape [n, 4], where n is the number of masks after NMS. shape (tuple): A tuple of integers representing the size of the input image in the format (h, w). upsample (bool): A flag to indicate whether to upsample the mask to the original image size. Default is False. Returns: (torch.Tensor): A binary mask tensor of shape [n, h, w], where n is the number of masks after NMS, and h and w are the height and width of the input image. The mask is applied to the bounding boxes. """ c, mh, mw = protos.shape # CHW ih, iw = shape masks = (masks_in @ protos.float().view(c, -1)).view(-1, mh, mw) # CHW downsampled_bboxes = bboxes.clone() downsampled_bboxes[:, 0] *= mw / iw downsampled_bboxes[:, 2] *= mw / iw downsampled_bboxes[:, 3] *= mh / ih downsampled_bboxes[:, 1] *= mh / ih masks = crop_mask(masks, downsampled_bboxes) # CHW if upsample: masks = F.interpolate(masks[None], shape, mode='bilinear', align_corners=False)[0] # CHW return masks.gt_(0) def process_mask_native(protos, masks_in, bboxes, shape): """ It takes the output of the mask head, and crops it after upsampling to the bounding boxes. Args: protos (torch.Tensor): [mask_dim, mask_h, mask_w] masks_in (torch.Tensor): [n, mask_dim], n is number of masks after nms bboxes (torch.Tensor): [n, 4], n is number of masks after nms shape (tuple): the size of the input image (h,w) Returns: masks (torch.Tensor): The returned masks with dimensions [h, w, n] """ c, mh, mw = protos.shape # CHW masks = (masks_in @ protos.float().view(c, -1)).view(-1, mh, mw) gain = min(mh / shape[0], mw / shape[1]) # gain = old / new pad = (mw - shape[1] * gain) / 2, (mh - shape[0] * gain) / 2 # wh padding top, left = int(pad[1]), int(pad[0]) # y, x bottom, right = int(mh - pad[1]), int(mw - pad[0]) masks = masks[:, top:bottom, left:right] masks = F.interpolate(masks[None], shape, mode='bilinear', align_corners=False)[0] # CHW masks = crop_mask(masks, bboxes) # CHW return masks.gt_(0) def scale_coords(img1_shape, coords, img0_shape, ratio_pad=None, normalize=False): """ Rescale segment coordinates (xyxy) from img1_shape to img0_shape Args: img1_shape (tuple): The shape of the image that the coords are from. coords (torch.Tensor): the coords to be scaled img0_shape (tuple): the shape of the image that the segmentation is being applied to ratio_pad (tuple): the ratio of the image size to the padded image size. normalize (bool): If True, the coordinates will be normalized to the range [0, 1]. Defaults to False Returns: coords (torch.Tensor): the segmented image. """ if ratio_pad is None: # calculate from img0_shape gain = min(img1_shape[0] / img0_shape[0], img1_shape[1] / img0_shape[1]) # gain = old / new pad = (img1_shape[1] - img0_shape[1] * gain) / 2, (img1_shape[0] - img0_shape[0] * gain) / 2 # wh padding else: gain = ratio_pad[0][0] pad = ratio_pad[1] coords[..., 0] -= pad[0] # x padding coords[..., 1] -= pad[1] # y padding coords[..., 0] /= gain coords[..., 1] /= gain clip_coords(coords, img0_shape) if normalize: coords[..., 0] /= img0_shape[1] # width coords[..., 1] /= img0_shape[0] # height return coords def masks2segments(masks, strategy='largest'): """ It takes a list of masks(n,h,w) and returns a list of segments(n,xy) Args: masks (torch.Tensor): the output of the model, which is a tensor of shape (batch_size, 160, 160) strategy (str): 'concat' or 'largest'. Defaults to largest Returns: segments (List): list of segment masks """ segments = [] for x in masks.int().cpu().numpy().astype('uint8'): c = cv2.findContours(x, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)[0] if c: if strategy == 'concat': # concatenate all segments c = np.concatenate([x.reshape(-1, 2) for x in c]) elif strategy == 'largest': # select largest segment c = np.array(c[np.array([len(x) for x in c]).argmax()]).reshape(-1, 2) else: c = np.zeros((0, 2)) # no segments found segments.append(c.astype('float32')) return segments def clean_str(s): """ Cleans a string by replacing special characters with underscore _ Args: s (str): a string needing special characters replaced Returns: (str): a string with special characters replaced by an underscore _ """ return re.sub(pattern='[|@#!¡·$€%&()=?¿^*;:,¨´><+]', repl='_', string=s) ================================================ FILE: ultralytics/yolo/utils/patches.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license """ Monkey patches to update/extend functionality of existing functions """ from pathlib import Path import cv2 import numpy as np import torch # OpenCV Multilanguage-friendly functions ------------------------------------------------------------------------------ _imshow = cv2.imshow # copy to avoid recursion errors def imread(filename, flags=cv2.IMREAD_COLOR): return cv2.imdecode(np.fromfile(filename, np.uint8), flags) def imwrite(filename, img): try: cv2.imencode(Path(filename).suffix, img)[1].tofile(filename) return True except Exception: return False def imshow(path, im): _imshow(path.encode('unicode_escape').decode(), im) # PyTorch functions ---------------------------------------------------------------------------------------------------- _torch_save = torch.save # copy to avoid recursion errors def torch_save(*args, **kwargs): # Use dill (if exists) to serialize the lambda functions where pickle does not do this try: import dill as pickle except ImportError: import pickle if 'pickle_module' not in kwargs: kwargs['pickle_module'] = pickle return _torch_save(*args, **kwargs) ================================================ FILE: ultralytics/yolo/utils/plotting.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import contextlib import math from pathlib import Path import cv2 import matplotlib.pyplot as plt import numpy as np import torch from PIL import Image, ImageDraw, ImageFont from PIL import __version__ as pil_version from scipy.ndimage import gaussian_filter1d from ultralytics.yolo.utils import LOGGER, TryExcept, plt_settings, threaded from .checks import check_font, check_version, is_ascii from .files import increment_path from .ops import clip_boxes, scale_image, xywh2xyxy, xyxy2xywh class Colors: # Ultralytics color palette https://ultralytics.com/ def __init__(self): """Initialize colors as hex = matplotlib.colors.TABLEAU_COLORS.values().""" hexs = ('FF3838', 'FF9D97', 'FF701F', 'FFB21D', 'CFD231', '48F90A', '92CC17', '3DDB86', '1A9334', '00D4BB', '2C99A8', '00C2FF', '344593', '6473FF', '0018EC', '8438FF', '520085', 'CB38FF', 'FF95C8', 'FF37C7') self.palette = [self.hex2rgb(f'#{c}') for c in hexs] self.n = len(self.palette) self.pose_palette = np.array([[255, 128, 0], [255, 153, 51], [255, 178, 102], [230, 230, 0], [255, 153, 255], [153, 204, 255], [255, 102, 255], [255, 51, 255], [102, 178, 255], [51, 153, 255], [255, 153, 153], [255, 102, 102], [255, 51, 51], [153, 255, 153], [102, 255, 102], [51, 255, 51], [0, 255, 0], [0, 0, 255], [255, 0, 0], [255, 255, 255]], dtype=np.uint8) def __call__(self, i, bgr=False): """Converts hex color codes to rgb values.""" c = self.palette[int(i) % self.n] return (c[2], c[1], c[0]) if bgr else c @staticmethod def hex2rgb(h): # rgb order (PIL) return tuple(int(h[1 + i:1 + i + 2], 16) for i in (0, 2, 4)) colors = Colors() # create instance for 'from utils.plots import colors' class Annotator: # YOLOv8 Annotator for train/val mosaics and jpgs and detect/hub inference annotations def __init__(self, im, line_width=None, font_size=None, font='Arial.ttf', pil=False, example='abc'): """Initialize the Annotator class with image and line width along with color palette for keypoints and limbs.""" assert im.data.contiguous, 'Image not contiguous. Apply np.ascontiguousarray(im) to Annotator() input images.' non_ascii = not is_ascii(example) # non-latin labels, i.e. asian, arabic, cyrillic self.pil = pil or non_ascii if self.pil: # use PIL self.im = im if isinstance(im, Image.Image) else Image.fromarray(im) self.draw = ImageDraw.Draw(self.im) try: font = check_font('Arial.Unicode.ttf' if non_ascii else font) size = font_size or max(round(sum(self.im.size) / 2 * 0.035), 12) self.font = ImageFont.truetype(str(font), size) except Exception: self.font = ImageFont.load_default() # Deprecation fix for w, h = getsize(string) -> _, _, w, h = getbox(string) if check_version(pil_version, '9.2.0'): self.font.getsize = lambda x: self.font.getbbox(x)[2:4] # text width, height else: # use cv2 self.im = im self.lw = line_width or max(round(sum(im.shape) / 2 * 0.003), 2) # line width # Pose self.skeleton = [[16, 14], [14, 12], [17, 15], [15, 13], [12, 13], [6, 12], [7, 13], [6, 7], [6, 8], [7, 9], [8, 10], [9, 11], [2, 3], [1, 2], [1, 3], [2, 4], [3, 5], [4, 6], [5, 7]] self.limb_color = colors.pose_palette[[9, 9, 9, 9, 7, 7, 7, 0, 0, 0, 0, 0, 16, 16, 16, 16, 16, 16, 16]] self.kpt_color = colors.pose_palette[[16, 16, 16, 16, 16, 0, 0, 0, 0, 0, 0, 9, 9, 9, 9, 9, 9]] def box_label(self, box, label='', color=(128, 128, 128), txt_color=(255, 255, 255)): """Add one xyxy box to image with label.""" if isinstance(box, torch.Tensor): box = box.tolist() if self.pil or not is_ascii(label): self.draw.rectangle(box, width=self.lw, outline=color) # box if label: w, h = self.font.getsize(label) # text width, height outside = box[1] - h >= 0 # label fits outside box self.draw.rectangle( (box[0], box[1] - h if outside else box[1], box[0] + w + 1, box[1] + 1 if outside else box[1] + h + 1), fill=color, ) # self.draw.text((box[0], box[1]), label, fill=txt_color, font=self.font, anchor='ls') # for PIL>8.0 self.draw.text((box[0], box[1] - h if outside else box[1]), label, fill=txt_color, font=self.font) else: # cv2 p1, p2 = (int(box[0]), int(box[1])), (int(box[2]), int(box[3])) cv2.rectangle(self.im, p1, p2, color, thickness=self.lw, lineType=cv2.LINE_AA) if label: tf = max(self.lw - 1, 1) # font thickness w, h = cv2.getTextSize(label, 0, fontScale=self.lw / 3, thickness=tf)[0] # text width, height outside = p1[1] - h >= 3 p2 = p1[0] + w, p1[1] - h - 3 if outside else p1[1] + h + 3 cv2.rectangle(self.im, p1, p2, color, -1, cv2.LINE_AA) # filled cv2.putText(self.im, label, (p1[0], p1[1] - 2 if outside else p1[1] + h + 2), 0, self.lw / 3, txt_color, thickness=tf, lineType=cv2.LINE_AA) def masks(self, masks, colors, im_gpu, alpha=0.5, retina_masks=False): """Plot masks at once. Args: masks (tensor): predicted masks on cuda, shape: [n, h, w] colors (List[List[Int]]): colors for predicted masks, [[r, g, b] * n] im_gpu (tensor): img is in cuda, shape: [3, h, w], range: [0, 1] alpha (float): mask transparency: 0.0 fully transparent, 1.0 opaque """ if self.pil: # Convert to numpy first self.im = np.asarray(self.im).copy() if len(masks) == 0: self.im[:] = im_gpu.permute(1, 2, 0).contiguous().cpu().numpy() * 255 if im_gpu.device != masks.device: im_gpu = im_gpu.to(masks.device) colors = torch.tensor(colors, device=masks.device, dtype=torch.float32) / 255.0 # shape(n,3) colors = colors[:, None, None] # shape(n,1,1,3) masks = masks.unsqueeze(3) # shape(n,h,w,1) masks_color = masks * (colors * alpha) # shape(n,h,w,3) inv_alph_masks = (1 - masks * alpha).cumprod(0) # shape(n,h,w,1) mcs = masks_color.max(dim=0).values # shape(n,h,w,3) im_gpu = im_gpu.flip(dims=[0]) # flip channel im_gpu = im_gpu.permute(1, 2, 0).contiguous() # shape(h,w,3) im_gpu = im_gpu * inv_alph_masks[-1] + mcs im_mask = (im_gpu * 255) im_mask_np = im_mask.byte().cpu().numpy() self.im[:] = im_mask_np if retina_masks else scale_image(im_mask_np, self.im.shape) if self.pil: # Convert im back to PIL and update draw self.fromarray(self.im) def kpts(self, kpts, shape=(640, 640), radius=5, kpt_line=True): """Plot keypoints on the image. Args: kpts (tensor): Predicted keypoints with shape [17, 3]. Each keypoint has (x, y, confidence). shape (tuple): Image shape as a tuple (h, w), where h is the height and w is the width. radius (int, optional): Radius of the drawn keypoints. Default is 5. kpt_line (bool, optional): If True, the function will draw lines connecting keypoints for human pose. Default is True. Note: `kpt_line=True` currently only supports human pose plotting. """ if self.pil: # Convert to numpy first self.im = np.asarray(self.im).copy() nkpt, ndim = kpts.shape is_pose = nkpt == 17 and ndim == 3 kpt_line &= is_pose # `kpt_line=True` for now only supports human pose plotting for i, k in enumerate(kpts): color_k = [int(x) for x in self.kpt_color[i]] if is_pose else colors(i) x_coord, y_coord = k[0], k[1] if x_coord % shape[1] != 0 and y_coord % shape[0] != 0: if len(k) == 3: conf = k[2] if conf < 0.5: continue cv2.circle(self.im, (int(x_coord), int(y_coord)), radius, color_k, -1, lineType=cv2.LINE_AA) if kpt_line: ndim = kpts.shape[-1] for i, sk in enumerate(self.skeleton): pos1 = (int(kpts[(sk[0] - 1), 0]), int(kpts[(sk[0] - 1), 1])) pos2 = (int(kpts[(sk[1] - 1), 0]), int(kpts[(sk[1] - 1), 1])) if ndim == 3: conf1 = kpts[(sk[0] - 1), 2] conf2 = kpts[(sk[1] - 1), 2] if conf1 < 0.5 or conf2 < 0.5: continue if pos1[0] % shape[1] == 0 or pos1[1] % shape[0] == 0 or pos1[0] < 0 or pos1[1] < 0: continue if pos2[0] % shape[1] == 0 or pos2[1] % shape[0] == 0 or pos2[0] < 0 or pos2[1] < 0: continue cv2.line(self.im, pos1, pos2, [int(x) for x in self.limb_color[i]], thickness=2, lineType=cv2.LINE_AA) if self.pil: # Convert im back to PIL and update draw self.fromarray(self.im) def rectangle(self, xy, fill=None, outline=None, width=1): """Add rectangle to image (PIL-only).""" self.draw.rectangle(xy, fill, outline, width) def text(self, xy, text, txt_color=(255, 255, 255), anchor='top', box_style=False): """Adds text to an image using PIL or cv2.""" if anchor == 'bottom': # start y from font bottom w, h = self.font.getsize(text) # text width, height xy[1] += 1 - h if self.pil: if box_style: w, h = self.font.getsize(text) self.draw.rectangle((xy[0], xy[1], xy[0] + w + 1, xy[1] + h + 1), fill=txt_color) # Using `txt_color` for background and draw fg with white color txt_color = (255, 255, 255) self.draw.text(xy, text, fill=txt_color, font=self.font) else: if box_style: tf = max(self.lw - 1, 1) # font thickness w, h = cv2.getTextSize(text, 0, fontScale=self.lw / 3, thickness=tf)[0] # text width, height outside = xy[1] - h >= 3 p2 = xy[0] + w, xy[1] - h - 3 if outside else xy[1] + h + 3 cv2.rectangle(self.im, xy, p2, txt_color, -1, cv2.LINE_AA) # filled # Using `txt_color` for background and draw fg with white color txt_color = (255, 255, 255) tf = max(self.lw - 1, 1) # font thickness cv2.putText(self.im, text, xy, 0, self.lw / 3, txt_color, thickness=tf, lineType=cv2.LINE_AA) def fromarray(self, im): """Update self.im from a numpy array.""" self.im = im if isinstance(im, Image.Image) else Image.fromarray(im) self.draw = ImageDraw.Draw(self.im) def result(self): """Return annotated image as array.""" return np.asarray(self.im) @TryExcept() # known issue https://github.com/ultralytics/yolov5/issues/5395 @plt_settings() def plot_labels(boxes, cls, names=(), save_dir=Path(''), on_plot=None): """Save and plot image with no axis or spines.""" import pandas as pd import seaborn as sn # Plot dataset labels LOGGER.info(f"Plotting labels to {save_dir / 'labels.jpg'}... ") b = boxes.transpose() # classes, boxes nc = int(cls.max() + 1) # number of classes x = pd.DataFrame(b.transpose(), columns=['x', 'y', 'width', 'height']) # Seaborn correlogram sn.pairplot(x, corner=True, diag_kind='auto', kind='hist', diag_kws=dict(bins=50), plot_kws=dict(pmax=0.9)) plt.savefig(save_dir / 'labels_correlogram.jpg', dpi=200) plt.close() # Matplotlib labels ax = plt.subplots(2, 2, figsize=(8, 8), tight_layout=True)[1].ravel() y = ax[0].hist(cls, bins=np.linspace(0, nc, nc + 1) - 0.5, rwidth=0.8) with contextlib.suppress(Exception): # color histogram bars by class [y[2].patches[i].set_color([x / 255 for x in colors(i)]) for i in range(nc)] # known issue #3195 ax[0].set_ylabel('instances') if 0 < len(names) < 30: ax[0].set_xticks(range(len(names))) ax[0].set_xticklabels(list(names.values()), rotation=90, fontsize=10) else: ax[0].set_xlabel('classes') sn.histplot(x, x='x', y='y', ax=ax[2], bins=50, pmax=0.9) sn.histplot(x, x='width', y='height', ax=ax[3], bins=50, pmax=0.9) # Rectangles boxes[:, 0:2] = 0.5 # center boxes = xywh2xyxy(boxes) * 1000 img = Image.fromarray(np.ones((1000, 1000, 3), dtype=np.uint8) * 255) for cls, box in zip(cls[:500], boxes[:500]): ImageDraw.Draw(img).rectangle(box, width=1, outline=colors(cls)) # plot ax[1].imshow(img) ax[1].axis('off') for a in [0, 1, 2, 3]: for s in ['top', 'right', 'left', 'bottom']: ax[a].spines[s].set_visible(False) fname = save_dir / 'labels.jpg' plt.savefig(fname, dpi=200) plt.close() if on_plot: on_plot(fname) def save_one_box(xyxy, im, file=Path('im.jpg'), gain=1.02, pad=10, square=False, BGR=False, save=True): """Save image crop as {file} with crop size multiple {gain} and {pad} pixels. Save and/or return crop.""" b = xyxy2xywh(xyxy.view(-1, 4)) # boxes if square: b[:, 2:] = b[:, 2:].max(1)[0].unsqueeze(1) # attempt rectangle to square b[:, 2:] = b[:, 2:] * gain + pad # box wh * gain + pad xyxy = xywh2xyxy(b).long() clip_boxes(xyxy, im.shape) crop = im[int(xyxy[0, 1]):int(xyxy[0, 3]), int(xyxy[0, 0]):int(xyxy[0, 2]), ::(1 if BGR else -1)] if save: file.parent.mkdir(parents=True, exist_ok=True) # make directory f = str(increment_path(file).with_suffix('.jpg')) # cv2.imwrite(f, crop) # save BGR, https://github.com/ultralytics/yolov5/issues/7007 chroma subsampling issue Image.fromarray(crop[..., ::-1]).save(f, quality=95, subsampling=0) # save RGB return crop @threaded def plot_images(images, batch_idx, cls, bboxes=np.zeros(0, dtype=np.float32), masks=np.zeros(0, dtype=np.uint8), kpts=np.zeros((0, 51), dtype=np.float32), paths=None, fname='images.jpg', names=None, on_plot=None): # Plot image grid with labels if isinstance(images, torch.Tensor): images = images.cpu().float().numpy() if isinstance(cls, torch.Tensor): cls = cls.cpu().numpy() if isinstance(bboxes, torch.Tensor): bboxes = bboxes.cpu().numpy() if isinstance(masks, torch.Tensor): masks = masks.cpu().numpy().astype(int) if isinstance(kpts, torch.Tensor): kpts = kpts.cpu().numpy() if isinstance(batch_idx, torch.Tensor): batch_idx = batch_idx.cpu().numpy() max_size = 1920 # max image size max_subplots = 16 # max image subplots, i.e. 4x4 bs, _, h, w = images.shape # batch size, _, height, width bs = min(bs, max_subplots) # limit plot images ns = np.ceil(bs ** 0.5) # number of subplots (square) if np.max(images[0]) <= 1: images *= 255 # de-normalise (optional) # Build Image mosaic = np.full((int(ns * h), int(ns * w), 3), 255, dtype=np.uint8) # init for i, im in enumerate(images): if i == max_subplots: # if last batch has fewer images than we expect break x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin im = im.transpose(1, 2, 0) mosaic[y:y + h, x:x + w, :] = im # Resize (optional) scale = max_size / ns / max(h, w) if scale < 1: h = math.ceil(scale * h) w = math.ceil(scale * w) mosaic = cv2.resize(mosaic, tuple(int(x * ns) for x in (w, h))) # Annotate fs = int((h + w) * ns * 0.01) # font size annotator = Annotator(mosaic, line_width=round(fs / 10), font_size=fs, pil=True, example=names) for i in range(i + 1): x, y = int(w * (i // ns)), int(h * (i % ns)) # block origin annotator.rectangle([x, y, x + w, y + h], None, (255, 255, 255), width=2) # borders if paths: annotator.text((x + 5, y + 5), text=Path(paths[i]).name[:40], txt_color=(220, 220, 220)) # filenames if len(cls) > 0: idx = batch_idx == i classes = cls[idx].astype('int') if len(bboxes): boxes = xywh2xyxy(bboxes[idx, :4]).T labels = bboxes.shape[1] == 4 # labels if no conf column conf = None if labels else bboxes[idx, 4] # check for confidence presence (label vs pred) if boxes.shape[1]: if boxes.max() <= 1.01: # if normalized with tolerance 0.01 boxes[[0, 2]] *= w # scale to pixels boxes[[1, 3]] *= h elif scale < 1: # absolute coords need scale if image scales boxes *= scale boxes[[0, 2]] += x boxes[[1, 3]] += y for j, box in enumerate(boxes.T.tolist()): c = classes[j] color = colors(c) c = names.get(c, c) if names else c if labels or conf[j] > 0.25: # 0.25 conf thresh label = f'{c}' if labels else f'{c} {conf[j]:.1f}' annotator.box_label(box, label, color=color) elif len(classes): for c in classes: color = colors(c) c = names.get(c, c) if names else c annotator.text((x, y), f'{c}', txt_color=color, box_style=True) # Plot keypoints if len(kpts): kpts_ = kpts[idx].copy() if len(kpts_): if kpts_[..., 0].max() <= 1.01 or kpts_[..., 1].max() <= 1.01: # if normalized with tolerance .01 kpts_[..., 0] *= w # scale to pixels kpts_[..., 1] *= h elif scale < 1: # absolute coords need scale if image scales kpts_ *= scale kpts_[..., 0] += x kpts_[..., 1] += y for j in range(len(kpts_)): if labels or conf[j] > 0.25: # 0.25 conf thresh annotator.kpts(kpts_[j]) # Plot masks if len(masks): if idx.shape[0] == masks.shape[0]: # overlap_masks=False image_masks = masks[idx] else: # overlap_masks=True image_masks = masks[[i]] # (1, 640, 640) nl = idx.sum() index = np.arange(nl).reshape((nl, 1, 1)) + 1 image_masks = np.repeat(image_masks, nl, axis=0) image_masks = np.where(image_masks == index, 1.0, 0.0) im = np.asarray(annotator.im).copy() for j, box in enumerate(boxes.T.tolist()): if labels or conf[j] > 0.25: # 0.25 conf thresh color = colors(classes[j]) mh, mw = image_masks[j].shape if mh != h or mw != w: mask = image_masks[j].astype(np.uint8) mask = cv2.resize(mask, (w, h)) mask = mask.astype(bool) else: mask = image_masks[j].astype(bool) with contextlib.suppress(Exception): im[y:y + h, x:x + w, :][mask] = im[y:y + h, x:x + w, :][mask] * 0.4 + np.array(color) * 0.6 annotator.fromarray(im) annotator.im.save(fname) # save if on_plot: on_plot(fname) @plt_settings() def plot_results(file='path/to/results.csv', dir='', segment=False, pose=False, classify=False, on_plot=None): """Plot training results.csv. Usage: from utils.plots import *; plot_results('path/to/results.csv').""" import pandas as pd save_dir = Path(file).parent if file else Path(dir) if classify: fig, ax = plt.subplots(2, 2, figsize=(6, 6), tight_layout=True) index = [1, 4, 2, 3] elif segment: fig, ax = plt.subplots(2, 8, figsize=(18, 6), tight_layout=True) index = [1, 2, 3, 4, 5, 6, 9, 10, 13, 14, 15, 16, 7, 8, 11, 12] elif pose: fig, ax = plt.subplots(2, 9, figsize=(21, 6), tight_layout=True) index = [1, 2, 3, 4, 5, 6, 7, 10, 11, 14, 15, 16, 17, 18, 8, 9, 12, 13] else: fig, ax = plt.subplots(2, 5, figsize=(12, 6), tight_layout=True) index = [1, 2, 3, 4, 5, 8, 9, 10, 6, 7] ax = ax.ravel() files = list(save_dir.glob('results*.csv')) assert len(files), f'No results.csv files found in {save_dir.resolve()}, nothing to plot.' for f in files: try: data = pd.read_csv(f) s = [x.strip() for x in data.columns] x = data.values[:, 0] for i, j in enumerate(index): y = data.values[:, j].astype('float') # y[y == 0] = np.nan # don't show zero values ax[i].plot(x, y, marker='.', label=f.stem, linewidth=2, markersize=8) # actual results ax[i].plot(x, gaussian_filter1d(y, sigma=3), ':', label='smooth', linewidth=2) # smoothing line ax[i].set_title(s[j], fontsize=12) # if j in [8, 9, 10]: # share train and val loss y axes # ax[i].get_shared_y_axes().join(ax[i], ax[i - 5]) except Exception as e: LOGGER.warning(f'WARNING: Plotting error for {f}: {e}') ax[1].legend() fname = save_dir / 'results.png' fig.savefig(fname, dpi=200) plt.close() if on_plot: on_plot(fname) def output_to_target(output, max_det=300): """Convert model output to target format [batch_id, class_id, x, y, w, h, conf] for plotting.""" targets = [] for i, o in enumerate(output): box, conf, cls = o[:max_det, :6].cpu().split((4, 1, 1), 1) j = torch.full((conf.shape[0], 1), i) targets.append(torch.cat((j, cls, xyxy2xywh(box), conf), 1)) targets = torch.cat(targets, 0).numpy() return targets[:, 0], targets[:, 1], targets[:, 2:] def feature_visualization(x, module_type, stage, n=32, save_dir=Path('runs/detect/exp')): """ Visualize feature maps of a given model module during inference. Args: x (torch.Tensor): Features to be visualized. module_type (str): Module type. stage (int): Module stage within the model. n (int, optional): Maximum number of feature maps to plot. Defaults to 32. save_dir (Path, optional): Directory to save results. Defaults to Path('runs/detect/exp'). """ for m in ['Detect', 'Pose', 'Segment']: if m in module_type: return batch, channels, height, width = x.shape # batch, channels, height, width if height > 1 and width > 1: f = save_dir / f"stage{stage}_{module_type.split('.')[-1]}_features.png" # filename blocks = torch.chunk(x[0].cpu(), channels, dim=0) # select batch index 0, block by channels n = min(n, channels) # number of plots fig, ax = plt.subplots(math.ceil(n / 8), 8, tight_layout=True) # 8 rows x n/8 cols ax = ax.ravel() plt.subplots_adjust(wspace=0.05, hspace=0.05) for i in range(n): ax[i].imshow(blocks[i].squeeze()) # cmap='gray' ax[i].axis('off') LOGGER.info(f'Saving {f}... ({n}/{channels})') plt.savefig(f, dpi=300, bbox_inches='tight') plt.close() np.save(str(f.with_suffix('.npy')), x[0].cpu().numpy()) # npy save ================================================ FILE: ultralytics/yolo/utils/tal.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import torch import torch.nn as nn from .checks import check_version from .metrics import bbox_iou TORCH_1_10 = check_version(torch.__version__, '1.10.0') def select_candidates_in_gts(xy_centers, gt_bboxes, eps=1e-9): """select the positive anchor center in gt Args: xy_centers (Tensor): shape(h*w, 4) gt_bboxes (Tensor): shape(b, n_boxes, 4) Return: (Tensor): shape(b, n_boxes, h*w) """ n_anchors = xy_centers.shape[0] bs, n_boxes, _ = gt_bboxes.shape lt, rb = gt_bboxes.view(-1, 1, 4).chunk(2, 2) # left-top, right-bottom bbox_deltas = torch.cat((xy_centers[None] - lt, rb - xy_centers[None]), dim=2).view(bs, n_boxes, n_anchors, -1) # return (bbox_deltas.min(3)[0] > eps).to(gt_bboxes.dtype) return bbox_deltas.amin(3).gt_(eps) def select_highest_overlaps(mask_pos, overlaps, n_max_boxes): """if an anchor box is assigned to multiple gts, the one with the highest iou will be selected. Args: mask_pos (Tensor): shape(b, n_max_boxes, h*w) overlaps (Tensor): shape(b, n_max_boxes, h*w) Return: target_gt_idx (Tensor): shape(b, h*w) fg_mask (Tensor): shape(b, h*w) mask_pos (Tensor): shape(b, n_max_boxes, h*w) """ # (b, n_max_boxes, h*w) -> (b, h*w) fg_mask = mask_pos.sum(-2) if fg_mask.max() > 1: # one anchor is assigned to multiple gt_bboxes mask_multi_gts = (fg_mask.unsqueeze(1) > 1).expand(-1, n_max_boxes, -1) # (b, n_max_boxes, h*w) max_overlaps_idx = overlaps.argmax(1) # (b, h*w) is_max_overlaps = torch.zeros(mask_pos.shape, dtype=mask_pos.dtype, device=mask_pos.device) is_max_overlaps.scatter_(1, max_overlaps_idx.unsqueeze(1), 1) mask_pos = torch.where(mask_multi_gts, is_max_overlaps, mask_pos).float() # (b, n_max_boxes, h*w) fg_mask = mask_pos.sum(-2) # Find each grid serve which gt(index) target_gt_idx = mask_pos.argmax(-2) # (b, h*w) return target_gt_idx, fg_mask, mask_pos class TaskAlignedAssigner(nn.Module): """ A task-aligned assigner for object detection. This class assigns ground-truth (gt) objects to anchors based on the task-aligned metric, which combines both classification and localization information. Attributes: topk (int): The number of top candidates to consider. num_classes (int): The number of object classes. alpha (float): The alpha parameter for the classification component of the task-aligned metric. beta (float): The beta parameter for the localization component of the task-aligned metric. eps (float): A small value to prevent division by zero. """ def __init__(self, topk=13, num_classes=80, alpha=1.0, beta=6.0, eps=1e-9): """Initialize a TaskAlignedAssigner object with customizable hyperparameters.""" super().__init__() self.topk = topk self.num_classes = num_classes self.bg_idx = num_classes self.alpha = alpha self.beta = beta self.eps = eps @torch.no_grad() def forward(self, pd_scores, pd_bboxes, anc_points, gt_labels, gt_bboxes, mask_gt): """ Compute the task-aligned assignment. Reference https://github.com/Nioolek/PPYOLOE_pytorch/blob/master/ppyoloe/assigner/tal_assigner.py Args: pd_scores (Tensor): shape(bs, num_total_anchors, num_classes) pd_bboxes (Tensor): shape(bs, num_total_anchors, 4) anc_points (Tensor): shape(num_total_anchors, 2) gt_labels (Tensor): shape(bs, n_max_boxes, 1) gt_bboxes (Tensor): shape(bs, n_max_boxes, 4) mask_gt (Tensor): shape(bs, n_max_boxes, 1) Returns: target_labels (Tensor): shape(bs, num_total_anchors) target_bboxes (Tensor): shape(bs, num_total_anchors, 4) target_scores (Tensor): shape(bs, num_total_anchors, num_classes) fg_mask (Tensor): shape(bs, num_total_anchors) target_gt_idx (Tensor): shape(bs, num_total_anchors) """ self.bs = pd_scores.size(0) self.n_max_boxes = gt_bboxes.size(1) if self.n_max_boxes == 0: device = gt_bboxes.device return (torch.full_like(pd_scores[..., 0], self.bg_idx).to(device), torch.zeros_like(pd_bboxes).to(device), torch.zeros_like(pd_scores).to(device), torch.zeros_like(pd_scores[..., 0]).to(device), torch.zeros_like(pd_scores[..., 0]).to(device)) mask_pos, align_metric, overlaps = self.get_pos_mask(pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points, mask_gt) target_gt_idx, fg_mask, mask_pos = select_highest_overlaps(mask_pos, overlaps, self.n_max_boxes) # Assigned target target_labels, target_bboxes, target_scores = self.get_targets(gt_labels, gt_bboxes, target_gt_idx, fg_mask) # Normalize align_metric *= mask_pos pos_align_metrics = align_metric.amax(axis=-1, keepdim=True) # b, max_num_obj pos_overlaps = (overlaps * mask_pos).amax(axis=-1, keepdim=True) # b, max_num_obj norm_align_metric = (align_metric * pos_overlaps / (pos_align_metrics + self.eps)).amax(-2).unsqueeze(-1) target_scores = target_scores * norm_align_metric return target_labels, target_bboxes, target_scores, fg_mask.bool(), target_gt_idx def get_pos_mask(self, pd_scores, pd_bboxes, gt_labels, gt_bboxes, anc_points, mask_gt): """Get in_gts mask, (b, max_num_obj, h*w).""" mask_in_gts = select_candidates_in_gts(anc_points, gt_bboxes) # Get anchor_align metric, (b, max_num_obj, h*w) align_metric, overlaps = self.get_box_metrics(pd_scores, pd_bboxes, gt_labels, gt_bboxes, mask_in_gts * mask_gt) # Get topk_metric mask, (b, max_num_obj, h*w) mask_topk = self.select_topk_candidates(align_metric, topk_mask=mask_gt.expand(-1, -1, self.topk).bool()) # Merge all mask to a final mask, (b, max_num_obj, h*w) mask_pos = mask_topk * mask_in_gts * mask_gt return mask_pos, align_metric, overlaps def get_box_metrics(self, pd_scores, pd_bboxes, gt_labels, gt_bboxes, mask_gt): """Compute alignment metric given predicted and ground truth bounding boxes.""" na = pd_bboxes.shape[-2] mask_gt = mask_gt.bool() # b, max_num_obj, h*w overlaps = torch.zeros([self.bs, self.n_max_boxes, na], dtype=pd_bboxes.dtype, device=pd_bboxes.device) bbox_scores = torch.zeros([self.bs, self.n_max_boxes, na], dtype=pd_scores.dtype, device=pd_scores.device) ind = torch.zeros([2, self.bs, self.n_max_boxes], dtype=torch.long) # 2, b, max_num_obj ind[0] = torch.arange(end=self.bs).view(-1, 1).expand(-1, self.n_max_boxes) # b, max_num_obj ind[1] = gt_labels.squeeze(-1) # b, max_num_obj # Get the scores of each grid for each gt cls bbox_scores[mask_gt] = pd_scores[ind[0], :, ind[1]][mask_gt] # b, max_num_obj, h*w # (b, max_num_obj, 1, 4), (b, 1, h*w, 4) pd_boxes = pd_bboxes.unsqueeze(1).expand(-1, self.n_max_boxes, -1, -1)[mask_gt] gt_boxes = gt_bboxes.unsqueeze(2).expand(-1, -1, na, -1)[mask_gt] overlaps[mask_gt] = bbox_iou(gt_boxes, pd_boxes, xywh=False, CIoU=True).squeeze(-1).clamp_(0) align_metric = bbox_scores.pow(self.alpha) * overlaps.pow(self.beta) return align_metric, overlaps def select_topk_candidates(self, metrics, largest=True, topk_mask=None): """ Select the top-k candidates based on the given metrics. Args: metrics (Tensor): A tensor of shape (b, max_num_obj, h*w), where b is the batch size, max_num_obj is the maximum number of objects, and h*w represents the total number of anchor points. largest (bool): If True, select the largest values; otherwise, select the smallest values. topk_mask (Tensor): An optional boolean tensor of shape (b, max_num_obj, topk), where topk is the number of top candidates to consider. If not provided, the top-k values are automatically computed based on the given metrics. Returns: (Tensor): A tensor of shape (b, max_num_obj, h*w) containing the selected top-k candidates. """ # (b, max_num_obj, topk) topk_metrics, topk_idxs = torch.topk(metrics, self.topk, dim=-1, largest=largest) if topk_mask is None: topk_mask = (topk_metrics.max(-1, keepdim=True)[0] > self.eps).expand_as(topk_idxs) # (b, max_num_obj, topk) topk_idxs.masked_fill_(~topk_mask, 0) # (b, max_num_obj, topk, h*w) -> (b, max_num_obj, h*w) count_tensor = torch.zeros(metrics.shape, dtype=torch.int8, device=topk_idxs.device) ones = torch.ones_like(topk_idxs[:, :, :1], dtype=torch.int8, device=topk_idxs.device) for k in range(self.topk): # Expand topk_idxs for each value of k and add 1 at the specified positions count_tensor.scatter_add_(-1, topk_idxs[:, :, k:k + 1], ones) # count_tensor.scatter_add_(-1, topk_idxs, torch.ones_like(topk_idxs, dtype=torch.int8, device=topk_idxs.device)) # filter invalid bboxes count_tensor.masked_fill_(count_tensor > 1, 0) return count_tensor.to(metrics.dtype) def get_targets(self, gt_labels, gt_bboxes, target_gt_idx, fg_mask): """ Compute target labels, target bounding boxes, and target scores for the positive anchor points. Args: gt_labels (Tensor): Ground truth labels of shape (b, max_num_obj, 1), where b is the batch size and max_num_obj is the maximum number of objects. gt_bboxes (Tensor): Ground truth bounding boxes of shape (b, max_num_obj, 4). target_gt_idx (Tensor): Indices of the assigned ground truth objects for positive anchor points, with shape (b, h*w), where h*w is the total number of anchor points. fg_mask (Tensor): A boolean tensor of shape (b, h*w) indicating the positive (foreground) anchor points. Returns: (Tuple[Tensor, Tensor, Tensor]): A tuple containing the following tensors: - target_labels (Tensor): Shape (b, h*w), containing the target labels for positive anchor points. - target_bboxes (Tensor): Shape (b, h*w, 4), containing the target bounding boxes for positive anchor points. - target_scores (Tensor): Shape (b, h*w, num_classes), containing the target scores for positive anchor points, where num_classes is the number of object classes. """ # Assigned target labels, (b, 1) batch_ind = torch.arange(end=self.bs, dtype=torch.int64, device=gt_labels.device)[..., None] target_gt_idx = target_gt_idx + batch_ind * self.n_max_boxes # (b, h*w) target_labels = gt_labels.long().flatten()[target_gt_idx] # (b, h*w) # Assigned target boxes, (b, max_num_obj, 4) -> (b, h*w) target_bboxes = gt_bboxes.view(-1, 4)[target_gt_idx] # Assigned target scores target_labels.clamp_(0) # 10x faster than F.one_hot() target_scores = torch.zeros((target_labels.shape[0], target_labels.shape[1], self.num_classes), dtype=torch.int64, device=target_labels.device) # (b, h*w, 80) target_scores.scatter_(2, target_labels.unsqueeze(-1), 1) fg_scores_mask = fg_mask[:, :, None].repeat(1, 1, self.num_classes) # (b, h*w, 80) target_scores = torch.where(fg_scores_mask > 0, target_scores, 0) return target_labels, target_bboxes, target_scores def make_anchors(feats, strides, grid_cell_offset=0.5): """Generate anchors from features.""" anchor_points, stride_tensor = [], [] assert feats is not None dtype, device = feats[0].dtype, feats[0].device for i, stride in enumerate(strides): _, _, h, w = feats[i].shape sx = torch.arange(end=w, device=device, dtype=dtype) + grid_cell_offset # shift x sy = torch.arange(end=h, device=device, dtype=dtype) + grid_cell_offset # shift y sy, sx = torch.meshgrid(sy, sx, indexing='ij') if TORCH_1_10 else torch.meshgrid(sy, sx) anchor_points.append(torch.stack((sx, sy), -1).view(-1, 2)) stride_tensor.append(torch.full((h * w, 1), stride, dtype=dtype, device=device)) return torch.cat(anchor_points), torch.cat(stride_tensor) def dist2bbox(distance, anchor_points, xywh=True, dim=-1): """Transform distance(ltrb) to box(xywh or xyxy).""" lt, rb = distance.chunk(2, dim) x1y1 = anchor_points - lt x2y2 = anchor_points + rb if xywh: c_xy = (x1y1 + x2y2) / 2 wh = x2y2 - x1y1 return torch.cat((c_xy, wh), dim) # xywh bbox return torch.cat((x1y1, x2y2), dim) # xyxy bbox def bbox2dist(anchor_points, bbox, reg_max): """Transform bbox(xyxy) to dist(ltrb).""" x1y1, x2y2 = bbox.chunk(2, -1) return torch.cat((anchor_points - x1y1, x2y2 - anchor_points), -1).clamp_(0, reg_max - 0.01) # dist (lt, rb) ================================================ FILE: ultralytics/yolo/utils/torch_utils.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import math import os import platform import random import time from contextlib import contextmanager from copy import deepcopy from pathlib import Path from typing import Union import numpy as np import torch import torch.distributed as dist import torch.nn as nn import torch.nn.functional as F import torchvision from ultralytics.yolo.utils import DEFAULT_CFG_DICT, DEFAULT_CFG_KEYS, LOGGER, RANK, __version__ from ultralytics.yolo.utils.checks import check_version try: import thop except ImportError: thop = None TORCHVISION_0_10 = check_version(torchvision.__version__, '0.10.0') TORCH_1_9 = check_version(torch.__version__, '1.9.0') TORCH_1_11 = check_version(torch.__version__, '1.11.0') TORCH_1_12 = check_version(torch.__version__, '1.12.0') TORCH_2_0 = check_version(torch.__version__, minimum='2.0') @contextmanager def torch_distributed_zero_first(local_rank: int): """Decorator to make all processes in distributed training wait for each local_master to do something.""" initialized = torch.distributed.is_available() and torch.distributed.is_initialized() if initialized and local_rank not in (-1, 0): dist.barrier(device_ids=[local_rank]) yield if initialized and local_rank == 0: dist.barrier(device_ids=[0]) def smart_inference_mode(): """Applies torch.inference_mode() decorator if torch>=1.9.0 else torch.no_grad() decorator.""" def decorate(fn): """Applies appropriate torch decorator for inference mode based on torch version.""" return (torch.inference_mode if TORCH_1_9 else torch.no_grad)()(fn) return decorate def select_device(device='', batch=0, newline=False, verbose=True): """Selects PyTorch Device. Options are device = None or 'cpu' or 0 or '0' or '0,1,2,3'.""" s = f'Ultralytics YOLOv{__version__} 🚀 Python-{platform.python_version()} torch-{torch.__version__} ' device = str(device).lower() for remove in 'cuda:', 'none', '(', ')', '[', ']', "'", ' ': device = device.replace(remove, '') # to string, 'cuda:0' -> '0' and '(0, 1)' -> '0,1' cpu = device == 'cpu' mps = device == 'mps' # Apple Metal Performance Shaders (MPS) if cpu or mps: os.environ['CUDA_VISIBLE_DEVICES'] = '-1' # force torch.cuda.is_available() = False elif device: # non-cpu device requested if device == 'cuda': device = '0' visible = os.environ.get('CUDA_VISIBLE_DEVICES', None) os.environ['CUDA_VISIBLE_DEVICES'] = device # set environment variable - must be before assert is_available() if not (torch.cuda.is_available() and torch.cuda.device_count() >= len(device.replace(',', ''))): LOGGER.info(s) install = 'See https://pytorch.org/get-started/locally/ for up-to-date torch install instructions if no ' \ 'CUDA devices are seen by torch.\n' if torch.cuda.device_count() == 0 else '' raise ValueError(f"Invalid CUDA 'device={device}' requested." f" Use 'device=cpu' or pass valid CUDA device(s) if available," f" i.e. 'device=0' or 'device=0,1,2,3' for Multi-GPU.\n" f'\ntorch.cuda.is_available(): {torch.cuda.is_available()}' f'\ntorch.cuda.device_count(): {torch.cuda.device_count()}' f"\nos.environ['CUDA_VISIBLE_DEVICES']: {visible}\n" f'{install}') if not cpu and not mps and torch.cuda.is_available(): # prefer GPU if available devices = device.split(',') if device else '0' # range(torch.cuda.device_count()) # i.e. 0,1,6,7 n = len(devices) # device count if n > 1 and batch > 0 and batch % n != 0: # check batch_size is divisible by device_count raise ValueError(f"'batch={batch}' must be a multiple of GPU count {n}. Try 'batch={batch // n * n}' or " f"'batch={batch // n * n + n}', the nearest batch sizes evenly divisible by {n}.") space = ' ' * (len(s) + 1) for i, d in enumerate(devices): p = torch.cuda.get_device_properties(i) s += f"{'' if i == 0 else space}CUDA:{d} ({p.name}, {p.total_memory / (1 << 20):.0f}MiB)\n" # bytes to MB arg = 'cuda:0' elif mps and getattr(torch, 'has_mps', False) and torch.backends.mps.is_available() and TORCH_2_0: # Prefer MPS if available s += 'MPS\n' arg = 'mps' else: # revert to CPU s += 'CPU\n' arg = 'cpu' if verbose and RANK == -1: LOGGER.info(s if newline else s.rstrip()) return torch.device(arg) def time_sync(): """PyTorch-accurate time.""" if torch.cuda.is_available(): torch.cuda.synchronize() return time.time() def fuse_conv_and_bn(conv, bn): """Fuse Conv2d() and BatchNorm2d() layers https://tehnokv.com/posts/fusing-batchnorm-and-conv/.""" fusedconv = nn.Conv2d(conv.in_channels, conv.out_channels, kernel_size=conv.kernel_size, stride=conv.stride, padding=conv.padding, dilation=conv.dilation, groups=conv.groups, bias=True).requires_grad_(False).to(conv.weight.device) # Prepare filters w_conv = conv.weight.clone().view(conv.out_channels, -1) w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) fusedconv.weight.copy_(torch.mm(w_bn, w_conv).view(fusedconv.weight.shape)) # Prepare spatial bias b_conv = torch.zeros(conv.weight.size(0), device=conv.weight.device) if conv.bias is None else conv.bias b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps)) fusedconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn) return fusedconv def fuse_deconv_and_bn(deconv, bn): """Fuse ConvTranspose2d() and BatchNorm2d() layers.""" fuseddconv = nn.ConvTranspose2d(deconv.in_channels, deconv.out_channels, kernel_size=deconv.kernel_size, stride=deconv.stride, padding=deconv.padding, output_padding=deconv.output_padding, dilation=deconv.dilation, groups=deconv.groups, bias=True).requires_grad_(False).to(deconv.weight.device) # Prepare filters w_deconv = deconv.weight.clone().view(deconv.out_channels, -1) w_bn = torch.diag(bn.weight.div(torch.sqrt(bn.eps + bn.running_var))) fuseddconv.weight.copy_(torch.mm(w_bn, w_deconv).view(fuseddconv.weight.shape)) # Prepare spatial bias b_conv = torch.zeros(deconv.weight.size(1), device=deconv.weight.device) if deconv.bias is None else deconv.bias b_bn = bn.bias - bn.weight.mul(bn.running_mean).div(torch.sqrt(bn.running_var + bn.eps)) fuseddconv.bias.copy_(torch.mm(w_bn, b_conv.reshape(-1, 1)).reshape(-1) + b_bn) return fuseddconv def model_info(model, detailed=False, verbose=True, imgsz=640): """Model information. imgsz may be int or list, i.e. imgsz=640 or imgsz=[640, 320].""" if not verbose: return n_p = get_num_params(model) # number of parameters n_g = get_num_gradients(model) # number of gradients n_l = len(list(model.modules())) # number of layers if detailed: LOGGER.info( f"{'layer':>5} {'name':>40} {'gradient':>9} {'parameters':>12} {'shape':>20} {'mu':>10} {'sigma':>10}") for i, (name, p) in enumerate(model.named_parameters()): name = name.replace('module_list.', '') LOGGER.info('%5g %40s %9s %12g %20s %10.3g %10.3g %10s' % (i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std(), p.dtype)) flops = get_flops(model, imgsz) fused = ' (fused)' if getattr(model, 'is_fused', lambda: False)() else '' fs = f', {flops:.1f} GFLOPs' if flops else '' yaml_file = getattr(model, 'yaml_file', '') or getattr(model, 'yaml', {}).get('yaml_file', '') model_name = Path(yaml_file).stem.replace('yolo', 'YOLO') or 'Model' LOGGER.info(f'{model_name} summary{fused}: {n_l} layers, {n_p} parameters, {n_g} gradients{fs}') return n_l, n_p, n_g, flops def get_num_params(model): """Return the total number of parameters in a YOLO model.""" return sum(x.numel() for x in model.parameters()) def get_num_gradients(model): """Return the total number of parameters with gradients in a YOLO model.""" return sum(x.numel() for x in model.parameters() if x.requires_grad) def model_info_for_loggers(trainer): """ Return model info dict with useful model information. Example for YOLOv8n: {'model/parameters': 3151904, 'model/GFLOPs': 8.746, 'model/speed_ONNX(ms)': 41.244, 'model/speed_TensorRT(ms)': 3.211, 'model/speed_PyTorch(ms)': 18.755} """ if trainer.args.profile: # profile ONNX and TensorRT times from ultralytics.yolo.utils.benchmarks import ProfileModels results = ProfileModels([trainer.last], device=trainer.device).profile()[0] results.pop('model/name') else: # only return PyTorch times from most recent validation results = { 'model/parameters': get_num_params(trainer.model), 'model/GFLOPs': round(get_flops(trainer.model), 3)} results['model/speed_PyTorch(ms)'] = round(trainer.validator.speed['inference'], 3) return results def get_flops(model, imgsz=640): """Return a YOLO model's FLOPs.""" try: model = de_parallel(model) p = next(model.parameters()) stride = max(int(model.stride.max()), 32) if hasattr(model, 'stride') else 32 # max stride im = torch.empty((1, p.shape[1], stride, stride), device=p.device) # input image in BCHW format flops = thop.profile(deepcopy(model), inputs=[im], verbose=False)[0] / 1E9 * 2 if thop else 0 # stride GFLOPs imgsz = imgsz if isinstance(imgsz, list) else [imgsz, imgsz] # expand if int/float return flops * imgsz[0] / stride * imgsz[1] / stride # 640x640 GFLOPs except Exception: return 0 def get_flops_with_torch_profiler(model, imgsz=640): # Compute model FLOPs (thop alternative) model = de_parallel(model) p = next(model.parameters()) stride = (max(int(model.stride.max()), 32) if hasattr(model, 'stride') else 32) * 2 # max stride im = torch.zeros((1, p.shape[1], stride, stride), device=p.device) # input image in BCHW format with torch.profiler.profile(with_flops=True) as prof: model(im) flops = sum(x.flops for x in prof.key_averages()) / 1E9 imgsz = imgsz if isinstance(imgsz, list) else [imgsz, imgsz] # expand if int/float flops = flops * imgsz[0] / stride * imgsz[1] / stride # 640x640 GFLOPs return flops def initialize_weights(model): """Initialize model weights to random values.""" for m in model.modules(): t = type(m) if t is nn.Conv2d: pass # nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') elif t is nn.BatchNorm2d: m.eps = 1e-3 m.momentum = 0.03 elif t in [nn.Hardswish, nn.LeakyReLU, nn.ReLU, nn.ReLU6, nn.SiLU]: m.inplace = True def scale_img(img, ratio=1.0, same_shape=False, gs=32): # img(16,3,256,416) # Scales img(bs,3,y,x) by ratio constrained to gs-multiple if ratio == 1.0: return img h, w = img.shape[2:] s = (int(h * ratio), int(w * ratio)) # new size img = F.interpolate(img, size=s, mode='bilinear', align_corners=False) # resize if not same_shape: # pad/crop img h, w = (math.ceil(x * ratio / gs) * gs for x in (h, w)) return F.pad(img, [0, w - s[1], 0, h - s[0]], value=0.447) # value = imagenet mean def make_divisible(x, divisor): """Returns nearest x divisible by divisor.""" if isinstance(divisor, torch.Tensor): divisor = int(divisor.max()) # to int return math.ceil(x / divisor) * divisor def copy_attr(a, b, include=(), exclude=()): """Copies attributes from object 'b' to object 'a', with options to include/exclude certain attributes.""" for k, v in b.__dict__.items(): if (len(include) and k not in include) or k.startswith('_') or k in exclude: continue else: setattr(a, k, v) def get_latest_opset(): """Return second-most (for maturity) recently supported ONNX opset by this version of torch.""" return max(int(k[14:]) for k in vars(torch.onnx) if 'symbolic_opset' in k) - 1 # opset def intersect_dicts(da, db, exclude=()): """Returns a dictionary of intersecting keys with matching shapes, excluding 'exclude' keys, using da values.""" return {k: v for k, v in da.items() if k in db and all(x not in k for x in exclude) and v.shape == db[k].shape} def is_parallel(model): """Returns True if model is of type DP or DDP.""" return isinstance(model, (nn.parallel.DataParallel, nn.parallel.DistributedDataParallel)) def de_parallel(model): """De-parallelize a model: returns single-GPU model if model is of type DP or DDP.""" return model.module if is_parallel(model) else model def one_cycle(y1=0.0, y2=1.0, steps=100): """Returns a lambda function for sinusoidal ramp from y1 to y2 https://arxiv.org/pdf/1812.01187.pdf.""" return lambda x: ((1 - math.cos(x * math.pi / steps)) / 2) * (y2 - y1) + y1 def init_seeds(seed=0, deterministic=False): """Initialize random number generator (RNG) seeds https://pytorch.org/docs/stable/notes/randomness.html.""" random.seed(seed) np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) # for Multi-GPU, exception safe # torch.backends.cudnn.benchmark = True # AutoBatch problem https://github.com/ultralytics/yolov5/issues/9287 if deterministic: # https://github.com/ultralytics/yolov5/pull/8213 if TORCH_2_0: torch.use_deterministic_algorithms(True) torch.backends.cudnn.deterministic = True os.environ['CUBLAS_WORKSPACE_CONFIG'] = ':4096:8' os.environ['PYTHONHASHSEED'] = str(seed) else: LOGGER.warning('WARNING ⚠️ Upgrade to torch>=2.0.0 for deterministic training.') else: torch.use_deterministic_algorithms(False) torch.backends.cudnn.deterministic = False class ModelEMA: """Updated Exponential Moving Average (EMA) from https://github.com/rwightman/pytorch-image-models Keeps a moving average of everything in the model state_dict (parameters and buffers) For EMA details see https://www.tensorflow.org/api_docs/python/tf/train/ExponentialMovingAverage To disable EMA set the `enabled` attribute to `False`. """ def __init__(self, model, decay=0.9999, tau=2000, updates=0): """Create EMA.""" self.ema = deepcopy(de_parallel(model)).eval() # FP32 EMA self.updates = updates # number of EMA updates self.decay = lambda x: decay * (1 - math.exp(-x / tau)) # decay exponential ramp (to help early epochs) for p in self.ema.parameters(): p.requires_grad_(False) self.enabled = True def update(self, model): """Update EMA parameters.""" if self.enabled: self.updates += 1 d = self.decay(self.updates) msd = de_parallel(model).state_dict() # model state_dict for k, v in self.ema.state_dict().items(): if v.dtype.is_floating_point: # true for FP16 and FP32 v *= d v += (1 - d) * msd[k].detach() # assert v.dtype == msd[k].dtype == torch.float32, f'{k}: EMA {v.dtype}, model {msd[k].dtype}' def update_attr(self, model, include=(), exclude=('process_group', 'reducer')): """Updates attributes and saves stripped model with optimizer removed.""" if self.enabled: copy_attr(self.ema, model, include, exclude) def strip_optimizer(f: Union[str, Path] = 'best.pt', s: str = '') -> None: """ Strip optimizer from 'f' to finalize training, optionally save as 's'. Args: f (str): file path to model to strip the optimizer from. Default is 'best.pt'. s (str): file path to save the model with stripped optimizer to. If not provided, 'f' will be overwritten. Returns: None Usage: from pathlib import Path from ultralytics.yolo.utils.torch_utils import strip_optimizer for f in Path('/Users/glennjocher/Downloads/weights').rglob('*.pt'): strip_optimizer(f) """ # Use dill (if exists) to serialize the lambda functions where pickle does not do this try: import dill as pickle except ImportError: import pickle x = torch.load(f, map_location=torch.device('cpu')) args = {**DEFAULT_CFG_DICT, **x['train_args']} if 'train_args' in x else None # combine args if x.get('ema'): x['model'] = x['ema'] # replace model with ema for k in 'optimizer', 'best_fitness', 'ema', 'updates': # keys x[k] = None x['epoch'] = -1 x['model'].half() # to FP16 for p in x['model'].parameters(): p.requires_grad = False x['train_args'] = {k: v for k, v in args.items() if k in DEFAULT_CFG_KEYS} # strip non-default keys # x['model'].args = x['train_args'] torch.save(x, s or f, pickle_module=pickle) mb = os.path.getsize(s or f) / 1E6 # filesize LOGGER.info(f"Optimizer stripped from {f},{f' saved as {s},' if s else ''} {mb:.1f}MB") def profile(input, ops, n=10, device=None): """ YOLOv8 speed/memory/FLOPs profiler Usage: input = torch.randn(16, 3, 640, 640) m1 = lambda x: x * torch.sigmoid(x) m2 = nn.SiLU() profile(input, [m1, m2], n=100) # profile over 100 iterations """ results = [] if not isinstance(device, torch.device): device = select_device(device) LOGGER.info(f"{'Params':>12s}{'GFLOPs':>12s}{'GPU_mem (GB)':>14s}{'forward (ms)':>14s}{'backward (ms)':>14s}" f"{'input':>24s}{'output':>24s}") for x in input if isinstance(input, list) else [input]: x = x.to(device) x.requires_grad = True for m in ops if isinstance(ops, list) else [ops]: m = m.to(device) if hasattr(m, 'to') else m # device m = m.half() if hasattr(m, 'half') and isinstance(x, torch.Tensor) and x.dtype is torch.float16 else m tf, tb, t = 0, 0, [0, 0, 0] # dt forward, backward try: flops = thop.profile(m, inputs=[x], verbose=False)[0] / 1E9 * 2 if thop else 0 # GFLOPs except Exception: flops = 0 try: for _ in range(n): t[0] = time_sync() y = m(x) t[1] = time_sync() try: _ = (sum(yi.sum() for yi in y) if isinstance(y, list) else y).sum().backward() t[2] = time_sync() except Exception: # no backward method # print(e) # for debug t[2] = float('nan') tf += (t[1] - t[0]) * 1000 / n # ms per op forward tb += (t[2] - t[1]) * 1000 / n # ms per op backward mem = torch.cuda.memory_reserved() / 1E9 if torch.cuda.is_available() else 0 # (GB) s_in, s_out = (tuple(x.shape) if isinstance(x, torch.Tensor) else 'list' for x in (x, y)) # shapes p = sum(x.numel() for x in m.parameters()) if isinstance(m, nn.Module) else 0 # parameters LOGGER.info(f'{p:12}{flops:12.4g}{mem:>14.3f}{tf:14.4g}{tb:14.4g}{str(s_in):>24s}{str(s_out):>24s}') results.append([p, flops, mem, tf, tb, s_in, s_out]) except Exception as e: LOGGER.info(e) results.append(None) torch.cuda.empty_cache() return results class EarlyStopping: """ Early stopping class that stops training when a specified number of epochs have passed without improvement. """ def __init__(self, patience=50): """ Initialize early stopping object Args: patience (int, optional): Number of epochs to wait after fitness stops improving before stopping. """ self.best_fitness = 0.0 # i.e. mAP self.best_epoch = 0 self.patience = patience or float('inf') # epochs to wait after fitness stops improving to stop self.possible_stop = False # possible stop may occur next epoch def __call__(self, epoch, fitness): """ Check whether to stop training Args: epoch (int): Current epoch of training fitness (float): Fitness value of current epoch Returns: (bool): True if training should stop, False otherwise """ if fitness is None: # check if fitness=None (happens when val=False) return False if fitness >= self.best_fitness: # >= 0 to allow for early zero-fitness stage of training self.best_epoch = epoch self.best_fitness = fitness delta = epoch - self.best_epoch # epochs without improvement self.possible_stop = delta >= (self.patience - 1) # possible stop may occur next epoch stop = delta >= self.patience # stop training if patience exceeded if stop: LOGGER.info(f'Stopping training early as no improvement observed in last {self.patience} epochs. ' f'Best results observed at epoch {self.best_epoch}, best model saved as best.pt.\n' f'To update EarlyStopping(patience={self.patience}) pass a new patience value, ' f'i.e. `patience=300` or use `patience=0` to disable EarlyStopping.') return stop ================================================ FILE: ultralytics/yolo/utils/tuner.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from ultralytics.yolo.utils import LOGGER try: from ray import tune from ray.air import RunConfig, session # noqa from ray.air.integrations.wandb import WandbLoggerCallback # noqa from ray.tune.schedulers import ASHAScheduler # noqa from ray.tune.schedulers import AsyncHyperBandScheduler as AHB # noqa except ImportError: LOGGER.info("Tuning hyperparameters requires ray/tune. Install using `pip install 'ray[tune]'`") tune = None default_space = { # 'optimizer': tune.choice(['SGD', 'Adam', 'AdamW', 'NAdam', 'RAdam', 'RMSProp']), 'lr0': tune.uniform(1e-5, 1e-1), 'lrf': tune.uniform(0.01, 1.0), # final OneCycleLR learning rate (lr0 * lrf) 'momentum': tune.uniform(0.6, 0.98), # SGD momentum/Adam beta1 'weight_decay': tune.uniform(0.0, 0.001), # optimizer weight decay 5e-4 'warmup_epochs': tune.uniform(0.0, 5.0), # warmup epochs (fractions ok) 'warmup_momentum': tune.uniform(0.0, 0.95), # warmup initial momentum 'box': tune.uniform(0.02, 0.2), # box loss gain 'cls': tune.uniform(0.2, 4.0), # cls loss gain (scale with pixels) 'hsv_h': tune.uniform(0.0, 0.1), # image HSV-Hue augmentation (fraction) 'hsv_s': tune.uniform(0.0, 0.9), # image HSV-Saturation augmentation (fraction) 'hsv_v': tune.uniform(0.0, 0.9), # image HSV-Value augmentation (fraction) 'degrees': tune.uniform(0.0, 45.0), # image rotation (+/- deg) 'translate': tune.uniform(0.0, 0.9), # image translation (+/- fraction) 'scale': tune.uniform(0.0, 0.9), # image scale (+/- gain) 'shear': tune.uniform(0.0, 10.0), # image shear (+/- deg) 'perspective': tune.uniform(0.0, 0.001), # image perspective (+/- fraction), range 0-0.001 'flipud': tune.uniform(0.0, 1.0), # image flip up-down (probability) 'fliplr': tune.uniform(0.0, 1.0), # image flip left-right (probability) 'mosaic': tune.uniform(0.0, 1.0), # image mixup (probability) 'mixup': tune.uniform(0.0, 1.0), # image mixup (probability) 'copy_paste': tune.uniform(0.0, 1.0)} # segment copy-paste (probability) task_metric_map = { 'detect': 'metrics/mAP50-95(B)', 'segment': 'metrics/mAP50-95(M)', 'classify': 'metrics/accuracy_top1', 'pose': 'metrics/mAP50-95(P)'} ================================================ FILE: ultralytics/yolo/v8/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from ultralytics.yolo.v8 import classify, detect, pose, segment __all__ = 'classify', 'segment', 'detect', 'pose' ================================================ FILE: ultralytics/yolo/v8/classify/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from ultralytics.yolo.v8.classify.predict import ClassificationPredictor, predict from ultralytics.yolo.v8.classify.train import ClassificationTrainer, train from ultralytics.yolo.v8.classify.val import ClassificationValidator, val __all__ = 'ClassificationPredictor', 'predict', 'ClassificationTrainer', 'train', 'ClassificationValidator', 'val' ================================================ FILE: ultralytics/yolo/v8/classify/predict.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import torch from ultralytics.yolo.engine.predictor import BasePredictor from ultralytics.yolo.engine.results import Results from ultralytics.yolo.utils import DEFAULT_CFG, ROOT class ClassificationPredictor(BasePredictor): def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): super().__init__(cfg, overrides, _callbacks) self.args.task = 'classify' def preprocess(self, img): """Converts input image to model-compatible data type.""" if not isinstance(img, torch.Tensor): img = torch.stack([self.transforms(im) for im in img], dim=0) img = (img if isinstance(img, torch.Tensor) else torch.from_numpy(img)).to(self.model.device) return img.half() if self.model.fp16 else img.float() # uint8 to fp16/32 def postprocess(self, preds, img, orig_imgs): """Postprocesses predictions to return Results objects.""" results = [] for i, pred in enumerate(preds): orig_img = orig_imgs[i] if isinstance(orig_imgs, list) else orig_imgs path = self.batch[0] img_path = path[i] if isinstance(path, list) else path results.append(Results(orig_img=orig_img, path=img_path, names=self.model.names, probs=pred)) return results def predict(cfg=DEFAULT_CFG, use_python=False): """Run YOLO model predictions on input images/videos.""" model = cfg.model or 'yolov8n-cls.pt' # or "resnet18" source = cfg.source if cfg.source is not None else ROOT / 'assets' if (ROOT / 'assets').exists() \ else 'https://ultralytics.com/images/bus.jpg' args = dict(model=model, source=source) if use_python: from ultralytics import YOLO YOLO(model)(**args) else: predictor = ClassificationPredictor(overrides=args) predictor.predict_cli() if __name__ == '__main__': predict() ================================================ FILE: ultralytics/yolo/v8/classify/train.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import torch import torchvision from ultralytics.nn.tasks import ClassificationModel, attempt_load_one_weight from ultralytics.yolo import v8 from ultralytics.yolo.data import ClassificationDataset, build_dataloader from ultralytics.yolo.engine.trainer import BaseTrainer from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER, RANK, colorstr from ultralytics.yolo.utils.plotting import plot_images, plot_results from ultralytics.yolo.utils.torch_utils import is_parallel, strip_optimizer, torch_distributed_zero_first class ClassificationTrainer(BaseTrainer): def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): """Initialize a ClassificationTrainer object with optional configuration overrides and callbacks.""" if overrides is None: overrides = {} overrides['task'] = 'classify' if overrides.get('imgsz') is None: overrides['imgsz'] = 224 super().__init__(cfg, overrides, _callbacks) def set_model_attributes(self): """Set the YOLO model's class names from the loaded dataset.""" self.model.names = self.data['names'] def get_model(self, cfg=None, weights=None, verbose=True): """Returns a modified PyTorch model configured for training YOLO.""" model = ClassificationModel(cfg, nc=self.data['nc'], verbose=verbose and RANK == -1) if weights: model.load(weights) for m in model.modules(): if not self.args.pretrained and hasattr(m, 'reset_parameters'): m.reset_parameters() if isinstance(m, torch.nn.Dropout) and self.args.dropout: m.p = self.args.dropout # set dropout for p in model.parameters(): p.requires_grad = True # for training return model def setup_model(self): """ load/create/download model for any task """ # Classification models require special handling if isinstance(self.model, torch.nn.Module): # if model is loaded beforehand. No setup needed return model = str(self.model) # Load a YOLO model locally, from torchvision, or from Ultralytics assets if model.endswith('.pt'): self.model, _ = attempt_load_one_weight(model, device='cpu') for p in self.model.parameters(): p.requires_grad = True # for training elif model.endswith('.yaml'): self.model = self.get_model(cfg=model) elif model in torchvision.models.__dict__: self.model = torchvision.models.__dict__[model](weights='IMAGENET1K_V1' if self.args.pretrained else None) else: FileNotFoundError(f'ERROR: model={model} not found locally or online. Please check model name.') ClassificationModel.reshape_outputs(self.model, self.data['nc']) return # dont return ckpt. Classification doesn't support resume def build_dataset(self, img_path, mode='train', batch=None): return ClassificationDataset(root=img_path, args=self.args, augment=mode == 'train') def get_dataloader(self, dataset_path, batch_size=16, rank=0, mode='train'): """Returns PyTorch DataLoader with transforms to preprocess images for inference.""" with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP dataset = self.build_dataset(dataset_path, mode) loader = build_dataloader(dataset, batch_size, self.args.workers, rank=rank) # Attach inference transforms if mode != 'train': if is_parallel(self.model): self.model.module.transforms = loader.dataset.torch_transforms else: self.model.transforms = loader.dataset.torch_transforms return loader def preprocess_batch(self, batch): """Preprocesses a batch of images and classes.""" batch['img'] = batch['img'].to(self.device) batch['cls'] = batch['cls'].to(self.device) return batch def progress_string(self): """Returns a formatted string showing training progress.""" return ('\n' + '%11s' * (4 + len(self.loss_names))) % \ ('Epoch', 'GPU_mem', *self.loss_names, 'Instances', 'Size') def get_validator(self): """Returns an instance of ClassificationValidator for validation.""" self.loss_names = ['loss'] return v8.classify.ClassificationValidator(self.test_loader, self.save_dir) def label_loss_items(self, loss_items=None, prefix='train'): """ Returns a loss dict with labelled training loss items tensor """ # Not needed for classification but necessary for segmentation & detection keys = [f'{prefix}/{x}' for x in self.loss_names] if loss_items is None: return keys loss_items = [round(float(loss_items), 5)] return dict(zip(keys, loss_items)) def resume_training(self, ckpt): """Resumes training from a given checkpoint.""" pass def plot_metrics(self): """Plots metrics from a CSV file.""" plot_results(file=self.csv, classify=True, on_plot=self.on_plot) # save results.png def final_eval(self): """Evaluate trained model and save validation results.""" for f in self.last, self.best: if f.exists(): strip_optimizer(f) # strip optimizers # TODO: validate best.pt after training completes # if f is self.best: # LOGGER.info(f'\nValidating {f}...') # self.validator.args.save_json = True # self.metrics = self.validator(model=f) # self.metrics.pop('fitness', None) # self.run_callbacks('on_fit_epoch_end') LOGGER.info(f"Results saved to {colorstr('bold', self.save_dir)}") def plot_training_samples(self, batch, ni): """Plots training samples with their annotations.""" plot_images(images=batch['img'], batch_idx=torch.arange(len(batch['img'])), cls=batch['cls'].squeeze(-1), fname=self.save_dir / f'train_batch{ni}.jpg', on_plot=self.on_plot) def train(cfg=DEFAULT_CFG, use_python=False): """Train the YOLO classification model.""" model = cfg.model or 'yolov8n-cls.pt' # or "resnet18" data = cfg.data or 'mnist160' # or yolo.ClassificationDataset("mnist") device = cfg.device if cfg.device is not None else '' args = dict(model=model, data=data, device=device) if use_python: from ultralytics import YOLO YOLO(model).train(**args) else: trainer = ClassificationTrainer(overrides=args) trainer.train() if __name__ == '__main__': train() ================================================ FILE: ultralytics/yolo/v8/classify/val.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import torch from ultralytics.yolo.data import ClassificationDataset, build_dataloader from ultralytics.yolo.engine.validator import BaseValidator from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER from ultralytics.yolo.utils.metrics import ClassifyMetrics, ConfusionMatrix from ultralytics.yolo.utils.plotting import plot_images class ClassificationValidator(BaseValidator): def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): """Initializes ClassificationValidator instance with args, dataloader, save_dir, and progress bar.""" super().__init__(dataloader, save_dir, pbar, args, _callbacks) self.args.task = 'classify' self.metrics = ClassifyMetrics() def get_desc(self): """Returns a formatted string summarizing classification metrics.""" return ('%22s' + '%11s' * 2) % ('classes', 'top1_acc', 'top5_acc') def init_metrics(self, model): """Initialize confusion matrix, class names, and top-1 and top-5 accuracy.""" self.names = model.names self.nc = len(model.names) self.confusion_matrix = ConfusionMatrix(nc=self.nc, task='classify') self.pred = [] self.targets = [] def preprocess(self, batch): """Preprocesses input batch and returns it.""" batch['img'] = batch['img'].to(self.device, non_blocking=True) batch['img'] = batch['img'].half() if self.args.half else batch['img'].float() batch['cls'] = batch['cls'].to(self.device) return batch def update_metrics(self, preds, batch): """Updates running metrics with model predictions and batch targets.""" n5 = min(len(self.model.names), 5) self.pred.append(preds.argsort(1, descending=True)[:, :n5]) self.targets.append(batch['cls']) def finalize_metrics(self, *args, **kwargs): """Finalizes metrics of the model such as confusion_matrix and speed.""" self.confusion_matrix.process_cls_preds(self.pred, self.targets) if self.args.plots: for normalize in True, False: self.confusion_matrix.plot(save_dir=self.save_dir, names=self.names.values(), normalize=normalize, on_plot=self.on_plot) self.metrics.speed = self.speed self.metrics.confusion_matrix = self.confusion_matrix def get_stats(self): """Returns a dictionary of metrics obtained by processing targets and predictions.""" self.metrics.process(self.targets, self.pred) return self.metrics.results_dict def build_dataset(self, img_path): return ClassificationDataset(root=img_path, args=self.args, augment=False) def get_dataloader(self, dataset_path, batch_size): """Builds and returns a data loader for classification tasks with given parameters.""" dataset = self.build_dataset(dataset_path) return build_dataloader(dataset, batch_size, self.args.workers, rank=-1) def print_results(self): """Prints evaluation metrics for YOLO object detection model.""" pf = '%22s' + '%11.3g' * len(self.metrics.keys) # print format LOGGER.info(pf % ('all', self.metrics.top1, self.metrics.top5)) def plot_val_samples(self, batch, ni): """Plot validation image samples.""" plot_images(images=batch['img'], batch_idx=torch.arange(len(batch['img'])), cls=batch['cls'].squeeze(-1), fname=self.save_dir / f'val_batch{ni}_labels.jpg', names=self.names, on_plot=self.on_plot) def plot_predictions(self, batch, preds, ni): """Plots predicted bounding boxes on input images and saves the result.""" plot_images(batch['img'], batch_idx=torch.arange(len(batch['img'])), cls=torch.argmax(preds, dim=1), fname=self.save_dir / f'val_batch{ni}_pred.jpg', names=self.names, on_plot=self.on_plot) # pred def val(cfg=DEFAULT_CFG, use_python=False): """Validate YOLO model using custom data.""" model = cfg.model or 'yolov8n-cls.pt' # or "resnet18" data = cfg.data or 'mnist160' args = dict(model=model, data=data) if use_python: from ultralytics import YOLO YOLO(model).val(**args) else: validator = ClassificationValidator(args=args) validator(model=args['model']) if __name__ == '__main__': val() ================================================ FILE: ultralytics/yolo/v8/detect/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from .predict import DetectionPredictor, predict from .train import DetectionTrainer, train from .val import DetectionValidator, val __all__ = 'DetectionPredictor', 'predict', 'DetectionTrainer', 'train', 'DetectionValidator', 'val' ================================================ FILE: ultralytics/yolo/v8/detect/predict.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import torch from ultralytics.yolo.engine.predictor import BasePredictor from ultralytics.yolo.engine.results import Results from ultralytics.yolo.utils import DEFAULT_CFG, ROOT, ops class DetectionPredictor(BasePredictor): def postprocess(self, preds, img, orig_imgs): """Postprocesses predictions and returns a list of Results objects.""" preds = ops.non_max_suppression(preds, self.args.conf, self.args.iou, agnostic=self.args.agnostic_nms, max_det=self.args.max_det, classes=self.args.classes) results = [] for i, pred in enumerate(preds): orig_img = orig_imgs[i] if isinstance(orig_imgs, list) else orig_imgs if not isinstance(orig_imgs, torch.Tensor): pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape) path = self.batch[0] img_path = path[i] if isinstance(path, list) else path results.append(Results(orig_img=orig_img, path=img_path, names=self.model.names, boxes=pred)) return results def predict(cfg=DEFAULT_CFG, use_python=False): """Runs YOLO model inference on input image(s).""" model = cfg.model or 'yolov8n.pt' source = cfg.source if cfg.source is not None else ROOT / 'assets' if (ROOT / 'assets').exists() \ else 'https://ultralytics.com/images/bus.jpg' args = dict(model=model, source=source) if use_python: from ultralytics import YOLO YOLO(model)(**args) else: predictor = DetectionPredictor(overrides=args) predictor.predict_cli() if __name__ == '__main__': predict() ================================================ FILE: ultralytics/yolo/v8/detect/train.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from copy import copy import numpy as np from ultralytics.nn.tasks import DetectionModel from ultralytics.yolo import v8 from ultralytics.yolo.data import build_dataloader, build_yolo_dataset from ultralytics.yolo.data.dataloaders.v5loader import create_dataloader from ultralytics.yolo.engine.trainer import BaseTrainer from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER, RANK, colorstr from ultralytics.yolo.utils.plotting import plot_images, plot_labels, plot_results from ultralytics.yolo.utils.torch_utils import de_parallel, torch_distributed_zero_first # BaseTrainer python usage class DetectionTrainer(BaseTrainer): def build_dataset(self, img_path, mode='train', batch=None): """Build YOLO Dataset Args: img_path (str): Path to the folder containing images. mode (str): `train` mode or `val` mode, users are able to customize different augmentations for each mode. batch (int, optional): Size of batches, this is for `rect`. Defaults to None. """ gs = max(int(de_parallel(self.model).stride.max() if self.model else 0), 32) return build_yolo_dataset(self.args, img_path, batch, self.data, mode=mode, rect=mode == 'val', stride=gs) def get_dataloader(self, dataset_path, batch_size=16, rank=0, mode='train'): """TODO: manage splits differently.""" # Calculate stride - check if model is initialized if self.args.v5loader: LOGGER.warning("WARNING ⚠️ 'v5loader' feature is deprecated and will be removed soon. You can train using " 'the default YOLOv8 dataloader instead, no argument is needed.') gs = max(int(de_parallel(self.model).stride.max() if self.model else 0), 32) return create_dataloader(path=dataset_path, imgsz=self.args.imgsz, batch_size=batch_size, stride=gs, hyp=vars(self.args), augment=mode == 'train', cache=self.args.cache, pad=0 if mode == 'train' else 0.5, rect=self.args.rect or mode == 'val', rank=rank, workers=self.args.workers, close_mosaic=self.args.close_mosaic != 0, prefix=colorstr(f'{mode}: '), shuffle=mode == 'train', seed=self.args.seed)[0] assert mode in ['train', 'val'] with torch_distributed_zero_first(rank): # init dataset *.cache only once if DDP dataset = self.build_dataset(dataset_path, mode, batch_size) shuffle = mode == 'train' if getattr(dataset, 'rect', False) and shuffle: LOGGER.warning("WARNING ⚠️ 'rect=True' is incompatible with DataLoader shuffle, setting shuffle=False") shuffle = False workers = self.args.workers if mode == 'train' else self.args.workers * 2 return build_dataloader(dataset, batch_size, workers, shuffle, rank) # return dataloader def preprocess_batch(self, batch): """Preprocesses a batch of images by scaling and converting to float.""" batch['img'] = batch['img'].to(self.device, non_blocking=True).float() / 255 return batch def set_model_attributes(self): """nl = de_parallel(self.model).model[-1].nl # number of detection layers (to scale hyps).""" # self.args.box *= 3 / nl # scale to layers # self.args.cls *= self.data["nc"] / 80 * 3 / nl # scale to classes and layers # self.args.cls *= (self.args.imgsz / 640) ** 2 * 3 / nl # scale to image size and layers self.model.nc = self.data['nc'] # attach number of classes to model self.model.names = self.data['names'] # attach class names to model self.model.args = self.args # attach hyperparameters to model # TODO: self.model.class_weights = labels_to_class_weights(dataset.labels, nc).to(device) * nc def get_model(self, cfg=None, weights=None, verbose=True): """Return a YOLO detection model.""" model = DetectionModel(cfg, nc=self.data['nc'], verbose=verbose and RANK == -1) if weights: model.load(weights) return model def get_validator(self): """Returns a DetectionValidator for YOLO model validation.""" self.loss_names = 'box_loss', 'cls_loss', 'dfl_loss' return v8.detect.DetectionValidator(self.test_loader, save_dir=self.save_dir, args=copy(self.args)) def label_loss_items(self, loss_items=None, prefix='train'): """ Returns a loss dict with labelled training loss items tensor """ # Not needed for classification but necessary for segmentation & detection keys = [f'{prefix}/{x}' for x in self.loss_names] if loss_items is not None: loss_items = [round(float(x), 5) for x in loss_items] # convert tensors to 5 decimal place floats return dict(zip(keys, loss_items)) else: return keys def progress_string(self): """Returns a formatted string of training progress with epoch, GPU memory, loss, instances and size.""" return ('\n' + '%11s' * (4 + len(self.loss_names))) % ('Epoch', 'GPU_mem', *self.loss_names, 'Instances', 'Size') def plot_training_samples(self, batch, ni): """Plots training samples with their annotations.""" plot_images(images=batch['img'], batch_idx=batch['batch_idx'], cls=batch['cls'].squeeze(-1), bboxes=batch['bboxes'], paths=batch['im_file'], fname=self.save_dir / f'train_batch{ni}.jpg', on_plot=self.on_plot) def plot_metrics(self): """Plots metrics from a CSV file.""" plot_results(file=self.csv, on_plot=self.on_plot) # save results.png def plot_training_labels(self): """Create a labeled training plot of the YOLO model.""" boxes = np.concatenate([lb['bboxes'] for lb in self.train_loader.dataset.labels], 0) cls = np.concatenate([lb['cls'] for lb in self.train_loader.dataset.labels], 0) plot_labels(boxes, cls.squeeze(), names=self.data['names'], save_dir=self.save_dir, on_plot=self.on_plot) def train(cfg=DEFAULT_CFG, use_python=False): """Train and optimize YOLO model given training data and device.""" model = cfg.model or 'yolov8n.pt' data = cfg.data or 'coco128.yaml' # or yolo.ClassificationDataset("mnist") device = cfg.device if cfg.device is not None else '' args = dict(model=model, data=data, device=device) if use_python: from ultralytics import YOLO YOLO(model).train(**args) else: trainer = DetectionTrainer(overrides=args) trainer.train() if __name__ == '__main__': train() ================================================ FILE: ultralytics/yolo/v8/detect/val.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import os from pathlib import Path import numpy as np import torch from ultralytics.yolo.data import build_dataloader, build_yolo_dataset from ultralytics.yolo.data.dataloaders.v5loader import create_dataloader from ultralytics.yolo.engine.validator import BaseValidator from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER, colorstr, ops from ultralytics.yolo.utils.checks import check_requirements from ultralytics.yolo.utils.metrics import ConfusionMatrix, DetMetrics, box_iou from ultralytics.yolo.utils.plotting import output_to_target, plot_images from ultralytics.yolo.utils.torch_utils import de_parallel class DetectionValidator(BaseValidator): def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): """Initialize detection model with necessary variables and settings.""" super().__init__(dataloader, save_dir, pbar, args, _callbacks) self.args.task = 'detect' self.is_coco = False self.class_map = None self.metrics = DetMetrics(save_dir=self.save_dir, on_plot=self.on_plot) self.iouv = torch.linspace(0.5, 0.95, 10) # iou vector for mAP@0.5:0.95 self.niou = self.iouv.numel() def preprocess(self, batch): """Preprocesses batch of images for YOLO training.""" batch['img'] = batch['img'].to(self.device, non_blocking=True) batch['img'] = (batch['img'].half() if self.args.half else batch['img'].float()) / 255 for k in ['batch_idx', 'cls', 'bboxes']: batch[k] = batch[k].to(self.device) nb = len(batch['img']) self.lb = [torch.cat([batch['cls'], batch['bboxes']], dim=-1)[batch['batch_idx'] == i] for i in range(nb)] if self.args.save_hybrid else [] # for autolabelling return batch def init_metrics(self, model): """Initialize evaluation metrics for YOLO.""" val = self.data.get(self.args.split, '') # validation path self.is_coco = isinstance(val, str) and 'coco' in val and val.endswith(f'{os.sep}val2017.txt') # is COCO self.class_map = ops.coco80_to_coco91_class() if self.is_coco else list(range(1000)) self.args.save_json |= self.is_coco and not self.training # run on final val if training COCO self.names = model.names self.nc = len(model.names) self.metrics.names = self.names self.metrics.plot = self.args.plots self.confusion_matrix = ConfusionMatrix(nc=self.nc) self.seen = 0 self.jdict = [] self.stats = [] def get_desc(self): """Return a formatted string summarizing class metrics of YOLO model.""" return ('%22s' + '%11s' * 6) % ('Class', 'Images', 'Instances', 'Box(P', 'R', 'mAP50', 'mAP50-95)') def postprocess(self, preds): """Apply Non-maximum suppression to prediction outputs.""" return ops.non_max_suppression(preds, self.args.conf, self.args.iou, labels=self.lb, multi_label=True, agnostic=self.args.single_cls, max_det=self.args.max_det) def update_metrics(self, preds, batch): """Metrics.""" for si, pred in enumerate(preds): idx = batch['batch_idx'] == si cls = batch['cls'][idx] bbox = batch['bboxes'][idx] nl, npr = cls.shape[0], pred.shape[0] # number of labels, predictions shape = batch['ori_shape'][si] correct_bboxes = torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device) # init self.seen += 1 if npr == 0: if nl: self.stats.append((correct_bboxes, *torch.zeros((2, 0), device=self.device), cls.squeeze(-1))) if self.args.plots: self.confusion_matrix.process_batch(detections=None, labels=cls.squeeze(-1)) continue # Predictions if self.args.single_cls: pred[:, 5] = 0 predn = pred.clone() ops.scale_boxes(batch['img'][si].shape[1:], predn[:, :4], shape, ratio_pad=batch['ratio_pad'][si]) # native-space pred # Evaluate if nl: height, width = batch['img'].shape[2:] tbox = ops.xywh2xyxy(bbox) * torch.tensor( (width, height, width, height), device=self.device) # target boxes ops.scale_boxes(batch['img'][si].shape[1:], tbox, shape, ratio_pad=batch['ratio_pad'][si]) # native-space labels labelsn = torch.cat((cls, tbox), 1) # native-space labels correct_bboxes = self._process_batch(predn, labelsn) # TODO: maybe remove these `self.` arguments as they already are member variable if self.args.plots: self.confusion_matrix.process_batch(predn, labelsn) self.stats.append((correct_bboxes, pred[:, 4], pred[:, 5], cls.squeeze(-1))) # (conf, pcls, tcls) # Save if self.args.save_json: self.pred_to_json(predn, batch['im_file'][si]) if self.args.save_txt: file = self.save_dir / 'labels' / f'{Path(batch["im_file"][si]).stem}.txt' self.save_one_txt(predn, self.args.save_conf, shape, file) def finalize_metrics(self, *args, **kwargs): """Set final values for metrics speed and confusion matrix.""" self.metrics.speed = self.speed self.metrics.confusion_matrix = self.confusion_matrix def get_stats(self): """Returns metrics statistics and results dictionary.""" stats = [torch.cat(x, 0).cpu().numpy() for x in zip(*self.stats)] # to numpy if len(stats) and stats[0].any(): self.metrics.process(*stats) self.nt_per_class = np.bincount(stats[-1].astype(int), minlength=self.nc) # number of targets per class return self.metrics.results_dict def print_results(self): """Prints training/validation set metrics per class.""" pf = '%22s' + '%11i' * 2 + '%11.3g' * len(self.metrics.keys) # print format LOGGER.info(pf % ('all', self.seen, self.nt_per_class.sum(), *self.metrics.mean_results())) if self.nt_per_class.sum() == 0: LOGGER.warning( f'WARNING ⚠️ no labels found in {self.args.task} set, can not compute metrics without labels') # Print results per class if self.args.verbose and not self.training and self.nc > 1 and len(self.stats): for i, c in enumerate(self.metrics.ap_class_index): LOGGER.info(pf % (self.names[c], self.seen, self.nt_per_class[c], *self.metrics.class_result(i))) if self.args.plots: for normalize in True, False: self.confusion_matrix.plot(save_dir=self.save_dir, names=self.names.values(), normalize=normalize, on_plot=self.on_plot) def _process_batch(self, detections, labels): """ Return correct prediction matrix Arguments: detections (array[N, 6]), x1, y1, x2, y2, conf, class labels (array[M, 5]), class, x1, y1, x2, y2 Returns: correct (array[N, 10]), for 10 IoU levels """ iou = box_iou(labels[:, 1:], detections[:, :4]) correct = np.zeros((detections.shape[0], self.iouv.shape[0])).astype(bool) correct_class = labels[:, 0:1] == detections[:, 5] for i in range(len(self.iouv)): x = torch.where((iou >= self.iouv[i]) & correct_class) # IoU > threshold and classes match if x[0].shape[0]: matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() # [label, detect, iou] if x[0].shape[0] > 1: matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 1], return_index=True)[1]] # matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 0], return_index=True)[1]] correct[matches[:, 1].astype(int), i] = True return torch.tensor(correct, dtype=torch.bool, device=detections.device) def build_dataset(self, img_path, mode='val', batch=None): """Build YOLO Dataset Args: img_path (str): Path to the folder containing images. mode (str): `train` mode or `val` mode, users are able to customize different augmentations for each mode. batch (int, optional): Size of batches, this is for `rect`. Defaults to None. """ gs = max(int(de_parallel(self.model).stride if self.model else 0), 32) return build_yolo_dataset(self.args, img_path, batch, self.data, mode=mode, stride=gs) def get_dataloader(self, dataset_path, batch_size): """TODO: manage splits differently.""" # Calculate stride - check if model is initialized if self.args.v5loader: LOGGER.warning("WARNING ⚠️ 'v5loader' feature is deprecated and will be removed soon. You can train using " 'the default YOLOv8 dataloader instead, no argument is needed.') gs = max(int(de_parallel(self.model).stride if self.model else 0), 32) return create_dataloader(path=dataset_path, imgsz=self.args.imgsz, batch_size=batch_size, stride=gs, hyp=vars(self.args), cache=False, pad=0.5, rect=self.args.rect, workers=self.args.workers, prefix=colorstr(f'{self.args.mode}: '), shuffle=False, seed=self.args.seed)[0] dataset = self.build_dataset(dataset_path, batch=batch_size, mode='val') dataloader = build_dataloader(dataset, batch_size, self.args.workers, shuffle=False, rank=-1) return dataloader def plot_val_samples(self, batch, ni): """Plot validation image samples.""" plot_images(batch['img'], batch['batch_idx'], batch['cls'].squeeze(-1), batch['bboxes'], paths=batch['im_file'], fname=self.save_dir / f'val_batch{ni}_labels.jpg', names=self.names, on_plot=self.on_plot) def plot_predictions(self, batch, preds, ni): """Plots predicted bounding boxes on input images and saves the result.""" plot_images(batch['img'], *output_to_target(preds, max_det=self.args.max_det), paths=batch['im_file'], fname=self.save_dir / f'val_batch{ni}_pred.jpg', names=self.names, on_plot=self.on_plot) # pred def save_one_txt(self, predn, save_conf, shape, file): """Save YOLO detections to a txt file in normalized coordinates in a specific format.""" gn = torch.tensor(shape)[[1, 0, 1, 0]] # normalization gain whwh for *xyxy, conf, cls in predn.tolist(): xywh = (ops.xyxy2xywh(torch.tensor(xyxy).view(1, 4)) / gn).view(-1).tolist() # normalized xywh line = (cls, *xywh, conf) if save_conf else (cls, *xywh) # label format with open(file, 'a') as f: f.write(('%g ' * len(line)).rstrip() % line + '\n') def pred_to_json(self, predn, filename): """Serialize YOLO predictions to COCO json format.""" stem = Path(filename).stem image_id = int(stem) if stem.isnumeric() else stem box = ops.xyxy2xywh(predn[:, :4]) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for p, b in zip(predn.tolist(), box.tolist()): self.jdict.append({ 'image_id': image_id, 'category_id': self.class_map[int(p[5])], 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5)}) def eval_json(self, stats): """Evaluates YOLO output in JSON format and returns performance statistics.""" if self.args.save_json and self.is_coco and len(self.jdict): anno_json = self.data['path'] / 'annotations/instances_val2017.json' # annotations pred_json = self.save_dir / 'predictions.json' # predictions LOGGER.info(f'\nEvaluating pycocotools mAP using {pred_json} and {anno_json}...') try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb check_requirements('pycocotools>=2.0.6') from pycocotools.coco import COCO # noqa from pycocotools.cocoeval import COCOeval # noqa for x in anno_json, pred_json: assert x.is_file(), f'{x} file not found' anno = COCO(str(anno_json)) # init annotations api pred = anno.loadRes(str(pred_json)) # init predictions api (must pass string, not Path) eval = COCOeval(anno, pred, 'bbox') if self.is_coco: eval.params.imgIds = [int(Path(x).stem) for x in self.dataloader.dataset.im_files] # images to eval eval.evaluate() eval.accumulate() eval.summarize() stats[self.metrics.keys[-1]], stats[self.metrics.keys[-2]] = eval.stats[:2] # update mAP50-95 and mAP50 except Exception as e: LOGGER.warning(f'pycocotools unable to run: {e}') return stats def val(cfg=DEFAULT_CFG, use_python=False): """Validate trained YOLO model on validation dataset.""" model = cfg.model or 'yolov8n.pt' data = cfg.data or 'coco128.yaml' args = dict(model=model, data=data) if use_python: from ultralytics import YOLO YOLO(model).val(**args) else: validator = DetectionValidator(args=args) validator(model=args['model']) if __name__ == '__main__': val() ================================================ FILE: ultralytics/yolo/v8/pose/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from .predict import PosePredictor, predict from .train import PoseTrainer, train from .val import PoseValidator, val __all__ = 'PoseTrainer', 'train', 'PoseValidator', 'val', 'PosePredictor', 'predict' ================================================ FILE: ultralytics/yolo/v8/pose/predict.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from ultralytics.yolo.engine.results import Results from ultralytics.yolo.utils import DEFAULT_CFG, ROOT, ops from ultralytics.yolo.v8.detect.predict import DetectionPredictor class PosePredictor(DetectionPredictor): def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): super().__init__(cfg, overrides, _callbacks) self.args.task = 'pose' def postprocess(self, preds, img, orig_imgs): """Return detection results for a given input image or list of images.""" preds = ops.non_max_suppression(preds, self.args.conf, self.args.iou, agnostic=self.args.agnostic_nms, max_det=self.args.max_det, classes=self.args.classes, nc=len(self.model.names)) results = [] for i, pred in enumerate(preds): orig_img = orig_imgs[i] if isinstance(orig_imgs, list) else orig_imgs shape = orig_img.shape pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], shape).round() pred_kpts = pred[:, 6:].view(len(pred), *self.model.kpt_shape) if len(pred) else pred[:, 6:] pred_kpts = ops.scale_coords(img.shape[2:], pred_kpts, shape) path = self.batch[0] img_path = path[i] if isinstance(path, list) else path results.append( Results(orig_img=orig_img, path=img_path, names=self.model.names, boxes=pred[:, :6], keypoints=pred_kpts)) return results def predict(cfg=DEFAULT_CFG, use_python=False): """Runs YOLO to predict objects in an image or video.""" model = cfg.model or 'yolov8n-pose.pt' source = cfg.source if cfg.source is not None else ROOT / 'assets' if (ROOT / 'assets').exists() \ else 'https://ultralytics.com/images/bus.jpg' args = dict(model=model, source=source) if use_python: from ultralytics import YOLO YOLO(model)(**args) else: predictor = PosePredictor(overrides=args) predictor.predict_cli() if __name__ == '__main__': predict() ================================================ FILE: ultralytics/yolo/v8/pose/train.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from copy import copy from ultralytics.nn.tasks import PoseModel from ultralytics.yolo import v8 from ultralytics.yolo.utils import DEFAULT_CFG from ultralytics.yolo.utils.plotting import plot_images, plot_results # BaseTrainer python usage class PoseTrainer(v8.detect.DetectionTrainer): def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): """Initialize a PoseTrainer object with specified configurations and overrides.""" if overrides is None: overrides = {} overrides['task'] = 'pose' super().__init__(cfg, overrides, _callbacks) def get_model(self, cfg=None, weights=None, verbose=True): """Get pose estimation model with specified configuration and weights.""" model = PoseModel(cfg, ch=3, nc=self.data['nc'], data_kpt_shape=self.data['kpt_shape'], verbose=verbose) if weights: model.load(weights) return model def set_model_attributes(self): """Sets keypoints shape attribute of PoseModel.""" super().set_model_attributes() self.model.kpt_shape = self.data['kpt_shape'] def get_validator(self): """Returns an instance of the PoseValidator class for validation.""" self.loss_names = 'box_loss', 'pose_loss', 'kobj_loss', 'cls_loss', 'dfl_loss' return v8.pose.PoseValidator(self.test_loader, save_dir=self.save_dir, args=copy(self.args)) def plot_training_samples(self, batch, ni): """Plot a batch of training samples with annotated class labels, bounding boxes, and keypoints.""" images = batch['img'] kpts = batch['keypoints'] cls = batch['cls'].squeeze(-1) bboxes = batch['bboxes'] paths = batch['im_file'] batch_idx = batch['batch_idx'] plot_images(images, batch_idx, cls, bboxes, kpts=kpts, paths=paths, fname=self.save_dir / f'train_batch{ni}.jpg', on_plot=self.on_plot) def plot_metrics(self): """Plots training/val metrics.""" plot_results(file=self.csv, pose=True, on_plot=self.on_plot) # save results.png def train(cfg=DEFAULT_CFG, use_python=False): """Train the YOLO model on the given data and device.""" model = cfg.model or 'yolov8n-pose.yaml' data = cfg.data or 'coco8-pose.yaml' device = cfg.device if cfg.device is not None else '' args = dict(model=model, data=data, device=device) if use_python: from ultralytics import YOLO YOLO(model).train(**args) else: trainer = PoseTrainer(overrides=args) trainer.train() if __name__ == '__main__': train() ================================================ FILE: ultralytics/yolo/v8/pose/val.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from pathlib import Path import numpy as np import torch from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER, ops from ultralytics.yolo.utils.checks import check_requirements from ultralytics.yolo.utils.metrics import OKS_SIGMA, PoseMetrics, box_iou, kpt_iou from ultralytics.yolo.utils.plotting import output_to_target, plot_images from ultralytics.yolo.v8.detect import DetectionValidator class PoseValidator(DetectionValidator): def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): """Initialize a 'PoseValidator' object with custom parameters and assigned attributes.""" super().__init__(dataloader, save_dir, pbar, args, _callbacks) self.args.task = 'pose' self.metrics = PoseMetrics(save_dir=self.save_dir, on_plot=self.on_plot) def preprocess(self, batch): """Preprocesses the batch by converting the 'keypoints' data into a float and moving it to the device.""" batch = super().preprocess(batch) batch['keypoints'] = batch['keypoints'].to(self.device).float() return batch def get_desc(self): """Returns description of evaluation metrics in string format.""" return ('%22s' + '%11s' * 10) % ('Class', 'Images', 'Instances', 'Box(P', 'R', 'mAP50', 'mAP50-95)', 'Pose(P', 'R', 'mAP50', 'mAP50-95)') def postprocess(self, preds): """Apply non-maximum suppression and return detections with high confidence scores.""" return ops.non_max_suppression(preds, self.args.conf, self.args.iou, labels=self.lb, multi_label=True, agnostic=self.args.single_cls, max_det=self.args.max_det, nc=self.nc) def init_metrics(self, model): """Initiate pose estimation metrics for YOLO model.""" super().init_metrics(model) self.kpt_shape = self.data['kpt_shape'] is_pose = self.kpt_shape == [17, 3] nkpt = self.kpt_shape[0] self.sigma = OKS_SIGMA if is_pose else np.ones(nkpt) / nkpt def update_metrics(self, preds, batch): """Metrics.""" for si, pred in enumerate(preds): idx = batch['batch_idx'] == si cls = batch['cls'][idx] bbox = batch['bboxes'][idx] kpts = batch['keypoints'][idx] nl, npr = cls.shape[0], pred.shape[0] # number of labels, predictions nk = kpts.shape[1] # number of keypoints shape = batch['ori_shape'][si] correct_kpts = torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device) # init correct_bboxes = torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device) # init self.seen += 1 if npr == 0: if nl: self.stats.append((correct_bboxes, correct_kpts, *torch.zeros( (2, 0), device=self.device), cls.squeeze(-1))) if self.args.plots: self.confusion_matrix.process_batch(detections=None, labels=cls.squeeze(-1)) continue # Predictions if self.args.single_cls: pred[:, 5] = 0 predn = pred.clone() ops.scale_boxes(batch['img'][si].shape[1:], predn[:, :4], shape, ratio_pad=batch['ratio_pad'][si]) # native-space pred pred_kpts = predn[:, 6:].view(npr, nk, -1) ops.scale_coords(batch['img'][si].shape[1:], pred_kpts, shape, ratio_pad=batch['ratio_pad'][si]) # Evaluate if nl: height, width = batch['img'].shape[2:] tbox = ops.xywh2xyxy(bbox) * torch.tensor( (width, height, width, height), device=self.device) # target boxes ops.scale_boxes(batch['img'][si].shape[1:], tbox, shape, ratio_pad=batch['ratio_pad'][si]) # native-space labels tkpts = kpts.clone() tkpts[..., 0] *= width tkpts[..., 1] *= height tkpts = ops.scale_coords(batch['img'][si].shape[1:], tkpts, shape, ratio_pad=batch['ratio_pad'][si]) labelsn = torch.cat((cls, tbox), 1) # native-space labels correct_bboxes = self._process_batch(predn[:, :6], labelsn) correct_kpts = self._process_batch(predn[:, :6], labelsn, pred_kpts, tkpts) if self.args.plots: self.confusion_matrix.process_batch(predn, labelsn) # Append correct_masks, correct_boxes, pconf, pcls, tcls self.stats.append((correct_bboxes, correct_kpts, pred[:, 4], pred[:, 5], cls.squeeze(-1))) # Save if self.args.save_json: self.pred_to_json(predn, batch['im_file'][si]) # if self.args.save_txt: # save_one_txt(predn, save_conf, shape, file=save_dir / 'labels' / f'{path.stem}.txt') def _process_batch(self, detections, labels, pred_kpts=None, gt_kpts=None): """ Return correct prediction matrix Arguments: detections (array[N, 6]), x1, y1, x2, y2, conf, class labels (array[M, 5]), class, x1, y1, x2, y2 pred_kpts (array[N, 51]), 51 = 17 * 3 gt_kpts (array[N, 51]) Returns: correct (array[N, 10]), for 10 IoU levels """ if pred_kpts is not None and gt_kpts is not None: # `0.53` is from https://github.com/jin-s13/xtcocoapi/blob/master/xtcocotools/cocoeval.py#L384 area = ops.xyxy2xywh(labels[:, 1:])[:, 2:].prod(1) * 0.53 iou = kpt_iou(gt_kpts, pred_kpts, sigma=self.sigma, area=area) else: # boxes iou = box_iou(labels[:, 1:], detections[:, :4]) correct = np.zeros((detections.shape[0], self.iouv.shape[0])).astype(bool) correct_class = labels[:, 0:1] == detections[:, 5] for i in range(len(self.iouv)): x = torch.where((iou >= self.iouv[i]) & correct_class) # IoU > threshold and classes match if x[0].shape[0]: matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() # [label, detect, iou] if x[0].shape[0] > 1: matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 1], return_index=True)[1]] # matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 0], return_index=True)[1]] correct[matches[:, 1].astype(int), i] = True return torch.tensor(correct, dtype=torch.bool, device=detections.device) def plot_val_samples(self, batch, ni): """Plots and saves validation set samples with predicted bounding boxes and keypoints.""" plot_images(batch['img'], batch['batch_idx'], batch['cls'].squeeze(-1), batch['bboxes'], kpts=batch['keypoints'], paths=batch['im_file'], fname=self.save_dir / f'val_batch{ni}_labels.jpg', names=self.names, on_plot=self.on_plot) def plot_predictions(self, batch, preds, ni): """Plots predictions for YOLO model.""" pred_kpts = torch.cat([p[:, 6:].view(-1, *self.kpt_shape) for p in preds], 0) plot_images(batch['img'], *output_to_target(preds, max_det=self.args.max_det), kpts=pred_kpts, paths=batch['im_file'], fname=self.save_dir / f'val_batch{ni}_pred.jpg', names=self.names, on_plot=self.on_plot) # pred def pred_to_json(self, predn, filename): """Converts YOLO predictions to COCO JSON format.""" stem = Path(filename).stem image_id = int(stem) if stem.isnumeric() else stem box = ops.xyxy2xywh(predn[:, :4]) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner for p, b in zip(predn.tolist(), box.tolist()): self.jdict.append({ 'image_id': image_id, 'category_id': self.class_map[int(p[5])], 'bbox': [round(x, 3) for x in b], 'keypoints': p[6:], 'score': round(p[4], 5)}) def eval_json(self, stats): """Evaluates object detection model using COCO JSON format.""" if self.args.save_json and self.is_coco and len(self.jdict): anno_json = self.data['path'] / 'annotations/person_keypoints_val2017.json' # annotations pred_json = self.save_dir / 'predictions.json' # predictions LOGGER.info(f'\nEvaluating pycocotools mAP using {pred_json} and {anno_json}...') try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb check_requirements('pycocotools>=2.0.6') from pycocotools.coco import COCO # noqa from pycocotools.cocoeval import COCOeval # noqa for x in anno_json, pred_json: assert x.is_file(), f'{x} file not found' anno = COCO(str(anno_json)) # init annotations api pred = anno.loadRes(str(pred_json)) # init predictions api (must pass string, not Path) for i, eval in enumerate([COCOeval(anno, pred, 'bbox'), COCOeval(anno, pred, 'keypoints')]): if self.is_coco: eval.params.imgIds = [int(Path(x).stem) for x in self.dataloader.dataset.im_files] # im to eval eval.evaluate() eval.accumulate() eval.summarize() idx = i * 4 + 2 stats[self.metrics.keys[idx + 1]], stats[ self.metrics.keys[idx]] = eval.stats[:2] # update mAP50-95 and mAP50 except Exception as e: LOGGER.warning(f'pycocotools unable to run: {e}') return stats def val(cfg=DEFAULT_CFG, use_python=False): """Performs validation on YOLO model using given data.""" model = cfg.model or 'yolov8n-pose.pt' data = cfg.data or 'coco8-pose.yaml' args = dict(model=model, data=data) if use_python: from ultralytics import YOLO YOLO(model).val(**args) else: validator = PoseValidator(args=args) validator(model=args['model']) if __name__ == '__main__': val() ================================================ FILE: ultralytics/yolo/v8/segment/__init__.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from .predict import SegmentationPredictor, predict from .train import SegmentationTrainer, train from .val import SegmentationValidator, val __all__ = 'SegmentationPredictor', 'predict', 'SegmentationTrainer', 'train', 'SegmentationValidator', 'val' ================================================ FILE: ultralytics/yolo/v8/segment/predict.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license import torch from ultralytics.yolo.engine.results import Results from ultralytics.yolo.utils import DEFAULT_CFG, ROOT, ops from ultralytics.yolo.v8.detect.predict import DetectionPredictor class SegmentationPredictor(DetectionPredictor): def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): super().__init__(cfg, overrides, _callbacks) self.args.task = 'segment' def postprocess(self, preds, img, orig_imgs): """TODO: filter by classes.""" p = ops.non_max_suppression(preds[0], self.args.conf, self.args.iou, agnostic=self.args.agnostic_nms, max_det=self.args.max_det, nc=len(self.model.names), classes=self.args.classes) results = [] proto = preds[1][-1] if len(preds[1]) == 3 else preds[1] # second output is len 3 if pt, but only 1 if exported for i, pred in enumerate(p): orig_img = orig_imgs[i] if isinstance(orig_imgs, list) else orig_imgs path = self.batch[0] img_path = path[i] if isinstance(path, list) else path if not len(pred): # save empty boxes results.append(Results(orig_img=orig_img, path=img_path, names=self.model.names, boxes=pred[:, :6])) continue if self.args.retina_masks: if not isinstance(orig_imgs, torch.Tensor): pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape) masks = ops.process_mask_native(proto[i], pred[:, 6:], pred[:, :4], orig_img.shape[:2]) # HWC else: masks = ops.process_mask(proto[i], pred[:, 6:], pred[:, :4], img.shape[2:], upsample=True) # HWC if not isinstance(orig_imgs, torch.Tensor): pred[:, :4] = ops.scale_boxes(img.shape[2:], pred[:, :4], orig_img.shape) results.append( Results(orig_img=orig_img, path=img_path, names=self.model.names, boxes=pred[:, :6], masks=masks)) return results def predict(cfg=DEFAULT_CFG, use_python=False): """Runs YOLO object detection on an image or video source.""" model = cfg.model or 'yolov8n-seg.pt' source = cfg.source if cfg.source is not None else ROOT / 'assets' if (ROOT / 'assets').exists() \ else 'https://ultralytics.com/images/bus.jpg' args = dict(model=model, source=source) if use_python: from ultralytics import YOLO YOLO(model)(**args) else: predictor = SegmentationPredictor(overrides=args) predictor.predict_cli() if __name__ == '__main__': predict() ================================================ FILE: ultralytics/yolo/v8/segment/train.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from copy import copy from ultralytics.nn.tasks import SegmentationModel from ultralytics.yolo import v8 from ultralytics.yolo.utils import DEFAULT_CFG, RANK from ultralytics.yolo.utils.plotting import plot_images, plot_results # BaseTrainer python usage class SegmentationTrainer(v8.detect.DetectionTrainer): def __init__(self, cfg=DEFAULT_CFG, overrides=None, _callbacks=None): """Initialize a SegmentationTrainer object with given arguments.""" if overrides is None: overrides = {} overrides['task'] = 'segment' super().__init__(cfg, overrides, _callbacks) def get_model(self, cfg=None, weights=None, verbose=True): """Return SegmentationModel initialized with specified config and weights.""" model = SegmentationModel(cfg, ch=3, nc=self.data['nc'], verbose=verbose and RANK == -1) if weights: model.load(weights) return model def get_validator(self): """Return an instance of SegmentationValidator for validation of YOLO model.""" self.loss_names = 'box_loss', 'seg_loss', 'cls_loss', 'dfl_loss' return v8.segment.SegmentationValidator(self.test_loader, save_dir=self.save_dir, args=copy(self.args)) def plot_training_samples(self, batch, ni): """Creates a plot of training sample images with labels and box coordinates.""" plot_images(batch['img'], batch['batch_idx'], batch['cls'].squeeze(-1), batch['bboxes'], batch['masks'], paths=batch['im_file'], fname=self.save_dir / f'train_batch{ni}.jpg', on_plot=self.on_plot) def plot_metrics(self): """Plots training/val metrics.""" plot_results(file=self.csv, segment=True, on_plot=self.on_plot) # save results.png def train(cfg=DEFAULT_CFG, use_python=False): """Train a YOLO segmentation model based on passed arguments.""" model = cfg.model or 'yolov8n-seg.pt' data = cfg.data or 'coco128-seg.yaml' # or yolo.ClassificationDataset("mnist") device = cfg.device if cfg.device is not None else '' args = dict(model=model, data=data, device=device) if use_python: from ultralytics import YOLO YOLO(model).train(**args) else: trainer = SegmentationTrainer(overrides=args) trainer.train() if __name__ == '__main__': train() ================================================ FILE: ultralytics/yolo/v8/segment/val.py ================================================ # Ultralytics YOLO 🚀, AGPL-3.0 license from multiprocessing.pool import ThreadPool from pathlib import Path import numpy as np import torch import torch.nn.functional as F from ultralytics.yolo.utils import DEFAULT_CFG, LOGGER, NUM_THREADS, ops from ultralytics.yolo.utils.checks import check_requirements from ultralytics.yolo.utils.metrics import SegmentMetrics, box_iou, mask_iou from ultralytics.yolo.utils.plotting import output_to_target, plot_images from ultralytics.yolo.v8.detect import DetectionValidator class SegmentationValidator(DetectionValidator): def __init__(self, dataloader=None, save_dir=None, pbar=None, args=None, _callbacks=None): """Initialize SegmentationValidator and set task to 'segment', metrics to SegmentMetrics.""" super().__init__(dataloader, save_dir, pbar, args, _callbacks) self.args.task = 'segment' self.metrics = SegmentMetrics(save_dir=self.save_dir, on_plot=self.on_plot) def preprocess(self, batch): """Preprocesses batch by converting masks to float and sending to device.""" batch = super().preprocess(batch) batch['masks'] = batch['masks'].to(self.device).float() return batch def init_metrics(self, model): """Initialize metrics and select mask processing function based on save_json flag.""" super().init_metrics(model) self.plot_masks = [] if self.args.save_json: check_requirements('pycocotools>=2.0.6') self.process = ops.process_mask_upsample # more accurate else: self.process = ops.process_mask # faster def get_desc(self): """Return a formatted description of evaluation metrics.""" return ('%22s' + '%11s' * 10) % ('Class', 'Images', 'Instances', 'Box(P', 'R', 'mAP50', 'mAP50-95)', 'Mask(P', 'R', 'mAP50', 'mAP50-95)') def postprocess(self, preds): """Postprocesses YOLO predictions and returns output detections with proto.""" p = ops.non_max_suppression(preds[0], self.args.conf, self.args.iou, labels=self.lb, multi_label=True, agnostic=self.args.single_cls, max_det=self.args.max_det, nc=self.nc) proto = preds[1][-1] if len(preds[1]) == 3 else preds[1] # second output is len 3 if pt, but only 1 if exported return p, proto def update_metrics(self, preds, batch): """Metrics.""" for si, (pred, proto) in enumerate(zip(preds[0], preds[1])): idx = batch['batch_idx'] == si cls = batch['cls'][idx] bbox = batch['bboxes'][idx] nl, npr = cls.shape[0], pred.shape[0] # number of labels, predictions shape = batch['ori_shape'][si] correct_masks = torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device) # init correct_bboxes = torch.zeros(npr, self.niou, dtype=torch.bool, device=self.device) # init self.seen += 1 if npr == 0: if nl: self.stats.append((correct_bboxes, correct_masks, *torch.zeros( (2, 0), device=self.device), cls.squeeze(-1))) if self.args.plots: self.confusion_matrix.process_batch(detections=None, labels=cls.squeeze(-1)) continue # Masks midx = [si] if self.args.overlap_mask else idx gt_masks = batch['masks'][midx] pred_masks = self.process(proto, pred[:, 6:], pred[:, :4], shape=batch['img'][si].shape[1:]) # Predictions if self.args.single_cls: pred[:, 5] = 0 predn = pred.clone() ops.scale_boxes(batch['img'][si].shape[1:], predn[:, :4], shape, ratio_pad=batch['ratio_pad'][si]) # native-space pred # Evaluate if nl: height, width = batch['img'].shape[2:] tbox = ops.xywh2xyxy(bbox) * torch.tensor( (width, height, width, height), device=self.device) # target boxes ops.scale_boxes(batch['img'][si].shape[1:], tbox, shape, ratio_pad=batch['ratio_pad'][si]) # native-space labels labelsn = torch.cat((cls, tbox), 1) # native-space labels correct_bboxes = self._process_batch(predn, labelsn) # TODO: maybe remove these `self.` arguments as they already are member variable correct_masks = self._process_batch(predn, labelsn, pred_masks, gt_masks, overlap=self.args.overlap_mask, masks=True) if self.args.plots: self.confusion_matrix.process_batch(predn, labelsn) # Append correct_masks, correct_boxes, pconf, pcls, tcls self.stats.append((correct_bboxes, correct_masks, pred[:, 4], pred[:, 5], cls.squeeze(-1))) pred_masks = torch.as_tensor(pred_masks, dtype=torch.uint8) if self.args.plots and self.batch_i < 3: self.plot_masks.append(pred_masks[:15].cpu()) # filter top 15 to plot # Save if self.args.save_json: pred_masks = ops.scale_image(pred_masks.permute(1, 2, 0).contiguous().cpu().numpy(), shape, ratio_pad=batch['ratio_pad'][si]) self.pred_to_json(predn, batch['im_file'][si], pred_masks) # if self.args.save_txt: # save_one_txt(predn, save_conf, shape, file=save_dir / 'labels' / f'{path.stem}.txt') def finalize_metrics(self, *args, **kwargs): """Sets speed and confusion matrix for evaluation metrics.""" self.metrics.speed = self.speed self.metrics.confusion_matrix = self.confusion_matrix def _process_batch(self, detections, labels, pred_masks=None, gt_masks=None, overlap=False, masks=False): """ Return correct prediction matrix Arguments: detections (array[N, 6]), x1, y1, x2, y2, conf, class labels (array[M, 5]), class, x1, y1, x2, y2 Returns: correct (array[N, 10]), for 10 IoU levels """ if masks: if overlap: nl = len(labels) index = torch.arange(nl, device=gt_masks.device).view(nl, 1, 1) + 1 gt_masks = gt_masks.repeat(nl, 1, 1) # shape(1,640,640) -> (n,640,640) gt_masks = torch.where(gt_masks == index, 1.0, 0.0) if gt_masks.shape[1:] != pred_masks.shape[1:]: gt_masks = F.interpolate(gt_masks[None], pred_masks.shape[1:], mode='bilinear', align_corners=False)[0] gt_masks = gt_masks.gt_(0.5) iou = mask_iou(gt_masks.view(gt_masks.shape[0], -1), pred_masks.view(pred_masks.shape[0], -1)) else: # boxes iou = box_iou(labels[:, 1:], detections[:, :4]) correct = np.zeros((detections.shape[0], self.iouv.shape[0])).astype(bool) correct_class = labels[:, 0:1] == detections[:, 5] for i in range(len(self.iouv)): x = torch.where((iou >= self.iouv[i]) & correct_class) # IoU > threshold and classes match if x[0].shape[0]: matches = torch.cat((torch.stack(x, 1), iou[x[0], x[1]][:, None]), 1).cpu().numpy() # [label, detect, iou] if x[0].shape[0] > 1: matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 1], return_index=True)[1]] # matches = matches[matches[:, 2].argsort()[::-1]] matches = matches[np.unique(matches[:, 0], return_index=True)[1]] correct[matches[:, 1].astype(int), i] = True return torch.tensor(correct, dtype=torch.bool, device=detections.device) def plot_val_samples(self, batch, ni): """Plots validation samples with bounding box labels.""" plot_images(batch['img'], batch['batch_idx'], batch['cls'].squeeze(-1), batch['bboxes'], batch['masks'], paths=batch['im_file'], fname=self.save_dir / f'val_batch{ni}_labels.jpg', names=self.names, on_plot=self.on_plot) def plot_predictions(self, batch, preds, ni): """Plots batch predictions with masks and bounding boxes.""" plot_images( batch['img'], *output_to_target(preds[0], max_det=15), # not set to self.args.max_det due to slow plotting speed torch.cat(self.plot_masks, dim=0) if len(self.plot_masks) else self.plot_masks, paths=batch['im_file'], fname=self.save_dir / f'val_batch{ni}_pred.jpg', names=self.names, on_plot=self.on_plot) # pred self.plot_masks.clear() def pred_to_json(self, predn, filename, pred_masks): """Save one JSON result.""" # Example result = {"image_id": 42, "category_id": 18, "bbox": [258.15, 41.29, 348.26, 243.78], "score": 0.236} from pycocotools.mask import encode # noqa def single_encode(x): """Encode predicted masks as RLE and append results to jdict.""" rle = encode(np.asarray(x[:, :, None], order='F', dtype='uint8'))[0] rle['counts'] = rle['counts'].decode('utf-8') return rle stem = Path(filename).stem image_id = int(stem) if stem.isnumeric() else stem box = ops.xyxy2xywh(predn[:, :4]) # xywh box[:, :2] -= box[:, 2:] / 2 # xy center to top-left corner pred_masks = np.transpose(pred_masks, (2, 0, 1)) with ThreadPool(NUM_THREADS) as pool: rles = pool.map(single_encode, pred_masks) for i, (p, b) in enumerate(zip(predn.tolist(), box.tolist())): self.jdict.append({ 'image_id': image_id, 'category_id': self.class_map[int(p[5])], 'bbox': [round(x, 3) for x in b], 'score': round(p[4], 5), 'segmentation': rles[i]}) def eval_json(self, stats): """Return COCO-style object detection evaluation metrics.""" if self.args.save_json and self.is_coco and len(self.jdict): anno_json = self.data['path'] / 'annotations/instances_val2017.json' # annotations pred_json = self.save_dir / 'predictions.json' # predictions LOGGER.info(f'\nEvaluating pycocotools mAP using {pred_json} and {anno_json}...') try: # https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocoEvalDemo.ipynb check_requirements('pycocotools>=2.0.6') from pycocotools.coco import COCO # noqa from pycocotools.cocoeval import COCOeval # noqa for x in anno_json, pred_json: assert x.is_file(), f'{x} file not found' anno = COCO(str(anno_json)) # init annotations api pred = anno.loadRes(str(pred_json)) # init predictions api (must pass string, not Path) for i, eval in enumerate([COCOeval(anno, pred, 'bbox'), COCOeval(anno, pred, 'segm')]): if self.is_coco: eval.params.imgIds = [int(Path(x).stem) for x in self.dataloader.dataset.im_files] # im to eval eval.evaluate() eval.accumulate() eval.summarize() idx = i * 4 + 2 stats[self.metrics.keys[idx + 1]], stats[ self.metrics.keys[idx]] = eval.stats[:2] # update mAP50-95 and mAP50 except Exception as e: LOGGER.warning(f'pycocotools unable to run: {e}') return stats def val(cfg=DEFAULT_CFG, use_python=False): """Validate trained YOLO model on validation data.""" model = cfg.model or 'yolov8n-seg.pt' data = cfg.data or 'coco128-seg.yaml' args = dict(model=model, data=data) if use_python: from ultralytics import YOLO YOLO(model).val(**args) else: validator = SegmentationValidator(args=args) validator(model=args['model']) if __name__ == '__main__': val() ================================================ FILE: utils/__init__.py ================================================ ================================================ FILE: utils/tools.py ================================================ import numpy as np from PIL import Image import matplotlib.pyplot as plt import cv2 import torch import os import sys def convert_box_xywh_to_xyxy(box): if len(box) == 4: return [box[0], box[1], box[0] + box[2], box[1] + box[3]] else: result = [] for b in box: b = convert_box_xywh_to_xyxy(b) result.append(b) return result def segment_image(image, bbox): image_array = np.array(image) segmented_image_array = np.zeros_like(image_array) x1, y1, x2, y2 = bbox segmented_image_array[y1:y2, x1:x2] = image_array[y1:y2, x1:x2] segmented_image = Image.fromarray(segmented_image_array) black_image = Image.new("RGB", image.size, (255, 255, 255)) # transparency_mask = np.zeros_like((), dtype=np.uint8) transparency_mask = np.zeros( (image_array.shape[0], image_array.shape[1]), dtype=np.uint8 ) transparency_mask[y1:y2, x1:x2] = 255 transparency_mask_image = Image.fromarray(transparency_mask, mode="L") black_image.paste(segmented_image, mask=transparency_mask_image) return black_image def format_results(result, filter=0): annotations = [] n = len(result.masks.data) for i in range(n): annotation = {} mask = result.masks.data[i] == 1.0 if torch.sum(mask) < filter: continue annotation["id"] = i annotation["segmentation"] = mask.cpu().numpy() annotation["bbox"] = result.boxes.data[i] annotation["score"] = result.boxes.conf[i] annotation["area"] = annotation["segmentation"].sum() annotations.append(annotation) return annotations def filter_masks(annotations): # filter the overlap mask annotations.sort(key=lambda x: x["area"], reverse=True) to_remove = set() for i in range(0, len(annotations)): a = annotations[i] for j in range(i + 1, len(annotations)): b = annotations[j] if i != j and j not in to_remove: # check if if b["area"] < a["area"]: if (a["segmentation"] & b["segmentation"]).sum() / b[ "segmentation" ].sum() > 0.8: to_remove.add(j) return [a for i, a in enumerate(annotations) if i not in to_remove], to_remove def get_bbox_from_mask(mask): mask = mask.astype(np.uint8) contours, hierarchy = cv2.findContours( mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE ) x1, y1, w, h = cv2.boundingRect(contours[0]) x2, y2 = x1 + w, y1 + h if len(contours) > 1: for b in contours: x_t, y_t, w_t, h_t = cv2.boundingRect(b) # 将多个bbox合并成一个 x1 = min(x1, x_t) y1 = min(y1, y_t) x2 = max(x2, x_t + w_t) y2 = max(y2, y_t + h_t) h = y2 - y1 w = x2 - x1 return [x1, y1, x2, y2] def fast_process( annotations, args, mask_random_color, bbox=None, points=None, edges=False ): if isinstance(annotations[0], dict): annotations = [annotation["segmentation"] for annotation in annotations] result_name = os.path.basename(args.img_path) image = cv2.imread(args.img_path) image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) original_h = image.shape[0] original_w = image.shape[1] if sys.platform == "darwin": plt.switch_backend("TkAgg") plt.figure(figsize=(original_w/100, original_h/100)) # Add subplot with no margin. plt.subplots_adjust(top=1, bottom=0, right=1, left=0, hspace=0, wspace=0) plt.margins(0, 0) plt.gca().xaxis.set_major_locator(plt.NullLocator()) plt.gca().yaxis.set_major_locator(plt.NullLocator()) plt.imshow(image) if args.better_quality == True: if isinstance(annotations[0], torch.Tensor): annotations = np.array(annotations.cpu()) for i, mask in enumerate(annotations): mask = cv2.morphologyEx( mask.astype(np.uint8), cv2.MORPH_CLOSE, np.ones((3, 3), np.uint8) ) annotations[i] = cv2.morphologyEx( mask.astype(np.uint8), cv2.MORPH_OPEN, np.ones((8, 8), np.uint8) ) if args.device == "cpu": annotations = np.array(annotations) fast_show_mask( annotations, plt.gca(), random_color=mask_random_color, bbox=bbox, points=points, point_label=args.point_label, retinamask=args.retina, target_height=original_h, target_width=original_w, ) else: if isinstance(annotations[0], np.ndarray): annotations = torch.from_numpy(annotations) fast_show_mask_gpu( annotations, plt.gca(), random_color=args.randomcolor, bbox=bbox, points=points, point_label=args.point_label, retinamask=args.retina, target_height=original_h, target_width=original_w, ) if isinstance(annotations, torch.Tensor): annotations = annotations.cpu().numpy() if args.withContours == True: contour_all = [] temp = np.zeros((original_h, original_w, 1)) for i, mask in enumerate(annotations): if type(mask) == dict: mask = mask["segmentation"] annotation = mask.astype(np.uint8) if args.retina == False: annotation = cv2.resize( annotation, (original_w, original_h), interpolation=cv2.INTER_NEAREST, ) contours, hierarchy = cv2.findContours( annotation, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE ) for contour in contours: contour_all.append(contour) cv2.drawContours(temp, contour_all, -1, (255, 255, 255), 2) color = np.array([0 / 255, 0 / 255, 255 / 255, 0.8]) contour_mask = temp / 255 * color.reshape(1, 1, -1) plt.imshow(contour_mask) save_path = args.output if not os.path.exists(save_path): os.makedirs(save_path) plt.axis("off") fig = plt.gcf() plt.draw() try: buf = fig.canvas.tostring_rgb() except AttributeError: fig.canvas.draw() buf = fig.canvas.tostring_rgb() cols, rows = fig.canvas.get_width_height() img_array = np.fromstring(buf, dtype=np.uint8).reshape(rows, cols, 3) cv2.imwrite(os.path.join(save_path, result_name), cv2.cvtColor(img_array, cv2.COLOR_RGB2BGR)) # CPU post process def fast_show_mask( annotation, ax, random_color=False, bbox=None, points=None, point_label=None, retinamask=True, target_height=960, target_width=960, ): msak_sum = annotation.shape[0] height = annotation.shape[1] weight = annotation.shape[2] # 将annotation 按照面积 排序 areas = np.sum(annotation, axis=(1, 2)) sorted_indices = np.argsort(areas) annotation = annotation[sorted_indices] index = (annotation != 0).argmax(axis=0) if random_color == True: color = np.random.random((msak_sum, 1, 1, 3)) else: color = np.ones((msak_sum, 1, 1, 3)) * np.array( [30 / 255, 144 / 255, 255 / 255] ) transparency = np.ones((msak_sum, 1, 1, 1)) * 0.6 visual = np.concatenate([color, transparency], axis=-1) mask_image = np.expand_dims(annotation, -1) * visual show = np.zeros((height, weight, 4)) h_indices, w_indices = np.meshgrid( np.arange(height), np.arange(weight), indexing="ij" ) indices = (index[h_indices, w_indices], h_indices, w_indices, slice(None)) # 使用向量化索引更新show的值 show[h_indices, w_indices, :] = mask_image[indices] if bbox is not None: x1, y1, x2, y2 = bbox ax.add_patch( plt.Rectangle( (x1, y1), x2 - x1, y2 - y1, fill=False, edgecolor="b", linewidth=1 ) ) # draw point if points is not None: plt.scatter( [point[0] for i, point in enumerate(points) if point_label[i] == 1], [point[1] for i, point in enumerate(points) if point_label[i] == 1], s=20, c="y", ) plt.scatter( [point[0] for i, point in enumerate(points) if point_label[i] == 0], [point[1] for i, point in enumerate(points) if point_label[i] == 0], s=20, c="m", ) if retinamask == False: show = cv2.resize( show, (target_width, target_height), interpolation=cv2.INTER_NEAREST ) ax.imshow(show) def fast_show_mask_gpu( annotation, ax, random_color=False, bbox=None, points=None, point_label=None, retinamask=True, target_height=960, target_width=960, ): msak_sum = annotation.shape[0] height = annotation.shape[1] weight = annotation.shape[2] areas = torch.sum(annotation, dim=(1, 2)) sorted_indices = torch.argsort(areas, descending=False) annotation = annotation[sorted_indices] # 找每个位置第一个非零值下标 index = (annotation != 0).to(torch.long).argmax(dim=0) if random_color == True: color = torch.rand((msak_sum, 1, 1, 3)).to(annotation.device) else: color = torch.ones((msak_sum, 1, 1, 3)).to(annotation.device) * torch.tensor( [30 / 255, 144 / 255, 255 / 255] ).to(annotation.device) transparency = torch.ones((msak_sum, 1, 1, 1)).to(annotation.device) * 0.6 visual = torch.cat([color, transparency], dim=-1) mask_image = torch.unsqueeze(annotation, -1) * visual # 按index取数，index指每个位置选哪个batch的数，把mask_image转成一个batch的形式 show = torch.zeros((height, weight, 4)).to(annotation.device) h_indices, w_indices = torch.meshgrid( torch.arange(height), torch.arange(weight), indexing="ij" ) indices = (index[h_indices, w_indices], h_indices, w_indices, slice(None)) # 使用向量化索引更新show的值 show[h_indices, w_indices, :] = mask_image[indices] show_cpu = show.cpu().numpy() if bbox is not None: x1, y1, x2, y2 = bbox ax.add_patch( plt.Rectangle( (x1, y1), x2 - x1, y2 - y1, fill=False, edgecolor="b", linewidth=1 ) ) # draw point if points is not None: plt.scatter( [point[0] for i, point in enumerate(points) if point_label[i] == 1], [point[1] for i, point in enumerate(points) if point_label[i] == 1], s=20, c="y", ) plt.scatter( [point[0] for i, point in enumerate(points) if point_label[i] == 0], [point[1] for i, point in enumerate(points) if point_label[i] == 0], s=20, c="m", ) if retinamask == False: show_cpu = cv2.resize( show_cpu, (target_width, target_height), interpolation=cv2.INTER_NEAREST ) ax.imshow(show_cpu) # clip @torch.no_grad() def retriev( model, preprocess, elements: [Image.Image], search_text: str, device ): preprocessed_images = [preprocess(image).to(device) for image in elements] import clip tokenized_text = clip.tokenize([search_text]).to(device) stacked_images = torch.stack(preprocessed_images) image_features = model.encode_image(stacked_images) text_features = model.encode_text(tokenized_text) image_features /= image_features.norm(dim=-1, keepdim=True) text_features /= text_features.norm(dim=-1, keepdim=True) probs = 100.0 * image_features @ text_features.T return probs[:, 0].softmax(dim=0) def crop_image(annotations, image_like): if isinstance(image_like, str): image = Image.open(image_like) else: image = image_like ori_w, ori_h = image.size mask_h, mask_w = annotations[0]["segmentation"].shape if ori_w != mask_w or ori_h != mask_h: image = image.resize((mask_w, mask_h)) cropped_boxes = [] cropped_images = [] not_crop = [] origin_id = [] for _, mask in enumerate(annotations): if np.sum(mask["segmentation"]) <= 100: continue origin_id.append(_) bbox = get_bbox_from_mask(mask["segmentation"]) # mask 的 bbox cropped_boxes.append(segment_image(image, bbox)) # 保存裁剪的图片 # cropped_boxes.append(segment_image(image,mask["segmentation"])) cropped_images.append(bbox) # 保存裁剪的图片的bbox return cropped_boxes, cropped_images, not_crop, origin_id, annotations def box_prompt(masks, bbox, target_height, target_width): h = masks.shape[1] w = masks.shape[2] if h != target_height or w != target_width: bbox = [ int(bbox[0] * w / target_width), int(bbox[1] * h / target_height), int(bbox[2] * w / target_width), int(bbox[3] * h / target_height), ] bbox[0] = round(bbox[0]) if round(bbox[0]) > 0 else 0 bbox[1] = round(bbox[1]) if round(bbox[1]) > 0 else 0 bbox[2] = round(bbox[2]) if round(bbox[2]) < w else w bbox[3] = round(bbox[3]) if round(bbox[3]) < h else h # IoUs = torch.zeros(len(masks), dtype=torch.float32) bbox_area = (bbox[3] - bbox[1]) * (bbox[2] - bbox[0]) masks_area = torch.sum(masks[:, bbox[1] : bbox[3], bbox[0] : bbox[2]], dim=(1, 2)) orig_masks_area = torch.sum(masks, dim=(1, 2)) union = bbox_area + orig_masks_area - masks_area IoUs = masks_area / union max_iou_index = torch.argmax(IoUs) return masks[max_iou_index].cpu().numpy(), max_iou_index def point_prompt(masks, points, point_label, target_height, target_width): # numpy 处理 h = masks[0]["segmentation"].shape[0] w = masks[0]["segmentation"].shape[1] if h != target_height or w != target_width: points = [ [int(point[0] * w / target_width), int(point[1] * h / target_height)] for point in points ] onemask = np.zeros((h, w)) masks = sorted(masks, key=lambda x: x['area'], reverse=True) for i, annotation in enumerate(masks): if type(annotation) == dict: mask = annotation['segmentation'] else: mask = annotation for i, point in enumerate(points): if mask[point[1], point[0]] == 1 and point_label[i] == 1: onemask[mask] = 1 if mask[point[1], point[0]] == 1 and point_label[i] == 0: onemask[mask] = 0 onemask = onemask >= 1 return onemask, 0 def text_prompt(annotations, text, img_path, device, wider=False, threshold=0.9): cropped_boxes, cropped_images, not_crop, origin_id, annotations_ = crop_image( annotations, img_path ) import clip clip_model, preprocess = clip.load("ViT-B/32", device=device) scores = retriev( clip_model, preprocess, cropped_boxes, text, device=device ) max_idx = scores.argsort() max_idx = max_idx[-1] max_idx = origin_id[int(max_idx)] # find the biggest mask which contains the mask with max score if wider: mask0 = annotations_[max_idx]["segmentation"] area0 = np.sum(mask0) areas = [(i, np.sum(mask["segmentation"])) for i, mask in enumerate(annotations_) if i in origin_id] areas = sorted(areas, key=lambda area: area[1], reverse=True) indices = [area[0] for area in areas] for index in indices: if index == max_idx or np.sum(annotations_[index]["segmentation"] & mask0) / area0 > threshold: max_idx = index break return annotations_[max_idx]["segmentation"], max_idx ================================================ FILE: utils/tools_gradio.py ================================================ import numpy as np from PIL import Image import matplotlib.pyplot as plt import cv2 import torch def fast_process( annotations, image, device, scale, better_quality=False, mask_random_color=True, bbox=None, use_retina=True, withContours=True, ): if isinstance(annotations[0], dict): annotations = [annotation['segmentation'] for annotation in annotations] original_h = image.height original_w = image.width if better_quality: if isinstance(annotations[0], torch.Tensor): annotations = np.array(annotations.cpu()) for i, mask in enumerate(annotations): mask = cv2.morphologyEx(mask.astype(np.uint8), cv2.MORPH_CLOSE, np.ones((3, 3), np.uint8)) annotations[i] = cv2.morphologyEx(mask.astype(np.uint8), cv2.MORPH_OPEN, np.ones((8, 8), np.uint8)) if device == 'cpu': annotations = np.array(annotations) inner_mask = fast_show_mask( annotations, plt.gca(), random_color=mask_random_color, bbox=bbox, retinamask=use_retina, target_height=original_h, target_width=original_w, ) else: if isinstance(annotations[0], np.ndarray): annotations = torch.from_numpy(annotations) inner_mask = fast_show_mask_gpu( annotations, plt.gca(), random_color=mask_random_color, bbox=bbox, retinamask=use_retina, target_height=original_h, target_width=original_w, ) if isinstance(annotations, torch.Tensor): annotations = annotations.cpu().numpy() if withContours: contour_all = [] temp = np.zeros((original_h, original_w, 1)) for i, mask in enumerate(annotations): if type(mask) == dict: mask = mask['segmentation'] annotation = mask.astype(np.uint8) if use_retina == False: annotation = cv2.resize( annotation, (original_w, original_h), interpolation=cv2.INTER_NEAREST, ) contours, _ = cv2.findContours(annotation, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) for contour in contours: contour_all.append(contour) cv2.drawContours(temp, contour_all, -1, (255, 255, 255), 2 // scale) color = np.array([0 / 255, 0 / 255, 255 / 255, 0.9]) contour_mask = temp / 255 * color.reshape(1, 1, -1) image = image.convert('RGBA') overlay_inner = Image.fromarray((inner_mask * 255).astype(np.uint8), 'RGBA') image.paste(overlay_inner, (0, 0), overlay_inner) if withContours: overlay_contour = Image.fromarray((contour_mask * 255).astype(np.uint8), 'RGBA') image.paste(overlay_contour, (0, 0), overlay_contour) return image # CPU post process def fast_show_mask( annotation, ax, random_color=False, bbox=None, retinamask=True, target_height=960, target_width=960, ): mask_sum = annotation.shape[0] height = annotation.shape[1] weight = annotation.shape[2] # 将annotation 按照面积 排序 areas = np.sum(annotation, axis=(1, 2)) sorted_indices = np.argsort(areas)[::1] annotation = annotation[sorted_indices] index = (annotation != 0).argmax(axis=0) if random_color: color = np.random.random((mask_sum, 1, 1, 3)) else: color = np.ones((mask_sum, 1, 1, 3)) * np.array([30 / 255, 144 / 255, 255 / 255]) transparency = np.ones((mask_sum, 1, 1, 1)) * 0.6 visual = np.concatenate([color, transparency], axis=-1) mask_image = np.expand_dims(annotation, -1) * visual mask = np.zeros((height, weight, 4)) h_indices, w_indices = np.meshgrid(np.arange(height), np.arange(weight), indexing='ij') indices = (index[h_indices, w_indices], h_indices, w_indices, slice(None)) mask[h_indices, w_indices, :] = mask_image[indices] if bbox is not None: x1, y1, x2, y2 = bbox ax.add_patch(plt.Rectangle((x1, y1), x2 - x1, y2 - y1, fill=False, edgecolor='b', linewidth=1)) if not retinamask: mask = cv2.resize(mask, (target_width, target_height), interpolation=cv2.INTER_NEAREST) return mask def fast_show_mask_gpu( annotation, ax, random_color=False, bbox=None, retinamask=True, target_height=960, target_width=960, ): device = annotation.device mask_sum = annotation.shape[0] height = annotation.shape[1] weight = annotation.shape[2] areas = torch.sum(annotation, dim=(1, 2)) sorted_indices = torch.argsort(areas, descending=False) annotation = annotation[sorted_indices] # 找每个位置第一个非零值下标 index = (annotation != 0).to(torch.long).argmax(dim=0) if random_color: color = torch.rand((mask_sum, 1, 1, 3)).to(device) else: color = torch.ones((mask_sum, 1, 1, 3)).to(device) * torch.tensor( [30 / 255, 144 / 255, 255 / 255] ).to(device) transparency = torch.ones((mask_sum, 1, 1, 1)).to(device) * 0.6 visual = torch.cat([color, transparency], dim=-1) mask_image = torch.unsqueeze(annotation, -1) * visual # 按index取数，index指每个位置选哪个batch的数，把mask_image转成一个batch的形式 mask = torch.zeros((height, weight, 4)).to(device) h_indices, w_indices = torch.meshgrid(torch.arange(height), torch.arange(weight)) indices = (index[h_indices, w_indices], h_indices, w_indices, slice(None)) # 使用向量化索引更新show的值 mask[h_indices, w_indices, :] = mask_image[indices] mask_cpu = mask.cpu().numpy() if bbox is not None: x1, y1, x2, y2 = bbox ax.add_patch( plt.Rectangle( (x1, y1), x2 - x1, y2 - y1, fill=False, edgecolor="b", linewidth=1 ) ) if not retinamask: mask_cpu = cv2.resize( mask_cpu, (target_width, target_height), interpolation=cv2.INTER_NEAREST ) return mask_cpu