Repository: jbwang1997/CrossKD Branch: master Commit: 87e156a46259 Files: 875 Total size: 31.0 MB Directory structure: gitextract_igmyqiaa/ ├── .circleci/ │ ├── config.yml │ ├── docker/ │ │ └── Dockerfile │ └── test.yml ├── .dev_scripts/ │ ├── batch_test_list.py │ ├── batch_train_list.txt │ ├── benchmark_filter.py │ ├── benchmark_full_models.txt │ ├── benchmark_inference_fps.py │ ├── benchmark_options.py │ ├── benchmark_test.py │ ├── benchmark_test_image.py │ ├── benchmark_train.py │ ├── benchmark_train_models.txt │ ├── benchmark_valid_flops.py │ ├── check_links.py │ ├── convert_test_benchmark_script.py │ ├── convert_train_benchmark_script.py │ ├── covignore.cfg │ ├── diff_coverage_test.sh │ ├── download_checkpoints.py │ ├── gather_models.py │ ├── gather_test_benchmark_metric.py │ ├── gather_train_benchmark_metric.py │ ├── linter.sh │ ├── test_benchmark.sh │ ├── test_init_backbone.py │ └── train_benchmark.sh ├── .github/ │ ├── CODE_OF_CONDUCT.md │ ├── CONTRIBUTING.md │ ├── ISSUE_TEMPLATE/ │ │ ├── config.yml │ │ ├── error-report.md │ │ ├── feature_request.md │ │ ├── general_questions.md │ │ └── reimplementation_questions.md │ ├── pull_request_template.md │ └── workflows/ │ └── deploy.yml ├── .gitignore ├── .owners.yml ├── .pre-commit-config-zh-cn.yaml ├── .pre-commit-config.yaml ├── .readthedocs.yml ├── CITATION.cff ├── LICENSE ├── MANIFEST.in ├── README.md ├── configs/ │ ├── _base_/ │ │ ├── datasets/ │ │ │ ├── cityscapes_detection.py │ │ │ ├── cityscapes_instance.py │ │ │ ├── coco_detection.py │ │ │ ├── coco_instance.py │ │ │ ├── coco_instance_semantic.py │ │ │ ├── coco_panoptic.py │ │ │ ├── deepfashion.py │ │ │ ├── lvis_v0.5_instance.py │ │ │ ├── lvis_v1_instance.py │ │ │ ├── objects365v1_detection.py │ │ │ ├── objects365v2_detection.py │ │ │ ├── openimages_detection.py │ │ │ ├── semi_coco_detection.py │ │ │ ├── voc0712.py │ │ │ └── wider_face.py │ │ ├── default_runtime.py │ │ ├── models/ │ │ │ ├── cascade-mask-rcnn_r50_fpn.py │ │ │ ├── cascade-rcnn_r50_fpn.py │ │ │ ├── fast-rcnn_r50_fpn.py │ │ │ ├── faster-rcnn_r50-caffe-c4.py │ │ │ ├── faster-rcnn_r50-caffe-dc5.py │ │ │ ├── faster-rcnn_r50_fpn.py │ │ │ ├── mask-rcnn_r50-caffe-c4.py │ │ │ ├── mask-rcnn_r50_fpn.py │ │ │ ├── retinanet_r50_fpn.py │ │ │ ├── rpn_r50-caffe-c4.py │ │ │ ├── rpn_r50_fpn.py │ │ │ └── ssd300.py │ │ └── schedules/ │ │ ├── schedule_1x.py │ │ ├── schedule_20e.py │ │ └── schedule_2x.py │ ├── atss/ │ │ ├── README.md │ │ ├── atss_r101_fpn_1x_coco.py │ │ ├── atss_r101_fpn_8xb8-amp-lsj-200e_coco.py │ │ ├── atss_r18_fpn_8xb8-amp-lsj-200e_coco.py │ │ ├── atss_r50_fpn_1x_coco.py │ │ ├── atss_r50_fpn_8xb8-amp-lsj-200e_coco.py │ │ └── metafile.yml │ ├── centernet/ │ │ ├── README.md │ │ ├── centernet-update_r101_fpn_8xb8-amp-lsj-200e_coco.py │ │ ├── centernet-update_r18_fpn_8xb8-amp-lsj-200e_coco.py │ │ ├── centernet-update_r50-caffe_fpn_ms-1x_coco.py │ │ ├── centernet-update_r50_fpn_8xb8-amp-lsj-200e_coco.py │ │ ├── centernet_r18-dcnv2_8xb16-crop512-140e_coco.py │ │ ├── centernet_r18_8xb16-crop512-140e_coco.py │ │ ├── centernet_tta.py │ │ └── metafile.yml │ ├── crosskd/ │ │ ├── crosskd_r18_gflv1_r50_fpn_1x_coco.py │ │ ├── crosskd_r18_retinanet_r50_fpn_1x_coco.py │ │ ├── crosskd_r50_atss_r101_fpn_1x_coco.py │ │ ├── crosskd_r50_fcos_r101-2x-ms_caffe_fpn_gn-head_2x_ms_coco.py │ │ ├── crosskd_r50_gflv1_r101-2x-ms_fpn_1x_coco.py │ │ ├── crosskd_r50_retinanet_r101_fpn_2x_coco.py │ │ └── crosskd_r50_retinanet_swint_fpn_1x_coco.py │ ├── crosskd+pkd/ │ │ ├── crosskd+pkd_r50_atss_r101_fpn_1x_coco.py │ │ ├── crosskd+pkd_r50_fcos_r101-2x-ms_caffe_fpn_gn-head_2x_ms_coco.py │ │ ├── crosskd+pkd_r50_gflv1_r101-2x-ms_fpn_1x_coco.py │ │ └── crosskd+pkd_r50_retinanet_r101_fpn_2x_coco.py │ ├── fcos/ │ │ ├── README.md │ │ ├── fcos_r101-caffe_fpn_gn-head-1x_coco.py │ │ ├── fcos_r101-caffe_fpn_gn-head_ms-640-800-2x_coco.py │ │ ├── fcos_r101_fpn_gn-head-center-normbbox-centeronreg-giou_8xb8-amp-lsj-200e_coco.py │ │ ├── fcos_r18_fpn_gn-head-center-normbbox-centeronreg-giou_8xb8-amp-lsj-200e_coco.py │ │ ├── fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py │ │ ├── fcos_r50-caffe_fpn_gn-head-center_1x_coco.py │ │ ├── fcos_r50-caffe_fpn_gn-head_1x_coco.py │ │ ├── fcos_r50-caffe_fpn_gn-head_4xb4-1x_coco.py │ │ ├── fcos_r50-caffe_fpn_gn-head_ms-640-800-2x_coco.py │ │ ├── fcos_r50-dcn-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py │ │ ├── fcos_r50_fpn_gn-head-center-normbbox-centeronreg-giou_8xb8-amp-lsj-200e_coco.py │ │ ├── fcos_x101-64x4d_fpn_gn-head_ms-640-800-2x_coco.py │ │ └── metafile.yml │ ├── gfl/ │ │ ├── README.md │ │ ├── gfl_r101-dconv-c3-c5_fpn_ms-2x_coco.py │ │ ├── gfl_r101_fpn_ms-2x_coco.py │ │ ├── gfl_r50_fpn_1x_coco.py │ │ ├── gfl_r50_fpn_ms-2x_coco.py │ │ ├── gfl_x101-32x4d-dconv-c4-c5_fpn_ms-2x_coco.py │ │ ├── gfl_x101-32x4d_fpn_ms-2x_coco.py │ │ └── metafile.yml │ ├── ld/ │ │ ├── README.md │ │ ├── ld_r101-gflv1-r101-dcn_fpn_2x_coco.py │ │ ├── ld_r18-gflv1-r101_fpn_1x_coco.py │ │ ├── ld_r34-gflv1-r101_fpn_1x_coco.py │ │ ├── ld_r50-gflv1-r101_fpn_1x_coco.py │ │ └── metafile.yml │ └── retinanet/ │ ├── README.md │ ├── metafile.yml │ ├── retinanet_r101-caffe_fpn_1x_coco.py │ ├── retinanet_r101-caffe_fpn_ms-3x_coco.py │ ├── retinanet_r101_fpn_1x_coco.py │ ├── retinanet_r101_fpn_2x_coco.py │ ├── retinanet_r101_fpn_8xb8-amp-lsj-200e_coco.py │ ├── retinanet_r101_fpn_ms-640-800-3x_coco.py │ ├── retinanet_r18_fpn_1x_coco.py │ ├── retinanet_r18_fpn_1xb8-1x_coco.py │ ├── retinanet_r18_fpn_8xb8-amp-lsj-200e_coco.py │ ├── retinanet_r50-caffe_fpn_1x_coco.py │ ├── retinanet_r50-caffe_fpn_ms-1x_coco.py │ ├── retinanet_r50-caffe_fpn_ms-2x_coco.py │ ├── retinanet_r50-caffe_fpn_ms-3x_coco.py │ ├── retinanet_r50_fpn_1x_coco.py │ ├── retinanet_r50_fpn_2x_coco.py │ ├── retinanet_r50_fpn_8xb8-amp-lsj-200e_coco.py │ ├── retinanet_r50_fpn_90k_coco.py │ ├── retinanet_r50_fpn_amp-1x_coco.py │ ├── retinanet_r50_fpn_ms-640-800-3x_coco.py │ ├── retinanet_swin-t-p4-w7_fpn_1x_coco.py │ ├── retinanet_tta.py │ ├── retinanet_x101-32x4d_fpn_1x_coco.py │ ├── retinanet_x101-32x4d_fpn_2x_coco.py │ ├── retinanet_x101-64x4d_fpn_1x_coco.py │ ├── retinanet_x101-64x4d_fpn_2x_coco.py │ └── retinanet_x101-64x4d_fpn_ms-640-800-3x_coco.py ├── demo/ │ ├── MMDet_InstanceSeg_Tutorial.ipynb │ ├── MMDet_Tutorial.ipynb │ ├── create_result_gif.py │ ├── image_demo.py │ ├── inference_demo.ipynb │ ├── video_demo.py │ ├── video_gpuaccel_demo.py │ └── webcam_demo.py ├── docker/ │ ├── Dockerfile │ ├── serve/ │ │ ├── Dockerfile │ │ ├── config.properties │ │ └── entrypoint.sh │ └── serve_cn/ │ └── Dockerfile ├── mmdet/ │ ├── __init__.py │ ├── apis/ │ │ ├── __init__.py │ │ ├── det_inferencer.py │ │ └── inference.py │ ├── datasets/ │ │ ├── __init__.py │ │ ├── api_wrappers/ │ │ │ ├── __init__.py │ │ │ └── coco_api.py │ │ ├── base_det_dataset.py │ │ ├── cityscapes.py │ │ ├── coco.py │ │ ├── coco_panoptic.py │ │ ├── crowdhuman.py │ │ ├── dataset_wrappers.py │ │ ├── deepfashion.py │ │ ├── lvis.py │ │ ├── objects365.py │ │ ├── openimages.py │ │ ├── samplers/ │ │ │ ├── __init__.py │ │ │ ├── batch_sampler.py │ │ │ ├── class_aware_sampler.py │ │ │ └── multi_source_sampler.py │ │ ├── transforms/ │ │ │ ├── __init__.py │ │ │ ├── augment_wrappers.py │ │ │ ├── colorspace.py │ │ │ ├── formatting.py │ │ │ ├── geometric.py │ │ │ ├── instaboost.py │ │ │ ├── loading.py │ │ │ ├── transforms.py │ │ │ └── wrappers.py │ │ ├── utils.py │ │ ├── voc.py │ │ ├── wider_face.py │ │ └── xml_style.py │ ├── engine/ │ │ ├── __init__.py │ │ ├── hooks/ │ │ │ ├── __init__.py │ │ │ ├── checkloss_hook.py │ │ │ ├── mean_teacher_hook.py │ │ │ ├── memory_profiler_hook.py │ │ │ ├── num_class_check_hook.py │ │ │ ├── pipeline_switch_hook.py │ │ │ ├── set_epoch_info_hook.py │ │ │ ├── sync_norm_hook.py │ │ │ ├── utils.py │ │ │ ├── visualization_hook.py │ │ │ └── yolox_mode_switch_hook.py │ │ ├── optimizers/ │ │ │ ├── __init__.py │ │ │ └── layer_decay_optimizer_constructor.py │ │ ├── runner/ │ │ │ ├── __init__.py │ │ │ └── loops.py │ │ └── schedulers/ │ │ ├── __init__.py │ │ └── quadratic_warmup.py │ ├── evaluation/ │ │ ├── __init__.py │ │ ├── functional/ │ │ │ ├── __init__.py │ │ │ ├── bbox_overlaps.py │ │ │ ├── class_names.py │ │ │ ├── mean_ap.py │ │ │ ├── panoptic_utils.py │ │ │ └── recall.py │ │ └── metrics/ │ │ ├── __init__.py │ │ ├── cityscapes_metric.py │ │ ├── coco_metric.py │ │ ├── coco_occluded_metric.py │ │ ├── coco_panoptic_metric.py │ │ ├── crowdhuman_metric.py │ │ ├── dump_det_results.py │ │ ├── dump_proposals_metric.py │ │ ├── lvis_metric.py │ │ ├── openimages_metric.py │ │ └── voc_metric.py │ ├── models/ │ │ ├── __init__.py │ │ ├── backbones/ │ │ │ ├── __init__.py │ │ │ ├── csp_darknet.py │ │ │ ├── cspnext.py │ │ │ ├── darknet.py │ │ │ ├── detectors_resnet.py │ │ │ ├── detectors_resnext.py │ │ │ ├── efficientnet.py │ │ │ ├── hourglass.py │ │ │ ├── hrnet.py │ │ │ ├── mobilenet_v2.py │ │ │ ├── pvt.py │ │ │ ├── regnet.py │ │ │ ├── res2net.py │ │ │ ├── resnest.py │ │ │ ├── resnet.py │ │ │ ├── resnext.py │ │ │ ├── ssd_vgg.py │ │ │ ├── swin.py │ │ │ └── trident_resnet.py │ │ ├── data_preprocessors/ │ │ │ ├── __init__.py │ │ │ └── data_preprocessor.py │ │ ├── dense_heads/ │ │ │ ├── __init__.py │ │ │ ├── anchor_free_head.py │ │ │ ├── anchor_head.py │ │ │ ├── atss_head.py │ │ │ ├── autoassign_head.py │ │ │ ├── base_dense_head.py │ │ │ ├── base_mask_head.py │ │ │ ├── boxinst_head.py │ │ │ ├── cascade_rpn_head.py │ │ │ ├── centernet_head.py │ │ │ ├── centernet_update_head.py │ │ │ ├── centripetal_head.py │ │ │ ├── condinst_head.py │ │ │ ├── conditional_detr_head.py │ │ │ ├── corner_head.py │ │ │ ├── dab_detr_head.py │ │ │ ├── ddod_head.py │ │ │ ├── deformable_detr_head.py │ │ │ ├── dense_test_mixins.py │ │ │ ├── detr_head.py │ │ │ ├── dino_head.py │ │ │ ├── embedding_rpn_head.py │ │ │ ├── fcos_head.py │ │ │ ├── fovea_head.py │ │ │ ├── free_anchor_retina_head.py │ │ │ ├── fsaf_head.py │ │ │ ├── ga_retina_head.py │ │ │ ├── ga_rpn_head.py │ │ │ ├── gfl_head.py │ │ │ ├── guided_anchor_head.py │ │ │ ├── lad_head.py │ │ │ ├── ld_head.py │ │ │ ├── mask2former_head.py │ │ │ ├── maskformer_head.py │ │ │ ├── nasfcos_head.py │ │ │ ├── paa_head.py │ │ │ ├── pisa_retinanet_head.py │ │ │ ├── pisa_ssd_head.py │ │ │ ├── reppoints_head.py │ │ │ ├── retina_head.py │ │ │ ├── retina_sepbn_head.py │ │ │ ├── rpn_head.py │ │ │ ├── rtmdet_head.py │ │ │ ├── rtmdet_ins_head.py │ │ │ ├── sabl_retina_head.py │ │ │ ├── solo_head.py │ │ │ ├── solov2_head.py │ │ │ ├── ssd_head.py │ │ │ ├── tood_head.py │ │ │ ├── vfnet_head.py │ │ │ ├── yolact_head.py │ │ │ ├── yolo_head.py │ │ │ ├── yolof_head.py │ │ │ └── yolox_head.py │ │ ├── detectors/ │ │ │ ├── __init__.py │ │ │ ├── atss.py │ │ │ ├── autoassign.py │ │ │ ├── base.py │ │ │ ├── base_detr.py │ │ │ ├── boxinst.py │ │ │ ├── cascade_rcnn.py │ │ │ ├── centernet.py │ │ │ ├── condinst.py │ │ │ ├── conditional_detr.py │ │ │ ├── cornernet.py │ │ │ ├── crosskd_atss.py │ │ │ ├── crosskd_fcos.py │ │ │ ├── crosskd_gfl.py │ │ │ ├── crosskd_retinanet.py │ │ │ ├── crosskd_single_stage.py │ │ │ ├── crowddet.py │ │ │ ├── d2_wrapper.py │ │ │ ├── dab_detr.py │ │ │ ├── ddod.py │ │ │ ├── deformable_detr.py │ │ │ ├── detr.py │ │ │ ├── dino.py │ │ │ ├── fast_rcnn.py │ │ │ ├── faster_rcnn.py │ │ │ ├── fcos.py │ │ │ ├── fovea.py │ │ │ ├── fsaf.py │ │ │ ├── gfl.py │ │ │ ├── grid_rcnn.py │ │ │ ├── htc.py │ │ │ ├── kd_one_stage.py │ │ │ ├── lad.py │ │ │ ├── mask2former.py │ │ │ ├── mask_rcnn.py │ │ │ ├── mask_scoring_rcnn.py │ │ │ ├── maskformer.py │ │ │ ├── nasfcos.py │ │ │ ├── paa.py │ │ │ ├── panoptic_fpn.py │ │ │ ├── panoptic_two_stage_segmentor.py │ │ │ ├── point_rend.py │ │ │ ├── queryinst.py │ │ │ ├── reppoints_detector.py │ │ │ ├── retinanet.py │ │ │ ├── rpn.py │ │ │ ├── rtmdet.py │ │ │ ├── scnet.py │ │ │ ├── semi_base.py │ │ │ ├── single_stage.py │ │ │ ├── single_stage_instance_seg.py │ │ │ ├── soft_teacher.py │ │ │ ├── solo.py │ │ │ ├── solov2.py │ │ │ ├── sparse_rcnn.py │ │ │ ├── tood.py │ │ │ ├── trident_faster_rcnn.py │ │ │ ├── two_stage.py │ │ │ ├── vfnet.py │ │ │ ├── yolact.py │ │ │ ├── yolo.py │ │ │ ├── yolof.py │ │ │ └── yolox.py │ │ ├── layers/ │ │ │ ├── __init__.py │ │ │ ├── activations.py │ │ │ ├── bbox_nms.py │ │ │ ├── brick_wrappers.py │ │ │ ├── conv_upsample.py │ │ │ ├── csp_layer.py │ │ │ ├── dropblock.py │ │ │ ├── ema.py │ │ │ ├── inverted_residual.py │ │ │ ├── matrix_nms.py │ │ │ ├── msdeformattn_pixel_decoder.py │ │ │ ├── normed_predictor.py │ │ │ ├── pixel_decoder.py │ │ │ ├── positional_encoding.py │ │ │ ├── res_layer.py │ │ │ ├── se_layer.py │ │ │ └── transformer/ │ │ │ ├── __init__.py │ │ │ ├── conditional_detr_layers.py │ │ │ ├── dab_detr_layers.py │ │ │ ├── deformable_detr_layers.py │ │ │ ├── detr_layers.py │ │ │ ├── dino_layers.py │ │ │ ├── mask2former_layers.py │ │ │ └── utils.py │ │ ├── losses/ │ │ │ ├── __init__.py │ │ │ ├── accuracy.py │ │ │ ├── ae_loss.py │ │ │ ├── balanced_l1_loss.py │ │ │ ├── cross_entropy_loss.py │ │ │ ├── dice_loss.py │ │ │ ├── focal_loss.py │ │ │ ├── gaussian_focal_loss.py │ │ │ ├── gfocal_loss.py │ │ │ ├── ghm_loss.py │ │ │ ├── iou_loss.py │ │ │ ├── kd_loss.py │ │ │ ├── mse_loss.py │ │ │ ├── pisa_loss.py │ │ │ ├── pkd_loss.py │ │ │ ├── seesaw_loss.py │ │ │ ├── smooth_l1_loss.py │ │ │ ├── utils.py │ │ │ └── varifocal_loss.py │ │ ├── necks/ │ │ │ ├── __init__.py │ │ │ ├── bfp.py │ │ │ ├── channel_mapper.py │ │ │ ├── cspnext_pafpn.py │ │ │ ├── ct_resnet_neck.py │ │ │ ├── dilated_encoder.py │ │ │ ├── dyhead.py │ │ │ ├── fpg.py │ │ │ ├── fpn.py │ │ │ ├── fpn_carafe.py │ │ │ ├── hrfpn.py │ │ │ ├── nas_fpn.py │ │ │ ├── nasfcos_fpn.py │ │ │ ├── pafpn.py │ │ │ ├── rfp.py │ │ │ ├── ssd_neck.py │ │ │ ├── ssh.py │ │ │ ├── yolo_neck.py │ │ │ └── yolox_pafpn.py │ │ ├── roi_heads/ │ │ │ ├── __init__.py │ │ │ ├── base_roi_head.py │ │ │ ├── bbox_heads/ │ │ │ │ ├── __init__.py │ │ │ │ ├── bbox_head.py │ │ │ │ ├── convfc_bbox_head.py │ │ │ │ ├── dii_head.py │ │ │ │ ├── double_bbox_head.py │ │ │ │ ├── multi_instance_bbox_head.py │ │ │ │ ├── sabl_head.py │ │ │ │ └── scnet_bbox_head.py │ │ │ ├── cascade_roi_head.py │ │ │ ├── double_roi_head.py │ │ │ ├── dynamic_roi_head.py │ │ │ ├── grid_roi_head.py │ │ │ ├── htc_roi_head.py │ │ │ ├── mask_heads/ │ │ │ │ ├── __init__.py │ │ │ │ ├── coarse_mask_head.py │ │ │ │ ├── dynamic_mask_head.py │ │ │ │ ├── fcn_mask_head.py │ │ │ │ ├── feature_relay_head.py │ │ │ │ ├── fused_semantic_head.py │ │ │ │ ├── global_context_head.py │ │ │ │ ├── grid_head.py │ │ │ │ ├── htc_mask_head.py │ │ │ │ ├── mask_point_head.py │ │ │ │ ├── maskiou_head.py │ │ │ │ ├── scnet_mask_head.py │ │ │ │ └── scnet_semantic_head.py │ │ │ ├── mask_scoring_roi_head.py │ │ │ ├── multi_instance_roi_head.py │ │ │ ├── pisa_roi_head.py │ │ │ ├── point_rend_roi_head.py │ │ │ ├── roi_extractors/ │ │ │ │ ├── __init__.py │ │ │ │ ├── base_roi_extractor.py │ │ │ │ ├── generic_roi_extractor.py │ │ │ │ └── single_level_roi_extractor.py │ │ │ ├── scnet_roi_head.py │ │ │ ├── shared_heads/ │ │ │ │ ├── __init__.py │ │ │ │ └── res_layer.py │ │ │ ├── sparse_roi_head.py │ │ │ ├── standard_roi_head.py │ │ │ ├── test_mixins.py │ │ │ └── trident_roi_head.py │ │ ├── seg_heads/ │ │ │ ├── __init__.py │ │ │ ├── base_semantic_head.py │ │ │ ├── panoptic_fpn_head.py │ │ │ └── panoptic_fusion_heads/ │ │ │ ├── __init__.py │ │ │ ├── base_panoptic_fusion_head.py │ │ │ ├── heuristic_fusion_head.py │ │ │ └── maskformer_fusion_head.py │ │ ├── task_modules/ │ │ │ ├── __init__.py │ │ │ ├── assigners/ │ │ │ │ ├── __init__.py │ │ │ │ ├── approx_max_iou_assigner.py │ │ │ │ ├── assign_result.py │ │ │ │ ├── atss_assigner.py │ │ │ │ ├── base_assigner.py │ │ │ │ ├── center_region_assigner.py │ │ │ │ ├── dynamic_soft_label_assigner.py │ │ │ │ ├── grid_assigner.py │ │ │ │ ├── hungarian_assigner.py │ │ │ │ ├── iou2d_calculator.py │ │ │ │ ├── match_cost.py │ │ │ │ ├── max_iou_assigner.py │ │ │ │ ├── multi_instance_assigner.py │ │ │ │ ├── point_assigner.py │ │ │ │ ├── region_assigner.py │ │ │ │ ├── sim_ota_assigner.py │ │ │ │ ├── task_aligned_assigner.py │ │ │ │ └── uniform_assigner.py │ │ │ ├── builder.py │ │ │ ├── coders/ │ │ │ │ ├── __init__.py │ │ │ │ ├── base_bbox_coder.py │ │ │ │ ├── bucketing_bbox_coder.py │ │ │ │ ├── delta_xywh_bbox_coder.py │ │ │ │ ├── distance_point_bbox_coder.py │ │ │ │ ├── legacy_delta_xywh_bbox_coder.py │ │ │ │ ├── pseudo_bbox_coder.py │ │ │ │ ├── tblr_bbox_coder.py │ │ │ │ └── yolo_bbox_coder.py │ │ │ ├── prior_generators/ │ │ │ │ ├── __init__.py │ │ │ │ ├── anchor_generator.py │ │ │ │ ├── point_generator.py │ │ │ │ └── utils.py │ │ │ └── samplers/ │ │ │ ├── __init__.py │ │ │ ├── base_sampler.py │ │ │ ├── combined_sampler.py │ │ │ ├── instance_balanced_pos_sampler.py │ │ │ ├── iou_balanced_neg_sampler.py │ │ │ ├── mask_pseudo_sampler.py │ │ │ ├── mask_sampling_result.py │ │ │ ├── multi_instance_random_sampler.py │ │ │ ├── multi_instance_sampling_result.py │ │ │ ├── ohem_sampler.py │ │ │ ├── pseudo_sampler.py │ │ │ ├── random_sampler.py │ │ │ ├── sampling_result.py │ │ │ └── score_hlr_sampler.py │ │ ├── test_time_augs/ │ │ │ ├── __init__.py │ │ │ ├── det_tta.py │ │ │ └── merge_augs.py │ │ └── utils/ │ │ ├── __init__.py │ │ ├── gaussian_target.py │ │ ├── make_divisible.py │ │ ├── misc.py │ │ ├── panoptic_gt_processing.py │ │ └── point_sample.py │ ├── registry.py │ ├── structures/ │ │ ├── __init__.py │ │ ├── bbox/ │ │ │ ├── __init__.py │ │ │ ├── base_boxes.py │ │ │ ├── bbox_overlaps.py │ │ │ ├── box_type.py │ │ │ ├── horizontal_boxes.py │ │ │ └── transforms.py │ │ ├── det_data_sample.py │ │ └── mask/ │ │ ├── __init__.py │ │ ├── mask_target.py │ │ ├── structures.py │ │ └── utils.py │ ├── testing/ │ │ ├── __init__.py │ │ ├── _fast_stop_training_hook.py │ │ └── _utils.py │ ├── utils/ │ │ ├── __init__.py │ │ ├── benchmark.py │ │ ├── collect_env.py │ │ ├── compat_config.py │ │ ├── contextmanagers.py │ │ ├── dist_utils.py │ │ ├── logger.py │ │ ├── memory.py │ │ ├── misc.py │ │ ├── profiling.py │ │ ├── replace_cfg_vals.py │ │ ├── setup_env.py │ │ ├── split_batch.py │ │ ├── typing_utils.py │ │ ├── util_mixins.py │ │ └── util_random.py │ ├── version.py │ └── visualization/ │ ├── __init__.py │ ├── local_visualizer.py │ └── palette.py ├── model-index.yml ├── projects/ │ ├── ConvNeXt-V2/ │ │ ├── README.md │ │ └── configs/ │ │ └── mask-rcnn_convnext-v2-b_fpn_lsj-3x-fcmae_coco.py │ ├── Detic/ │ │ ├── README.md │ │ ├── configs/ │ │ │ └── detic_centernet2_swin-b_fpn_4x_lvis-coco-in21k.py │ │ ├── demo.py │ │ └── detic/ │ │ ├── __init__.py │ │ ├── centernet_rpn_head.py │ │ ├── detic_bbox_head.py │ │ ├── detic_roi_head.py │ │ ├── text_encoder.py │ │ ├── utils.py │ │ └── zero_shot_classifier.py │ ├── DiffusionDet/ │ │ ├── README.md │ │ ├── configs/ │ │ │ └── diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py │ │ ├── diffusiondet/ │ │ │ ├── __init__.py │ │ │ ├── diffusiondet.py │ │ │ ├── head.py │ │ │ └── loss.py │ │ └── model_converters/ │ │ └── diffusiondet_resnet_to_mmdet.py │ ├── EfficientDet/ │ │ ├── README.md │ │ ├── configs/ │ │ │ └── efficientdet_effb0_bifpn_16xb8-crop512-300e_coco.py │ │ ├── convert_tf_to_pt.py │ │ └── efficientdet/ │ │ ├── __init__.py │ │ ├── anchor_generator.py │ │ ├── api_wrappers/ │ │ │ ├── __init__.py │ │ │ └── coco_api.py │ │ ├── bifpn.py │ │ ├── coco_90class.py │ │ ├── coco_90metric.py │ │ ├── efficientdet.py │ │ ├── efficientdet_head.py │ │ ├── trans_max_iou_assigner.py │ │ ├── utils.py │ │ └── yxyx_bbox_coder.py │ ├── SparseInst/ │ │ ├── README.md │ │ ├── configs/ │ │ │ └── sparseinst_r50_iam_8xb8-ms-270k_coco.py │ │ └── sparseinst/ │ │ ├── __init__.py │ │ ├── decoder.py │ │ ├── encoder.py │ │ ├── loss.py │ │ └── sparseinst.py │ └── example_project/ │ ├── README.md │ ├── configs/ │ │ └── faster-rcnn_dummy-resnet_fpn_1x_coco.py │ └── dummy/ │ ├── __init__.py │ └── dummy_resnet.py ├── pytest.ini ├── requirements/ │ ├── albu.txt │ ├── build.txt │ ├── docs.txt │ ├── mminstall.txt │ ├── optional.txt │ ├── readthedocs.txt │ ├── runtime.txt │ └── tests.txt ├── requirements.txt ├── setup.cfg ├── setup.py ├── tests/ │ ├── test_apis/ │ │ ├── test_det_inferencer.py │ │ └── test_inference.py │ ├── test_datasets/ │ │ ├── test_cityscapes.py │ │ ├── test_coco.py │ │ ├── test_coco_api_wrapper.py │ │ ├── test_coco_panoptic.py │ │ ├── test_crowdhuman.py │ │ ├── test_lvis.py │ │ ├── test_objects365.py │ │ ├── test_openimages.py │ │ ├── test_pascal_voc.py │ │ ├── test_samplers/ │ │ │ ├── test_batch_sampler.py │ │ │ └── test_multi_source_sampler.py │ │ ├── test_transforms/ │ │ │ ├── __init__.py │ │ │ ├── test_augment_wrappers.py │ │ │ ├── test_colorspace.py │ │ │ ├── test_formatting.py │ │ │ ├── test_geometric.py │ │ │ ├── test_instaboost.py │ │ │ ├── test_loading.py │ │ │ ├── test_transforms.py │ │ │ ├── test_wrappers.py │ │ │ └── utils.py │ │ └── test_tta.py │ ├── test_engine/ │ │ ├── __init__.py │ │ ├── test_hooks/ │ │ │ ├── test_checkloss_hook.py │ │ │ ├── test_mean_teacher_hook.py │ │ │ ├── test_memory_profiler_hook.py │ │ │ ├── test_num_class_check_hook.py │ │ │ ├── test_sync_norm_hook.py │ │ │ ├── test_visualization_hook.py │ │ │ └── test_yolox_mode_switch_hook.py │ │ ├── test_optimizers/ │ │ │ ├── __init__.py │ │ │ └── test_layer_decay_optimizer_constructor.py │ │ ├── test_runner/ │ │ │ └── test_loops.py │ │ └── test_schedulers/ │ │ └── test_quadratic_warmup.py │ ├── test_evaluation/ │ │ └── test_metrics/ │ │ ├── __init__.py │ │ ├── test_cityscapes_metric.py │ │ ├── test_coco_metric.py │ │ ├── test_coco_occluded_metric.py │ │ ├── test_coco_panoptic_metric.py │ │ ├── test_crowdhuman_metric.py │ │ ├── test_dump_det_results.py │ │ ├── test_lvis_metric.py │ │ └── test_openimages_metric.py │ ├── test_models/ │ │ ├── test_backbones/ │ │ │ ├── __init__.py │ │ │ ├── test_csp_darknet.py │ │ │ ├── test_detectors_resnet.py │ │ │ ├── test_efficientnet.py │ │ │ ├── test_hourglass.py │ │ │ ├── test_hrnet.py │ │ │ ├── test_mobilenet_v2.py │ │ │ ├── test_pvt.py │ │ │ ├── test_regnet.py │ │ │ ├── test_renext.py │ │ │ ├── test_res2net.py │ │ │ ├── test_resnest.py │ │ │ ├── test_resnet.py │ │ │ ├── test_swin.py │ │ │ ├── test_trident_resnet.py │ │ │ └── utils.py │ │ ├── test_data_preprocessors/ │ │ │ ├── test_batch_resize.py │ │ │ ├── test_boxinst_preprocessor.py │ │ │ └── test_data_preprocessor.py │ │ ├── test_dense_heads/ │ │ │ ├── test_anchor_head.py │ │ │ ├── test_atss_head.py │ │ │ ├── test_autoassign_head.py │ │ │ ├── test_boxinst_head.py │ │ │ ├── test_cascade_rpn_head.py │ │ │ ├── test_centernet_head.py │ │ │ ├── test_centernet_update_head.py │ │ │ ├── test_centripetal_head.py │ │ │ ├── test_condinst_head.py │ │ │ ├── test_corner_head.py │ │ │ ├── test_ddod_head.py │ │ │ ├── test_embedding_rpn_head.py │ │ │ ├── test_fcos_head.py │ │ │ ├── test_fovea_head.py │ │ │ ├── test_free_anchor_head.py │ │ │ ├── test_fsaf_head.py │ │ │ ├── test_ga_retina_head.py │ │ │ ├── test_ga_rpn_head.py │ │ │ ├── test_gfl_head.py │ │ │ ├── test_guided_anchor_head.py │ │ │ ├── test_lad_head.py │ │ │ ├── test_ld_head.py │ │ │ ├── test_nasfcos_head.py │ │ │ ├── test_paa_head.py │ │ │ ├── test_pisa_retinanet_head.py │ │ │ ├── test_pisa_ssd_head.py │ │ │ ├── test_reppoints_head.py │ │ │ ├── test_retina_sepBN_head.py │ │ │ ├── test_rpn_head.py │ │ │ ├── test_sabl_retina_head.py │ │ │ ├── test_solo_head.py │ │ │ ├── test_solov2_head.py │ │ │ ├── test_ssd_head.py │ │ │ ├── test_tood_head.py │ │ │ ├── test_vfnet_head.py │ │ │ ├── test_yolo_head.py │ │ │ ├── test_yolof_head.py │ │ │ └── test_yolox_head.py │ │ ├── test_detectors/ │ │ │ ├── test_conditional_detr.py │ │ │ ├── test_cornernet.py │ │ │ ├── test_dab_detr.py │ │ │ ├── test_deformable_detr.py │ │ │ ├── test_detr.py │ │ │ ├── test_dino.py │ │ │ ├── test_kd_single_stage.py │ │ │ ├── test_maskformer.py │ │ │ ├── test_panoptic_two_stage_segmentor.py │ │ │ ├── test_rpn.py │ │ │ ├── test_semi_base.py │ │ │ ├── test_single_stage.py │ │ │ ├── test_single_stage_instance_seg.py │ │ │ └── test_two_stage.py │ │ ├── test_layers/ │ │ │ ├── __init__.py │ │ │ ├── test_brick_wrappers.py │ │ │ ├── test_conv_upsample.py │ │ │ ├── test_ema.py │ │ │ ├── test_inverted_residual.py │ │ │ ├── test_plugins.py │ │ │ ├── test_position_encoding.py │ │ │ ├── test_se_layer.py │ │ │ └── test_transformer.py │ │ ├── test_losses/ │ │ │ ├── test_gaussian_focal_loss.py │ │ │ └── test_loss.py │ │ ├── test_necks/ │ │ │ ├── test_ct_resnet_neck.py │ │ │ └── test_necks.py │ │ ├── test_roi_heads/ │ │ │ ├── test_bbox_heads/ │ │ │ │ ├── test_bbox_head.py │ │ │ │ ├── test_double_bbox_head.py │ │ │ │ ├── test_multi_instance_bbox_head.py │ │ │ │ ├── test_sabl_bbox_head.py │ │ │ │ └── test_scnet_bbox_head.py │ │ │ ├── test_cascade_roi_head.py │ │ │ ├── test_dynamic_roi_head.py │ │ │ ├── test_grid_roi_head.py │ │ │ ├── test_htc_roi_head.py │ │ │ ├── test_mask_heads/ │ │ │ │ ├── test_coarse_mask_head.py │ │ │ │ ├── test_fcn_mask_head.py │ │ │ │ ├── test_feature_relay_head.py │ │ │ │ ├── test_fused_semantic_head.py │ │ │ │ ├── test_global_context_head.py │ │ │ │ ├── test_grid_head.py │ │ │ │ ├── test_htc_mask_head.py │ │ │ │ ├── test_maskiou_head.py │ │ │ │ ├── test_scnet_mask_head.py │ │ │ │ └── test_scnet_semantic_head.py │ │ │ ├── test_mask_scoring_roI_head.py │ │ │ ├── test_multi_instance_roi_head.py │ │ │ ├── test_pisa_roi_head.py │ │ │ ├── test_point_rend_roi_head.py │ │ │ ├── test_roi_extractors/ │ │ │ │ ├── test_generic_roi_extractor.py │ │ │ │ └── test_single_level_roi_extractor.py │ │ │ ├── test_scnet_roi_head.py │ │ │ ├── test_sparse_roi_head.py │ │ │ ├── test_standard_roi_head.py │ │ │ └── test_trident_roi_head.py │ │ ├── test_seg_heads/ │ │ │ ├── test_heuristic_fusion_head.py │ │ │ ├── test_maskformer_fusion_head.py │ │ │ └── test_panoptic_fpn_head.py │ │ ├── test_task_modules/ │ │ │ ├── __init__.py │ │ │ ├── test_assigners/ │ │ │ │ ├── test_approx_max_iou_assigner.py │ │ │ │ ├── test_atss_assigner.py │ │ │ │ ├── test_center_region_assigner.py │ │ │ │ ├── test_dynamic_soft_label_assigner.py │ │ │ │ ├── test_grid_assigner.py │ │ │ │ ├── test_hungarian_assigner.py │ │ │ │ ├── test_max_iou_assigner.py │ │ │ │ ├── test_point_assigner.py │ │ │ │ ├── test_region_assigner.py │ │ │ │ ├── test_simota_assigner.py │ │ │ │ ├── test_task_aligned_assigner.py │ │ │ │ └── test_task_uniform_assigner.py │ │ │ ├── test_coder/ │ │ │ │ └── test_delta_xywh_bbox_coder.py │ │ │ ├── test_iou2d_calculator.py │ │ │ ├── test_prior_generators/ │ │ │ │ └── test_anchor_generator.py │ │ │ └── test_samplers/ │ │ │ └── test_pesudo_sampler.py │ │ ├── test_tta/ │ │ │ └── test_det_tta.py │ │ └── test_utils/ │ │ ├── test_misc.py │ │ └── test_model_misc.py │ ├── test_structures/ │ │ ├── __init__.py │ │ ├── test_bbox/ │ │ │ ├── __init__.py │ │ │ ├── test_base_boxes.py │ │ │ ├── test_box_type.py │ │ │ ├── test_horizontal_boxes.py │ │ │ └── utils.py │ │ ├── test_det_data_sample.py │ │ └── test_mask/ │ │ └── test_mask_structures.py │ ├── test_utils/ │ │ ├── test_benchmark.py │ │ ├── test_memory.py │ │ ├── test_replace_cfg_vals.py │ │ └── test_setup_env.py │ └── test_visualization/ │ ├── test_local_visualizer.py │ └── test_palette.py └── tools/ ├── analysis_tools/ │ ├── analyze_logs.py │ ├── analyze_results.py │ ├── benchmark.py │ ├── browse_dataset.py │ ├── coco_error_analysis.py │ ├── coco_occluded_separated_recall.py │ ├── confusion_matrix.py │ ├── eval_metric.py │ ├── get_flops.py │ ├── optimize_anchors.py │ ├── robustness_eval.py │ └── test_robustness.py ├── dataset_converters/ │ ├── cityscapes.py │ ├── images2coco.py │ └── pascal_voc.py ├── deployment/ │ ├── mmdet2torchserve.py │ ├── mmdet_handler.py │ └── test_torchserver.py ├── dist_test.sh ├── dist_train.sh ├── misc/ │ ├── download_dataset.py │ ├── gen_coco_panoptic_test_info.py │ ├── get_crowdhuman_id_hw.py │ ├── get_image_metas.py │ ├── print_config.py │ └── split_coco.py ├── model_converters/ │ ├── detectron2_to_mmdet.py │ ├── detectron2pytorch.py │ ├── publish_model.py │ ├── regnet2mmdet.py │ ├── selfsup2mmdet.py │ ├── upgrade_model_version.py │ └── upgrade_ssd_version.py ├── slurm_test.sh ├── slurm_train.sh ├── test.py └── train.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .circleci/config.yml ================================================ version: 2.1 # this allows you to use CircleCI's dynamic configuration feature setup: true # the path-filtering orb is required to continue a pipeline based on # the path of an updated fileset orbs: path-filtering: circleci/path-filtering@0.1.2 workflows: # the always-run workflow is always triggered, regardless of the pipeline parameters. always-run: jobs: # the path-filtering/filter job determines which pipeline # parameters to update. - path-filtering/filter: name: check-updated-files # 3-column, whitespace-delimited mapping. One mapping per # line: # mapping: | mmdet/.* lint_only false requirements/.* lint_only false tests/.* lint_only false tools/.* lint_only false configs/.* lint_only false .circleci/.* lint_only false base-revision: dev-3.x # this is the path of the configuration we should trigger once # path filtering and pipeline parameter value updates are # complete. In this case, we are using the parent dynamic # configuration itself. config-path: .circleci/test.yml ================================================ FILE: .circleci/docker/Dockerfile ================================================ ARG PYTORCH="1.8.1" ARG CUDA="10.2" ARG CUDNN="7" FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel # To fix GPG key error when running apt-get update RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub RUN apt-get update && apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx ================================================ FILE: .circleci/test.yml ================================================ version: 2.1 # the default pipeline parameters, which will be updated according to # the results of the path-filtering orb parameters: lint_only: type: boolean default: true jobs: lint: docker: - image: cimg/python:3.7.4 steps: - checkout - run: name: Install pre-commit hook command: | pip install pre-commit pre-commit install - run: name: Linting command: pre-commit run --all-files - run: name: Check docstring coverage command: | pip install interrogate interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-magic --ignore-regex "__repr__" --fail-under 85 mmdet build_cpu: parameters: # The python version must match available image tags in # https://circleci.com/developer/images/image/cimg/python python: type: string torch: type: string torchvision: type: string docker: - image: cimg/python:<< parameters.python >> resource_class: large steps: - checkout - run: name: Install Libraries command: | sudo apt-get update sudo apt-get install -y ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 libgl1-mesa-glx libjpeg-dev zlib1g-dev libtinfo-dev libncurses5 - run: name: Configure Python & pip command: | pip install --upgrade pip pip install wheel - run: name: Install PyTorch command: | python -V python -m pip install torch==<< parameters.torch >>+cpu torchvision==<< parameters.torchvision >>+cpu -f https://download.pytorch.org/whl/torch_stable.html - when: condition: equal: ["3.9.0", << parameters.python >>] steps: - run: pip install "protobuf <= 3.20.1" && sudo apt-get update && sudo apt-get -y install libprotobuf-dev protobuf-compiler cmake - run: name: Install mmdet dependencies # numpy may be downgraded after building pycocotools, which causes `ImportError: numpy.core.multiarray failed to import` # force reinstall pycocotools to ensure pycocotools being built under the currenct numpy command: | python -m pip install git+ssh://git@github.com/open-mmlab/mmengine.git@main pip install -U openmim mim install 'mmcv >= 2.0.0rc4' pip install -r requirements/tests.txt -r requirements/optional.txt pip install --force-reinstall pycocotools pip install albumentations>=0.3.2 --no-binary imgaug,albumentations pip install git+https://github.com/cocodataset/panopticapi.git - run: name: Build and install command: | pip install -e . - run: name: Run unittests command: | python -m coverage run --branch --source mmdet -m pytest tests/ python -m coverage xml python -m coverage report -m build_cuda: parameters: torch: type: string cuda: type: enum enum: ["10.1", "10.2", "11.1"] cudnn: type: integer default: 7 machine: image: ubuntu-2004-cuda-11.4:202110-01 # docker_layer_caching: true resource_class: gpu.nvidia.small steps: - checkout - run: # CLoning repos in VM since Docker doesn't have access to the private key name: Clone Repos command: | git clone -b main --depth 1 ssh://git@github.com/open-mmlab/mmengine.git /home/circleci/mmengine - run: name: Build Docker image command: | docker build .circleci/docker -t mmdetection:gpu --build-arg PYTORCH=<< parameters.torch >> --build-arg CUDA=<< parameters.cuda >> --build-arg CUDNN=<< parameters.cudnn >> docker run --gpus all -t -d -v /home/circleci/project:/mmdetection -v /home/circleci/mmengine:/mmengine -w /mmdetection --name mmdetection mmdetection:gpu docker exec mmdetection apt-get install -y git - run: name: Install mmdet dependencies command: | docker exec mmdetection pip install -e /mmengine docker exec mmdetection pip install -U openmim docker exec mmdetection mim install 'mmcv >= 2.0.0rc4' docker exec mmdetection pip install -r requirements/tests.txt -r requirements/optional.txt docker exec mmdetection pip install pycocotools docker exec mmdetection pip install albumentations>=0.3.2 --no-binary imgaug,albumentations docker exec mmdetection pip install git+https://github.com/cocodataset/panopticapi.git docker exec mmdetection python -c 'import mmcv; print(mmcv.__version__)' - run: name: Build and install command: | docker exec mmdetection pip install -e . - run: name: Run unittests command: | docker exec mmdetection python -m pytest tests/ workflows: pr_stage_lint: when: << pipeline.parameters.lint_only >> jobs: - lint: name: lint filters: branches: ignore: - dev-3.x pr_stage_test: when: not: << pipeline.parameters.lint_only >> jobs: - lint: name: lint filters: branches: ignore: - dev-3.x - build_cpu: name: minimum_version_cpu torch: 1.6.0 torchvision: 0.7.0 python: 3.7.4 # The lowest python 3.7.x version available on CircleCI images requires: - lint - build_cpu: name: maximum_version_cpu torch: 1.13.0 torchvision: 0.14.0 python: 3.9.0 requires: - minimum_version_cpu - hold: type: approval requires: - maximum_version_cpu - build_cuda: name: mainstream_version_gpu torch: 1.8.1 # Use double quotation mark to explicitly specify its type # as string instead of number cuda: "10.2" requires: - hold merge_stage_test: when: not: << pipeline.parameters.lint_only >> jobs: - build_cuda: name: minimum_version_gpu torch: 1.6.0 cuda: "10.1" filters: branches: only: - dev-3.x ================================================ FILE: .dev_scripts/batch_test_list.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # missing wider_face/timm_example/strong_baselines/simple_copy_paste/ # selfsup_pretrain/seesaw_loss/pascal_voc/openimages/lvis/ld/lad/cityscapes/deepfashion # yapf: disable atss = dict( config='configs/atss/atss_r50_fpn_1x_coco.py', checkpoint='atss_r50_fpn_1x_coco_20200209-985f7bd0.pth', url='https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r50_fpn_1x_coco/atss_r50_fpn_1x_coco_20200209-985f7bd0.pth', # noqa eval='bbox', metric=dict(bbox_mAP=39.4), ) autoassign = dict( config='configs/autoassign/autoassign_r50-caffe_fpn_1x_coco.py', checkpoint='auto_assign_r50_fpn_1x_coco_20210413_115540-5e17991f.pth', url='https://download.openmmlab.com/mmdetection/v2.0/autoassign/auto_assign_r50_fpn_1x_coco/auto_assign_r50_fpn_1x_coco_20210413_115540-5e17991f.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.4), ) carafe = dict( config='configs/carafe/faster-rcnn_r50_fpn-carafe_1x_coco.py', checkpoint='faster_rcnn_r50_fpn_carafe_1x_coco_bbox_mAP-0.386_20200504_175733-385a75b7.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/carafe/faster_rcnn_r50_fpn_carafe_1x_coco/faster_rcnn_r50_fpn_carafe_1x_coco_bbox_mAP-0.386_20200504_175733-385a75b7.pth', # noqa eval='bbox', metric=dict(bbox_mAP=38.6), ) cascade_rcnn = [ dict( config='configs/cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py', checkpoint='cascade_rcnn_r50_fpn_1x_coco_20200316-3dc56deb.pth', eval='bbox', url='https://download.openmmlab.com/mmdetection/v2.0/cascade_rcnn/cascade_rcnn_r50_fpn_1x_coco/cascade_rcnn_r50_fpn_1x_coco_20200316-3dc56deb.pth', # noqa metric=dict(bbox_mAP=40.3), ), dict( config='configs/cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py', checkpoint='cascade_mask_rcnn_r50_fpn_1x_coco_20200203-9d4dcb24.pth', url='https://download.openmmlab.com/mmdetection/v2.0/cascade_rcnn/cascade_mask_rcnn_r50_fpn_1x_coco/cascade_mask_rcnn_r50_fpn_1x_coco_20200203-9d4dcb24.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=41.2, segm_mAP=35.9), ), ] cascade_rpn = dict( config='configs/cascade_rpn/cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco.py', # noqa checkpoint='crpn_faster_rcnn_r50_caffe_fpn_1x_coco-c8283cca.pth', url='https://download.openmmlab.com/mmdetection/v2.0/cascade_rpn/crpn_faster_rcnn_r50_caffe_fpn_1x_coco/crpn_faster_rcnn_r50_caffe_fpn_1x_coco-c8283cca.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.4), ) centernet = dict( config='configs/centernet/centernet_r18-dcnv2_8xb16-crop512-140e_coco.py', checkpoint='centernet_resnet18_dcnv2_140e_coco_20210702_155131-c8cd631f.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/centernet/centernet_resnet18_dcnv2_140e_coco/centernet_resnet18_dcnv2_140e_coco_20210702_155131-c8cd631f.pth', # noqa eval='bbox', metric=dict(bbox_mAP=29.5), ) centripetalnet = dict( config='configs/centripetalnet/centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco.py', # noqa checkpoint='centripetalnet_hourglass104_mstest_16x6_210e_coco_20200915_204804-3ccc61e5.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/centripetalnet/centripetalnet_hourglass104_mstest_16x6_210e_coco/centripetalnet_hourglass104_mstest_16x6_210e_coco_20200915_204804-3ccc61e5.pth', # noqa eval='bbox', metric=dict(bbox_mAP=44.7), ) convnext = dict( config='configs/convnext/cascade-mask-rcnn_convnext-s-p4-w7_fpn_4conv1fc-giou_amp-ms-crop-3x_coco.py', # noqa checkpoint='cascade_mask_rcnn_convnext-s_p4_w7_fpn_giou_4conv1f_fp16_ms-crop_3x_coco_20220510_201004-3d24f5a4.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/convnext/cascade_mask_rcnn_convnext-s_p4_w7_fpn_giou_4conv1f_fp16_ms-crop_3x_coco/cascade_mask_rcnn_convnext-s_p4_w7_fpn_giou_4conv1f_fp16_ms-crop_3x_coco_20220510_201004-3d24f5a4.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=51.8, segm_mAP=44.8), ) cornernet = dict( config='configs/cornernet/cornernet_hourglass104_8xb6-210e-mstest_coco.py', checkpoint='cornernet_hourglass104_mstest_8x6_210e_coco_20200825_150618-79b44c30.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/cornernet/cornernet_hourglass104_mstest_8x6_210e_coco/cornernet_hourglass104_mstest_8x6_210e_coco_20200825_150618-79b44c30.pth', # noqa eval='bbox', metric=dict(bbox_mAP=41.2), ) dcn = dict( config='configs/dcn/faster-rcnn_r50-dconv-c3-c5_fpn_1x_coco.py', checkpoint='faster_rcnn_r50_fpn_dconv_c3-c5_1x_coco_20200130-d68aed1e.pth', url='https://download.openmmlab.com/mmdetection/v2.0/dcn/faster_rcnn_r50_fpn_dconv_c3-c5_1x_coco/faster_rcnn_r50_fpn_dconv_c3-c5_1x_coco_20200130-d68aed1e.pth', # noqa eval='bbox', metric=dict(bbox_mAP=41.3), ) dcnv2 = dict( config='configs/dcnv2/faster-rcnn_r50_fpn_mdpool_1x_coco.py', checkpoint='faster_rcnn_r50_fpn_mdpool_1x_coco_20200307-c0df27ff.pth', url='https://download.openmmlab.com/mmdetection/v2.0/dcn/faster_rcnn_r50_fpn_mdpool_1x_coco/faster_rcnn_r50_fpn_mdpool_1x_coco_20200307-c0df27ff.pth', # noqa eval='bbox', metric=dict(bbox_mAP=38.7), ) ddod = dict( config='configs/ddod/ddod_r50_fpn_1x_coco.py', checkpoint='ddod_r50_fpn_1x_coco_20220523_223737-29b2fc67.pth', url='https://download.openmmlab.com/mmdetection/v2.0/ddod/ddod_r50_fpn_1x_coco/ddod_r50_fpn_1x_coco_20220523_223737-29b2fc67.pth', # noqa eval='bbox', metric=dict(bbox_mAP=41.7), ) deformable_detr = dict( config='configs/deformable_detr/deformable-detr_r50_16xb2-50e_coco.py', checkpoint='deformable_detr_r50_16x2_50e_coco_20210419_220030-a12b9512.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/deformable_detr/deformable_detr_r50_16x2_50e_coco/deformable_detr_r50_16x2_50e_coco_20210419_220030-a12b9512.pth', # noqa eval='bbox', metric=dict(bbox_mAP=44.5), ) detectors = dict( config='configs/detectors/detectors_htc-r50_1x_coco.py', checkpoint='detectors_htc_r50_1x_coco-329b1453.pth', url='https://download.openmmlab.com/mmdetection/v2.0/detectors/detectors_htc_r50_1x_coco/detectors_htc_r50_1x_coco-329b1453.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=49.1, segm_mAP=42.6), ) detr = dict( config='configs/detr/detr_r50_8xb2-150e_coco.py', checkpoint='detr_r50_8x2_150e_coco_20201130_194835-2c4b8974.pth', url='https://download.openmmlab.com/mmdetection/v2.0/detr/detr_r50_8x2_150e_coco/detr_r50_8x2_150e_coco_20201130_194835-2c4b8974.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.1), ) double_heads = dict( config='configs/double_heads/dh-faster-rcnn_r50_fpn_1x_coco.py', checkpoint='dh_faster_rcnn_r50_fpn_1x_coco_20200130-586b67df.pth', url='https://download.openmmlab.com/mmdetection/v2.0/double_heads/dh_faster_rcnn_r50_fpn_1x_coco/dh_faster_rcnn_r50_fpn_1x_coco_20200130-586b67df.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.0), ) dyhead = dict( config='configs/dyhead/atss_r50_fpn_dyhead_1x_coco.py', checkpoint='atss_r50_fpn_dyhead_4x4_1x_coco_20211219_023314-eaa620c6.pth', url='https://download.openmmlab.com/mmdetection/v2.0/dyhead/atss_r50_fpn_dyhead_4x4_1x_coco/atss_r50_fpn_dyhead_4x4_1x_coco_20211219_023314-eaa620c6.pth', # noqa eval='bbox', metric=dict(bbox_mAP=43.3), ) dynamic_rcnn = dict( config='configs/dynamic_rcnn/dynamic-rcnn_r50_fpn_1x_coco.py', checkpoint='dynamic_rcnn_r50_fpn_1x-62a3f276.pth', url='https://download.openmmlab.com/mmdetection/v2.0/dynamic_rcnn/dynamic_rcnn_r50_fpn_1x/dynamic_rcnn_r50_fpn_1x-62a3f276.pth', # noqa eval='bbox', metric=dict(bbox_mAP=38.9), ) efficientnet = dict( config='configs/efficientnet/retinanet_effb3_fpn_8xb4-crop896-1x_coco.py', checkpoint='retinanet_effb3_fpn_crop896_8x4_1x_coco_20220322_234806-615a0dda.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/efficientnet/retinanet_effb3_fpn_crop896_8x4_1x_coco/retinanet_effb3_fpn_crop896_8x4_1x_coco_20220322_234806-615a0dda.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.5), ) empirical_attention = dict( config='configs/empirical_attention/faster-rcnn_r50-attn1111_fpn_1x_coco.py', # noqa checkpoint='faster_rcnn_r50_fpn_attention_1111_1x_coco_20200130-403cccba.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/empirical_attention/faster_rcnn_r50_fpn_attention_1111_1x_coco/faster_rcnn_r50_fpn_attention_1111_1x_coco_20200130-403cccba.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.0), ) faster_rcnn = dict( config='configs/faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py', checkpoint='faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth', url='https://download.openmmlab.com/mmdetection/v2.0/faster_rcnn/faster_rcnn_r50_fpn_1x_coco/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth', # noqa eval='bbox', metric=dict(bbox_mAP=37.4), ) fcos = dict( config='configs/fcos/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py', # noqa checkpoint='fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_1x_coco-0a0d75a8.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_1x_coco/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_1x_coco-0a0d75a8.pth', # noqa eval='bbox', metric=dict(bbox_mAP=38.7), ) foveabox = dict( config='configs/foveabox/fovea_r50_fpn_gn-head-align_4xb4-2x_coco.py', checkpoint='fovea_align_r50_fpn_gn-head_4x4_2x_coco_20200203-8987880d.pth', url='https://download.openmmlab.com/mmdetection/v2.0/foveabox/fovea_align_r50_fpn_gn-head_4x4_2x_coco/fovea_align_r50_fpn_gn-head_4x4_2x_coco_20200203-8987880d.pth', # noqa eval='bbox', metric=dict(bbox_mAP=37.9), ) fpg = dict( config='configs/fpg/mask-rcnn_r50_fpg_crop640-50e_coco.py', checkpoint='mask_rcnn_r50_fpg_crop640_50e_coco_20220311_011857-233b8334.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/fpg/mask_rcnn_r50_fpg_crop640_50e_coco/mask_rcnn_r50_fpg_crop640_50e_coco_20220311_011857-233b8334.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=43.0, segm_mAP=38.1), ) free_anchor = dict( config='configs/free_anchor/freeanchor_r50_fpn_1x_coco.py', checkpoint='retinanet_free_anchor_r50_fpn_1x_coco_20200130-0f67375f.pth', url='https://download.openmmlab.com/mmdetection/v2.0/free_anchor/retinanet_free_anchor_r50_fpn_1x_coco/retinanet_free_anchor_r50_fpn_1x_coco_20200130-0f67375f.pth', # noqa eval='bbox', metric=dict(bbox_mAP=38.7), ) fsaf = dict( config='configs/fsaf/fsaf_r50_fpn_1x_coco.py', checkpoint='fsaf_r50_fpn_1x_coco-94ccc51f.pth', url='https://download.openmmlab.com/mmdetection/v2.0/fsaf/fsaf_r50_fpn_1x_coco/fsaf_r50_fpn_1x_coco-94ccc51f.pth', # noqa eval='bbox', metric=dict(bbox_mAP=37.4), ) gcnet = dict( config='configs/gcnet/mask-rcnn_r50-syncbn-gcb-r16-c3-c5_fpn_1x_coco.py', # noqa checkpoint='mask_rcnn_r50_fpn_syncbn-backbone_r16_gcb_c3-c5_1x_coco_20200202-587b99aa.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/gcnet/mask_rcnn_r50_fpn_syncbn-backbone_r16_gcb_c3-c5_1x_coco/mask_rcnn_r50_fpn_syncbn-backbone_r16_gcb_c3-c5_1x_coco_20200202-587b99aa.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=40.4, segm_mAP=36.2), ) gfl = dict( config='configs/gfl/gfl_r50_fpn_1x_coco.py', checkpoint='gfl_r50_fpn_1x_coco_20200629_121244-25944287.pth', url='https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r50_fpn_1x_coco/gfl_r50_fpn_1x_coco_20200629_121244-25944287.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.2), ) ghm = dict( config='configs/ghm/retinanet_r50_fpn_ghm-1x_coco.py', checkpoint='retinanet_ghm_r50_fpn_1x_coco_20200130-a437fda3.pth', url='https://download.openmmlab.com/mmdetection/v2.0/ghm/retinanet_ghm_r50_fpn_1x_coco/retinanet_ghm_r50_fpn_1x_coco_20200130-a437fda3.pth', # noqa eval='bbox', metric=dict(bbox_mAP=37.0), ) gn = dict( config='configs/gn/mask-rcnn_r50_fpn_gn-all_2x_coco.py', checkpoint='mask_rcnn_r50_fpn_gn-all_2x_coco_20200206-8eee02a6.pth', url='https://download.openmmlab.com/mmdetection/v2.0/gn/mask_rcnn_r50_fpn_gn-all_2x_coco/mask_rcnn_r50_fpn_gn-all_2x_coco_20200206-8eee02a6.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=40.1, segm_mAP=36.4), ) gn_ws = dict( config='configs/gn+ws/faster-rcnn_r50_fpn_gn-ws-all_1x_coco.py', checkpoint='faster_rcnn_r50_fpn_gn_ws-all_1x_coco_20200130-613d9fe2.pth', url='https://download.openmmlab.com/mmdetection/v2.0/gn%2Bws/faster_rcnn_r50_fpn_gn_ws-all_1x_coco/faster_rcnn_r50_fpn_gn_ws-all_1x_coco_20200130-613d9fe2.pth', # noqa eval='bbox', metric=dict(bbox_mAP=39.7), ) grid_rcnn = dict( config='configs/grid_rcnn/grid-rcnn_r50_fpn_gn-head_2x_coco.py', checkpoint='grid_rcnn_r50_fpn_gn-head_2x_coco_20200130-6cca8223.pth', url='https://download.openmmlab.com/mmdetection/v2.0/grid_rcnn/grid_rcnn_r50_fpn_gn-head_2x_coco/grid_rcnn_r50_fpn_gn-head_2x_coco_20200130-6cca8223.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.4), ) groie = dict( config='configs/groie/faste-rcnn_r50_fpn_groie_1x_coco.py', checkpoint='faster_rcnn_r50_fpn_groie_1x_coco_20200604_211715-66ee9516.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/groie/faster_rcnn_r50_fpn_groie_1x_coco/faster_rcnn_r50_fpn_groie_1x_coco_20200604_211715-66ee9516.pth', # noqa eval='bbox', metric=dict(bbox_mAP=38.3), ) guided_anchoring = dict( config='configs/guided_anchoring/ga-retinanet_r50-caffe_fpn_1x_coco.py', # noqa checkpoint='ga_retinanet_r50_caffe_fpn_1x_coco_20201020-39581c6f.pth', url='https://download.openmmlab.com/mmdetection/v2.0/guided_anchoring/ga_retinanet_r50_caffe_fpn_1x_coco/ga_retinanet_r50_caffe_fpn_1x_coco_20201020-39581c6f.pth', # noqa eval='bbox', metric=dict(bbox_mAP=36.9), ) hrnet = dict( config='configs/hrnet/faster-rcnn_hrnetv2p-w18-1x_coco.py', checkpoint='faster_rcnn_hrnetv2p_w18_1x_coco_20200130-56651a6d.pth', url='https://download.openmmlab.com/mmdetection/v2.0/hrnet/faster_rcnn_hrnetv2p_w18_1x_coco/faster_rcnn_hrnetv2p_w18_1x_coco_20200130-56651a6d.pth', # noqa eval='bbox', metric=dict(bbox_mAP=36.9), ) htc = dict( config='configs/htc/htc_r50_fpn_1x_coco.py', checkpoint='htc_r50_fpn_1x_coco_20200317-7332cf16.pth', url='https://download.openmmlab.com/mmdetection/v2.0/htc/htc_r50_fpn_1x_coco/htc_r50_fpn_1x_coco_20200317-7332cf16.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=42.3, segm_mAP=37.4), ) instaboost = dict( config='configs/instaboost/mask-rcnn_r50_fpn_instaboost-4x_coco.py', checkpoint='mask_rcnn_r50_fpn_instaboost_4x_coco_20200307-d025f83a.pth', url='https://download.openmmlab.com/mmdetection/v2.0/instaboost/mask_rcnn_r50_fpn_instaboost_4x_coco/mask_rcnn_r50_fpn_instaboost_4x_coco_20200307-d025f83a.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=40.6, segm_mAP=36.6), ) libra_rcnn = dict( config='configs/libra_rcnn/libra-faster-rcnn_r50_fpn_1x_coco.py', checkpoint='libra_faster_rcnn_r50_fpn_1x_coco_20200130-3afee3a9.pth', url='https://download.openmmlab.com/mmdetection/v2.0/libra_rcnn/libra_faster_rcnn_r50_fpn_1x_coco/libra_faster_rcnn_r50_fpn_1x_coco_20200130-3afee3a9.pth', # noqa eval='bbox', metric=dict(bbox_mAP=38.3), ) mask2former = dict( config='configs/mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py', checkpoint='mask2former_r50_lsj_8x2_50e_coco-panoptic_20220326_224516-11a44721.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/mask2former/mask2former_r50_lsj_8x2_50e_coco-panoptic/mask2former_r50_lsj_8x2_50e_coco-panoptic_20220326_224516-11a44721.pth', # noqa eval=['bbox', 'segm', 'PQ'], metric=dict(PQ=51.9, bbox_mAP=44.8, segm_mAP=41.9), ) mask_rcnn = dict( config='configs/mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py', checkpoint='mask_rcnn_r50_fpn_1x_coco_20200205-d4b0c5d6.pth', url='https://download.openmmlab.com/mmdetection/v2.0/mask_rcnn/mask_rcnn_r50_fpn_1x_coco/mask_rcnn_r50_fpn_1x_coco_20200205-d4b0c5d6.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=38.2, segm_mAP=34.7), ) maskformer = dict( config='configs/maskformer/maskformer_r50_ms-16xb1-75e_coco.py', checkpoint='maskformer_r50_mstrain_16x1_75e_coco_20220221_141956-bc2699cb.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/maskformer/maskformer_r50_mstrain_16x1_75e_coco/maskformer_r50_mstrain_16x1_75e_coco_20220221_141956-bc2699cb.pth', # noqa eval='PQ', metric=dict(PQ=46.9), ) ms_rcnn = dict( config='configs/ms_rcnn/ms-rcnn_r50-caffe_fpn_1x_coco.py', checkpoint='ms_rcnn_r50_caffe_fpn_1x_coco_20200702_180848-61c9355e.pth', url='https://download.openmmlab.com/mmdetection/v2.0/ms_rcnn/ms_rcnn_r50_caffe_fpn_1x_coco/ms_rcnn_r50_caffe_fpn_1x_coco_20200702_180848-61c9355e.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=38.2, segm_mAP=36.0), ) nas_fcos = dict( config='configs/nas_fcos/nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco.py', # noqa checkpoint='nas_fcos_nashead_r50_caffe_fpn_gn-head_4x4_1x_coco_20200520-1bdba3ce.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/nas_fcos/nas_fcos_nashead_r50_caffe_fpn_gn-head_4x4_1x_coco/nas_fcos_nashead_r50_caffe_fpn_gn-head_4x4_1x_coco_20200520-1bdba3ce.pth', # noqa eval='bbox', metric=dict(bbox_mAP=39.4), ) nas_fpn = dict( config='configs/nas_fpn/retinanet_r50_nasfpn_crop640-50e_coco.py', checkpoint='retinanet_r50_nasfpn_crop640_50e_coco-0ad1f644.pth', url='https://download.openmmlab.com/mmdetection/v2.0/nas_fpn/retinanet_r50_nasfpn_crop640_50e_coco/retinanet_r50_nasfpn_crop640_50e_coco-0ad1f644.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.5), ) paa = dict( config='configs/paa/paa_r50_fpn_1x_coco.py', checkpoint='paa_r50_fpn_1x_coco_20200821-936edec3.pth', url='https://download.openmmlab.com/mmdetection/v2.0/paa/paa_r50_fpn_1x_coco/paa_r50_fpn_1x_coco_20200821-936edec3.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.4), ) pafpn = dict( config='configs/pafpn/faster-rcnn_r50_pafpn_1x_coco.py', checkpoint='faster_rcnn_r50_pafpn_1x_coco_bbox_mAP-0.375_20200503_105836-b7b4b9bd.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/pafpn/faster_rcnn_r50_pafpn_1x_coco/faster_rcnn_r50_pafpn_1x_coco_bbox_mAP-0.375_20200503_105836-b7b4b9bd.pth', # noqa eval='bbox', metric=dict(bbox_mAP=37.5), ) panoptic_fpn = dict( config='configs/panoptic_fpn/panoptic-fpn_r50_fpn_1x_coco.py', checkpoint='panoptic_fpn_r50_fpn_1x_coco_20210821_101153-9668fd13.pth', url='https://download.openmmlab.com/mmdetection/v2.0/panoptic_fpn/panoptic_fpn_r50_fpn_1x_coco/panoptic_fpn_r50_fpn_1x_coco_20210821_101153-9668fd13.pth', # noqa eval='PQ', metric=dict(PQ=40.2), ) pisa = dict( config='configs/pisa/faster-rcnn_r50_fpn_pisa_1x_coco.py', checkpoint='pisa_faster_rcnn_r50_fpn_1x_coco-dea93523.pth', url='https://download.openmmlab.com/mmdetection/v2.0/pisa/pisa_faster_rcnn_r50_fpn_1x_coco/pisa_faster_rcnn_r50_fpn_1x_coco-dea93523.pth', # noqa eval='bbox', metric=dict(bbox_mAP=38.4), ) point_rend = dict( config='configs/point_rend/point-rend_r50-caffe_fpn_ms-1x_coco.py', checkpoint='point_rend_r50_caffe_fpn_mstrain_1x_coco-1bcb5fb4.pth', url='https://download.openmmlab.com/mmdetection/v2.0/point_rend/point_rend_r50_caffe_fpn_mstrain_1x_coco/point_rend_r50_caffe_fpn_mstrain_1x_coco-1bcb5fb4.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=38.4, segm_mAP=36.3), ) pvt = dict( config='configs/pvt/retinanet_pvt-s_fpn_1x_coco.py', checkpoint='retinanet_pvt-s_fpn_1x_coco_20210906_142921-b6c94a5b.pth', url='https://download.openmmlab.com/mmdetection/v2.0/pvt/retinanet_pvt-s_fpn_1x_coco/retinanet_pvt-s_fpn_1x_coco_20210906_142921-b6c94a5b.pth', # noqa eval='bbox', metric=dict(bbox_mAP=40.4), ) queryinst = dict( config='configs/queryinst/queryinst_r50_fpn_1x_coco.py', checkpoint='queryinst_r50_fpn_1x_coco_20210907_084916-5a8f1998.pth', url='https://download.openmmlab.com/mmdetection/v2.0/queryinst/queryinst_r50_fpn_1x_coco/queryinst_r50_fpn_1x_coco_20210907_084916-5a8f1998.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=42.0, segm_mAP=37.5), ) regnet = dict( config='configs/regnet/mask-rcnn_regnetx-3.2GF_fpn_1x_coco.py', checkpoint='mask_rcnn_regnetx-3.2GF_fpn_1x_coco_20200520_163141-2a9d1814.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/regnet/mask_rcnn_regnetx-3.2GF_fpn_1x_coco/mask_rcnn_regnetx-3.2GF_fpn_1x_coco_20200520_163141-2a9d1814.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=40.4, segm_mAP=36.7), ) reppoints = dict( config='configs/reppoints/reppoints-moment_r50_fpn_1x_coco.py', checkpoint='reppoints_moment_r50_fpn_1x_coco_20200330-b73db8d1.pth', url='https://download.openmmlab.com/mmdetection/v2.0/reppoints/reppoints_moment_r50_fpn_1x_coco/reppoints_moment_r50_fpn_1x_coco_20200330-b73db8d1.pth', # noqa eval='bbox', metric=dict(bbox_mAP=37.0), ) res2net = dict( config='configs/res2net/faster-rcnn_res2net-101_fpn_2x_coco.py', checkpoint='faster_rcnn_r2_101_fpn_2x_coco-175f1da6.pth', url='https://download.openmmlab.com/mmdetection/v2.0/res2net/faster_rcnn_r2_101_fpn_2x_coco/faster_rcnn_r2_101_fpn_2x_coco-175f1da6.pth', # noqa eval='bbox', metric=dict(bbox_mAP=43.0), ) resnest = dict( config='configs/resnest/faster-rcnn_s50_fpn_syncbn-backbone+head_ms-range-1x_coco.py', # noqa checkpoint='faster_rcnn_s50_fpn_syncbn-backbone+head_mstrain-range_1x_coco_20200926_125502-20289c16.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/resnest/faster_rcnn_s50_fpn_syncbn-backbone%2Bhead_mstrain-range_1x_coco/faster_rcnn_s50_fpn_syncbn-backbone%2Bhead_mstrain-range_1x_coco_20200926_125502-20289c16.pth', # noqa eval='bbox', metric=dict(bbox_mAP=42.0), ) resnet_strikes_back = dict( config='configs/resnet_strikes_back/mask-rcnn_r50-rsb-pre_fpn_1x_coco.py', # noqa checkpoint='mask_rcnn_r50_fpn_rsb-pretrain_1x_coco_20220113_174054-06ce8ba0.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/resnet_strikes_back/mask_rcnn_r50_fpn_rsb-pretrain_1x_coco/mask_rcnn_r50_fpn_rsb-pretrain_1x_coco_20220113_174054-06ce8ba0.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=41.2, segm_mAP=38.2), ) retinanet = dict( config='configs/retinanet/retinanet_r50_fpn_1x_coco.py', checkpoint='retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth', url='https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_1x_coco/retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth', # noqa eval='bbox', metric=dict(bbox_mAP=36.5), ) rpn = dict( config='configs/rpn/rpn_r50_fpn_1x_coco.py', checkpoint='rpn_r50_fpn_1x_coco_20200218-5525fa2e.pth', url='https://download.openmmlab.com/mmdetection/v2.0/rpn/rpn_r50_fpn_1x_coco/rpn_r50_fpn_1x_coco_20200218-5525fa2e.pth', # noqa eval='proposal_fast', metric=dict(AR_1000=58.2), ) sabl = [ dict( config='configs/sabl/sabl-retinanet_r50_fpn_1x_coco.py', checkpoint='sabl_retinanet_r50_fpn_1x_coco-6c54fd4f.pth', url='https://download.openmmlab.com/mmdetection/v2.0/sabl/sabl_retinanet_r50_fpn_1x_coco/sabl_retinanet_r50_fpn_1x_coco-6c54fd4f.pth', # noqa eval='bbox', metric=dict(bbox_mAP=37.7), ), dict( config='configs/sabl/sabl-faster-rcnn_r50_fpn_1x_coco.py', checkpoint='sabl_faster_rcnn_r50_fpn_1x_coco-e867595b.pth', url='https://download.openmmlab.com/mmdetection/v2.0/sabl/sabl_faster_rcnn_r50_fpn_1x_coco/sabl_faster_rcnn_r50_fpn_1x_coco-e867595b.pth', # noqa eval='bbox', metric=dict(bbox_mAP=39.9), ), ] scnet = dict( config='configs/scnet/scnet_r50_fpn_1x_coco.py', checkpoint='scnet_r50_fpn_1x_coco-c3f09857.pth', url='https://download.openmmlab.com/mmdetection/v2.0/scnet/scnet_r50_fpn_1x_coco/scnet_r50_fpn_1x_coco-c3f09857.pth', # noqa eval='bbox', metric=dict(bbox_mAP=43.5), ) scratch = dict( config='configs/scratch/mask-rcnn_r50-scratch_fpn_gn-all_6x_coco.py', checkpoint='scratch_mask_rcnn_r50_fpn_gn_6x_bbox_mAP-0.412__segm_mAP-0.374_20200201_193051-1e190a40.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/scratch/mask_rcnn_r50_fpn_gn-all_scratch_6x_coco/scratch_mask_rcnn_r50_fpn_gn_6x_bbox_mAP-0.412__segm_mAP-0.374_20200201_193051-1e190a40.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=41.2, segm_mAP=37.4), ) solo = dict( config='configs/solo/decoupled-solo_r50_fpn_1x_coco.py', checkpoint='decoupled_solo_r50_fpn_1x_coco_20210820_233348-6337c589.pth', url='https://download.openmmlab.com/mmdetection/v2.0/solo/decoupled_solo_r50_fpn_1x_coco/decoupled_solo_r50_fpn_1x_coco_20210820_233348-6337c589.pth', # noqa eval='segm', metric=dict(segm_mAP=33.9), ) solov2 = dict( config='configs/solov2/solov2_r50_fpn_1x_coco.py', checkpoint='solov2_r50_fpn_1x_coco_20220512_125858-a357fa23.pth', url='https://download.openmmlab.com/mmdetection/v2.0/solov2/solov2_r50_fpn_1x_coco/solov2_r50_fpn_1x_coco_20220512_125858-a357fa23.pth', # noqa eval='segm', metric=dict(segm_mAP=34.8), ) sparse_rcnn = dict( config='configs/sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py', checkpoint='sparse_rcnn_r50_fpn_1x_coco_20201222_214453-dc79b137.pth', url='https://download.openmmlab.com/mmdetection/v2.0/sparse_rcnn/sparse_rcnn_r50_fpn_1x_coco/sparse_rcnn_r50_fpn_1x_coco_20201222_214453-dc79b137.pth', # noqa eval='bbox', metric=dict(bbox_mAP=37.9), ) ssd = [ dict( config='configs/ssd/ssd300_coco.py', checkpoint='ssd300_coco_20210803_015428-d231a06e.pth', url='https://download.openmmlab.com/mmdetection/v2.0/ssd/ssd300_coco/ssd300_coco_20210803_015428-d231a06e.pth', # noqa eval='bbox', metric=dict(bbox_mAP=25.5), ), dict( config='configs/ssd/ssdlite_mobilenetv2-scratch_8xb24-600e_coco.py', checkpoint='ssdlite_mobilenetv2_scratch_600e_coco_20210629_110627-974d9307.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/ssd/ssdlite_mobilenetv2_scratch_600e_coco/ssdlite_mobilenetv2_scratch_600e_coco_20210629_110627-974d9307.pth', # noqa eval='bbox', metric=dict(bbox_mAP=21.3), ), ] swin = dict( config='configs/swin/mask-rcnn_swin-t-p4-w7_fpn_1x_coco.py', checkpoint='mask_rcnn_swin-t-p4-w7_fpn_1x_coco_20210902_120937-9d6b7cfa.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/swin/mask_rcnn_swin-t-p4-w7_fpn_1x_coco/mask_rcnn_swin-t-p4-w7_fpn_1x_coco_20210902_120937-9d6b7cfa.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=42.7, segm_mAP=39.3), ) tood = dict( config='configs/tood/tood_r50_fpn_1x_coco.py', checkpoint='tood_r50_fpn_1x_coco_20211210_103425-20e20746.pth', url='https://download.openmmlab.com/mmdetection/v2.0/tood/tood_r50_fpn_1x_coco/tood_r50_fpn_1x_coco_20211210_103425-20e20746.pth', # noqa eval='bbox', metric=dict(bbox_mAP=42.4), ) tridentnet = dict( config='configs/tridentnet/tridentnet_r50-caffe_1x_coco.py', checkpoint='tridentnet_r50_caffe_1x_coco_20201230_141838-2ec0b530.pth', url='https://download.openmmlab.com/mmdetection/v2.0/tridentnet/tridentnet_r50_caffe_1x_coco/tridentnet_r50_caffe_1x_coco_20201230_141838-2ec0b530.pth', # noqa eval='bbox', metric=dict(bbox_mAP=37.6), ) vfnet = dict( config='configs/vfnet/vfnet_r50_fpn_1x_coco.py', checkpoint='vfnet_r50_fpn_1x_coco_20201027-38db6f58.pth', url='https://download.openmmlab.com/mmdetection/v2.0/vfnet/vfnet_r50_fpn_1x_coco/vfnet_r50_fpn_1x_coco_20201027-38db6f58.pth', # noqa eval='bbox', metric=dict(bbox_mAP=41.6), ) yolact = dict( config='configs/yolact/yolact_r50_1xb8-55e_coco.py', checkpoint='yolact_r50_1x8_coco_20200908-f38d58df.pth', url='https://download.openmmlab.com/mmdetection/v2.0/yolact/yolact_r50_1x8_coco/yolact_r50_1x8_coco_20200908-f38d58df.pth', # noqa eval=['bbox', 'segm'], metric=dict(bbox_mAP=31.2, segm_mAP=29.0), ) yolo = dict( config='configs/yolo/yolov3_d53_8xb8-320-273e_coco.py', checkpoint='yolov3_d53_320_273e_coco-421362b6.pth', url='https://download.openmmlab.com/mmdetection/v2.0/yolo/yolov3_d53_320_273e_coco/yolov3_d53_320_273e_coco-421362b6.pth', # noqa eval='bbox', metric=dict(bbox_mAP=27.9), ) yolof = dict( config='configs/yolof/yolof_r50-c5_8xb8-1x_coco.py', checkpoint='yolof_r50_c5_8x8_1x_coco_20210425_024427-8e864411.pth', url='https://download.openmmlab.com/mmdetection/v2.0/yolof/yolof_r50_c5_8x8_1x_coco/yolof_r50_c5_8x8_1x_coco_20210425_024427-8e864411.pth', # noqa eval='bbox', metric=dict(bbox_mAP=37.5), ) yolox = dict( config='configs/yolox/yolox_tiny_8xb8-300e_coco.py', checkpoint='yolox_tiny_8x8_300e_coco_20211124_171234-b4047906.pth', # noqa url='https://download.openmmlab.com/mmdetection/v2.0/yolox/yolox_tiny_8x8_300e_coco/yolox_tiny_8x8_300e_coco_20211124_171234-b4047906.pth', # noqa eval='bbox', metric=dict(bbox_mAP=31.8), ) # yapf: enable ================================================ FILE: .dev_scripts/batch_train_list.txt ================================================ configs/albu_example/mask-rcnn_r50_fpn_albu_1x_coco.py configs/atss/atss_r50_fpn_1x_coco.py configs/autoassign/autoassign_r50-caffe_fpn_1x_coco.py configs/carafe/faster-rcnn_r50_fpn-carafe_1x_coco.py configs/cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py configs/cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py configs/cascade_rpn/cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco.py configs/centernet/centernet_r18-dcnv2_8xb16-crop512-140e_coco.py configs/centernet/centernet-update_r50-caffe_fpn_ms-1x_coco.py configs/centripetalnet/centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco.py configs/cornernet/cornernet_hourglass104_8xb6-210e-mstest_coco.py configs/convnext/mask-rcnn_convnext-t-p4-w7_fpn_amp-ms-crop-3x_coco.py configs/dcn/faster-rcnn_r50-dconv-c3-c5_fpn_1x_coco.py configs/dcnv2/faster-rcnn_r50_fpn_mdpool_1x_coco.py configs/ddod/ddod_r50_fpn_1x_coco.py configs/detectors/detectors_htc-r50_1x_coco.py configs/deformable_detr/deformable-detr_r50_16xb2-50e_coco.py configs/detr/detr_r50_8xb2-150e_coco.py configs/double_heads/dh-faster-rcnn_r50_fpn_1x_coco.py configs/dynamic_rcnn/dynamic-rcnn_r50_fpn_1x_coco.py configs/dyhead/atss_r50_fpn_dyhead_1x_coco.py configs/efficientnet/retinanet_effb3_fpn_8xb4-crop896-1x_coco.py configs/empirical_attention/faster-rcnn_r50-attn1111_fpn_1x_coco.py configs/faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py configs/faster_rcnn/faster-rcnn_r50-caffe-dc5_ms-1x_coco.py configs/fcos/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py configs/foveabox/fovea_r50_fpn_gn-head-align_4xb4-2x_coco.py configs/fpg/mask-rcnn_r50_fpg_crop640-50e_coco.py configs/free_anchor/freeanchor_r50_fpn_1x_coco.py configs/fsaf/fsaf_r50_fpn_1x_coco.py configs/gcnet/mask-rcnn_r50-syncbn-gcb-r16-c3-c5_fpn_1x_coco.py configs/gfl/gfl_r50_fpn_1x_coco.py configs/ghm/retinanet_r50_fpn_ghm-1x_coco.py configs/gn/mask-rcnn_r50_fpn_gn-all_2x_coco.py configs/gn+ws/faster-rcnn_r50_fpn_gn-ws-all_1x_coco.py configs/grid_rcnn/grid-rcnn_r50_fpn_gn-head_2x_coco.py configs/groie/faste-rcnn_r50_fpn_groie_1x_coco.py configs/guided_anchoring/ga-retinanet_r50-caffe_fpn_1x_coco.py configs/hrnet/faster-rcnn_hrnetv2p-w18-1x_coco.py configs/htc/htc_r50_fpn_1x_coco.py configs/instaboost/mask-rcnn_r50_fpn_instaboost-4x_coco.py configs/lad/lad_r50-paa-r101_fpn_2xb8_coco_1x.py configs/ld/ld_r18-gflv1-r101_fpn_1x_coco.py configs/libra_rcnn/libra-faster-rcnn_r50_fpn_1x_coco.py configs/mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py configs/mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py configs/maskformer/maskformer_r50_ms-16xb1-75e_coco.py configs/ms_rcnn/ms-rcnn_r50-caffe_fpn_1x_coco.py configs/nas_fcos/nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco.py configs/nas_fpn/retinanet_r50_nasfpn_crop640-50e_coco.py configs/paa/paa_r50_fpn_1x_coco.py configs/pafpn/faster-rcnn_r50_pafpn_1x_coco.py configs/panoptic_fpn/panoptic-fpn_r50_fpn_1x_coco.py configs/pisa/mask-rcnn_r50_fpn_pisa_1x_coco.py configs/point_rend/point-rend_r50-caffe_fpn_ms-1x_coco.py configs/pvt/retinanet_pvt-t_fpn_1x_coco.py configs/queryinst/queryinst_r50_fpn_1x_coco.py configs/regnet/retinanet_regnetx-800MF_fpn_1x_coco.py configs/reppoints/reppoints-moment_r50_fpn_1x_coco.py configs/res2net/faster-rcnn_res2net-101_fpn_2x_coco.py configs/resnest/faster-rcnn_s50_fpn_syncbn-backbone+head_ms-range-1x_coco.py configs/resnet_strikes_back/retinanet_r50-rsb-pre_fpn_1x_coco.py configs/retinanet/retinanet_r50-caffe_fpn_1x_coco.py configs/rpn/rpn_r50_fpn_1x_coco.py configs/sabl/sabl-retinanet_r50_fpn_1x_coco.py configs/scnet/scnet_r50_fpn_1x_coco.py configs/scratch/faster-rcnn_r50-scratch_fpn_gn-all_6x_coco.py configs/solo/solo_r50_fpn_1x_coco.py configs/solov2/solov2_r50_fpn_1x_coco.py configs/sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py configs/ssd/ssd300_coco.py configs/ssd/ssdlite_mobilenetv2-scratch_8xb24-600e_coco.py configs/swin/mask-rcnn_swin-t-p4-w7_fpn_1x_coco.py configs/tood/tood_r50_fpn_1x_coco.py 'configs/tridentnet/tridentnet_r50-caffe_1x_coco.py configs/vfnet/vfnet_r50_fpn_1x_coco.py configs/yolact/yolact_r50_8xb8-55e_coco.py configs/yolo/yolov3_d53_8xb8-320-273e_coco.py configs/yolof/yolof_r50-c5_8xb8-1x_coco.py configs/yolox/yolox_tiny_8xb8-300e_coco.py ================================================ FILE: .dev_scripts/benchmark_filter.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os import os.path as osp def parse_args(): parser = argparse.ArgumentParser(description='Filter configs to train') parser.add_argument( '--basic-arch', action='store_true', help='to train models in basic arch') parser.add_argument( '--datasets', action='store_true', help='to train models in dataset') parser.add_argument( '--data-pipeline', action='store_true', help='to train models related to data pipeline, e.g. augmentations') parser.add_argument( '--nn-module', action='store_true', help='to train models related to neural network modules') parser.add_argument( '--model-options', nargs='+', help='custom options to special model benchmark') parser.add_argument( '--out', type=str, default='batch_train_list.txt', help='output path of gathered metrics to be stored') args = parser.parse_args() return args basic_arch_root = [ 'atss', 'autoassign', 'cascade_rcnn', 'cascade_rpn', 'centripetalnet', 'cornernet', 'detectors', 'deformable_detr', 'detr', 'double_heads', 'dynamic_rcnn', 'faster_rcnn', 'fcos', 'foveabox', 'fp16', 'free_anchor', 'fsaf', 'gfl', 'ghm', 'grid_rcnn', 'guided_anchoring', 'htc', 'ld', 'libra_rcnn', 'mask_rcnn', 'ms_rcnn', 'nas_fcos', 'paa', 'pisa', 'point_rend', 'reppoints', 'retinanet', 'rpn', 'sabl', 'ssd', 'tridentnet', 'vfnet', 'yolact', 'yolo', 'sparse_rcnn', 'scnet', 'yolof', 'centernet' ] datasets_root = [ 'wider_face', 'pascal_voc', 'cityscapes', 'lvis', 'deepfashion' ] data_pipeline_root = ['albu_example', 'instaboost'] nn_module_root = [ 'carafe', 'dcn', 'empirical_attention', 'gcnet', 'gn', 'gn+ws', 'hrnet', 'pafpn', 'nas_fpn', 'regnet', 'resnest', 'res2net', 'groie' ] benchmark_pool = [ 'configs/albu_example/mask_rcnn_r50_fpn_albu_1x_coco.py', 'configs/atss/atss_r50_fpn_1x_coco.py', 'configs/autoassign/autoassign_r50_fpn_8x2_1x_coco.py', 'configs/carafe/mask_rcnn_r50_fpn_carafe_1x_coco.py', 'configs/cascade_rcnn/cascade_mask_rcnn_r50_fpn_1x_coco.py', 'configs/cascade_rpn/crpn_faster_rcnn_r50_caffe_fpn_1x_coco.py', 'configs/centernet/centernet_resnet18_dcnv2_140e_coco.py', 'configs/centripetalnet/' 'centripetalnet_hourglass104_mstest_16x6_210e_coco.py', 'configs/cityscapes/mask_rcnn_r50_fpn_1x_cityscapes.py', 'configs/cornernet/' 'cornernet_hourglass104_mstest_8x6_210e_coco.py', 'configs/dcn/mask_rcnn_r50_fpn_mdconv_c3-c5_1x_coco.py', 'configs/dcn/faster_rcnn_r50_fpn_dpool_1x_coco.py', 'configs/dcn/faster_rcnn_r50_fpn_mdpool_1x_coco.py', 'configs/dcn/mask_rcnn_r50_fpn_dconv_c3-c5_1x_coco.py', 'configs/deformable_detr/deformable_detr_r50_16x2_50e_coco.py', 'configs/detectors/detectors_htc_r50_1x_coco.py', 'configs/detr/detr_r50_8x2_150e_coco.py', 'configs/double_heads/dh_faster_rcnn_r50_fpn_1x_coco.py', 'configs/dynamic_rcnn/dynamic_rcnn_r50_fpn_1x_coco.py', 'configs/empirical_attention/faster_rcnn_r50_fpn_attention_1111_dcn_1x_coco.py', # noqa 'configs/faster_rcnn/faster_rcnn_r50_fpn_1x_coco.py', 'configs/faster_rcnn/faster_rcnn_r50_fpn_ohem_1x_coco.py', 'configs/faster_rcnn/faster_rcnn_r50_caffe_fpn_1x_coco.py', 'configs/faster_rcnn/faster_rcnn_r50_caffe_fpn_mstrain_1x_coco.py', 'configs/faster_rcnn/faster_rcnn_r50_caffe_dc5_mstrain_1x_coco.py', 'configs/fcos/fcos_center_r50_caffe_fpn_gn-head_4x4_1x_coco.py', 'configs/foveabox/fovea_align_r50_fpn_gn-head_4x4_2x_coco.py', 'configs/retinanet/retinanet_r50_fpn_fp16_1x_coco.py', 'configs/mask_rcnn/mask_rcnn_r50_fpn_fp16_1x_coco.py', 'configs/free_anchor/retinanet_free_anchor_r50_fpn_1x_coco.py', 'configs/fsaf/fsaf_r50_fpn_1x_coco.py', 'configs/gcnet/mask_rcnn_r50_fpn_r4_gcb_c3-c5_1x_coco.py', 'configs/gfl/gfl_r50_fpn_1x_coco.py', 'configs/ghm/retinanet_ghm_r50_fpn_1x_coco.py', 'configs/gn/mask_rcnn_r50_fpn_gn-all_2x_coco.py', 'configs/gn+ws/mask_rcnn_r50_fpn_gn_ws-all_2x_coco.py', 'configs/grid_rcnn/grid_rcnn_r50_fpn_gn-head_2x_coco.py', 'configs/groie/faster_rcnn_r50_fpn_groie_1x_coco.py', 'configs/guided_anchoring/ga_faster_r50_caffe_fpn_1x_coco.py', 'configs/hrnet/mask_rcnn_hrnetv2p_w18_1x_coco.py', 'configs/htc/htc_r50_fpn_1x_coco.py', 'configs/instaboost/mask_rcnn_r50_fpn_instaboost_4x_coco.py', 'configs/ld/ld_r18_gflv1_r101_fpn_coco_1x.py', 'configs/libra_rcnn/libra_faster_rcnn_r50_fpn_1x_coco.py', 'configs/lvis/mask_rcnn_r50_fpn_sample1e-3_mstrain_1x_lvis_v1.py', 'configs/mask_rcnn/mask_rcnn_r50_caffe_fpn_mstrain-poly_1x_coco.py', 'configs/ms_rcnn/ms_rcnn_r50_caffe_fpn_1x_coco.py', 'configs/nas_fcos/nas_fcos_nashead_r50_caffe_fpn_gn-head_4x4_1x_coco.py', 'configs/nas_fpn/retinanet_r50_nasfpn_crop640_50e_coco.py', 'configs/paa/paa_r50_fpn_1x_coco.py', 'configs/pafpn/faster_rcnn_r50_pafpn_1x_coco.py', 'configs/pisa/pisa_mask_rcnn_r50_fpn_1x_coco.py', 'configs/point_rend/point_rend_r50_caffe_fpn_mstrain_1x_coco.py', 'configs/regnet/mask_rcnn_regnetx-3.2GF_fpn_1x_coco.py', 'configs/reppoints/reppoints_moment_r50_fpn_gn-neck+head_1x_coco.py', 'configs/res2net/faster_rcnn_r2_101_fpn_2x_coco.py', 'configs/resnest/' 'mask_rcnn_s50_fpn_syncbn-backbone+head_mstrain_1x_coco.py', 'configs/retinanet/retinanet_r50_caffe_fpn_1x_coco.py', 'configs/rpn/rpn_r50_fpn_1x_coco.py', 'configs/sabl/sabl_retinanet_r50_fpn_1x_coco.py', 'configs/ssd/ssd300_coco.py', 'configs/tridentnet/tridentnet_r50_caffe_1x_coco.py', 'configs/vfnet/vfnet_r50_fpn_1x_coco.py', 'configs/yolact/yolact_r50_1x8_coco.py', 'configs/yolo/yolov3_d53_320_273e_coco.py', 'configs/sparse_rcnn/sparse_rcnn_r50_fpn_1x_coco.py', 'configs/scnet/scnet_r50_fpn_1x_coco.py', 'configs/yolof/yolof_r50_c5_8x8_1x_coco.py', ] def main(): args = parse_args() benchmark_type = [] if args.basic_arch: benchmark_type += basic_arch_root if args.datasets: benchmark_type += datasets_root if args.data_pipeline: benchmark_type += data_pipeline_root if args.nn_module: benchmark_type += nn_module_root special_model = args.model_options if special_model is not None: benchmark_type += special_model config_dpath = 'configs/' benchmark_configs = [] for cfg_root in benchmark_type: cfg_dir = osp.join(config_dpath, cfg_root) configs = os.scandir(cfg_dir) for cfg in configs: config_path = osp.join(cfg_dir, cfg.name) if (config_path in benchmark_pool and config_path not in benchmark_configs): benchmark_configs.append(config_path) print(f'Totally found {len(benchmark_configs)} configs to benchmark') with open(args.out, 'w') as f: for config in benchmark_configs: f.write(config + '\n') if __name__ == '__main__': main() ================================================ FILE: .dev_scripts/benchmark_full_models.txt ================================================ albu_example/mask-rcnn_r50_fpn_albu_1x_coco.py atss/atss_r50_fpn_1x_coco.py autoassign/autoassign_r50-caffe_fpn_1x_coco.py boxinst/boxinst_r50_fpn_ms-90k_coco.py carafe/faster-rcnn_r50_fpn-carafe_1x_coco.py cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py cascade_rpn/cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco.py centernet/centernet-update_r50-caffe_fpn_ms-1x_coco.py centripetalnet/centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco.py condinst/condinst_r50_fpn_ms-poly-90k_coco_instance.py conditional_detr/conditional-detr_r50_8xb2-50e_coco.py convnext/mask-rcnn_convnext-t-p4-w7_fpn_amp-ms-crop-3x_coco.py cornernet/cornernet_hourglass104_8xb6-210e-mstest_coco.py dab_detr/dab-detr_r50_8xb2-50e_coco.py dcn/mask-rcnn_r50-dconv-c3-c5_fpn_1x_coco.py dcnv2/faster-rcnn_r50_fpn_mdpool_1x_coco.py ddod/ddod_r50_fpn_1x_coco.py deformable_detr/deformable-detr_r50_16xb2-50e_coco.py detectors/detectors_htc-r50_1x_coco.py detr/detr_r50_8xb2-150e_coco.py dino/dino-4scale_r50_8xb2-12e_coco.py double_heads/dh-faster-rcnn_r50_fpn_1x_coco.py dyhead/atss_r50_fpn_dyhead_1x_coco.py dynamic_rcnn/dynamic-rcnn_r50_fpn_1x_coco.py efficientnet/retinanet_effb3_fpn_8xb4-crop896-1x_coco.py empirical_attention/faster-rcnn_r50-attn0010-dcn_fpn_1x_coco.py faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py fcos/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py foveabox/fovea_r50_fpn_gn-head-align_4xb4-2x_coco.py fpg/retinanet_r50_fpg_crop640_50e_coco.py free_anchor/freeanchor_r50_fpn_1x_coco.py fsaf/fsaf_r50_fpn_1x_coco.py gcnet/mask-rcnn_r50-gcb-r4-c3-c5_fpn_1x_coco.py gfl/gfl_r50_fpn_1x_coco.py ghm/retinanet_r50_fpn_ghm-1x_coco.py gn/mask-rcnn_r50_fpn_gn-all_2x_coco.py gn+ws/faster-rcnn_r50_fpn_gn-ws-all_1x_coco.py grid_rcnn/grid-rcnn_r50_fpn_gn-head_1x_coco.py groie/faste-rcnn_r50_fpn_groie_1x_coco.py guided_anchoring/ga-faster-rcnn_r50_fpn_1x_coco.py hrnet/htc_hrnetv2p-w18_20e_coco.py htc/htc_r50_fpn_1x_coco.py instaboost/mask-rcnn_r50_fpn_instaboost-4x_coco.py lad/lad_r50-paa-r101_fpn_2xb8_coco_1x.py ld/ld_r18-gflv1-r101_fpn_1x_coco.py libra_rcnn/libra-faster-rcnn_r50_fpn_1x_coco.py lvis/mask-rcnn_r50_fpn_sample1e-3_ms-1x_lvis-v1.py mask2former/mask2former_r50_8xb2-lsj-50e_coco.py mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py maskformer/maskformer_r50_ms-16xb1-75e_coco.py ms_rcnn/ms-rcnn_r50_fpn_1x_coco.py nas_fcos/nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco.py nas_fpn/retinanet_r50_nasfpn_crop640-50e_coco.py paa/paa_r50_fpn_1x_coco.py pafpn/faster-rcnn_r50_pafpn_1x_coco.py panoptic_fpn/panoptic-fpn_r50_fpn_1x_coco.py pisa/faster-rcnn_r50_fpn_pisa_1x_coco.py point_rend/point-rend_r50-caffe_fpn_ms-1x_coco.py pvt/retinanet_pvtv2-b0_fpn_1x_coco.py queryinst/queryinst_r50_fpn_1x_coco.py regnet/mask-rcnn_regnetx-3.2GF_fpn_1x_coco.py reppoints/reppoints-moment_r50_fpn-gn_head-gn_1x_coco.py res2net/faster-rcnn_res2net-101_fpn_2x_coco.py resnest/mask-rcnn_s50_fpn_syncbn-backbone+head_ms-1x_coco.py resnet_strikes_back/faster-rcnn_r50-rsb-pre_fpn_1x_coco.py retinanet/retinanet_r50_fpn_1x_coco.py rpn/rpn_r50_fpn_1x_coco.py rtmdet/rtmdet_s_8xb32-300e_coco.py rtmdet/rtmdet-ins_s_8xb32-300e_coco.py sabl/sabl-retinanet_r50_fpn_1x_coco.py scnet/scnet_r50_fpn_1x_coco.py scratch/faster-rcnn_r50-scratch_fpn_gn-all_6x_coco.py seesaw_loss/mask-rcnn_r50_fpn_seesaw-loss_random-ms-2x_lvis-v1.py simple_copy_paste/mask-rcnn_r50_fpn_rpn-2conv_4conv1fc_syncbn-all_32xb2-ssj-scp-90k_coco.py soft_teacher/soft-teacher_faster-rcnn_r50-caffe_fpn_180k_semi-0.1-coco.py solo/solo_r50_fpn_1x_coco.py solov2/solov2_r50_fpn_1x_coco.py sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py ssd/ssd300_coco.py strong_baselines/mask-rcnn_r50-caffe_fpn_rpn-2conv_4conv1fc_syncbn-all_amp-lsj-100e_coco.py swin/mask-rcnn_swin-t-p4-w7_fpn_1x_coco.py timm_example/retinanet_timm-tv-resnet50_fpn_1x_coco.py tood/tood_r50_fpn_1x_coco.py tridentnet/tridentnet_r50-caffe_1x_coco.py vfnet/vfnet_r50_fpn_1x_coco.py yolact/yolact_r50_8xb8-55e_coco.py yolo/yolov3_d53_8xb8-320-273e_coco.py yolof/yolof_r50-c5_8xb8-1x_coco.py yolox/yolox_s_8xb8-300e_coco.py ================================================ FILE: .dev_scripts/benchmark_inference_fps.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os import os.path as osp from mmengine.config import Config, DictAction from mmengine.dist import init_dist from mmengine.fileio import dump from mmengine.utils import mkdir_or_exist from terminaltables import GithubFlavoredMarkdownTable from tools.analysis_tools.benchmark import repeat_measure_inference_speed def parse_args(): parser = argparse.ArgumentParser( description='MMDet benchmark a model of FPS') parser.add_argument('config', help='test config file path') parser.add_argument('checkpoint_root', help='Checkpoint file root path') parser.add_argument( '--round-num', type=int, default=1, help='round a number to a given precision in decimal digits') parser.add_argument( '--repeat-num', type=int, default=1, help='number of repeat times of measurement for averaging the results') parser.add_argument( '--out', type=str, help='output path of gathered fps to be stored') parser.add_argument( '--max-iter', type=int, default=2000, help='num of max iter') parser.add_argument( '--log-interval', type=int, default=50, help='interval of logging') parser.add_argument( '--fuse-conv-bn', action='store_true', help='Whether to fuse conv and bn, this will slightly increase' 'the inference speed') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') parser.add_argument( '--launcher', choices=['none', 'pytorch', 'slurm', 'mpi'], default='none', help='job launcher') parser.add_argument('--local_rank', type=int, default=0) args = parser.parse_args() if 'LOCAL_RANK' not in os.environ: os.environ['LOCAL_RANK'] = str(args.local_rank) return args def results2markdown(result_dict): table_data = [] is_multiple_results = False for cfg_name, value in result_dict.items(): name = cfg_name.replace('configs/', '') fps = value['fps'] ms_times_pre_image = value['ms_times_pre_image'] if isinstance(fps, list): is_multiple_results = True mean_fps = value['mean_fps'] mean_times_pre_image = value['mean_times_pre_image'] fps_str = ','.join([str(s) for s in fps]) ms_times_pre_image_str = ','.join( [str(s) for s in ms_times_pre_image]) table_data.append([ name, fps_str, mean_fps, ms_times_pre_image_str, mean_times_pre_image ]) else: table_data.append([name, fps, ms_times_pre_image]) if is_multiple_results: table_data.insert(0, [ 'model', 'fps', 'mean_fps', 'times_pre_image(ms)', 'mean_times_pre_image(ms)' ]) else: table_data.insert(0, ['model', 'fps', 'times_pre_image(ms)']) table = GithubFlavoredMarkdownTable(table_data) print(table.table, flush=True) if __name__ == '__main__': args = parse_args() assert args.round_num >= 0 assert args.repeat_num >= 1 config = Config.fromfile(args.config) if args.launcher == 'none': raise NotImplementedError('Only supports distributed mode') else: init_dist(args.launcher) result_dict = {} for model_key in config: model_infos = config[model_key] if not isinstance(model_infos, list): model_infos = [model_infos] for model_info in model_infos: record_metrics = model_info['metric'] cfg_path = model_info['config'].strip() cfg = Config.fromfile(cfg_path) checkpoint = osp.join(args.checkpoint_root, model_info['checkpoint'].strip()) try: fps = repeat_measure_inference_speed(cfg, checkpoint, args.max_iter, args.log_interval, args.fuse_conv_bn, args.repeat_num) if args.repeat_num > 1: fps_list = [round(fps_, args.round_num) for fps_ in fps] times_pre_image_list = [ round(1000 / fps_, args.round_num) for fps_ in fps ] mean_fps = round( sum(fps_list) / len(fps_list), args.round_num) mean_times_pre_image = round( sum(times_pre_image_list) / len(times_pre_image_list), args.round_num) print( f'{cfg_path} ' f'Overall fps: {fps_list}[{mean_fps}] img / s, ' f'times per image: ' f'{times_pre_image_list}[{mean_times_pre_image}] ' f'ms / img', flush=True) result_dict[cfg_path] = dict( fps=fps_list, mean_fps=mean_fps, ms_times_pre_image=times_pre_image_list, mean_times_pre_image=mean_times_pre_image) else: print( f'{cfg_path} fps : {fps:.{args.round_num}f} img / s, ' f'times per image: {1000 / fps:.{args.round_num}f} ' f'ms / img', flush=True) result_dict[cfg_path] = dict( fps=round(fps, args.round_num), ms_times_pre_image=round(1000 / fps, args.round_num)) except Exception as e: print(f'{cfg_path} error: {repr(e)}') if args.repeat_num > 1: result_dict[cfg_path] = dict( fps=[0], mean_fps=0, ms_times_pre_image=[0], mean_times_pre_image=0) else: result_dict[cfg_path] = dict(fps=0, ms_times_pre_image=0) if args.out: mkdir_or_exist(args.out) dump(result_dict, osp.join(args.out, 'batch_inference_fps.json')) results2markdown(result_dict) ================================================ FILE: .dev_scripts/benchmark_options.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. third_part_libs = [ 'pip install -r ../requirements/albu.txt', 'pip install instaboostfast', 'pip install git+https://github.com/cocodataset/panopticapi.git', 'pip install timm', 'pip install mmcls>=1.0.0rc0', 'pip install git+https://github.com/lvis-dataset/lvis-api.git', ] default_floating_range = 0.5 model_floating_ranges = {'atss/atss_r50_fpn_1x_coco.py': 0.3} ================================================ FILE: .dev_scripts/benchmark_test.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import logging import os import os.path as osp from argparse import ArgumentParser from mmengine.config import Config, DictAction from mmengine.logging import MMLogger from mmengine.registry import RUNNERS from mmengine.runner import Runner from mmdet.testing import replace_to_ceph from mmdet.utils import register_all_modules, replace_cfg_vals def parse_args(): parser = ArgumentParser() parser.add_argument('config', help='test config file path') parser.add_argument('checkpoint_root', help='Checkpoint file root path') parser.add_argument('--work-dir', help='the dir to save logs') parser.add_argument('--ceph', action='store_true') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') parser.add_argument( '--launcher', choices=['none', 'pytorch', 'slurm', 'mpi'], default='none', help='job launcher') parser.add_argument('--local_rank', type=int, default=0) args = parser.parse_args() if 'LOCAL_RANK' not in os.environ: os.environ['LOCAL_RANK'] = str(args.local_rank) args = parser.parse_args() return args # TODO: Need to refactor test.py so that it can be reused. def fast_test_model(config_name, checkpoint, args, logger=None): cfg = Config.fromfile(config_name) cfg = replace_cfg_vals(cfg) cfg.launcher = args.launcher if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) # work_dir is determined in this priority: CLI > segment in file > filename if args.work_dir is not None: # update configs according to CLI args if args.work_dir is not None cfg.work_dir = osp.join(args.work_dir, osp.splitext(osp.basename(config_name))[0]) elif cfg.get('work_dir', None) is None: # use config filename as default work_dir if cfg.work_dir is None cfg.work_dir = osp.join('./work_dirs', osp.splitext(osp.basename(config_name))[0]) if args.ceph: replace_to_ceph(cfg) cfg.load_from = checkpoint # TODO: temporary plan if 'visualizer' in cfg: if 'name' in cfg.visualizer: del cfg.visualizer.name # build the runner from config if 'runner_type' not in cfg: # build the default runner runner = Runner.from_cfg(cfg) else: # build customized runner from the registry # if 'runner_type' is set in the cfg runner = RUNNERS.build(cfg) runner.test() # Sample test whether the inference code is correct def main(args): # register all modules in mmdet into the registries register_all_modules(init_default_scope=False) config = Config.fromfile(args.config) # test all model logger = MMLogger.get_instance( name='MMLogger', log_file='benchmark_test.log', log_level=logging.ERROR) for model_key in config: model_infos = config[model_key] if not isinstance(model_infos, list): model_infos = [model_infos] for model_info in model_infos: print('processing: ', model_info['config'], flush=True) config_name = model_info['config'].strip() checkpoint = osp.join(args.checkpoint_root, model_info['checkpoint'].strip()) try: fast_test_model(config_name, checkpoint, args, logger) except Exception as e: logger.error(f'{config_name} " : {repr(e)}') if __name__ == '__main__': args = parse_args() main(args) ================================================ FILE: .dev_scripts/benchmark_test_image.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import logging import os.path as osp from argparse import ArgumentParser import mmcv from mmengine.config import Config from mmengine.logging import MMLogger from mmengine.utils import mkdir_or_exist from mmdet.apis import inference_detector, init_detector from mmdet.registry import VISUALIZERS from mmdet.utils import register_all_modules def parse_args(): parser = ArgumentParser() parser.add_argument('config', help='test config file path') parser.add_argument('checkpoint_root', help='Checkpoint file root path') parser.add_argument('--img', default='demo/demo.jpg', help='Image file') parser.add_argument('--aug', action='store_true', help='aug test') parser.add_argument('--model-name', help='model name to inference') parser.add_argument('--show', action='store_true', help='show results') parser.add_argument('--out-dir', default=None, help='Dir to output file') parser.add_argument( '--wait-time', type=float, default=1, help='the interval of show (s), 0 is block') parser.add_argument( '--device', default='cuda:0', help='Device used for inference') parser.add_argument( '--palette', default='coco', choices=['coco', 'voc', 'citys', 'random'], help='Color palette used for visualization') parser.add_argument( '--score-thr', type=float, default=0.3, help='bbox score threshold') args = parser.parse_args() return args def inference_model(config_name, checkpoint, visualizer, args, logger=None): cfg = Config.fromfile(config_name) if args.aug: raise NotImplementedError() model = init_detector( cfg, checkpoint, palette=args.palette, device=args.device) visualizer.dataset_meta = model.dataset_meta # test a single image result = inference_detector(model, args.img) # show the results if args.show or args.out_dir is not None: img = mmcv.imread(args.img) img = mmcv.imconvert(img, 'bgr', 'rgb') out_file = None if args.out_dir is not None: out_dir = args.out_dir mkdir_or_exist(out_dir) out_file = osp.join( out_dir, config_name.split('/')[-1].replace('py', 'jpg')) visualizer.add_datasample( 'result', img, data_sample=result, draw_gt=False, show=args.show, wait_time=args.wait_time, out_file=out_file, pred_score_thr=args.score_thr) return result # Sample test whether the inference code is correct def main(args): # register all modules in mmdet into the registries register_all_modules() config = Config.fromfile(args.config) # init visualizer visualizer_cfg = dict(type='DetLocalVisualizer', name='visualizer') visualizer = VISUALIZERS.build(visualizer_cfg) # test single model if args.model_name: if args.model_name in config: model_infos = config[args.model_name] if not isinstance(model_infos, list): model_infos = [model_infos] model_info = model_infos[0] config_name = model_info['config'].strip() print(f'processing: {config_name}', flush=True) checkpoint = osp.join(args.checkpoint_root, model_info['checkpoint'].strip()) # build the model from a config file and a checkpoint file inference_model(config_name, checkpoint, visualizer, args) return else: raise RuntimeError('model name input error.') # test all model logger = MMLogger.get_instance( name='MMLogger', log_file='benchmark_test_image.log', log_level=logging.ERROR) for model_key in config: model_infos = config[model_key] if not isinstance(model_infos, list): model_infos = [model_infos] for model_info in model_infos: print('processing: ', model_info['config'], flush=True) config_name = model_info['config'].strip() checkpoint = osp.join(args.checkpoint_root, model_info['checkpoint'].strip()) try: # build the model from a config file and a checkpoint file inference_model(config_name, checkpoint, visualizer, args, logger) except Exception as e: logger.error(f'{config_name} " : {repr(e)}') if __name__ == '__main__': args = parse_args() main(args) ================================================ FILE: .dev_scripts/benchmark_train.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import logging import os import os.path as osp from argparse import ArgumentParser from mmengine.config import Config, DictAction from mmengine.logging import MMLogger, print_log from mmengine.registry import RUNNERS from mmengine.runner import Runner from mmdet.testing import replace_to_ceph from mmdet.utils import register_all_modules, replace_cfg_vals def parse_args(): parser = ArgumentParser() parser.add_argument('config', help='test config file path') parser.add_argument('--work-dir', help='the dir to save logs and models') parser.add_argument('--ceph', action='store_true') parser.add_argument('--save-ckpt', action='store_true') parser.add_argument( '--amp', action='store_true', default=False, help='enable automatic-mixed-precision training') parser.add_argument( '--auto-scale-lr', action='store_true', help='enable automatically scaling LR.') parser.add_argument( '--resume', action='store_true', help='resume from the latest checkpoint in the work_dir automatically') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') parser.add_argument( '--launcher', choices=['none', 'pytorch', 'slurm', 'mpi'], default='none', help='job launcher') parser.add_argument('--local_rank', type=int, default=0) args = parser.parse_args() if 'LOCAL_RANK' not in os.environ: os.environ['LOCAL_RANK'] = str(args.local_rank) args = parser.parse_args() return args # TODO: Need to refactor train.py so that it can be reused. def fast_train_model(config_name, args, logger=None): cfg = Config.fromfile(config_name) cfg = replace_cfg_vals(cfg) cfg.launcher = args.launcher if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) # work_dir is determined in this priority: CLI > segment in file > filename if args.work_dir is not None: # update configs according to CLI args if args.work_dir is not None cfg.work_dir = osp.join(args.work_dir, osp.splitext(osp.basename(config_name))[0]) elif cfg.get('work_dir', None) is None: # use config filename as default work_dir if cfg.work_dir is None cfg.work_dir = osp.join('./work_dirs', osp.splitext(osp.basename(config_name))[0]) ckpt_hook = cfg.default_hooks.checkpoint by_epoch = ckpt_hook.get('by_epoch', True) fast_stop_hook = dict(type='FastStopTrainingHook') fast_stop_hook['by_epoch'] = by_epoch if args.save_ckpt: if by_epoch: interval = 1 stop_iter_or_epoch = 2 else: interval = 4 stop_iter_or_epoch = 10 fast_stop_hook['stop_iter_or_epoch'] = stop_iter_or_epoch fast_stop_hook['save_ckpt'] = True ckpt_hook.interval = interval if 'custom_hooks' in cfg: cfg.custom_hooks.append(fast_stop_hook) else: custom_hooks = [fast_stop_hook] cfg.custom_hooks = custom_hooks # TODO: temporary plan if 'visualizer' in cfg: if 'name' in cfg.visualizer: del cfg.visualizer.name # enable automatic-mixed-precision training if args.amp is True: optim_wrapper = cfg.optim_wrapper.type if optim_wrapper == 'AmpOptimWrapper': print_log( 'AMP training is already enabled in your config.', logger='current', level=logging.WARNING) else: assert optim_wrapper == 'OptimWrapper', ( '`--amp` is only supported when the optimizer wrapper type is ' f'`OptimWrapper` but got {optim_wrapper}.') cfg.optim_wrapper.type = 'AmpOptimWrapper' cfg.optim_wrapper.loss_scale = 'dynamic' # enable automatically scaling LR if args.auto_scale_lr: if 'auto_scale_lr' in cfg and \ 'enable' in cfg.auto_scale_lr and \ 'base_batch_size' in cfg.auto_scale_lr: cfg.auto_scale_lr.enable = True else: raise RuntimeError('Can not find "auto_scale_lr" or ' '"auto_scale_lr.enable" or ' '"auto_scale_lr.base_batch_size" in your' ' configuration file.') if args.ceph: replace_to_ceph(cfg) cfg.resume = args.resume # build the runner from config if 'runner_type' not in cfg: # build the default runner runner = Runner.from_cfg(cfg) else: # build customized runner from the registry # if 'runner_type' is set in the cfg runner = RUNNERS.build(cfg) runner.train() # Sample test whether the train code is correct def main(args): # register all modules in mmdet into the registries register_all_modules(init_default_scope=False) config = Config.fromfile(args.config) # test all model logger = MMLogger.get_instance( name='MMLogger', log_file='benchmark_train.log', log_level=logging.ERROR) for model_key in config: model_infos = config[model_key] if not isinstance(model_infos, list): model_infos = [model_infos] for model_info in model_infos: print('processing: ', model_info['config'], flush=True) config_name = model_info['config'].strip() try: fast_train_model(config_name, args, logger) except RuntimeError as e: # quick exit is the normal exit message if 'quick exit' not in repr(e): logger.error(f'{config_name} " : {repr(e)}') except Exception as e: logger.error(f'{config_name} " : {repr(e)}') if __name__ == '__main__': args = parse_args() main(args) ================================================ FILE: .dev_scripts/benchmark_train_models.txt ================================================ atss/atss_r50_fpn_1x_coco.py faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py panoptic_fpn/panoptic-fpn_r50_fpn_1x_coco.py retinanet/retinanet_r50_fpn_1x_coco.py rtmdet/rtmdet_s_8xb32-300e_coco.py rtmdet/rtmdet-ins_s_8xb32-300e_coco.py deformable_detr/deformable-detr_r50_16xb2-50e_coco.py fcos/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py centernet/centernet-update_r50-caffe_fpn_ms-1x_coco.py dino/dino-4scale_r50_8xb2-12e_coco.py htc/htc_r50_fpn_1x_coco.py mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py swin/mask-rcnn_swin-t-p4-w7_fpn_1x_coco.py condinst/condinst_r50_fpn_ms-poly-90k_coco_instance.py lvis/mask-rcnn_r50_fpn_sample1e-3_ms-1x_lvis-v1.py convnext/mask-rcnn_convnext-t-p4-w7_fpn_amp-ms-crop-3x_coco.py ================================================ FILE: .dev_scripts/benchmark_valid_flops.py ================================================ import logging import re import tempfile from argparse import ArgumentParser from collections import OrderedDict from functools import partial from pathlib import Path import numpy as np import pandas as pd import torch from mmengine import Config, DictAction from mmengine.analysis import get_model_complexity_info from mmengine.analysis.print_helper import _format_size from mmengine.fileio import FileClient from mmengine.logging import MMLogger from mmengine.model import revert_sync_batchnorm from mmengine.runner import Runner from modelindex.load_model_index import load from rich.console import Console from rich.table import Table from rich.text import Text from tqdm import tqdm from mmdet.registry import MODELS from mmdet.utils import register_all_modules console = Console() MMDET_ROOT = Path(__file__).absolute().parents[1] def parse_args(): parser = ArgumentParser(description='Valid all models in model-index.yml') parser.add_argument( '--shape', type=int, nargs='+', default=[1280, 800], help='input image size') parser.add_argument( '--checkpoint_root', help='Checkpoint file root path. If set, load checkpoint before test.') parser.add_argument('--img', default='demo/demo.jpg', help='Image file') parser.add_argument('--models', nargs='+', help='models name to inference') parser.add_argument( '--batch-size', type=int, default=1, help='The batch size during the inference.') parser.add_argument( '--flops', action='store_true', help='Get Flops and Params of models') parser.add_argument( '--flops-str', action='store_true', help='Output FLOPs and params counts in a string form.') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') parser.add_argument( '--size_divisor', type=int, default=32, help='Pad the input image, the minimum size that is divisible ' 'by size_divisor, -1 means do not pad the image.') args = parser.parse_args() return args def inference(config_file, checkpoint, work_dir, args, exp_name): logger = MMLogger.get_instance(name='MMLogger') logger.warning('if you want test flops, please make sure torch>=1.12') cfg = Config.fromfile(config_file) cfg.work_dir = work_dir cfg.load_from = checkpoint cfg.log_level = 'WARN' cfg.experiment_name = exp_name if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) # forward the model result = {'model': config_file.stem} if args.flops: if len(args.shape) == 1: h = w = args.shape[0] elif len(args.shape) == 2: h, w = args.shape else: raise ValueError('invalid input shape') divisor = args.size_divisor if divisor > 0: h = int(np.ceil(h / divisor)) * divisor w = int(np.ceil(w / divisor)) * divisor input_shape = (3, h, w) result['resolution'] = input_shape try: cfg = Config.fromfile(config_file) if hasattr(cfg, 'head_norm_cfg'): cfg['head_norm_cfg'] = dict(type='SyncBN', requires_grad=True) cfg['model']['roi_head']['bbox_head']['norm_cfg'] = dict( type='SyncBN', requires_grad=True) cfg['model']['roi_head']['mask_head']['norm_cfg'] = dict( type='SyncBN', requires_grad=True) if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) model = MODELS.build(cfg.model) input = torch.rand(1, *input_shape) if torch.cuda.is_available(): model.cuda() input = input.cuda() model = revert_sync_batchnorm(model) inputs = (input, ) model.eval() outputs = get_model_complexity_info( model, input_shape, inputs, show_table=False, show_arch=False) flops = outputs['flops'] params = outputs['params'] activations = outputs['activations'] result['Get Types'] = 'direct' except: # noqa 772 logger = MMLogger.get_instance(name='MMLogger') logger.warning( 'Direct get flops failed, try to get flops with data') cfg = Config.fromfile(config_file) if hasattr(cfg, 'head_norm_cfg'): cfg['head_norm_cfg'] = dict(type='SyncBN', requires_grad=True) cfg['model']['roi_head']['bbox_head']['norm_cfg'] = dict( type='SyncBN', requires_grad=True) cfg['model']['roi_head']['mask_head']['norm_cfg'] = dict( type='SyncBN', requires_grad=True) data_loader = Runner.build_dataloader(cfg.val_dataloader) data_batch = next(iter(data_loader)) model = MODELS.build(cfg.model) if torch.cuda.is_available(): model = model.cuda() model = revert_sync_batchnorm(model) model.eval() _forward = model.forward data = model.data_preprocessor(data_batch) del data_loader model.forward = partial( _forward, data_samples=data['data_samples']) outputs = get_model_complexity_info( model, input_shape, data['inputs'], show_table=False, show_arch=False) flops = outputs['flops'] params = outputs['params'] activations = outputs['activations'] result['Get Types'] = 'dataloader' if args.flops_str: flops = _format_size(flops) params = _format_size(params) activations = _format_size(activations) result['flops'] = flops result['params'] = params return result def show_summary(summary_data, args): table = Table(title='Validation Benchmark Regression Summary') table.add_column('Model') table.add_column('Validation') table.add_column('Resolution (c, h, w)') if args.flops: table.add_column('Flops', justify='right', width=11) table.add_column('Params', justify='right') for model_name, summary in summary_data.items(): row = [model_name] valid = summary['valid'] color = 'green' if valid == 'PASS' else 'red' row.append(f'[{color}]{valid}[/{color}]') if valid == 'PASS': row.append(str(summary['resolution'])) if args.flops: row.append(str(summary['flops'])) row.append(str(summary['params'])) table.add_row(*row) console.print(table) table_data = { x.header: [Text.from_markup(y).plain for y in x.cells] for x in table.columns } table_pd = pd.DataFrame(table_data) table_pd.to_csv('./mmdetection_flops.csv') # Sample test whether the inference code is correct def main(args): register_all_modules() model_index_file = MMDET_ROOT / 'model-index.yml' model_index = load(str(model_index_file)) model_index.build_models_with_collections() models = OrderedDict({model.name: model for model in model_index.models}) logger = MMLogger( 'validation', logger_name='validation', log_file='benchmark_test_image.log', log_level=logging.INFO) if args.models: patterns = [ re.compile(pattern.replace('+', '_')) for pattern in args.models ] filter_models = {} for k, v in models.items(): k = k.replace('+', '_') if any([re.match(pattern, k) for pattern in patterns]): filter_models[k] = v if len(filter_models) == 0: print('No model found, please specify models in:') print('\n'.join(models.keys())) return models = filter_models summary_data = {} tmpdir = tempfile.TemporaryDirectory() for model_name, model_info in tqdm(models.items()): if model_info.config is None: continue model_info.config = model_info.config.replace('%2B', '+') config = Path(model_info.config) try: config.exists() except: # noqa 722 logger.error(f'{model_name}: {config} not found.') continue logger.info(f'Processing: {model_name}') http_prefix = 'https://download.openmmlab.com/mmdetection/' if args.checkpoint_root is not None: root = args.checkpoint_root if 's3://' in args.checkpoint_root: from petrel_client.common.exception import AccessDeniedError file_client = FileClient.infer_client(uri=root) checkpoint = file_client.join_path( root, model_info.weights[len(http_prefix):]) try: exists = file_client.exists(checkpoint) except AccessDeniedError: exists = False else: checkpoint = Path(root) / model_info.weights[len(http_prefix):] exists = checkpoint.exists() if exists: checkpoint = str(checkpoint) else: print(f'WARNING: {model_name}: {checkpoint} not found.') checkpoint = None else: checkpoint = None try: # build the model from a config file and a checkpoint file result = inference(MMDET_ROOT / config, checkpoint, tmpdir.name, args, model_name) result['valid'] = 'PASS' except Exception: # noqa 722 import traceback logger.error(f'"{config}" :\n{traceback.format_exc()}') result = {'valid': 'FAIL'} summary_data[model_name] = result tmpdir.cleanup() show_summary(summary_data, args) if __name__ == '__main__': args = parse_args() main(args) ================================================ FILE: .dev_scripts/check_links.py ================================================ # Modified from: # https://github.com/allenai/allennlp/blob/main/scripts/check_links.py import argparse import logging import os import pathlib import re import sys from multiprocessing.dummy import Pool from typing import NamedTuple, Optional, Tuple import requests from mmengine.logging import MMLogger def parse_args(): parser = argparse.ArgumentParser( description='Goes through all the inline-links ' 'in markdown files and reports the breakages') parser.add_argument( '--num-threads', type=int, default=100, help='Number of processes to confirm the link') parser.add_argument('--https-proxy', type=str, help='https proxy') parser.add_argument( '--out', type=str, default='link_reports.txt', help='output path of reports') args = parser.parse_args() return args OK_STATUS_CODES = ( 200, 401, # the resource exists but may require some sort of login. 403, # ^ same 405, # HEAD method not allowed. # the resource exists, but our default 'Accept-' header may not # match what the server can provide. 406, ) class MatchTuple(NamedTuple): source: str name: str link: str def check_link( match_tuple: MatchTuple, http_session: requests.Session, logger: logging = None) -> Tuple[MatchTuple, bool, Optional[str]]: reason: Optional[str] = None if match_tuple.link.startswith('http'): result_ok, reason = check_url(match_tuple, http_session) else: result_ok = check_path(match_tuple) if logger is None: print(f" {'✓' if result_ok else '✗'} {match_tuple.link}") else: logger.info(f" {'✓' if result_ok else '✗'} {match_tuple.link}") return match_tuple, result_ok, reason def check_url(match_tuple: MatchTuple, http_session: requests.Session) -> Tuple[bool, str]: """Check if a URL is reachable.""" try: result = http_session.head( match_tuple.link, timeout=5, allow_redirects=True) return ( result.ok or result.status_code in OK_STATUS_CODES, f'status code = {result.status_code}', ) except (requests.ConnectionError, requests.Timeout): return False, 'connection error' def check_path(match_tuple: MatchTuple) -> bool: """Check if a file in this repository exists.""" relative_path = match_tuple.link.split('#')[0] full_path = os.path.join( os.path.dirname(str(match_tuple.source)), relative_path) return os.path.exists(full_path) def main(): args = parse_args() # setup logger logger = MMLogger.get_instance(name='mmdet', log_file=args.out) # setup https_proxy if args.https_proxy: os.environ['https_proxy'] = args.https_proxy # setup http_session http_session = requests.Session() for resource_prefix in ('http://', 'https://'): http_session.mount( resource_prefix, requests.adapters.HTTPAdapter( max_retries=5, pool_connections=20, pool_maxsize=args.num_threads), ) logger.info('Finding all markdown files in the current directory...') project_root = (pathlib.Path(__file__).parent / '..').resolve() markdown_files = project_root.glob('**/*.md') all_matches = set() url_regex = re.compile(r'\[([^!][^\]]+)\]\(([^)(]+)\)') for markdown_file in markdown_files: with open(markdown_file) as handle: for line in handle.readlines(): matches = url_regex.findall(line) for name, link in matches: if 'localhost' not in link: all_matches.add( MatchTuple( source=str(markdown_file), name=name, link=link)) logger.info(f' {len(all_matches)} markdown files found') logger.info('Checking to make sure we can retrieve each link...') with Pool(processes=args.num_threads) as pool: results = pool.starmap(check_link, [(match, http_session, logger) for match in list(all_matches)]) # collect unreachable results unreachable_results = [(match_tuple, reason) for match_tuple, success, reason in results if not success] if unreachable_results: logger.info('================================================') logger.info(f'Unreachable links ({len(unreachable_results)}):') for match_tuple, reason in unreachable_results: logger.info(' > Source: ' + match_tuple.source) logger.info(' Name: ' + match_tuple.name) logger.info(' Link: ' + match_tuple.link) if reason is not None: logger.info(' Reason: ' + reason) sys.exit(1) logger.info('No Unreachable link found.') if __name__ == '__main__': main() ================================================ FILE: .dev_scripts/convert_test_benchmark_script.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os import os.path as osp from mmengine import Config def parse_args(): parser = argparse.ArgumentParser( description='Convert benchmark model list to script') parser.add_argument('config', help='test config file path') parser.add_argument('--port', type=int, default=29666, help='dist port') parser.add_argument( '--run', action='store_true', help='run script directly') parser.add_argument( '--out', type=str, help='path to save model benchmark script') args = parser.parse_args() return args def process_model_info(model_info, work_dir): config = model_info['config'].strip() fname, _ = osp.splitext(osp.basename(config)) job_name = fname work_dir = '$WORK_DIR/' + fname checkpoint = model_info['checkpoint'].strip() return dict( config=config, job_name=job_name, work_dir=work_dir, checkpoint=checkpoint) def create_test_bash_info(commands, model_test_dict, port, script_name, partition): config = model_test_dict['config'] job_name = model_test_dict['job_name'] checkpoint = model_test_dict['checkpoint'] work_dir = model_test_dict['work_dir'] echo_info = f' \necho \'{config}\' &' commands.append(echo_info) commands.append('\n') command_info = f'GPUS=8 GPUS_PER_NODE=8 ' \ f'CPUS_PER_TASK=$CPUS_PRE_TASK {script_name} ' command_info += f'{partition} ' command_info += f'{job_name} ' command_info += f'{config} ' command_info += f'$CHECKPOINT_DIR/{checkpoint} ' command_info += f'--work-dir {work_dir} ' command_info += f'--cfg-option env_cfg.dist_cfg.port={port} ' command_info += ' &' commands.append(command_info) def main(): args = parse_args() if args.out: out_suffix = args.out.split('.')[-1] assert args.out.endswith('.sh'), \ f'Expected out file path suffix is .sh, but get .{out_suffix}' assert args.out or args.run, \ ('Please specify at least one operation (save/run/ the ' 'script) with the argument "--out" or "--run"') commands = [] partition_name = 'PARTITION=$1 ' commands.append(partition_name) commands.append('\n') checkpoint_root = 'CHECKPOINT_DIR=$2 ' commands.append(checkpoint_root) commands.append('\n') work_dir = 'WORK_DIR=$3 ' commands.append(work_dir) commands.append('\n') cpus_pre_task = 'CPUS_PER_TASK=${4:-2} ' commands.append(cpus_pre_task) commands.append('\n') script_name = osp.join('tools', 'slurm_test.sh') port = args.port cfg = Config.fromfile(args.config) for model_key in cfg: model_infos = cfg[model_key] if not isinstance(model_infos, list): model_infos = [model_infos] for model_info in model_infos: print('processing: ', model_info['config']) model_test_dict = process_model_info(model_info, work_dir) create_test_bash_info(commands, model_test_dict, port, script_name, '$PARTITION') port += 1 command_str = ''.join(commands) if args.out: with open(args.out, 'w') as f: f.write(command_str) if args.run: os.system(command_str) if __name__ == '__main__': main() ================================================ FILE: .dev_scripts/convert_train_benchmark_script.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os import os.path as osp def parse_args(): parser = argparse.ArgumentParser( description='Convert benchmark model json to script') parser.add_argument( 'txt_path', type=str, help='txt path output by benchmark_filter') parser.add_argument( '--run', action='store_true', help='run script directly') parser.add_argument( '--out', type=str, help='path to save model benchmark script') args = parser.parse_args() return args def determine_gpus(cfg_name): gpus = 8 gpus_pre_node = 8 if cfg_name.find('16x') >= 0: gpus = 16 elif cfg_name.find('4xb4') >= 0: gpus = 4 gpus_pre_node = 4 elif 'lad' in cfg_name: gpus = 2 gpus_pre_node = 2 return gpus, gpus_pre_node def main(): args = parse_args() if args.out: out_suffix = args.out.split('.')[-1] assert args.out.endswith('.sh'), \ f'Expected out file path suffix is .sh, but get .{out_suffix}' assert args.out or args.run, \ ('Please specify at least one operation (save/run/ the ' 'script) with the argument "--out" or "--run"') root_name = './tools' train_script_name = osp.join(root_name, 'slurm_train.sh') commands = [] partition_name = 'PARTITION=$1 ' commands.append(partition_name) commands.append('\n') work_dir = 'WORK_DIR=$2 ' commands.append(work_dir) commands.append('\n') cpus_pre_task = 'CPUS_PER_TASK=${3:-4} ' commands.append(cpus_pre_task) commands.append('\n') commands.append('\n') with open(args.txt_path, 'r') as f: model_cfgs = f.readlines() for i, cfg in enumerate(model_cfgs): cfg = cfg.strip() if len(cfg) == 0: continue # print cfg name echo_info = f'echo \'{cfg}\' &' commands.append(echo_info) commands.append('\n') fname, _ = osp.splitext(osp.basename(cfg)) out_fname = '$WORK_DIR/' + fname gpus, gpus_pre_node = determine_gpus(cfg) command_info = f'GPUS={gpus} GPUS_PER_NODE={gpus_pre_node} ' \ f'CPUS_PER_TASK=$CPUS_PRE_TASK {train_script_name} ' command_info += '$PARTITION ' command_info += f'{fname} ' command_info += f'{cfg} ' command_info += f'{out_fname} ' command_info += '--cfg-options default_hooks.checkpoint.' \ 'max_keep_ckpts=1 ' command_info += '&' commands.append(command_info) if i < len(model_cfgs): commands.append('\n') command_str = ''.join(commands) if args.out: with open(args.out, 'w') as f: f.write(command_str) if args.run: os.system(command_str) if __name__ == '__main__': main() ================================================ FILE: .dev_scripts/covignore.cfg ================================================ # Each line should be the relative path to the root directory # of this repo. Support regular expression as well. # For example: .*/__init__.py ================================================ FILE: .dev_scripts/diff_coverage_test.sh ================================================ #!/bin/bash readarray -t IGNORED_FILES < $( dirname "$0" )/covignore.cfg REUSE_COVERAGE_REPORT=${REUSE_COVERAGE_REPORT:-0} REPO=${1:-"origin"} BRANCH=${2:-"refactor_dev"} git fetch $REPO $BRANCH PY_FILES="" for FILE_NAME in $(git diff --name-only ${REPO}/${BRANCH}); do # Only test python files in mmdet/ existing in current branch, and not ignored in covignore.cfg if [ ${FILE_NAME: -3} == ".py" ] && [ ${FILE_NAME:0:6} == "mmdet/" ] && [ -f "$FILE_NAME" ]; then IGNORED=false for IGNORED_FILE_NAME in "${IGNORED_FILES[@]}"; do # Skip blank lines if [ -z "$IGNORED_FILE_NAME" ]; then continue fi if [ "${IGNORED_FILE_NAME::1}" != "#" ] && [[ "$FILE_NAME" =~ $IGNORED_FILE_NAME ]]; then echo "Ignoring $FILE_NAME" IGNORED=true break fi done if [ "$IGNORED" = false ]; then PY_FILES="$PY_FILES $FILE_NAME" fi fi done # Only test the coverage when PY_FILES are not empty, otherwise they will test the entire project if [ ! -z "${PY_FILES}" ] then if [ "$REUSE_COVERAGE_REPORT" == "0" ]; then coverage run --branch --source mmdet -m pytest tests/ fi coverage report --fail-under 80 -m $PY_FILES interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --ignore-magic --ignore-regex "__repr__" --fail-under 95 $PY_FILES fi ================================================ FILE: .dev_scripts/download_checkpoints.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import math import os import os.path as osp from multiprocessing import Pool import torch from mmengine.config import Config from mmengine.utils import mkdir_or_exist def download(url, out_file, min_bytes=math.pow(1024, 2), progress=True): # math.pow(1024, 2) is mean 1 MB assert_msg = f"Downloaded url '{url}' does not exist " \ f'or size is < min_bytes={min_bytes}' try: print(f'Downloading {url} to {out_file}...') torch.hub.download_url_to_file(url, str(out_file), progress=progress) assert osp.exists( out_file) and osp.getsize(out_file) > min_bytes, assert_msg except Exception as e: if osp.exists(out_file): os.remove(out_file) print(f'ERROR: {e}\nRe-attempting {url} to {out_file} ...') os.system(f"curl -L '{url}' -o '{out_file}' --retry 3 -C -" ) # curl download, retry and resume on fail finally: if osp.exists(out_file) and osp.getsize(out_file) < min_bytes: os.remove(out_file) # remove partial downloads if not osp.exists(out_file): print(f'ERROR: {assert_msg}\n') print('=========================================\n') def parse_args(): parser = argparse.ArgumentParser(description='Download checkpoints') parser.add_argument('config', help='test config file path') parser.add_argument( 'out', type=str, help='output dir of checkpoints to be stored') parser.add_argument( '--nproc', type=int, default=16, help='num of Processes') parser.add_argument( '--intranet', action='store_true', help='switch to internal network url') args = parser.parse_args() return args if __name__ == '__main__': args = parse_args() mkdir_or_exist(args.out) cfg = Config.fromfile(args.config) checkpoint_url_list = [] checkpoint_out_list = [] for model in cfg: model_infos = cfg[model] if not isinstance(model_infos, list): model_infos = [model_infos] for model_info in model_infos: checkpoint = model_info['checkpoint'] out_file = osp.join(args.out, checkpoint) if not osp.exists(out_file): url = model_info['url'] if args.intranet is True: url = url.replace('.com', '.sensetime.com') url = url.replace('https', 'http') checkpoint_url_list.append(url) checkpoint_out_list.append(out_file) if len(checkpoint_url_list) > 0: pool = Pool(min(os.cpu_count(), args.nproc)) pool.starmap(download, zip(checkpoint_url_list, checkpoint_out_list)) else: print('No files to download!') ================================================ FILE: .dev_scripts/gather_models.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import glob import json import os.path as osp import shutil import subprocess from collections import OrderedDict import torch import yaml from mmengine.config import Config from mmengine.fileio import dump from mmengine.utils import mkdir_or_exist, scandir def ordered_yaml_dump(data, stream=None, Dumper=yaml.SafeDumper, **kwds): class OrderedDumper(Dumper): pass def _dict_representer(dumper, data): return dumper.represent_mapping( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, data.items()) OrderedDumper.add_representer(OrderedDict, _dict_representer) return yaml.dump(data, stream, OrderedDumper, **kwds) def process_checkpoint(in_file, out_file): checkpoint = torch.load(in_file, map_location='cpu') # remove optimizer for smaller file size if 'optimizer' in checkpoint: del checkpoint['optimizer'] # remove ema state_dict for key in list(checkpoint['state_dict']): if key.startswith('ema_'): checkpoint['state_dict'].pop(key) # if it is necessary to remove some sensitive data in checkpoint['meta'], # add the code here. if torch.__version__ >= '1.6': torch.save(checkpoint, out_file, _use_new_zipfile_serialization=False) else: torch.save(checkpoint, out_file) sha = subprocess.check_output(['sha256sum', out_file]).decode() final_file = out_file.rstrip('.pth') + '-{}.pth'.format(sha[:8]) subprocess.Popen(['mv', out_file, final_file]) return final_file def is_by_epoch(config): cfg = Config.fromfile('./configs/' + config) return cfg.runner.type == 'EpochBasedRunner' def get_final_epoch_or_iter(config): cfg = Config.fromfile('./configs/' + config) if cfg.runner.type == 'EpochBasedRunner': return cfg.runner.max_epochs else: return cfg.runner.max_iters def get_best_epoch_or_iter(exp_dir): best_epoch_iter_full_path = list( sorted(glob.glob(osp.join(exp_dir, 'best_*.pth'))))[-1] best_epoch_or_iter_model_path = best_epoch_iter_full_path.split('/')[-1] best_epoch_or_iter = best_epoch_or_iter_model_path.\ split('_')[-1].split('.')[0] return best_epoch_or_iter_model_path, int(best_epoch_or_iter) def get_real_epoch_or_iter(config): cfg = Config.fromfile('./configs/' + config) if cfg.runner.type == 'EpochBasedRunner': epoch = cfg.runner.max_epochs if cfg.data.train.type == 'RepeatDataset': epoch *= cfg.data.train.times return epoch else: return cfg.runner.max_iters def get_final_results(log_json_path, epoch_or_iter, results_lut, by_epoch=True): result_dict = dict() last_val_line = None last_train_line = None last_val_line_idx = -1 last_train_line_idx = -1 with open(log_json_path, 'r') as f: for i, line in enumerate(f.readlines()): log_line = json.loads(line) if 'mode' not in log_line.keys(): continue if by_epoch: if (log_line['mode'] == 'train' and log_line['epoch'] == epoch_or_iter): result_dict['memory'] = log_line['memory'] if (log_line['mode'] == 'val' and log_line['epoch'] == epoch_or_iter): result_dict.update({ key: log_line[key] for key in results_lut if key in log_line }) return result_dict else: if log_line['mode'] == 'train': last_train_line_idx = i last_train_line = log_line if log_line and log_line['mode'] == 'val': last_val_line_idx = i last_val_line = log_line # bug: max_iters = 768, last_train_line['iter'] = 750 assert last_val_line_idx == last_train_line_idx + 1, \ 'Log file is incomplete' result_dict['memory'] = last_train_line['memory'] result_dict.update({ key: last_val_line[key] for key in results_lut if key in last_val_line }) return result_dict def get_dataset_name(config): # If there are more dataset, add here. name_map = dict( CityscapesDataset='Cityscapes', CocoDataset='COCO', CocoPanopticDataset='COCO', DeepFashionDataset='Deep Fashion', LVISV05Dataset='LVIS v0.5', LVISV1Dataset='LVIS v1', VOCDataset='Pascal VOC', WIDERFaceDataset='WIDER Face', OpenImagesDataset='OpenImagesDataset', OpenImagesChallengeDataset='OpenImagesChallengeDataset', Objects365V1Dataset='Objects365 v1', Objects365V2Dataset='Objects365 v2') cfg = Config.fromfile('./configs/' + config) return name_map[cfg.dataset_type] def convert_model_info_to_pwc(model_infos): pwc_files = {} for model in model_infos: cfg_folder_name = osp.split(model['config'])[-2] pwc_model_info = OrderedDict() pwc_model_info['Name'] = osp.split(model['config'])[-1].split('.')[0] pwc_model_info['In Collection'] = 'Please fill in Collection name' pwc_model_info['Config'] = osp.join('configs', model['config']) # get metadata memory = round(model['results']['memory'] / 1024, 1) meta_data = OrderedDict() meta_data['Training Memory (GB)'] = memory if 'epochs' in model: meta_data['Epochs'] = get_real_epoch_or_iter(model['config']) else: meta_data['Iterations'] = get_real_epoch_or_iter(model['config']) pwc_model_info['Metadata'] = meta_data # get dataset name dataset_name = get_dataset_name(model['config']) # get results results = [] # if there are more metrics, add here. if 'bbox_mAP' in model['results']: metric = round(model['results']['bbox_mAP'] * 100, 1) results.append( OrderedDict( Task='Object Detection', Dataset=dataset_name, Metrics={'box AP': metric})) if 'segm_mAP' in model['results']: metric = round(model['results']['segm_mAP'] * 100, 1) results.append( OrderedDict( Task='Instance Segmentation', Dataset=dataset_name, Metrics={'mask AP': metric})) if 'PQ' in model['results']: metric = round(model['results']['PQ'], 1) results.append( OrderedDict( Task='Panoptic Segmentation', Dataset=dataset_name, Metrics={'PQ': metric})) pwc_model_info['Results'] = results link_string = 'https://download.openmmlab.com/mmdetection/v2.0/' link_string += '{}/{}'.format(model['config'].rstrip('.py'), osp.split(model['model_path'])[-1]) pwc_model_info['Weights'] = link_string if cfg_folder_name in pwc_files: pwc_files[cfg_folder_name].append(pwc_model_info) else: pwc_files[cfg_folder_name] = [pwc_model_info] return pwc_files def parse_args(): parser = argparse.ArgumentParser(description='Gather benchmarked models') parser.add_argument( 'root', type=str, help='root path of benchmarked models to be gathered') parser.add_argument( 'out', type=str, help='output path of gathered models to be stored') parser.add_argument( '--best', action='store_true', help='whether to gather the best model.') args = parser.parse_args() return args def main(): args = parse_args() models_root = args.root models_out = args.out mkdir_or_exist(models_out) # find all models in the root directory to be gathered raw_configs = list(scandir('./configs', '.py', recursive=True)) # filter configs that is not trained in the experiments dir used_configs = [] for raw_config in raw_configs: if osp.exists(osp.join(models_root, raw_config)): used_configs.append(raw_config) print(f'Find {len(used_configs)} models to be gathered') # find final_ckpt and log file for trained each config # and parse the best performance model_infos = [] for used_config in used_configs: exp_dir = osp.join(models_root, used_config) by_epoch = is_by_epoch(used_config) # check whether the exps is finished if args.best is True: final_model, final_epoch_or_iter = get_best_epoch_or_iter(exp_dir) else: final_epoch_or_iter = get_final_epoch_or_iter(used_config) final_model = '{}_{}.pth'.format('epoch' if by_epoch else 'iter', final_epoch_or_iter) model_path = osp.join(exp_dir, final_model) # skip if the model is still training if not osp.exists(model_path): continue # get the latest logs log_json_path = list( sorted(glob.glob(osp.join(exp_dir, '*.log.json'))))[-1] log_txt_path = list(sorted(glob.glob(osp.join(exp_dir, '*.log'))))[-1] cfg = Config.fromfile('./configs/' + used_config) results_lut = cfg.evaluation.metric if not isinstance(results_lut, list): results_lut = [results_lut] # case when using VOC, the evaluation key is only 'mAP' # when using Panoptic Dataset, the evaluation key is 'PQ'. for i, key in enumerate(results_lut): if 'mAP' not in key and 'PQ' not in key: results_lut[i] = key + '_mAP' model_performance = get_final_results(log_json_path, final_epoch_or_iter, results_lut, by_epoch) if model_performance is None: continue model_time = osp.split(log_txt_path)[-1].split('.')[0] model_info = dict( config=used_config, results=model_performance, model_time=model_time, final_model=final_model, log_json_path=osp.split(log_json_path)[-1]) model_info['epochs' if by_epoch else 'iterations'] =\ final_epoch_or_iter model_infos.append(model_info) # publish model for each checkpoint publish_model_infos = [] for model in model_infos: model_publish_dir = osp.join(models_out, model['config'].rstrip('.py')) mkdir_or_exist(model_publish_dir) model_name = osp.split(model['config'])[-1].split('.')[0] model_name += '_' + model['model_time'] publish_model_path = osp.join(model_publish_dir, model_name) trained_model_path = osp.join(models_root, model['config'], model['final_model']) # convert model final_model_path = process_checkpoint(trained_model_path, publish_model_path) # copy log shutil.copy( osp.join(models_root, model['config'], model['log_json_path']), osp.join(model_publish_dir, f'{model_name}.log.json')) shutil.copy( osp.join(models_root, model['config'], model['log_json_path'].rstrip('.json')), osp.join(model_publish_dir, f'{model_name}.log')) # copy config to guarantee reproducibility config_path = model['config'] config_path = osp.join( 'configs', config_path) if 'configs' not in config_path else config_path target_config_path = osp.split(config_path)[-1] shutil.copy(config_path, osp.join(model_publish_dir, target_config_path)) model['model_path'] = final_model_path publish_model_infos.append(model) models = dict(models=publish_model_infos) print(f'Totally gathered {len(publish_model_infos)} models') dump(models, osp.join(models_out, 'model_info.json')) pwc_files = convert_model_info_to_pwc(publish_model_infos) for name in pwc_files: with open(osp.join(models_out, name + '_metafile.yml'), 'w') as f: ordered_yaml_dump(pwc_files[name], f, encoding='utf-8') if __name__ == '__main__': main() ================================================ FILE: .dev_scripts/gather_test_benchmark_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import glob import os.path as osp from mmengine.config import Config from mmengine.fileio import dump, load from mmengine.utils import mkdir_or_exist def parse_args(): parser = argparse.ArgumentParser( description='Gather benchmarked models metric') parser.add_argument('config', help='test config file path') parser.add_argument( 'root', type=str, help='root path of benchmarked models to be gathered') parser.add_argument( '--out', type=str, help='output path of gathered metrics to be stored') parser.add_argument( '--not-show', action='store_true', help='not show metrics') parser.add_argument( '--show-all', action='store_true', help='show all model metrics') args = parser.parse_args() return args if __name__ == '__main__': args = parse_args() root_path = args.root metrics_out = args.out result_dict = {} cfg = Config.fromfile(args.config) for model_key in cfg: model_infos = cfg[model_key] if not isinstance(model_infos, list): model_infos = [model_infos] for model_info in model_infos: record_metrics = model_info['metric'] config = model_info['config'].strip() fname, _ = osp.splitext(osp.basename(config)) metric_json_dir = osp.join(root_path, fname) if osp.exists(metric_json_dir): json_list = glob.glob(osp.join(metric_json_dir, '*.json')) if len(json_list) > 0: log_json_path = list(sorted(json_list))[-1] metric = load(log_json_path) if config in metric.get('config', {}): new_metrics = dict() for record_metric_key in record_metrics: record_metric_key_bk = record_metric_key old_metric = record_metrics[record_metric_key] if record_metric_key == 'AR_1000': record_metric_key = 'AR@1000' if record_metric_key not in metric['metric']: raise KeyError( 'record_metric_key not exist, please ' 'check your config') new_metric = round( metric['metric'][record_metric_key] * 100, 1) new_metrics[record_metric_key_bk] = new_metric if args.show_all: result_dict[config] = dict( before=record_metrics, after=new_metrics) else: for record_metric_key in record_metrics: old_metric = record_metrics[record_metric_key] new_metric = new_metrics[record_metric_key] if old_metric != new_metric: result_dict[config] = dict( before=record_metrics, after=new_metrics) break else: print(f'{config} not included in: {log_json_path}') else: print(f'{config} not exist file: {metric_json_dir}') else: print(f'{config} not exist dir: {metric_json_dir}') if metrics_out: mkdir_or_exist(metrics_out) dump(result_dict, osp.join(metrics_out, 'batch_test_metric_info.json')) if not args.not_show: print('===================================') for config_name, metrics in result_dict.items(): print(config_name, metrics) print('===================================') ================================================ FILE: .dev_scripts/gather_train_benchmark_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import glob import os.path as osp from gather_models import get_final_results from mmengine.config import Config from mmengine.fileio import dump from mmengine.utils import mkdir_or_exist try: import xlrd except ImportError: xlrd = None try: import xlutils from xlutils.copy import copy except ImportError: xlutils = None def parse_args(): parser = argparse.ArgumentParser( description='Gather benchmarked models metric') parser.add_argument( 'root', type=str, help='root path of benchmarked models to be gathered') parser.add_argument( 'txt_path', type=str, help='txt path output by benchmark_filter') parser.add_argument( '--out', type=str, help='output path of gathered metrics to be stored') parser.add_argument( '--not-show', action='store_true', help='not show metrics') parser.add_argument( '--excel', type=str, help='input path of excel to be recorded') parser.add_argument( '--ncol', type=int, help='Number of column to be modified or appended') args = parser.parse_args() return args if __name__ == '__main__': args = parse_args() if args.excel: assert args.ncol, 'Please specify "--excel" and "--ncol" ' \ 'at the same time' if xlrd is None: raise RuntimeError( 'xlrd is not installed,' 'Please use “pip install xlrd==1.2.0” to install') if xlutils is None: raise RuntimeError( 'xlutils is not installed,' 'Please use “pip install xlutils==2.0.0” to install') readbook = xlrd.open_workbook(args.excel) sheet = readbook.sheet_by_name('Sheet1') sheet_info = {} total_nrows = sheet.nrows for i in range(3, sheet.nrows): sheet_info[sheet.row_values(i)[0]] = i xlrw = copy(readbook) table = xlrw.get_sheet(0) root_path = args.root metrics_out = args.out result_dict = {} with open(args.txt_path, 'r') as f: model_cfgs = f.readlines() for i, config in enumerate(model_cfgs): config = config.strip() if len(config) == 0: continue config_name = osp.split(config)[-1] config_name = osp.splitext(config_name)[0] result_path = osp.join(root_path, config_name) if osp.exists(result_path): # 1 read config cfg = Config.fromfile(config) total_epochs = cfg.runner.max_epochs final_results = cfg.evaluation.metric if not isinstance(final_results, list): final_results = [final_results] final_results_out = [] for key in final_results: if 'proposal_fast' in key: final_results_out.append('AR@1000') # RPN elif 'mAP' not in key: final_results_out.append(key + '_mAP') # 2 determine whether total_epochs ckpt exists ckpt_path = f'epoch_{total_epochs}.pth' if osp.exists(osp.join(result_path, ckpt_path)): log_json_path = list( sorted(glob.glob(osp.join(result_path, '*.log.json'))))[-1] # 3 read metric model_performance = get_final_results( log_json_path, total_epochs, final_results_out) if model_performance is None: print(f'log file error: {log_json_path}') continue for performance in model_performance: if performance in ['AR@1000', 'bbox_mAP', 'segm_mAP']: metric = round( model_performance[performance] * 100, 1) model_performance[performance] = metric result_dict[config] = model_performance # update and append excel content if args.excel: if 'AR@1000' in model_performance: metrics = f'{model_performance["AR@1000"]}' \ f'(AR@1000)' elif 'segm_mAP' in model_performance: metrics = f'{model_performance["bbox_mAP"]}/' \ f'{model_performance["segm_mAP"]}' else: metrics = f'{model_performance["bbox_mAP"]}' row_num = sheet_info.get(config, None) if row_num: table.write(row_num, args.ncol, metrics) else: table.write(total_nrows, 0, config) table.write(total_nrows, args.ncol, metrics) total_nrows += 1 else: print(f'{config} not exist: {ckpt_path}') else: print(f'not exist: {config}') # 4 save or print results if metrics_out: mkdir_or_exist(metrics_out) dump(result_dict, osp.join(metrics_out, 'model_metric_info.json')) if not args.not_show: print('===================================') for config_name, metrics in result_dict.items(): print(config_name, metrics) print('===================================') if args.excel: filename, sufflx = osp.splitext(args.excel) xlrw.save(f'{filename}_o{sufflx}') print(f'>>> Output {filename}_o{sufflx}') ================================================ FILE: .dev_scripts/linter.sh ================================================ yapf -r -i mmdet/ configs/ tests/ tools/ isort -rc mmdet/ configs/ tests/ tools/ flake8 . ================================================ FILE: .dev_scripts/test_benchmark.sh ================================================ PARTITION=$1 CHECKPOINT_DIR=$2 WORK_DIR=$3 CPUS_PER_TASK=${4:-2} echo 'configs/atss/atss_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION atss_r50_fpn_1x_coco configs/atss/atss_r50_fpn_1x_coco.py $CHECKPOINT_DIR/atss_r50_fpn_1x_coco_20200209-985f7bd0.pth --work-dir $WORK_DIR/atss_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29666 & echo 'configs/autoassign/autoassign_r50-caffe_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION autoassign_r50-caffe_fpn_1x_coco configs/autoassign/autoassign_r50-caffe_fpn_1x_coco.py $CHECKPOINT_DIR/auto_assign_r50_fpn_1x_coco_20210413_115540-5e17991f.pth --work-dir $WORK_DIR/autoassign_r50-caffe_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29667 & echo 'configs/carafe/faster-rcnn_r50_fpn-carafe_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_r50_fpn-carafe_1x_coco configs/carafe/faster-rcnn_r50_fpn-carafe_1x_coco.py $CHECKPOINT_DIR/faster_rcnn_r50_fpn_carafe_1x_coco_bbox_mAP-0.386_20200504_175733-385a75b7.pth --work-dir $WORK_DIR/faster-rcnn_r50_fpn-carafe_1x_coco --cfg-option env_cfg.dist_cfg.port=29668 & echo 'configs/cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION cascade-rcnn_r50_fpn_1x_coco configs/cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/cascade_rcnn_r50_fpn_1x_coco_20200316-3dc56deb.pth --work-dir $WORK_DIR/cascade-rcnn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29669 & echo 'configs/cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION cascade-mask-rcnn_r50_fpn_1x_coco configs/cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/cascade_mask_rcnn_r50_fpn_1x_coco_20200203-9d4dcb24.pth --work-dir $WORK_DIR/cascade-mask-rcnn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29670 & echo 'configs/cascade_rpn/cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco configs/cascade_rpn/cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco.py $CHECKPOINT_DIR/crpn_faster_rcnn_r50_caffe_fpn_1x_coco-c8283cca.pth --work-dir $WORK_DIR/cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29671 & echo 'configs/centernet/centernet_r18-dcnv2_8xb16-crop512-140e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION centernet_r18-dcnv2_8xb16-crop512-140e_coco configs/centernet/centernet_r18-dcnv2_8xb16-crop512-140e_coco.py $CHECKPOINT_DIR/centernet_resnet18_dcnv2_140e_coco_20210702_155131-c8cd631f.pth --work-dir $WORK_DIR/centernet_r18-dcnv2_8xb16-crop512-140e_coco --cfg-option env_cfg.dist_cfg.port=29672 & echo 'configs/centripetalnet/centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco configs/centripetalnet/centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco.py $CHECKPOINT_DIR/centripetalnet_hourglass104_mstest_16x6_210e_coco_20200915_204804-3ccc61e5.pth --work-dir $WORK_DIR/centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco --cfg-option env_cfg.dist_cfg.port=29673 & echo 'configs/convnext/cascade-mask-rcnn_convnext-s-p4-w7_fpn_4conv1fc-giou_amp-ms-crop-3x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION cascade-mask-rcnn_convnext-s-p4-w7_fpn_4conv1fc-giou_amp-ms-crop-3x_coco configs/convnext/cascade-mask-rcnn_convnext-s-p4-w7_fpn_4conv1fc-giou_amp-ms-crop-3x_coco.py $CHECKPOINT_DIR/cascade_mask_rcnn_convnext-s_p4_w7_fpn_giou_4conv1f_fp16_ms-crop_3x_coco_20220510_201004-3d24f5a4.pth --work-dir $WORK_DIR/cascade-mask-rcnn_convnext-s-p4-w7_fpn_4conv1fc-giou_amp-ms-crop-3x_coco --cfg-option env_cfg.dist_cfg.port=29674 & echo 'configs/cornernet/cornernet_hourglass104_8xb6-210e-mstest_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION cornernet_hourglass104_8xb6-210e-mstest_coco configs/cornernet/cornernet_hourglass104_8xb6-210e-mstest_coco.py $CHECKPOINT_DIR/cornernet_hourglass104_mstest_8x6_210e_coco_20200825_150618-79b44c30.pth --work-dir $WORK_DIR/cornernet_hourglass104_8xb6-210e-mstest_coco --cfg-option env_cfg.dist_cfg.port=29675 & echo 'configs/dcn/faster-rcnn_r50-dconv-c3-c5_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_r50-dconv-c3-c5_fpn_1x_coco configs/dcn/faster-rcnn_r50-dconv-c3-c5_fpn_1x_coco.py $CHECKPOINT_DIR/faster_rcnn_r50_fpn_dconv_c3-c5_1x_coco_20200130-d68aed1e.pth --work-dir $WORK_DIR/faster-rcnn_r50-dconv-c3-c5_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29676 & echo 'configs/dcnv2/faster-rcnn_r50_fpn_mdpool_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_r50_fpn_mdpool_1x_coco configs/dcnv2/faster-rcnn_r50_fpn_mdpool_1x_coco.py $CHECKPOINT_DIR/faster_rcnn_r50_fpn_mdpool_1x_coco_20200307-c0df27ff.pth --work-dir $WORK_DIR/faster-rcnn_r50_fpn_mdpool_1x_coco --cfg-option env_cfg.dist_cfg.port=29677 & echo 'configs/ddod/ddod_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION ddod_r50_fpn_1x_coco configs/ddod/ddod_r50_fpn_1x_coco.py $CHECKPOINT_DIR/ddod_r50_fpn_1x_coco_20220523_223737-29b2fc67.pth --work-dir $WORK_DIR/ddod_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29678 & echo 'configs/deformable_detr/deformable-detr_r50_16xb2-50e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION deformable-detr_r50_16xb2-50e_coco configs/deformable_detr/deformable-detr_r50_16xb2-50e_coco.py $CHECKPOINT_DIR/deformable_detr_r50_16x2_50e_coco_20210419_220030-a12b9512.pth --work-dir $WORK_DIR/deformable-detr_r50_16xb2-50e_coco --cfg-option env_cfg.dist_cfg.port=29679 & echo 'configs/detectors/detectors_htc-r50_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION detectors_htc-r50_1x_coco configs/detectors/detectors_htc-r50_1x_coco.py $CHECKPOINT_DIR/detectors_htc_r50_1x_coco-329b1453.pth --work-dir $WORK_DIR/detectors_htc-r50_1x_coco --cfg-option env_cfg.dist_cfg.port=29680 & echo 'configs/detr/detr_r50_8xb2-150e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION detr_r50_8xb2-150e_coco configs/detr/detr_r50_8xb2-150e_coco.py $CHECKPOINT_DIR/detr_r50_8x2_150e_coco_20201130_194835-2c4b8974.pth --work-dir $WORK_DIR/detr_r50_8xb2-150e_coco --cfg-option env_cfg.dist_cfg.port=29681 & echo 'configs/double_heads/dh-faster-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION dh-faster-rcnn_r50_fpn_1x_coco configs/double_heads/dh-faster-rcnn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/dh_faster_rcnn_r50_fpn_1x_coco_20200130-586b67df.pth --work-dir $WORK_DIR/dh-faster-rcnn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29682 & echo 'configs/dyhead/atss_r50_fpn_dyhead_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION atss_r50_fpn_dyhead_1x_coco configs/dyhead/atss_r50_fpn_dyhead_1x_coco.py $CHECKPOINT_DIR/atss_r50_fpn_dyhead_4x4_1x_coco_20211219_023314-eaa620c6.pth --work-dir $WORK_DIR/atss_r50_fpn_dyhead_1x_coco --cfg-option env_cfg.dist_cfg.port=29683 & echo 'configs/dynamic_rcnn/dynamic-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION dynamic-rcnn_r50_fpn_1x_coco configs/dynamic_rcnn/dynamic-rcnn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/dynamic_rcnn_r50_fpn_1x-62a3f276.pth --work-dir $WORK_DIR/dynamic-rcnn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29684 & echo 'configs/efficientnet/retinanet_effb3_fpn_8xb4-crop896-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION retinanet_effb3_fpn_8xb4-crop896-1x_coco configs/efficientnet/retinanet_effb3_fpn_8xb4-crop896-1x_coco.py $CHECKPOINT_DIR/retinanet_effb3_fpn_crop896_8x4_1x_coco_20220322_234806-615a0dda.pth --work-dir $WORK_DIR/retinanet_effb3_fpn_8xb4-crop896-1x_coco --cfg-option env_cfg.dist_cfg.port=29685 & echo 'configs/empirical_attention/faster-rcnn_r50-attn1111_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_r50-attn1111_fpn_1x_coco configs/empirical_attention/faster-rcnn_r50-attn1111_fpn_1x_coco.py $CHECKPOINT_DIR/faster_rcnn_r50_fpn_attention_1111_1x_coco_20200130-403cccba.pth --work-dir $WORK_DIR/faster-rcnn_r50-attn1111_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29686 & echo 'configs/faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_r50_fpn_1x_coco configs/faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/faster_rcnn_r50_fpn_1x_coco_20200130-047c8118.pth --work-dir $WORK_DIR/faster-rcnn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29687 & echo 'configs/fcos/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco configs/fcos/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py $CHECKPOINT_DIR/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_1x_coco-0a0d75a8.pth --work-dir $WORK_DIR/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco --cfg-option env_cfg.dist_cfg.port=29688 & echo 'configs/foveabox/fovea_r50_fpn_gn-head-align_4xb4-2x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION fovea_r50_fpn_gn-head-align_4xb4-2x_coco configs/foveabox/fovea_r50_fpn_gn-head-align_4xb4-2x_coco.py $CHECKPOINT_DIR/fovea_align_r50_fpn_gn-head_4x4_2x_coco_20200203-8987880d.pth --work-dir $WORK_DIR/fovea_r50_fpn_gn-head-align_4xb4-2x_coco --cfg-option env_cfg.dist_cfg.port=29689 & echo 'configs/fpg/mask-rcnn_r50_fpg_crop640-50e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION mask-rcnn_r50_fpg_crop640-50e_coco configs/fpg/mask-rcnn_r50_fpg_crop640-50e_coco.py $CHECKPOINT_DIR/mask_rcnn_r50_fpg_crop640_50e_coco_20220311_011857-233b8334.pth --work-dir $WORK_DIR/mask-rcnn_r50_fpg_crop640-50e_coco --cfg-option env_cfg.dist_cfg.port=29690 & echo 'configs/free_anchor/freeanchor_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION freeanchor_r50_fpn_1x_coco configs/free_anchor/freeanchor_r50_fpn_1x_coco.py $CHECKPOINT_DIR/retinanet_free_anchor_r50_fpn_1x_coco_20200130-0f67375f.pth --work-dir $WORK_DIR/freeanchor_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29691 & echo 'configs/fsaf/fsaf_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION fsaf_r50_fpn_1x_coco configs/fsaf/fsaf_r50_fpn_1x_coco.py $CHECKPOINT_DIR/fsaf_r50_fpn_1x_coco-94ccc51f.pth --work-dir $WORK_DIR/fsaf_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29692 & echo 'configs/gcnet/mask-rcnn_r50-syncbn-gcb-r16-c3-c5_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION mask-rcnn_r50-syncbn-gcb-r16-c3-c5_fpn_1x_coco configs/gcnet/mask-rcnn_r50-syncbn-gcb-r16-c3-c5_fpn_1x_coco.py $CHECKPOINT_DIR/mask_rcnn_r50_fpn_syncbn-backbone_r16_gcb_c3-c5_1x_coco_20200202-587b99aa.pth --work-dir $WORK_DIR/mask-rcnn_r50-syncbn-gcb-r16-c3-c5_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29693 & echo 'configs/gfl/gfl_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION gfl_r50_fpn_1x_coco configs/gfl/gfl_r50_fpn_1x_coco.py $CHECKPOINT_DIR/gfl_r50_fpn_1x_coco_20200629_121244-25944287.pth --work-dir $WORK_DIR/gfl_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29694 & echo 'configs/ghm/retinanet_r50_fpn_ghm-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION retinanet_r50_fpn_ghm-1x_coco configs/ghm/retinanet_r50_fpn_ghm-1x_coco.py $CHECKPOINT_DIR/retinanet_ghm_r50_fpn_1x_coco_20200130-a437fda3.pth --work-dir $WORK_DIR/retinanet_r50_fpn_ghm-1x_coco --cfg-option env_cfg.dist_cfg.port=29695 & echo 'configs/gn/mask-rcnn_r50_fpn_gn-all_2x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION mask-rcnn_r50_fpn_gn-all_2x_coco configs/gn/mask-rcnn_r50_fpn_gn-all_2x_coco.py $CHECKPOINT_DIR/mask_rcnn_r50_fpn_gn-all_2x_coco_20200206-8eee02a6.pth --work-dir $WORK_DIR/mask-rcnn_r50_fpn_gn-all_2x_coco --cfg-option env_cfg.dist_cfg.port=29696 & echo 'configs/gn+ws/faster-rcnn_r50_fpn_gn-ws-all_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_r50_fpn_gn-ws-all_1x_coco configs/gn+ws/faster-rcnn_r50_fpn_gn-ws-all_1x_coco.py $CHECKPOINT_DIR/faster_rcnn_r50_fpn_gn_ws-all_1x_coco_20200130-613d9fe2.pth --work-dir $WORK_DIR/faster-rcnn_r50_fpn_gn-ws-all_1x_coco --cfg-option env_cfg.dist_cfg.port=29697 & echo 'configs/grid_rcnn/grid-rcnn_r50_fpn_gn-head_2x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION grid-rcnn_r50_fpn_gn-head_2x_coco configs/grid_rcnn/grid-rcnn_r50_fpn_gn-head_2x_coco.py $CHECKPOINT_DIR/grid_rcnn_r50_fpn_gn-head_2x_coco_20200130-6cca8223.pth --work-dir $WORK_DIR/grid-rcnn_r50_fpn_gn-head_2x_coco --cfg-option env_cfg.dist_cfg.port=29698 & echo 'configs/groie/faste-rcnn_r50_fpn_groie_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faste-rcnn_r50_fpn_groie_1x_coco configs/groie/faste-rcnn_r50_fpn_groie_1x_coco.py $CHECKPOINT_DIR/faster_rcnn_r50_fpn_groie_1x_coco_20200604_211715-66ee9516.pth --work-dir $WORK_DIR/faste-rcnn_r50_fpn_groie_1x_coco --cfg-option env_cfg.dist_cfg.port=29699 & echo 'configs/guided_anchoring/ga-retinanet_r50-caffe_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION ga-retinanet_r50-caffe_fpn_1x_coco configs/guided_anchoring/ga-retinanet_r50-caffe_fpn_1x_coco.py $CHECKPOINT_DIR/ga_retinanet_r50_caffe_fpn_1x_coco_20201020-39581c6f.pth --work-dir $WORK_DIR/ga-retinanet_r50-caffe_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29700 & echo 'configs/hrnet/faster-rcnn_hrnetv2p-w18-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_hrnetv2p-w18-1x_coco configs/hrnet/faster-rcnn_hrnetv2p-w18-1x_coco.py $CHECKPOINT_DIR/faster_rcnn_hrnetv2p_w18_1x_coco_20200130-56651a6d.pth --work-dir $WORK_DIR/faster-rcnn_hrnetv2p-w18-1x_coco --cfg-option env_cfg.dist_cfg.port=29701 & echo 'configs/htc/htc_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION htc_r50_fpn_1x_coco configs/htc/htc_r50_fpn_1x_coco.py $CHECKPOINT_DIR/htc_r50_fpn_1x_coco_20200317-7332cf16.pth --work-dir $WORK_DIR/htc_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29702 & echo 'configs/instaboost/mask-rcnn_r50_fpn_instaboost-4x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION mask-rcnn_r50_fpn_instaboost-4x_coco configs/instaboost/mask-rcnn_r50_fpn_instaboost-4x_coco.py $CHECKPOINT_DIR/mask_rcnn_r50_fpn_instaboost_4x_coco_20200307-d025f83a.pth --work-dir $WORK_DIR/mask-rcnn_r50_fpn_instaboost-4x_coco --cfg-option env_cfg.dist_cfg.port=29703 & echo 'configs/libra_rcnn/libra-faster-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION libra-faster-rcnn_r50_fpn_1x_coco configs/libra_rcnn/libra-faster-rcnn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/libra_faster_rcnn_r50_fpn_1x_coco_20200130-3afee3a9.pth --work-dir $WORK_DIR/libra-faster-rcnn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29704 & echo 'configs/mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION mask2former_r50_8xb2-lsj-50e_coco-panoptic configs/mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py $CHECKPOINT_DIR/mask2former_r50_lsj_8x2_50e_coco-panoptic_20220326_224516-11a44721.pth --work-dir $WORK_DIR/mask2former_r50_8xb2-lsj-50e_coco-panoptic --cfg-option env_cfg.dist_cfg.port=29705 & echo 'configs/mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION mask-rcnn_r50_fpn_1x_coco configs/mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/mask_rcnn_r50_fpn_1x_coco_20200205-d4b0c5d6.pth --work-dir $WORK_DIR/mask-rcnn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29706 & echo 'configs/maskformer/maskformer_r50_ms-16xb1-75e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION maskformer_r50_ms-16xb1-75e_coco configs/maskformer/maskformer_r50_ms-16xb1-75e_coco.py $CHECKPOINT_DIR/maskformer_r50_mstrain_16x1_75e_coco_20220221_141956-bc2699cb.pth --work-dir $WORK_DIR/maskformer_r50_ms-16xb1-75e_coco --cfg-option env_cfg.dist_cfg.port=29707 & echo 'configs/ms_rcnn/ms-rcnn_r50-caffe_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION ms-rcnn_r50-caffe_fpn_1x_coco configs/ms_rcnn/ms-rcnn_r50-caffe_fpn_1x_coco.py $CHECKPOINT_DIR/ms_rcnn_r50_caffe_fpn_1x_coco_20200702_180848-61c9355e.pth --work-dir $WORK_DIR/ms-rcnn_r50-caffe_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29708 & echo 'configs/nas_fcos/nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco configs/nas_fcos/nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco.py $CHECKPOINT_DIR/nas_fcos_nashead_r50_caffe_fpn_gn-head_4x4_1x_coco_20200520-1bdba3ce.pth --work-dir $WORK_DIR/nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco --cfg-option env_cfg.dist_cfg.port=29709 & echo 'configs/nas_fpn/retinanet_r50_nasfpn_crop640-50e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION retinanet_r50_nasfpn_crop640-50e_coco configs/nas_fpn/retinanet_r50_nasfpn_crop640-50e_coco.py $CHECKPOINT_DIR/retinanet_r50_nasfpn_crop640_50e_coco-0ad1f644.pth --work-dir $WORK_DIR/retinanet_r50_nasfpn_crop640-50e_coco --cfg-option env_cfg.dist_cfg.port=29710 & echo 'configs/paa/paa_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION paa_r50_fpn_1x_coco configs/paa/paa_r50_fpn_1x_coco.py $CHECKPOINT_DIR/paa_r50_fpn_1x_coco_20200821-936edec3.pth --work-dir $WORK_DIR/paa_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29711 & echo 'configs/pafpn/faster-rcnn_r50_pafpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_r50_pafpn_1x_coco configs/pafpn/faster-rcnn_r50_pafpn_1x_coco.py $CHECKPOINT_DIR/faster_rcnn_r50_pafpn_1x_coco_bbox_mAP-0.375_20200503_105836-b7b4b9bd.pth --work-dir $WORK_DIR/faster-rcnn_r50_pafpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29712 & echo 'configs/panoptic_fpn/panoptic-fpn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION panoptic-fpn_r50_fpn_1x_coco configs/panoptic_fpn/panoptic-fpn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/panoptic_fpn_r50_fpn_1x_coco_20210821_101153-9668fd13.pth --work-dir $WORK_DIR/panoptic-fpn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29713 & echo 'configs/pisa/faster-rcnn_r50_fpn_pisa_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_r50_fpn_pisa_1x_coco configs/pisa/faster-rcnn_r50_fpn_pisa_1x_coco.py $CHECKPOINT_DIR/pisa_faster_rcnn_r50_fpn_1x_coco-dea93523.pth --work-dir $WORK_DIR/faster-rcnn_r50_fpn_pisa_1x_coco --cfg-option env_cfg.dist_cfg.port=29714 & echo 'configs/point_rend/point-rend_r50-caffe_fpn_ms-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION point-rend_r50-caffe_fpn_ms-1x_coco configs/point_rend/point-rend_r50-caffe_fpn_ms-1x_coco.py $CHECKPOINT_DIR/point_rend_r50_caffe_fpn_mstrain_1x_coco-1bcb5fb4.pth --work-dir $WORK_DIR/point-rend_r50-caffe_fpn_ms-1x_coco --cfg-option env_cfg.dist_cfg.port=29715 & echo 'configs/pvt/retinanet_pvt-s_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION retinanet_pvt-s_fpn_1x_coco configs/pvt/retinanet_pvt-s_fpn_1x_coco.py $CHECKPOINT_DIR/retinanet_pvt-s_fpn_1x_coco_20210906_142921-b6c94a5b.pth --work-dir $WORK_DIR/retinanet_pvt-s_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29716 & echo 'configs/queryinst/queryinst_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION queryinst_r50_fpn_1x_coco configs/queryinst/queryinst_r50_fpn_1x_coco.py $CHECKPOINT_DIR/queryinst_r50_fpn_1x_coco_20210907_084916-5a8f1998.pth --work-dir $WORK_DIR/queryinst_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29717 & echo 'configs/regnet/mask-rcnn_regnetx-3.2GF_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION mask-rcnn_regnetx-3.2GF_fpn_1x_coco configs/regnet/mask-rcnn_regnetx-3.2GF_fpn_1x_coco.py $CHECKPOINT_DIR/mask_rcnn_regnetx-3.2GF_fpn_1x_coco_20200520_163141-2a9d1814.pth --work-dir $WORK_DIR/mask-rcnn_regnetx-3.2GF_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29718 & echo 'configs/reppoints/reppoints-moment_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION reppoints-moment_r50_fpn_1x_coco configs/reppoints/reppoints-moment_r50_fpn_1x_coco.py $CHECKPOINT_DIR/reppoints_moment_r50_fpn_1x_coco_20200330-b73db8d1.pth --work-dir $WORK_DIR/reppoints-moment_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29719 & echo 'configs/res2net/faster-rcnn_res2net-101_fpn_2x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_res2net-101_fpn_2x_coco configs/res2net/faster-rcnn_res2net-101_fpn_2x_coco.py $CHECKPOINT_DIR/faster_rcnn_r2_101_fpn_2x_coco-175f1da6.pth --work-dir $WORK_DIR/faster-rcnn_res2net-101_fpn_2x_coco --cfg-option env_cfg.dist_cfg.port=29720 & echo 'configs/resnest/faster-rcnn_s50_fpn_syncbn-backbone+head_ms-range-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION faster-rcnn_s50_fpn_syncbn-backbone+head_ms-range-1x_coco configs/resnest/faster-rcnn_s50_fpn_syncbn-backbone+head_ms-range-1x_coco.py $CHECKPOINT_DIR/faster_rcnn_s50_fpn_syncbn-backbone+head_mstrain-range_1x_coco_20200926_125502-20289c16.pth --work-dir $WORK_DIR/faster-rcnn_s50_fpn_syncbn-backbone+head_ms-range-1x_coco --cfg-option env_cfg.dist_cfg.port=29721 & echo 'configs/resnet_strikes_back/mask-rcnn_r50-rsb-pre_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION mask-rcnn_r50-rsb-pre_fpn_1x_coco configs/resnet_strikes_back/mask-rcnn_r50-rsb-pre_fpn_1x_coco.py $CHECKPOINT_DIR/mask_rcnn_r50_fpn_rsb-pretrain_1x_coco_20220113_174054-06ce8ba0.pth --work-dir $WORK_DIR/mask-rcnn_r50-rsb-pre_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29722 & echo 'configs/retinanet/retinanet_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION retinanet_r50_fpn_1x_coco configs/retinanet/retinanet_r50_fpn_1x_coco.py $CHECKPOINT_DIR/retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth --work-dir $WORK_DIR/retinanet_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29723 & echo 'configs/rpn/rpn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION rpn_r50_fpn_1x_coco configs/rpn/rpn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/rpn_r50_fpn_1x_coco_20200218-5525fa2e.pth --work-dir $WORK_DIR/rpn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29724 & echo 'configs/sabl/sabl-retinanet_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION sabl-retinanet_r50_fpn_1x_coco configs/sabl/sabl-retinanet_r50_fpn_1x_coco.py $CHECKPOINT_DIR/sabl_retinanet_r50_fpn_1x_coco-6c54fd4f.pth --work-dir $WORK_DIR/sabl-retinanet_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29725 & echo 'configs/sabl/sabl-faster-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION sabl-faster-rcnn_r50_fpn_1x_coco configs/sabl/sabl-faster-rcnn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/sabl_faster_rcnn_r50_fpn_1x_coco-e867595b.pth --work-dir $WORK_DIR/sabl-faster-rcnn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29726 & echo 'configs/scnet/scnet_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION scnet_r50_fpn_1x_coco configs/scnet/scnet_r50_fpn_1x_coco.py $CHECKPOINT_DIR/scnet_r50_fpn_1x_coco-c3f09857.pth --work-dir $WORK_DIR/scnet_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29727 & echo 'configs/scratch/mask-rcnn_r50-scratch_fpn_gn-all_6x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION mask-rcnn_r50-scratch_fpn_gn-all_6x_coco configs/scratch/mask-rcnn_r50-scratch_fpn_gn-all_6x_coco.py $CHECKPOINT_DIR/scratch_mask_rcnn_r50_fpn_gn_6x_bbox_mAP-0.412__segm_mAP-0.374_20200201_193051-1e190a40.pth --work-dir $WORK_DIR/mask-rcnn_r50-scratch_fpn_gn-all_6x_coco --cfg-option env_cfg.dist_cfg.port=29728 & echo 'configs/solo/decoupled-solo_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION decoupled-solo_r50_fpn_1x_coco configs/solo/decoupled-solo_r50_fpn_1x_coco.py $CHECKPOINT_DIR/decoupled_solo_r50_fpn_1x_coco_20210820_233348-6337c589.pth --work-dir $WORK_DIR/decoupled-solo_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29729 & echo 'configs/solov2/solov2_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION solov2_r50_fpn_1x_coco configs/solov2/solov2_r50_fpn_1x_coco.py $CHECKPOINT_DIR/solov2_r50_fpn_1x_coco_20220512_125858-a357fa23.pth --work-dir $WORK_DIR/solov2_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29730 & echo 'configs/sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION sparse-rcnn_r50_fpn_1x_coco configs/sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py $CHECKPOINT_DIR/sparse_rcnn_r50_fpn_1x_coco_20201222_214453-dc79b137.pth --work-dir $WORK_DIR/sparse-rcnn_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29731 & echo 'configs/ssd/ssd300_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION ssd300_coco configs/ssd/ssd300_coco.py $CHECKPOINT_DIR/ssd300_coco_20210803_015428-d231a06e.pth --work-dir $WORK_DIR/ssd300_coco --cfg-option env_cfg.dist_cfg.port=29732 & echo 'configs/ssd/ssdlite_mobilenetv2-scratch_8xb24-600e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION ssdlite_mobilenetv2-scratch_8xb24-600e_coco configs/ssd/ssdlite_mobilenetv2-scratch_8xb24-600e_coco.py $CHECKPOINT_DIR/ssdlite_mobilenetv2_scratch_600e_coco_20210629_110627-974d9307.pth --work-dir $WORK_DIR/ssdlite_mobilenetv2-scratch_8xb24-600e_coco --cfg-option env_cfg.dist_cfg.port=29733 & echo 'configs/swin/mask-rcnn_swin-t-p4-w7_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION mask-rcnn_swin-t-p4-w7_fpn_1x_coco configs/swin/mask-rcnn_swin-t-p4-w7_fpn_1x_coco.py $CHECKPOINT_DIR/mask_rcnn_swin-t-p4-w7_fpn_1x_coco_20210902_120937-9d6b7cfa.pth --work-dir $WORK_DIR/mask-rcnn_swin-t-p4-w7_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29734 & echo 'configs/tood/tood_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION tood_r50_fpn_1x_coco configs/tood/tood_r50_fpn_1x_coco.py $CHECKPOINT_DIR/tood_r50_fpn_1x_coco_20211210_103425-20e20746.pth --work-dir $WORK_DIR/tood_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29735 & echo 'configs/tridentnet/tridentnet_r50-caffe_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION tridentnet_r50-caffe_1x_coco configs/tridentnet/tridentnet_r50-caffe_1x_coco.py $CHECKPOINT_DIR/tridentnet_r50_caffe_1x_coco_20201230_141838-2ec0b530.pth --work-dir $WORK_DIR/tridentnet_r50-caffe_1x_coco --cfg-option env_cfg.dist_cfg.port=29736 & echo 'configs/vfnet/vfnet_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION vfnet_r50_fpn_1x_coco configs/vfnet/vfnet_r50_fpn_1x_coco.py $CHECKPOINT_DIR/vfnet_r50_fpn_1x_coco_20201027-38db6f58.pth --work-dir $WORK_DIR/vfnet_r50_fpn_1x_coco --cfg-option env_cfg.dist_cfg.port=29737 & echo 'configs/yolact/yolact_r50_1xb8-55e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION yolact_r50_1xb8-55e_coco configs/yolact/yolact_r50_1xb8-55e_coco.py $CHECKPOINT_DIR/yolact_r50_1x8_coco_20200908-f38d58df.pth --work-dir $WORK_DIR/yolact_r50_1xb8-55e_coco --cfg-option env_cfg.dist_cfg.port=29738 & echo 'configs/yolo/yolov3_d53_8xb8-320-273e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION yolov3_d53_8xb8-320-273e_coco configs/yolo/yolov3_d53_8xb8-320-273e_coco.py $CHECKPOINT_DIR/yolov3_d53_320_273e_coco-421362b6.pth --work-dir $WORK_DIR/yolov3_d53_8xb8-320-273e_coco --cfg-option env_cfg.dist_cfg.port=29739 & echo 'configs/yolof/yolof_r50-c5_8xb8-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION yolof_r50-c5_8xb8-1x_coco configs/yolof/yolof_r50-c5_8xb8-1x_coco.py $CHECKPOINT_DIR/yolof_r50_c5_8x8_1x_coco_20210425_024427-8e864411.pth --work-dir $WORK_DIR/yolof_r50-c5_8xb8-1x_coco --cfg-option env_cfg.dist_cfg.port=29740 & echo 'configs/yolox/yolox_tiny_8xb8-300e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK tools/slurm_test.sh $PARTITION yolox_tiny_8xb8-300e_coco configs/yolox/yolox_tiny_8xb8-300e_coco.py $CHECKPOINT_DIR/yolox_tiny_8x8_300e_coco_20211124_171234-b4047906.pth --work-dir $WORK_DIR/yolox_tiny_8xb8-300e_coco --cfg-option env_cfg.dist_cfg.port=29741 & ================================================ FILE: .dev_scripts/test_init_backbone.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """Check out backbone whether successfully load pretrained checkpoint.""" import copy import os from os.path import dirname, exists, join import pytest from mmengine.config import Config from mmengine.runner import CheckpointLoader from mmengine.utils import ProgressBar from mmdet.registry import MODELS def _get_config_directory(): """Find the predefined detector config directory.""" try: # Assume we are running in the source mmdetection repo repo_dpath = dirname(dirname(__file__)) except NameError: # For IPython development when this __file__ is not defined import mmdet repo_dpath = dirname(dirname(mmdet.__file__)) config_dpath = join(repo_dpath, 'configs') if not exists(config_dpath): raise Exception('Cannot find config path') return config_dpath def _get_config_module(fname): """Load a configuration as a python module.""" config_dpath = _get_config_directory() config_fpath = join(config_dpath, fname) config_mod = Config.fromfile(config_fpath) return config_mod def _get_detector_cfg(fname): """Grab configs necessary to create a detector. These are deep copied to allow for safe modification of parameters without influencing other tests. """ config = _get_config_module(fname) model = copy.deepcopy(config.model) return model def _traversed_config_file(): """We traversed all potential config files under the `config` file. If you need to print details or debug code, you can use this function. If the `backbone.init_cfg` is None (do not use `Pretrained` init way), you need add the folder name in `ignores_folder` (if the config files in this folder all set backbone.init_cfg is None) or add config name in `ignores_file` (if the config file set backbone.init_cfg is None) """ config_path = _get_config_directory() check_cfg_names = [] # `base`, `legacy_1.x` and `common` ignored by default. ignores_folder = ['_base_', 'legacy_1.x', 'common'] # 'ld' need load teacher model, if want to check 'ld', # please check teacher_config path first. ignores_folder += ['ld'] # `selfsup_pretrain` need convert model, if want to check this model, # need to convert the model first. ignores_folder += ['selfsup_pretrain'] # the `init_cfg` in 'centripetalnet', 'cornernet', 'cityscapes', # 'scratch' is None. # the `init_cfg` in ssdlite(`ssdlite_mobilenetv2_scratch_600e_coco.py`) # is None # Please confirm `bockbone.init_cfg` is None first. ignores_folder += ['centripetalnet', 'cornernet', 'cityscapes', 'scratch'] ignores_file = ['ssdlite_mobilenetv2_scratch_600e_coco.py'] for config_file_name in os.listdir(config_path): if config_file_name not in ignores_folder: config_file = join(config_path, config_file_name) if os.path.isdir(config_file): for config_sub_file in os.listdir(config_file): if config_sub_file.endswith('py') and \ config_sub_file not in ignores_file: name = join(config_file, config_sub_file) check_cfg_names.append(name) return check_cfg_names def _check_backbone(config, print_cfg=True): """Check out backbone whether successfully load pretrained model, by using `backbone.init_cfg`. First, using `CheckpointLoader.load_checkpoint` to load the checkpoint without loading models. Then, using `MODELS.build` to build models, and using `model.init_weights()` to initialize the parameters. Finally, assert weights and bias of each layer loaded from pretrained checkpoint are equal to the weights and bias of original checkpoint. For the convenience of comparison, we sum up weights and bias of each loaded layer separately. Args: config (str): Config file path. print_cfg (bool): Whether print logger and return the result. Returns: results (str or None): If backbone successfully load pretrained checkpoint, return None; else, return config file path. """ if print_cfg: print('-' * 15 + 'loading ', config) cfg = Config.fromfile(config) init_cfg = None try: init_cfg = cfg.model.backbone.init_cfg init_flag = True except AttributeError: init_flag = False if init_cfg is None or init_cfg.get('type') != 'Pretrained': init_flag = False if init_flag: checkpoint = CheckpointLoader.load_checkpoint(init_cfg.checkpoint) if 'state_dict' in checkpoint: state_dict = checkpoint['state_dict'] else: state_dict = checkpoint model = MODELS.build(cfg.model) model.init_weights() checkpoint_layers = state_dict.keys() for name, value in model.backbone.state_dict().items(): if name in checkpoint_layers: assert value.equal(state_dict[name]) if print_cfg: print('-' * 10 + 'Successfully load checkpoint' + '-' * 10 + '\n', ) return None else: if print_cfg: print(config + '\n' + '-' * 10 + 'config file do not have init_cfg' + '-' * 10 + '\n') return config @pytest.mark.parametrize('config', _traversed_config_file()) def test_load_pretrained(config): """Check out backbone whether successfully load pretrained model by using `backbone.init_cfg`. Details please refer to `_check_backbone` """ _check_backbone(config, print_cfg=False) def _test_load_pretrained(): """We traversed all potential config files under the `config` file. If you need to print details or debug code, you can use this function. Returns: check_cfg_names (list[str]): Config files that backbone initialized from pretrained checkpoint might be problematic. Need to recheck the config file. The output including the config files that the backbone.init_cfg is None """ check_cfg_names = _traversed_config_file() need_check_cfg = [] prog_bar = ProgressBar(len(check_cfg_names)) for config in check_cfg_names: init_cfg_name = _check_backbone(config) if init_cfg_name is not None: need_check_cfg.append(init_cfg_name) prog_bar.update() print('These config files need to be checked again') print(need_check_cfg) ================================================ FILE: .dev_scripts/train_benchmark.sh ================================================ PARTITION=$1 WORK_DIR=$2 CPUS_PER_TASK=${3:-4} echo 'configs/albu_example/mask-rcnn_r50_fpn_albu_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION mask-rcnn_r50_fpn_albu_1x_coco configs/albu_example/mask-rcnn_r50_fpn_albu_1x_coco.py $WORK_DIR/mask-rcnn_r50_fpn_albu_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/atss/atss_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION atss_r50_fpn_1x_coco configs/atss/atss_r50_fpn_1x_coco.py $WORK_DIR/atss_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/autoassign/autoassign_r50-caffe_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION autoassign_r50-caffe_fpn_1x_coco configs/autoassign/autoassign_r50-caffe_fpn_1x_coco.py $WORK_DIR/autoassign_r50-caffe_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/carafe/faster-rcnn_r50_fpn-carafe_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_r50_fpn-carafe_1x_coco configs/carafe/faster-rcnn_r50_fpn-carafe_1x_coco.py $WORK_DIR/faster-rcnn_r50_fpn-carafe_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION cascade-rcnn_r50_fpn_1x_coco configs/cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py $WORK_DIR/cascade-rcnn_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION cascade-mask-rcnn_r50_fpn_1x_coco configs/cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py $WORK_DIR/cascade-mask-rcnn_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/cascade_rpn/cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco configs/cascade_rpn/cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco.py $WORK_DIR/cascade-rpn_faster-rcnn_r50-caffe_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/centernet/centernet_r18-dcnv2_8xb16-crop512-140e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION centernet_r18-dcnv2_8xb16-crop512-140e_coco configs/centernet/centernet_r18-dcnv2_8xb16-crop512-140e_coco.py $WORK_DIR/centernet_r18-dcnv2_8xb16-crop512-140e_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/centernet/centernet-update_r50-caffe_fpn_ms-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION centernet-update_r50-caffe_fpn_ms-1x_coco configs/centernet/centernet-update_r50-caffe_fpn_ms-1x_coco.py $WORK_DIR/centernet-update_r50-caffe_fpn_ms-1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/centripetalnet/centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco.py' & GPUS=16 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco configs/centripetalnet/centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco.py $WORK_DIR/centripetalnet_hourglass104_16xb6-crop511-210e-mstest_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/cornernet/cornernet_hourglass104_8xb6-210e-mstest_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION cornernet_hourglass104_8xb6-210e-mstest_coco configs/cornernet/cornernet_hourglass104_8xb6-210e-mstest_coco.py $WORK_DIR/cornernet_hourglass104_8xb6-210e-mstest_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/convnext/mask-rcnn_convnext-t-p4-w7_fpn_amp-ms-crop-3x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION mask-rcnn_convnext-t-p4-w7_fpn_amp-ms-crop-3x_coco configs/convnext/mask-rcnn_convnext-t-p4-w7_fpn_amp-ms-crop-3x_coco.py $WORK_DIR/mask-rcnn_convnext-t-p4-w7_fpn_amp-ms-crop-3x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/dcn/faster-rcnn_r50-dconv-c3-c5_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_r50-dconv-c3-c5_fpn_1x_coco configs/dcn/faster-rcnn_r50-dconv-c3-c5_fpn_1x_coco.py $WORK_DIR/faster-rcnn_r50-dconv-c3-c5_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/dcnv2/faster-rcnn_r50_fpn_mdpool_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_r50_fpn_mdpool_1x_coco configs/dcnv2/faster-rcnn_r50_fpn_mdpool_1x_coco.py $WORK_DIR/faster-rcnn_r50_fpn_mdpool_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/ddod/ddod_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION ddod_r50_fpn_1x_coco configs/ddod/ddod_r50_fpn_1x_coco.py $WORK_DIR/ddod_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/detectors/detectors_htc-r50_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION detectors_htc-r50_1x_coco configs/detectors/detectors_htc-r50_1x_coco.py $WORK_DIR/detectors_htc-r50_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/deformable_detr/deformable-detr_r50_16xb2-50e_coco.py' & GPUS=16 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION deformable-detr_r50_16xb2-50e_coco configs/deformable_detr/deformable-detr_r50_16xb2-50e_coco.py $WORK_DIR/deformable-detr_r50_16xb2-50e_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/detr/detr_r50_8xb2-150e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION detr_r50_8xb2-150e_coco configs/detr/detr_r50_8xb2-150e_coco.py $WORK_DIR/detr_r50_8xb2-150e_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/double_heads/dh-faster-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION dh-faster-rcnn_r50_fpn_1x_coco configs/double_heads/dh-faster-rcnn_r50_fpn_1x_coco.py $WORK_DIR/dh-faster-rcnn_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/dynamic_rcnn/dynamic-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION dynamic-rcnn_r50_fpn_1x_coco configs/dynamic_rcnn/dynamic-rcnn_r50_fpn_1x_coco.py $WORK_DIR/dynamic-rcnn_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/dyhead/atss_r50_fpn_dyhead_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION atss_r50_fpn_dyhead_1x_coco configs/dyhead/atss_r50_fpn_dyhead_1x_coco.py $WORK_DIR/atss_r50_fpn_dyhead_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/efficientnet/retinanet_effb3_fpn_8xb4-crop896-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION retinanet_effb3_fpn_8xb4-crop896-1x_coco configs/efficientnet/retinanet_effb3_fpn_8xb4-crop896-1x_coco.py $WORK_DIR/retinanet_effb3_fpn_8xb4-crop896-1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/empirical_attention/faster-rcnn_r50-attn1111_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_r50-attn1111_fpn_1x_coco configs/empirical_attention/faster-rcnn_r50-attn1111_fpn_1x_coco.py $WORK_DIR/faster-rcnn_r50-attn1111_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_r50_fpn_1x_coco configs/faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py $WORK_DIR/faster-rcnn_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/faster_rcnn/faster-rcnn_r50-caffe-dc5_ms-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_r50-caffe-dc5_ms-1x_coco configs/faster_rcnn/faster-rcnn_r50-caffe-dc5_ms-1x_coco.py $WORK_DIR/faster-rcnn_r50-caffe-dc5_ms-1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/fcos/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco configs/fcos/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py $WORK_DIR/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/foveabox/fovea_r50_fpn_gn-head-align_4xb4-2x_coco.py' & GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION fovea_r50_fpn_gn-head-align_4xb4-2x_coco configs/foveabox/fovea_r50_fpn_gn-head-align_4xb4-2x_coco.py $WORK_DIR/fovea_r50_fpn_gn-head-align_4xb4-2x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/fpg/mask-rcnn_r50_fpg_crop640-50e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION mask-rcnn_r50_fpg_crop640-50e_coco configs/fpg/mask-rcnn_r50_fpg_crop640-50e_coco.py $WORK_DIR/mask-rcnn_r50_fpg_crop640-50e_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/free_anchor/freeanchor_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION freeanchor_r50_fpn_1x_coco configs/free_anchor/freeanchor_r50_fpn_1x_coco.py $WORK_DIR/freeanchor_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/fsaf/fsaf_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION fsaf_r50_fpn_1x_coco configs/fsaf/fsaf_r50_fpn_1x_coco.py $WORK_DIR/fsaf_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/gcnet/mask-rcnn_r50-syncbn-gcb-r16-c3-c5_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION mask-rcnn_r50-syncbn-gcb-r16-c3-c5_fpn_1x_coco configs/gcnet/mask-rcnn_r50-syncbn-gcb-r16-c3-c5_fpn_1x_coco.py $WORK_DIR/mask-rcnn_r50-syncbn-gcb-r16-c3-c5_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/gfl/gfl_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION gfl_r50_fpn_1x_coco configs/gfl/gfl_r50_fpn_1x_coco.py $WORK_DIR/gfl_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/ghm/retinanet_r50_fpn_ghm-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION retinanet_r50_fpn_ghm-1x_coco configs/ghm/retinanet_r50_fpn_ghm-1x_coco.py $WORK_DIR/retinanet_r50_fpn_ghm-1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/gn/mask-rcnn_r50_fpn_gn-all_2x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION mask-rcnn_r50_fpn_gn-all_2x_coco configs/gn/mask-rcnn_r50_fpn_gn-all_2x_coco.py $WORK_DIR/mask-rcnn_r50_fpn_gn-all_2x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/gn+ws/faster-rcnn_r50_fpn_gn-ws-all_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_r50_fpn_gn-ws-all_1x_coco configs/gn+ws/faster-rcnn_r50_fpn_gn-ws-all_1x_coco.py $WORK_DIR/faster-rcnn_r50_fpn_gn-ws-all_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/grid_rcnn/grid-rcnn_r50_fpn_gn-head_2x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION grid-rcnn_r50_fpn_gn-head_2x_coco configs/grid_rcnn/grid-rcnn_r50_fpn_gn-head_2x_coco.py $WORK_DIR/grid-rcnn_r50_fpn_gn-head_2x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/groie/faste-rcnn_r50_fpn_groie_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faste-rcnn_r50_fpn_groie_1x_coco configs/groie/faste-rcnn_r50_fpn_groie_1x_coco.py $WORK_DIR/faste-rcnn_r50_fpn_groie_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/guided_anchoring/ga-retinanet_r50-caffe_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION ga-retinanet_r50-caffe_fpn_1x_coco configs/guided_anchoring/ga-retinanet_r50-caffe_fpn_1x_coco.py $WORK_DIR/ga-retinanet_r50-caffe_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/hrnet/faster-rcnn_hrnetv2p-w18-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_hrnetv2p-w18-1x_coco configs/hrnet/faster-rcnn_hrnetv2p-w18-1x_coco.py $WORK_DIR/faster-rcnn_hrnetv2p-w18-1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/htc/htc_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION htc_r50_fpn_1x_coco configs/htc/htc_r50_fpn_1x_coco.py $WORK_DIR/htc_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/instaboost/mask-rcnn_r50_fpn_instaboost-4x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION mask-rcnn_r50_fpn_instaboost-4x_coco configs/instaboost/mask-rcnn_r50_fpn_instaboost-4x_coco.py $WORK_DIR/mask-rcnn_r50_fpn_instaboost-4x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/lad/lad_r50-paa-r101_fpn_2xb8_coco_1x.py' & GPUS=2 GPUS_PER_NODE=2 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION lad_r50-paa-r101_fpn_2xb8_coco_1x configs/lad/lad_r50-paa-r101_fpn_2xb8_coco_1x.py $WORK_DIR/lad_r50-paa-r101_fpn_2xb8_coco_1x --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/ld/ld_r18-gflv1-r101_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION ld_r18-gflv1-r101_fpn_1x_coco configs/ld/ld_r18-gflv1-r101_fpn_1x_coco.py $WORK_DIR/ld_r18-gflv1-r101_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/libra_rcnn/libra-faster-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION libra-faster-rcnn_r50_fpn_1x_coco configs/libra_rcnn/libra-faster-rcnn_r50_fpn_1x_coco.py $WORK_DIR/libra-faster-rcnn_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION mask2former_r50_8xb2-lsj-50e_coco-panoptic configs/mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py $WORK_DIR/mask2former_r50_8xb2-lsj-50e_coco-panoptic --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION mask-rcnn_r50_fpn_1x_coco configs/mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py $WORK_DIR/mask-rcnn_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/maskformer/maskformer_r50_ms-16xb1-75e_coco.py' & GPUS=16 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION maskformer_r50_ms-16xb1-75e_coco configs/maskformer/maskformer_r50_ms-16xb1-75e_coco.py $WORK_DIR/maskformer_r50_ms-16xb1-75e_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/ms_rcnn/ms-rcnn_r50-caffe_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION ms-rcnn_r50-caffe_fpn_1x_coco configs/ms_rcnn/ms-rcnn_r50-caffe_fpn_1x_coco.py $WORK_DIR/ms-rcnn_r50-caffe_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/nas_fcos/nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco.py' & GPUS=4 GPUS_PER_NODE=4 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco configs/nas_fcos/nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco.py $WORK_DIR/nas-fcos_r50-caffe_fpn_nashead-gn-head_4xb4-1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/nas_fpn/retinanet_r50_nasfpn_crop640-50e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION retinanet_r50_nasfpn_crop640-50e_coco configs/nas_fpn/retinanet_r50_nasfpn_crop640-50e_coco.py $WORK_DIR/retinanet_r50_nasfpn_crop640-50e_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/paa/paa_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION paa_r50_fpn_1x_coco configs/paa/paa_r50_fpn_1x_coco.py $WORK_DIR/paa_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/pafpn/faster-rcnn_r50_pafpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_r50_pafpn_1x_coco configs/pafpn/faster-rcnn_r50_pafpn_1x_coco.py $WORK_DIR/faster-rcnn_r50_pafpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/panoptic_fpn/panoptic-fpn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION panoptic-fpn_r50_fpn_1x_coco configs/panoptic_fpn/panoptic-fpn_r50_fpn_1x_coco.py $WORK_DIR/panoptic-fpn_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/pisa/mask-rcnn_r50_fpn_pisa_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION mask-rcnn_r50_fpn_pisa_1x_coco configs/pisa/mask-rcnn_r50_fpn_pisa_1x_coco.py $WORK_DIR/mask-rcnn_r50_fpn_pisa_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/point_rend/point-rend_r50-caffe_fpn_ms-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION point-rend_r50-caffe_fpn_ms-1x_coco configs/point_rend/point-rend_r50-caffe_fpn_ms-1x_coco.py $WORK_DIR/point-rend_r50-caffe_fpn_ms-1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/pvt/retinanet_pvt-t_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION retinanet_pvt-t_fpn_1x_coco configs/pvt/retinanet_pvt-t_fpn_1x_coco.py $WORK_DIR/retinanet_pvt-t_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/queryinst/queryinst_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION queryinst_r50_fpn_1x_coco configs/queryinst/queryinst_r50_fpn_1x_coco.py $WORK_DIR/queryinst_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/regnet/retinanet_regnetx-800MF_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION retinanet_regnetx-800MF_fpn_1x_coco configs/regnet/retinanet_regnetx-800MF_fpn_1x_coco.py $WORK_DIR/retinanet_regnetx-800MF_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/reppoints/reppoints-moment_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION reppoints-moment_r50_fpn_1x_coco configs/reppoints/reppoints-moment_r50_fpn_1x_coco.py $WORK_DIR/reppoints-moment_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/res2net/faster-rcnn_res2net-101_fpn_2x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_res2net-101_fpn_2x_coco configs/res2net/faster-rcnn_res2net-101_fpn_2x_coco.py $WORK_DIR/faster-rcnn_res2net-101_fpn_2x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/resnest/faster-rcnn_s50_fpn_syncbn-backbone+head_ms-range-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_s50_fpn_syncbn-backbone+head_ms-range-1x_coco configs/resnest/faster-rcnn_s50_fpn_syncbn-backbone+head_ms-range-1x_coco.py $WORK_DIR/faster-rcnn_s50_fpn_syncbn-backbone+head_ms-range-1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/resnet_strikes_back/retinanet_r50-rsb-pre_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION retinanet_r50-rsb-pre_fpn_1x_coco configs/resnet_strikes_back/retinanet_r50-rsb-pre_fpn_1x_coco.py $WORK_DIR/retinanet_r50-rsb-pre_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/retinanet/retinanet_r50-caffe_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION retinanet_r50-caffe_fpn_1x_coco configs/retinanet/retinanet_r50-caffe_fpn_1x_coco.py $WORK_DIR/retinanet_r50-caffe_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/rpn/rpn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION rpn_r50_fpn_1x_coco configs/rpn/rpn_r50_fpn_1x_coco.py $WORK_DIR/rpn_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/sabl/sabl-retinanet_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION sabl-retinanet_r50_fpn_1x_coco configs/sabl/sabl-retinanet_r50_fpn_1x_coco.py $WORK_DIR/sabl-retinanet_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/scnet/scnet_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION scnet_r50_fpn_1x_coco configs/scnet/scnet_r50_fpn_1x_coco.py $WORK_DIR/scnet_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/scratch/faster-rcnn_r50-scratch_fpn_gn-all_6x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION faster-rcnn_r50-scratch_fpn_gn-all_6x_coco configs/scratch/faster-rcnn_r50-scratch_fpn_gn-all_6x_coco.py $WORK_DIR/faster-rcnn_r50-scratch_fpn_gn-all_6x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/solo/solo_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION solo_r50_fpn_1x_coco configs/solo/solo_r50_fpn_1x_coco.py $WORK_DIR/solo_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/solov2/solov2_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION solov2_r50_fpn_1x_coco configs/solov2/solov2_r50_fpn_1x_coco.py $WORK_DIR/solov2_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION sparse-rcnn_r50_fpn_1x_coco configs/sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py $WORK_DIR/sparse-rcnn_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/ssd/ssd300_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION ssd300_coco configs/ssd/ssd300_coco.py $WORK_DIR/ssd300_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/ssd/ssdlite_mobilenetv2-scratch_8xb24-600e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION ssdlite_mobilenetv2-scratch_8xb24-600e_coco configs/ssd/ssdlite_mobilenetv2-scratch_8xb24-600e_coco.py $WORK_DIR/ssdlite_mobilenetv2-scratch_8xb24-600e_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/swin/mask-rcnn_swin-t-p4-w7_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION mask-rcnn_swin-t-p4-w7_fpn_1x_coco configs/swin/mask-rcnn_swin-t-p4-w7_fpn_1x_coco.py $WORK_DIR/mask-rcnn_swin-t-p4-w7_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/tood/tood_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION tood_r50_fpn_1x_coco configs/tood/tood_r50_fpn_1x_coco.py $WORK_DIR/tood_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo ''configs/tridentnet/tridentnet_r50-caffe_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION tridentnet_r50-caffe_1x_coco 'configs/tridentnet/tridentnet_r50-caffe_1x_coco.py $WORK_DIR/tridentnet_r50-caffe_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/vfnet/vfnet_r50_fpn_1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION vfnet_r50_fpn_1x_coco configs/vfnet/vfnet_r50_fpn_1x_coco.py $WORK_DIR/vfnet_r50_fpn_1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/yolact/yolact_r50_8xb8-55e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION yolact_r50_8xb8-55e_coco configs/yolact/yolact_r50_8xb8-55e_coco.py $WORK_DIR/yolact_r50_8xb8-55e_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/yolo/yolov3_d53_8xb8-320-273e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION yolov3_d53_8xb8-320-273e_coco configs/yolo/yolov3_d53_8xb8-320-273e_coco.py $WORK_DIR/yolov3_d53_8xb8-320-273e_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/yolof/yolof_r50-c5_8xb8-1x_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION yolof_r50-c5_8xb8-1x_coco configs/yolof/yolof_r50-c5_8xb8-1x_coco.py $WORK_DIR/yolof_r50-c5_8xb8-1x_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & echo 'configs/yolox/yolox_tiny_8xb8-300e_coco.py' & GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=$CPUS_PRE_TASK ./tools/slurm_train.sh $PARTITION yolox_tiny_8xb8-300e_coco configs/yolox/yolox_tiny_8xb8-300e_coco.py $WORK_DIR/yolox_tiny_8xb8-300e_coco --cfg-options default_hooks.checkpoint.max_keep_ckpts=1 & ================================================ FILE: .github/CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behavior that contributes to creating a positive environment include: - Using welcoming and inclusive language - Being respectful of differing viewpoints and experiences - Gracefully accepting constructive criticism - Focusing on what is best for the community - Showing empathy towards other community members Examples of unacceptable behavior by participants include: - The use of sexualized language or imagery and unwelcome sexual attention or advances - Trolling, insulting/derogatory comments, and personal or political attacks - Public or private harassment - Publishing others' private information, such as a physical or electronic address, without explicit permission - Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at chenkaidev@gmail.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq [homepage]: https://www.contributor-covenant.org ================================================ FILE: .github/CONTRIBUTING.md ================================================ We appreciate all contributions to improve MMDetection. Please refer to [CONTRIBUTING.md](https://github.com/open-mmlab/mmcv/blob/master/CONTRIBUTING.md) in MMCV for more details about the contributing guideline. ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: Common Issues url: https://mmdetection.readthedocs.io/en/latest/faq.html about: Check if your issue already has solutions - name: MMDetection Documentation url: https://mmdetection.readthedocs.io/en/latest/ about: Check if your question is answered in docs ================================================ FILE: .github/ISSUE_TEMPLATE/error-report.md ================================================ --- name: Error report about: Create a report to help us improve title: '' labels: '' assignees: '' --- Thanks for your error report and we appreciate it a lot. **Checklist** 1. I have searched related issues but cannot get the expected help. 2. I have read the [FAQ documentation](https://mmdetection.readthedocs.io/en/latest/faq.html) but cannot get the expected help. 3. The bug has not been fixed in the latest version. **Describe the bug** A clear and concise description of what the bug is. **Reproduction** 1. What command or script did you run? ```none A placeholder for the command. ``` 2. Did you make any modifications on the code or config? Did you understand what you have modified? 3. What dataset did you use? **Environment** 1. Please run `python mmdet/utils/collect_env.py` to collect necessary environment information and paste it here. 2. You may add addition that may be helpful for locating the problem, such as - How you installed PyTorch \[e.g., pip, conda, source\] - Other environment variables that may be related (such as `$PATH`, `$LD_LIBRARY_PATH`, `$PYTHONPATH`, etc.) **Error traceback** If applicable, paste the error trackback here. ```none A placeholder for trackback. ``` **Bug fix** If you have already identified the reason, you can provide the information here. If you are willing to create a PR to fix it, please also leave a comment here and that would be much appreciated! ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request about: Suggest an idea for this project title: '' labels: '' assignees: '' --- **Describe the feature** **Motivation** A clear and concise description of the motivation of the feature. Ex1. It is inconvenient when \[....\]. Ex2. There is a recent paper \[....\], which is very helpful for \[....\]. **Related resources** If there is an official code release or third-party implementations, please also provide the information here, which would be very helpful. **Additional context** Add any other context or screenshots about the feature request here. If you would like to implement the feature and create a PR, please leave a comment here and that would be much appreciated. ================================================ FILE: .github/ISSUE_TEMPLATE/general_questions.md ================================================ --- name: General questions about: Ask general questions to get help title: '' labels: '' assignees: '' --- ================================================ FILE: .github/ISSUE_TEMPLATE/reimplementation_questions.md ================================================ --- name: Reimplementation Questions about: Ask about questions during model reimplementation title: '' labels: reimplementation assignees: '' --- **Notice** There are several common situations in the reimplementation issues as below 1. Reimplement a model in the model zoo using the provided configs 2. Reimplement a model in the model zoo on other dataset (e.g., custom datasets) 3. Reimplement a custom model but all the components are implemented in MMDetection 4. Reimplement a custom model with new modules implemented by yourself There are several things to do for different cases as below. - For case 1 & 3, please follow the steps in the following sections thus we could help to quick identify the issue. - For case 2 & 4, please understand that we are not able to do much help here because we usually do not know the full code and the users should be responsible to the code they write. - One suggestion for case 2 & 4 is that the users should first check whether the bug lies in the self-implemented code or the original code. For example, users can first make sure that the same model runs well on supported datasets. If you still need help, please describe what you have done and what you obtain in the issue, and follow the steps in the following sections and try as clear as possible so that we can better help you. **Checklist** 1. I have searched related issues but cannot get the expected help. 2. The issue has not been fixed in the latest version. **Describe the issue** A clear and concise description of what the problem you meet and what have you done. **Reproduction** 1. What command or script did you run? ```none A placeholder for the command. ``` 2. What config dir you run? ```none A placeholder for the config. ``` 3. Did you make any modifications on the code or config? Did you understand what you have modified? 4. What dataset did you use? **Environment** 1. Please run `python mmdet/utils/collect_env.py` to collect necessary environment information and paste it here. 2. You may add addition that may be helpful for locating the problem, such as 1. How you installed PyTorch \[e.g., pip, conda, source\] 2. Other environment variables that may be related (such as `$PATH`, `$LD_LIBRARY_PATH`, `$PYTHONPATH`, etc.) **Results** If applicable, paste the related results here, e.g., what you expect and what you get. ```none A placeholder for results comparison ``` **Issue fix** If you have already identified the reason, you can provide the information here. If you are willing to create a PR to fix it, please also leave a comment here and that would be much appreciated! ================================================ FILE: .github/pull_request_template.md ================================================ Thanks for your contribution and we appreciate it a lot. The following instructions would make your pull request more healthy and more easily get feedback. If you do not understand some items, don't worry, just make the pull request and seek help from maintainers. ## Motivation Please describe the motivation of this PR and the goal you want to achieve through this PR. ## Modification Please briefly describe what modification is made in this PR. ## BC-breaking (Optional) Does the modification introduce changes that break the backward-compatibility of the downstream repos? If so, please describe how it breaks the compatibility and how the downstream projects should modify their code to keep compatibility with this PR. ## Use cases (Optional) If this PR introduces a new feature, it is better to list some use cases here, and update the documentation. ## Checklist 1. Pre-commit or other linting tools are used to fix the potential lint issues. 2. The modification is covered by complete unit tests. If not, please add more unit test to ensure the correctness. 3. If the modification has potential influence on downstream projects, this PR should be tested with downstream projects, like MMDet or MMCls. 4. The documentation has been modified accordingly, like docstring or example tutorials. ================================================ FILE: .github/workflows/deploy.yml ================================================ name: deploy on: push concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: build-n-publish: runs-on: ubuntu-latest if: startsWith(github.event.ref, 'refs/tags') steps: - uses: actions/checkout@v2 - name: Set up Python 3.7 uses: actions/setup-python@v2 with: python-version: 3.7 - name: Install torch run: pip install torch - name: Install wheel run: pip install wheel - name: Build MMDetection run: python setup.py sdist bdist_wheel - name: Publish distribution to PyPI run: | pip install twine twine upload dist/* -u __token__ -p ${{ secrets.pypi_password }} ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/en/_build/ docs/zh_cn/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # pyenv .python-version # celery beat schedule file celerybeat-schedule # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ data/ data .vscode .idea .DS_Store # custom *.pkl *.pkl.json *.log.json docs/modelzoo_statistics.md mmdet/.mim work_dirs/ # Pytorch *.pth *.py~ *.sh~ ================================================ FILE: .owners.yml ================================================ assign: strategy: # random daily-shift-based scedule: '*/1 * * * *' assignees: - Czm369 - hhaAndroid - jbwang1997 - RangiLyu - BIGWangYuDong - chhluo - ZwwWayne ================================================ FILE: .pre-commit-config-zh-cn.yaml ================================================ exclude: ^tests/data/ repos: - repo: https://gitee.com/openmmlab/mirrors-flake8 rev: 5.0.4 hooks: - id: flake8 - repo: https://gitee.com/openmmlab/mirrors-isort rev: 5.11.5 hooks: - id: isort - repo: https://gitee.com/openmmlab/mirrors-yapf rev: v0.32.0 hooks: - id: yapf - repo: https://gitee.com/openmmlab/mirrors-pre-commit-hooks rev: v4.3.0 hooks: - id: trailing-whitespace - id: check-yaml - id: end-of-file-fixer - id: requirements-txt-fixer - id: double-quote-string-fixer - id: check-merge-conflict - id: fix-encoding-pragma args: ["--remove"] - id: mixed-line-ending args: ["--fix=lf"] - repo: https://gitee.com/openmmlab/mirrors-mdformat rev: 0.7.9 hooks: - id: mdformat args: ["--number"] additional_dependencies: - mdformat-openmmlab - mdformat_frontmatter - linkify-it-py - repo: https://gitee.com/openmmlab/mirrors-codespell rev: v2.2.1 hooks: - id: codespell - repo: https://gitee.com/openmmlab/mirrors-docformatter rev: v1.3.1 hooks: - id: docformatter args: ["--in-place", "--wrap-descriptions", "79"] - repo: https://gitee.com/openmmlab/mirrors-pyupgrade rev: v3.0.0 hooks: - id: pyupgrade args: ["--py36-plus"] - repo: https://gitee.com/open-mmlab/pre-commit-hooks rev: v0.2.0 hooks: - id: check-algo-readme - id: check-copyright args: ["mmdet"] # - repo: https://gitee.com/openmmlab/mirrors-mypy # rev: v0.812 # hooks: # - id: mypy # exclude: "docs" ================================================ FILE: .pre-commit-config.yaml ================================================ repos: - repo: https://github.com/PyCQA/flake8 rev: 5.0.4 hooks: - id: flake8 - repo: https://github.com/PyCQA/isort rev: 5.11.5 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-yapf rev: v0.32.0 hooks: - id: yapf - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: - id: trailing-whitespace - id: check-yaml - id: end-of-file-fixer - id: requirements-txt-fixer - id: double-quote-string-fixer - id: check-merge-conflict - id: fix-encoding-pragma args: ["--remove"] - id: mixed-line-ending args: ["--fix=lf"] - repo: https://github.com/codespell-project/codespell rev: v2.2.1 hooks: - id: codespell - repo: https://github.com/executablebooks/mdformat rev: 0.7.9 hooks: - id: mdformat args: ["--number"] additional_dependencies: - mdformat-openmmlab - mdformat_frontmatter - linkify-it-py - repo: https://github.com/myint/docformatter rev: v1.3.1 hooks: - id: docformatter args: ["--in-place", "--wrap-descriptions", "79"] - repo: https://github.com/open-mmlab/pre-commit-hooks rev: v0.2.0 # Use the ref you want to point at hooks: - id: check-algo-readme - id: check-copyright args: ["mmdet"] # replace the dir_to_check with your expected directory to check ================================================ FILE: .readthedocs.yml ================================================ version: 2 formats: all python: version: 3.7 install: - requirements: requirements/docs.txt - requirements: requirements/readthedocs.txt ================================================ FILE: CITATION.cff ================================================ cff-version: 1.2.0 message: "If you use this software, please cite it as below." authors: - name: "MMDetection Contributors" title: "OpenMMLab Detection Toolbox and Benchmark" date-released: 2018-08-22 url: "https://github.com/open-mmlab/mmdetection" license: Apache-2.0 ================================================ FILE: LICENSE ================================================ ## creative commons # Attribution-NonCommercial 4.0 International Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. ### Using Creative Commons Public Licenses Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. * __Considerations for licensors:__ Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. [More considerations for licensors](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensors). * __Considerations for the public:__ By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. [More considerations for the public](http://wiki.creativecommons.org/Considerations_for_licensors_and_licensees#Considerations_for_licensees). ## Creative Commons Attribution-NonCommercial 4.0 International Public License By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution-NonCommercial 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. ### Section 1 – Definitions. a. __Adapted Material__ means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. b. __Adapter's License__ means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. c. __Copyright and Similar Rights__ means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. d. __Effective Technological Measures__ means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. e. __Exceptions and Limitations__ means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. f. __Licensed Material__ means the artistic or literary work, database, or other material to which the Licensor applied this Public License. g. __Licensed Rights__ means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. h. __Licensor__ means the individual(s) or entity(ies) granting rights under this Public License. i. __NonCommercial__ means not primarily intended for or directed towards commercial advantage or monetary compensation. For purposes of this Public License, the exchange of the Licensed Material for other material subject to Copyright and Similar Rights by digital file-sharing or similar means is NonCommercial provided there is no payment of monetary compensation in connection with the exchange. j. __Share__ means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. k. __Sui Generis Database Rights__ means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. l. __You__ means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning. ### Section 2 – Scope. a. ___License grant.___ 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: A. reproduce and Share the Licensed Material, in whole or in part, for NonCommercial purposes only; and B. produce, reproduce, and Share Adapted Material for NonCommercial purposes only. 2. __Exceptions and Limitations.__ For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. 3. __Term.__ The term of this Public License is specified in Section 6(a). 4. __Media and formats; technical modifications allowed.__ The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. 5. __Downstream recipients.__ A. __Offer from the Licensor – Licensed Material.__ Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. B. __No downstream restrictions.__ You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. 6. __No endorsement.__ Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). b. ___Other rights.___ 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. 2. Patent and trademark rights are not licensed under this Public License. 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties, including when the Licensed Material is used other than for NonCommercial purposes. ### Section 3 – License Conditions. Your exercise of the Licensed Rights is expressly made subject to the following conditions. a. ___Attribution.___ 1. If You Share the Licensed Material (including in modified form), You must: A. retain the following if it is supplied by the Licensor with the Licensed Material: i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); ii. a copyright notice; iii. a notice that refers to this Public License; iv. a notice that refers to the disclaimer of warranties; v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. 4. If You Share Adapted Material You produce, the Adapter's License You apply must not prevent recipients of the Adapted Material from complying with this Public License. ### Section 4 – Sui Generis Database Rights. Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database for NonCommercial purposes only; b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material; and c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. ### Section 5 – Disclaimer of Warranties and Limitation of Liability. a. __Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You.__ b. __To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You.__ c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. ### Section 6 – Term and Termination. a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or 2. upon express reinstatement by the Licensor. For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. c. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. d. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. ### Section 7 – Other Terms and Conditions. a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. ### Section 8 – Interpretation. a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. > Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at [creativecommons.org/policies](http://creativecommons.org/policies), Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. > > Creative Commons may be contacted at creativecommons.org Copyright (c) 2022 MCG-NKU ================================================ FILE: MANIFEST.in ================================================ include requirements/*.txt include mmdet/VERSION include mmdet/.mim/model-index.yml include mmdet/.mim/demo/*/* recursive-include mmdet/.mim/configs *.py *.yml recursive-include mmdet/.mim/tools *.sh *.py ================================================ FILE: README.md ================================================ #

🌟 `CrossKD: Cross-Head Knowledge Distillation for Dense Object Detection` 🌟

![Python 3.8](https://img.shields.io/badge/python-3.8-g) ![pytorch 1.12.1](https://img.shields.io/badge/pytorch-1.12.0-blue.svg) This repository contains the official implementation of the following paper: > **CrossKD: Cross-Head Knowledge Distillation for Dense Object Detection**
> [Jiabao Wang](https://scholar.google.co.uk/citations?hl=en&user=S9ErhhEAAAAJ)\*, [Yuming Chen](https://github.com/FishAndWasabi/)\*, [Zhaohui Zheng](https://scholar.google.co.uk/citations?hl=en&user=0X71NDYAAAAJ)，[Xiang Li](http://implus.github.io/), [Ming-Ming Cheng](https://mmcheng.net/cmm), [Qibin Hou](https://houqb.github.io/)\*
> (\* denotes equal contribution)
> VCIP, School of Computer Science, Nankai University
[[Arxiv Paper](https://arxiv.org/abs/2306.11369)] ## Introduction Knowledge Distillation (KD) has been validated as an effective model compression technique for learning compact object detectors. Existing state-of-the-art KD methods for object detection are mostly based on feature imitation, which is generally observed to be better than prediction mimicking. In this paper, we show that the inconsistency of the optimization objectives between the ground-truth signals and distillation targets is the key reason for the inefficiency of prediction mimicking. To alleviate this issue, we present a simple yet effective distillation scheme, termed CrossKD, which delivers the intermediate features of the student's detection head to the teacher's detection head. The resulting cross-head predictions are then forced to mimic the teacher's predictions. Such a distillation manner relieves the student's head from receiving contradictory supervision signals from the ground-truth annotations and the teacher's predictions, greatly improving the student's detection performance. On MS COCO, with only prediction mimicking losses applied, our CrossKD boosts the average precision of GFL ResNet-50 with 1x training schedule from 40.2 to 43.7, outperforming all existing KD methods for object detection. ![struture](assets/structure.png) ## Get Started ### 1. Prerequisites **Dependencies** - Ubuntu >= 20.04 - CUDA >= 11.3 - pytorch==1.12.1 - torchvision=0.13.1 - mmcv==2.0.0rc4 - mmengine==0.7.3 Our implementation based on MMDetection==3.0.0rc6. For more information about installation, please see the [official instructions](https://mmdetection.readthedocs.io/en/3.x/). **Step 0.** Create Conda Environment ```shell conda create --name openmmlab python=3.8 -y conda activate openmmlab ``` **Step 1.** Install [Pytorch](https://pytorch.org) ```shell conda install pytorch==1.12.1 torchvision==0.13.1 torchaudio==0.12.1 cudatoolkit=11.3 -c pytorch ``` **Step 2.** Install [MMEngine](https://github.com/open-mmlab/mmengine) and [MMCV](https://github.com/open-mmlab/mmcv) using [MIM](https://github.com/open-mmlab/mim). ```shell pip install -U openmim mim install "mmengine==0.7.3" mim install "mmcv==2.0.0rc4" ``` **Step 3.** Install [CrossKD](https://github.com/jbwang1997/CrossKD.git). ```shell git clone https://github.com/jbwang1997/CrossKD cd CrossKD pip install -v -e . # "-v" means verbose, or more output # "-e" means installing a project in editable mode, # thus any local modifications made to the code will take effect without reinstallation. ``` **Step 4.** Prepare dataset follow the [official instructions](https://mmdetection.readthedocs.io/en/3.x/user_guides/dataset_prepare.html). ### 2. Training **Single GPU** ```shell python tools/train.py configs/crosskd/${CONFIG_FILE} [optional arguments] ``` **Multi GPU** ```shell CUDA_VISIBLE_DEVICES=x,x,x,x python tools/dist_train.sh \ configs/crosskd/${CONFIG_FILE} ${GPU_NUM} [optional arguments] ``` ### 3. Evaluation ```shell python tools/test.py configs/crosskd/${CONFIG_FILE} ${CHECKPOINT_FILE} ``` ## Results ### 1. GFL | **Method** | schedule | AP | Config | Download | |:------------------:|:--------:|:-----------:|:--------------------------------------------------------------------------------:|:----------------------------------------------------------------------------------------------------------------------------------------------------:| | **GFL-Res101 (T)** | 2x+ms | 44.9 | [config]() | [model](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_mstrain_2x_coco/gfl_r101_fpn_mstrain_2x_coco_20200629_200126-dd12f847.pth) | | **GFL-Res50 (S)** | 1x | 40.2 | [config]() | [model](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r50_fpn_1x_coco/gfl_r50_fpn_1x_coco_20200629_121244-25944287.pth) | | **CrossKD** | 1x | 43.7 (+3.5) | [config]() | [model](https://drive.google.com/file/d/1S7fyDkFSAauJry0ZGS-ZW-P3CJb7RlsO/view?usp=drive_link) | | **CrossKD+PKD** | 1x | 43.9 (+3.7) | [config]() | [model](https://drive.google.com/file/d/1LJZ27al2omdXb3cUty-RX37pMLp8L-4B/view?usp=drive_link) | ### 2. RetinaNet | **Method** | schedule | AP | Config | Download | |:------------------------:|:--------:|:-----------:|:------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------:| | **RetineNet-Res101 (T)** | 2x | 38.9 | [config]() | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_2x_coco/retinanet_r101_fpn_2x_coco_20200131-5560aee8.pth) | | **RetineNet-Res50 (S)** | 2x | 37.4 | [config]() | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_2x_coco/retinanet_r50_fpn_2x_coco_20200131-fdb43119.pth) | | **CrossKD** | 2x | 39.7 (+2.3) | [config]() | [model](https://drive.google.com/file/d/1fjwtuoKd4a_b5CHf6X0tKDmSNlwzYfWb/view?usp=drive_link) | | **CrossKD+PKD** | 2x | 39.8 (+2.4) | [config]() | [model](https://drive.google.com/file/d/1Ha9r5DrzaZ_9tz8x9PVxOkGaKAApIBGd/view?usp=drive_link) | ### 3. FCOS | **Method** | schedule | AP | Config | Download | |:-------------------:|:--------:|:-----------:|:------------------------------------------------------------------------------------------------:|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| | **FCOS-Res101 (T)** | 2x+ms | 40.8 | [config]() | [model](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco-511424d6.pth) | | **FCOS-Res50 (S)** | 2x+ms | 38.5 | [config]() | [model](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r50_caffe_fpn_gn-head_mstrain_640-800_2x_coco/fcos_r50_caffe_fpn_gn-head_mstrain_640-800_2x_coco-d92ceeea.pth) | | **CrossKD** | 2x+ms | 41.1 (+2.6) | [config]() | [model](https://drive.google.com/file/d/1ll5vOGFMEfOsNCkgbPuqh0uMNFnfICbE/view?usp=drive_link) | | **CrossKD+PKD** | 2x+ms | 41.3 (+2.8) | [config]() | [model](https://drive.google.com/file/d/1r-UzxAOYOfPJFIV5e7Rd3P3uC9gXP09v/view?usp=drive_link) | ### 4. ATSS | **Method** | schedule | AP | Config | Download | |:-------------------:|:--------:|:----------:|:-------------------------------------------------------------------------:|:------------------------------------------------------------------------------------------------------------------------------:| | **ATSS-Res101 (T)** | 1x | 41.5 | [config]() | [model](https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r101_fpn_1x_coco/atss_r101_fpn_1x_20200825-dfcadd6f.pth) | | **ATSS-Res50 (S)** | 1x | 39.4 | [config]() | [model](https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r50_fpn_1x_coco/atss_r50_fpn_1x_coco_20200209-985f7bd0.pth) | | **CrossKD** | 1x | 41.8(+2.4) | [config]() | [model](https://drive.google.com/file/d/1qyxOMaxQrwJ20tEgIwU8pi31O8A1hsEG/view?usp=drive_link) | | **CrossKD+PKD** | 1x | 41.8(+2.4) | [config]() | [model](https://drive.google.com/file/d/1LkuKau1Na843ZPSNz77DqV8v8111b2_y/view?usp=drive_link) | ## Heterogeneous Results ### 1. Swin-Tiny | **Method** | schedule | AP | Config | Download | |:-------------------:|:--------:|:----------:|:-------------------------------------------------------------------------:|:------------------------------------------------------------------------------------------------------------------------------:| | **RetinaNet-SwinT (T)** | 1x | 37.3 | [config]() | [model](https://drive.google.com/file/d/1W2KGR77XfQ5SRomgIyxCjqUNGJichcgh/view?usp=drive_link) | | **RetinaNet-Res50 (S)** | 1x | 36.5 | [config]() | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_1x_coco/retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth) | | **CrossKD** | 1x | 38.0 (+1.5) | [config]() | [model](https://drive.google.com/file/d/17rbkcXgqKfVUweRgzK7NtqFcLC-PohKX/view?usp=drive_link) | ## Citation If you find our repo useful for your research, please cite us: ``` @misc{wang2023crosskd, title={CrossKD: Cross-Head Knowledge Distillation for Dense Object Detection}, author={Jiabao Wang and Yuming Chen and Zhaohui Zheng and Xiang Li and Ming-Ming Cheng and Qibin Hou}, year={2023}, eprint={2306.11369}, archivePrefix={arXiv}, primaryClass={cs.CV} } ``` This project is based on the open source codebase [MMDetection](https://github.com/open-mmlab/mmdetection). ``` @article{mmdetection, title = {{MMDetection}: Open MMLab Detection Toolbox and Benchmark}, author = {Chen, Kai and Wang, Jiaqi and Pang, Jiangmiao and Cao, Yuhang and Xiong, Yu and Li, Xiaoxiao and Sun, Shuyang and Feng, Wansen and Liu, Ziwei and Xu, Jiarui and Zhang, Zheng and Cheng, Dazhi and Zhu, Chenchen and Cheng, Tianheng and Zhao, Qijie and Li, Buyu and Lu, Xin and Zhu, Rui and Wu, Yue and Dai, Jifeng and Wang, Jingdong and Shi, Jianping and Ouyang, Wanli and Loy, Chen Change and Lin, Dahua}, journal= {arXiv preprint arXiv:1906.07155}, year={2019} } ``` ## License Licensed under a [Creative Commons Attribution-NonCommercial 4.0 International](https://creativecommons.org/licenses/by-nc/4.0/) for Non-commercial use only. Any commercial use should get formal permission first. ## Contact For technical questions, please contact `jbwang@mail.nankai.edu.cn` and `chenyuming@mail.nankai.edu.cn`. ## Acknowledgement This repo is modified from open source object detection codebase [MMDetection](https://github.com/open-mmlab/mmdetection). ================================================ FILE: configs/_base_/datasets/cityscapes_detection.py ================================================ # dataset settings dataset_type = 'CityscapesDataset' data_root = 'data/cityscapes/' train_pipeline = [ dict(type='LoadImageFromFile'), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomResize', scale=[(2048, 800), (2048, 1024)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile'), dict(type='Resize', scale=(2048, 1024), keep_ratio=True), # If you don't have a gt annotation, delete the pipeline dict(type='LoadAnnotations', with_bbox=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type='RepeatDataset', times=8, dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instancesonly_filtered_gtFine_train.json', data_prefix=dict(img='leftImg8bit/train/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline))) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instancesonly_filtered_gtFine_val.json', data_prefix=dict(img='leftImg8bit/val/'), test_mode=True, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict( type='CocoMetric', ann_file=data_root + 'annotations/instancesonly_filtered_gtFine_val.json', metric='bbox') test_evaluator = val_evaluator ================================================ FILE: configs/_base_/datasets/cityscapes_instance.py ================================================ # dataset settings dataset_type = 'CityscapesDataset' data_root = 'data/cityscapes/' train_pipeline = [ dict(type='LoadImageFromFile'), dict(type='LoadAnnotations', with_bbox=True, with_mask=True), dict( type='RandomResize', scale=[(2048, 800), (2048, 1024)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile'), dict(type='Resize', scale=(2048, 1024), keep_ratio=True), # If you don't have a gt annotation, delete the pipeline dict(type='LoadAnnotations', with_bbox=True, with_mask=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type='RepeatDataset', times=8, dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instancesonly_filtered_gtFine_train.json', data_prefix=dict(img='leftImg8bit/train/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline))) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instancesonly_filtered_gtFine_val.json', data_prefix=dict(img='leftImg8bit/val/'), test_mode=True, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = [ dict( type='CocoMetric', ann_file=data_root + 'annotations/instancesonly_filtered_gtFine_val.json', metric=['bbox', 'segm']), dict( type='CityScapesMetric', ann_file=data_root + 'annotations/instancesonly_filtered_gtFine_val.json', seg_prefix=data_root + '/gtFine/val', outfile_prefix='./work_dirs/cityscapes_metric/instance') ] test_evaluator = val_evaluator # inference on test dataset and # format the output results for submission. # test_dataloader = dict( # batch_size=1, # num_workers=2, # persistent_workers=True, # drop_last=False, # sampler=dict(type='DefaultSampler', shuffle=False), # dataset=dict( # type=dataset_type, # data_root=data_root, # ann_file='annotations/instancesonly_filtered_gtFine_test.json', # data_prefix=dict(img='leftImg8bit/test/'), # test_mode=True, # filter_cfg=dict(filter_empty_gt=True, min_size=32), # pipeline=test_pipeline)) # test_evaluator = dict( # type='CityScapesMetric', # format_only=True, # outfile_prefix='./work_dirs/cityscapes_metric/test') ================================================ FILE: configs/_base_/datasets/coco_detection.py ================================================ # dataset settings dataset_type = 'CocoDataset' data_root = 'data/coco/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadAnnotations', with_bbox=True), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(1333, 800), keep_ratio=True), # If you don't have a gt annotation, delete the pipeline dict(type='LoadAnnotations', with_bbox=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=2, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instances_train2017.json', data_prefix=dict(img='train2017/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline)) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instances_val2017.json', data_prefix=dict(img='val2017/'), test_mode=True, pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict( type='CocoMetric', ann_file=data_root + 'annotations/instances_val2017.json', metric='bbox', format_only=False) test_evaluator = val_evaluator # inference on test dataset and # format the output results for submission. # test_dataloader = dict( # batch_size=1, # num_workers=2, # persistent_workers=True, # drop_last=False, # sampler=dict(type='DefaultSampler', shuffle=False), # dataset=dict( # type=dataset_type, # data_root=data_root, # ann_file=data_root + 'annotations/image_info_test-dev2017.json', # data_prefix=dict(img='test2017/'), # test_mode=True, # pipeline=test_pipeline)) # test_evaluator = dict( # type='CocoMetric', # metric='bbox', # format_only=True, # ann_file=data_root + 'annotations/image_info_test-dev2017.json', # outfile_prefix='./work_dirs/coco_detection/test') ================================================ FILE: configs/_base_/datasets/coco_instance.py ================================================ # dataset settings dataset_type = 'CocoDataset' data_root = 'data/coco/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadAnnotations', with_bbox=True, with_mask=True), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(1333, 800), keep_ratio=True), # If you don't have a gt annotation, delete the pipeline dict(type='LoadAnnotations', with_bbox=True, with_mask=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=2, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instances_train2017.json', data_prefix=dict(img='train2017/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline)) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instances_val2017.json', data_prefix=dict(img='val2017/'), test_mode=True, pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict( type='CocoMetric', ann_file=data_root + 'annotations/instances_val2017.json', metric=['bbox', 'segm'], format_only=False) test_evaluator = val_evaluator # inference on test dataset and # format the output results for submission. # test_dataloader = dict( # batch_size=1, # num_workers=2, # persistent_workers=True, # drop_last=False, # sampler=dict(type='DefaultSampler', shuffle=False), # dataset=dict( # type=dataset_type, # data_root=data_root, # ann_file=data_root + 'annotations/image_info_test-dev2017.json', # data_prefix=dict(img='test2017/'), # test_mode=True, # pipeline=test_pipeline)) # test_evaluator = dict( # type='CocoMetric', # metric=['bbox', 'segm'], # format_only=True, # ann_file=data_root + 'annotations/image_info_test-dev2017.json', # outfile_prefix='./work_dirs/coco_instance/test') ================================================ FILE: configs/_base_/datasets/coco_instance_semantic.py ================================================ # dataset settings dataset_type = 'CocoDataset' data_root = 'data/coco/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict( type='LoadAnnotations', with_bbox=True, with_mask=True, with_seg=True), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(1333, 800), keep_ratio=True), # If you don't have a gt annotation, delete the pipeline dict( type='LoadAnnotations', with_bbox=True, with_mask=True, with_seg=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=2, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instances_train2017.json', data_prefix=dict(img='train2017/', seg='stuffthingmaps/train2017/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline)) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instances_val2017.json', data_prefix=dict(img='val2017/'), test_mode=True, pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict( type='CocoMetric', ann_file=data_root + 'annotations/instances_val2017.json', metric=['bbox', 'segm'], format_only=False) test_evaluator = val_evaluator ================================================ FILE: configs/_base_/datasets/coco_panoptic.py ================================================ # dataset settings dataset_type = 'CocoPanopticDataset' data_root = 'data/coco/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadPanopticAnnotations', file_client_args=file_client_args), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict(type='LoadPanopticAnnotations', file_client_args=file_client_args), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=2, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/panoptic_train2017.json', data_prefix=dict( img='train2017/', seg='annotations/panoptic_train2017/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline)) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/panoptic_val2017.json', data_prefix=dict(img='val2017/', seg='annotations/panoptic_val2017/'), test_mode=True, pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict( type='CocoPanopticMetric', ann_file=data_root + 'annotations/panoptic_val2017.json', seg_prefix=data_root + 'annotations/panoptic_val2017/', file_client_args=file_client_args, ) test_evaluator = val_evaluator # inference on test dataset and # format the output results for submission. # test_dataloader = dict( # batch_size=1, # num_workers=1, # persistent_workers=True, # drop_last=False, # sampler=dict(type='DefaultSampler', shuffle=False), # dataset=dict( # type=dataset_type, # data_root=data_root, # ann_file='annotations/panoptic_image_info_test-dev2017.json', # data_prefix=dict(img='test2017/'), # test_mode=True, # pipeline=test_pipeline)) # test_evaluator = dict( # type='CocoPanopticMetric', # format_only=True, # ann_file=data_root + 'annotations/panoptic_image_info_test-dev2017.json', # outfile_prefix='./work_dirs/coco_panoptic/test') ================================================ FILE: configs/_base_/datasets/deepfashion.py ================================================ # dataset settings dataset_type = 'DeepFashionDataset' data_root = 'data/DeepFashion/In-shop/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadAnnotations', with_bbox=True, with_mask=True), dict(type='Resize', scale=(750, 1101), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(750, 1101), keep_ratio=True), dict(type='LoadAnnotations', with_bbox=True, with_mask=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=2, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type='RepeatDataset', times=2, dataset=dict( type=dataset_type, data_root=data_root, ann_file='Anno/segmentation/DeepFashion_segmentation_train.json', data_prefix=dict(img='Img/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline))) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='Anno/segmentation/DeepFashion_segmentation_query.json', data_prefix=dict(img='Img/'), test_mode=True, pipeline=test_pipeline)) test_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='Anno/segmentation/DeepFashion_segmentation_gallery.json', data_prefix=dict(img='Img/'), test_mode=True, pipeline=test_pipeline)) val_evaluator = dict( type='CocoMetric', ann_file=data_root + 'Anno/segmentation/DeepFashion_segmentation_query.json', metric=['bbox', 'segm'], format_only=False) test_evaluator = dict( type='CocoMetric', ann_file=data_root + 'Anno/segmentation/DeepFashion_segmentation_gallery.json', metric=['bbox', 'segm'], format_only=False) ================================================ FILE: configs/_base_/datasets/lvis_v0.5_instance.py ================================================ # dataset settings dataset_type = 'LVISV05Dataset' data_root = 'data/lvis_v0.5/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadAnnotations', with_bbox=True, with_mask=True), dict( type='RandomChoiceResize', scales=[(1333, 640), (1333, 672), (1333, 704), (1333, 736), (1333, 768), (1333, 800)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict(type='LoadAnnotations', with_bbox=True, with_mask=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=2, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type='ClassBalancedDataset', oversample_thr=1e-3, dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/lvis_v0.5_train.json', data_prefix=dict(img='train2017/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline))) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/lvis_v0.5_val.json', data_prefix=dict(img='val2017/'), test_mode=True, pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict( type='LVISMetric', ann_file=data_root + 'annotations/lvis_v0.5_val.json', metric=['bbox', 'segm']) test_evaluator = val_evaluator ================================================ FILE: configs/_base_/datasets/lvis_v1_instance.py ================================================ # dataset settings _base_ = 'lvis_v0.5_instance.py' dataset_type = 'LVISV1Dataset' data_root = 'data/lvis_v1/' train_dataloader = dict( dataset=dict( dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/lvis_v1_train.json', data_prefix=dict(img='')))) val_dataloader = dict( dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/lvis_v1_val.json', data_prefix=dict(img=''))) test_dataloader = val_dataloader val_evaluator = dict(ann_file=data_root + 'annotations/lvis_v1_val.json') test_evaluator = val_evaluator ================================================ FILE: configs/_base_/datasets/objects365v1_detection.py ================================================ # dataset settings dataset_type = 'Objects365V1Dataset' data_root = 'data/Objects365/Obj365_v1/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadAnnotations', with_bbox=True), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(1333, 800), keep_ratio=True), # If you don't have a gt annotation, delete the pipeline dict(type='LoadAnnotations', with_bbox=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=2, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/objects365_train.json', data_prefix=dict(img='train/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline)) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/objects365_val.json', data_prefix=dict(img='val/'), test_mode=True, pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict( type='CocoMetric', ann_file=data_root + 'annotations/objects365_val.json', metric='bbox', sort_categories=True, format_only=False) test_evaluator = val_evaluator ================================================ FILE: configs/_base_/datasets/objects365v2_detection.py ================================================ # dataset settings dataset_type = 'Objects365V2Dataset' data_root = 'data/Objects365/Obj365_v2/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadAnnotations', with_bbox=True), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(1333, 800), keep_ratio=True), # If you don't have a gt annotation, delete the pipeline dict(type='LoadAnnotations', with_bbox=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=2, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/zhiyuan_objv2_train.json', data_prefix=dict(img='train/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline)) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/zhiyuan_objv2_val.json', data_prefix=dict(img='val/'), test_mode=True, pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict( type='CocoMetric', ann_file=data_root + 'annotations/zhiyuan_objv2_val.json', metric='bbox', format_only=False) test_evaluator = val_evaluator ================================================ FILE: configs/_base_/datasets/openimages_detection.py ================================================ # dataset settings dataset_type = 'OpenImagesDataset' data_root = 'data/OpenImages/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadAnnotations', with_bbox=True), dict(type='Resize', scale=(1024, 800), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(1024, 800), keep_ratio=True), # avoid bboxes being resized dict(type='LoadAnnotations', with_bbox=True), # TODO: find a better way to collect image_level_labels dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'instances', 'image_level_labels')) ] train_dataloader = dict( batch_size=2, num_workers=0, # workers_per_gpu > 0 may occur out of memory persistent_workers=False, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/oidv6-train-annotations-bbox.csv', data_prefix=dict(img='OpenImages/train/'), label_file='annotations/class-descriptions-boxable.csv', hierarchy_file='annotations/bbox_labels_600_hierarchy.json', meta_file='annotations/train-image-metas.pkl', pipeline=train_pipeline)) val_dataloader = dict( batch_size=1, num_workers=0, persistent_workers=False, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/validation-annotations-bbox.csv', data_prefix=dict(img='OpenImages/validation/'), label_file='annotations/class-descriptions-boxable.csv', hierarchy_file='annotations/bbox_labels_600_hierarchy.json', meta_file='annotations/validation-image-metas.pkl', image_level_ann_file='annotations/validation-' 'annotations-human-imagelabels-boxable.csv', pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict( type='OpenImagesMetric', iou_thrs=0.5, ioa_thrs=0.5, use_group_of=True, get_supercategory=True) test_evaluator = val_evaluator ================================================ FILE: configs/_base_/datasets/semi_coco_detection.py ================================================ # dataset settings dataset_type = 'CocoDataset' data_root = 'data/coco/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') color_space = [ [dict(type='ColorTransform')], [dict(type='AutoContrast')], [dict(type='Equalize')], [dict(type='Sharpness')], [dict(type='Posterize')], [dict(type='Solarize')], [dict(type='Color')], [dict(type='Contrast')], [dict(type='Brightness')], ] geometric = [ [dict(type='Rotate')], [dict(type='ShearX')], [dict(type='ShearY')], [dict(type='TranslateX')], [dict(type='TranslateY')], ] scale = [(1333, 400), (1333, 1200)] branch_field = ['sup', 'unsup_teacher', 'unsup_student'] # pipeline used to augment labeled data, # which will be sent to student model for supervised training. sup_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadAnnotations', with_bbox=True), dict(type='RandomResize', scale=scale, keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='RandAugment', aug_space=color_space, aug_num=1), dict(type='FilterAnnotations', min_gt_bbox_wh=(1e-2, 1e-2)), dict( type='MultiBranch', branch_field=branch_field, sup=dict(type='PackDetInputs')) ] # pipeline used to augment unlabeled data weakly, # which will be sent to teacher model for predicting pseudo instances. weak_pipeline = [ dict(type='RandomResize', scale=scale, keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix')), ] # pipeline used to augment unlabeled data strongly, # which will be sent to student model for unsupervised training. strong_pipeline = [ dict(type='RandomResize', scale=scale, keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='RandomOrder', transforms=[ dict(type='RandAugment', aug_space=color_space, aug_num=1), dict(type='RandAugment', aug_space=geometric, aug_num=1), ]), dict(type='RandomErasing', n_patches=(1, 5), ratio=(0, 0.2)), dict(type='FilterAnnotations', min_gt_bbox_wh=(1e-2, 1e-2)), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix')), ] # pipeline used to augment unlabeled data into different views unsup_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadEmptyAnnotations'), dict( type='MultiBranch', branch_field=branch_field, unsup_teacher=weak_pipeline, unsup_student=strong_pipeline, ) ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] batch_size = 5 num_workers = 5 # There are two common semi-supervised learning settings on the coco dataset： # (1) Divide the train2017 into labeled and unlabeled datasets # by a fixed percentage, such as 1%, 2%, 5% and 10%. # The format of labeled_ann_file and unlabeled_ann_file are # instances_train2017.{fold}@{percent}.json, and # instances_train2017.{fold}@{percent}-unlabeled.json # `fold` is used for cross-validation, and `percent` represents # the proportion of labeled data in the train2017. # (2) Choose the train2017 as the labeled dataset # and unlabeled2017 as the unlabeled dataset. # The labeled_ann_file and unlabeled_ann_file are # instances_train2017.json and image_info_unlabeled2017.json # We use this configuration by default. labeled_dataset = dict( type=dataset_type, data_root=data_root, ann_file='annotations/instances_train2017.json', data_prefix=dict(img='train2017/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=sup_pipeline) unlabeled_dataset = dict( type=dataset_type, data_root=data_root, ann_file='annotations/instances_unlabeled2017.json', data_prefix=dict(img='unlabeled2017/'), filter_cfg=dict(filter_empty_gt=False), pipeline=unsup_pipeline) train_dataloader = dict( batch_size=batch_size, num_workers=num_workers, persistent_workers=True, sampler=dict( type='GroupMultiSourceSampler', batch_size=batch_size, source_ratio=[1, 4]), dataset=dict( type='ConcatDataset', datasets=[labeled_dataset, unlabeled_dataset])) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instances_val2017.json', data_prefix=dict(img='val2017/'), test_mode=True, pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict( type='CocoMetric', ann_file=data_root + 'annotations/instances_val2017.json', metric='bbox', format_only=False) test_evaluator = val_evaluator ================================================ FILE: configs/_base_/datasets/voc0712.py ================================================ # dataset settings dataset_type = 'VOCDataset' data_root = 'data/VOCdevkit/' # file_client_args = dict( # backend='petrel', # path_mapping=dict({ # './data/': 's3://openmmlab/datasets/detection/', # 'data/': 's3://openmmlab/datasets/detection/' # })) file_client_args = dict(backend='disk') train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='LoadAnnotations', with_bbox=True), dict(type='Resize', scale=(1000, 600), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict(type='LoadImageFromFile', file_client_args=file_client_args), dict(type='Resize', scale=(1000, 600), keep_ratio=True), # avoid bboxes being resized dict(type='LoadAnnotations', with_bbox=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=2, num_workers=2, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), batch_sampler=dict(type='AspectRatioBatchSampler'), dataset=dict( type='RepeatDataset', times=3, dataset=dict( type='ConcatDataset', # VOCDataset will add different `dataset_type` in dataset.metainfo, # which will get error if using ConcatDataset. Adding # `ignore_keys` can avoid this error. ignore_keys=['dataset_type'], datasets=[ dict( type=dataset_type, data_root=data_root, ann_file='VOC2007/ImageSets/Main/trainval.txt', data_prefix=dict(sub_data_root='VOC2007/'), filter_cfg=dict( filter_empty_gt=True, min_size=32, bbox_min_size=32), pipeline=train_pipeline), dict( type=dataset_type, data_root=data_root, ann_file='VOC2012/ImageSets/Main/trainval.txt', data_prefix=dict(sub_data_root='VOC2012/'), filter_cfg=dict( filter_empty_gt=True, min_size=32, bbox_min_size=32), pipeline=train_pipeline) ]))) val_dataloader = dict( batch_size=1, num_workers=2, persistent_workers=True, drop_last=False, sampler=dict(type='DefaultSampler', shuffle=False), dataset=dict( type=dataset_type, data_root=data_root, ann_file='VOC2007/ImageSets/Main/test.txt', data_prefix=dict(sub_data_root='VOC2007/'), test_mode=True, pipeline=test_pipeline)) test_dataloader = val_dataloader # Pascal VOC2007 uses `11points` as default evaluate mode, while PASCAL # VOC2012 defaults to use 'area'. val_evaluator = dict(type='VOCMetric', metric='mAP', eval_mode='11points') test_evaluator = val_evaluator ================================================ FILE: configs/_base_/datasets/wider_face.py ================================================ # dataset settings dataset_type = 'WIDERFaceDataset' data_root = 'data/WIDERFace/' img_norm_cfg = dict(mean=[123.675, 116.28, 103.53], std=[1, 1, 1], to_rgb=True) train_pipeline = [ dict(type='LoadImageFromFile', to_float32=True), dict(type='LoadAnnotations', with_bbox=True), dict( type='PhotoMetricDistortion', brightness_delta=32, contrast_range=(0.5, 1.5), saturation_range=(0.5, 1.5), hue_delta=18), dict( type='Expand', mean=img_norm_cfg['mean'], to_rgb=img_norm_cfg['to_rgb'], ratio_range=(1, 4)), dict( type='MinIoURandomCrop', min_ious=(0.1, 0.3, 0.5, 0.7, 0.9), min_crop_size=0.3), dict(type='Resize', img_scale=(300, 300), keep_ratio=False), dict(type='Normalize', **img_norm_cfg), dict(type='RandomFlip', flip_ratio=0.5), dict(type='DefaultFormatBundle'), dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']), ] test_pipeline = [ dict(type='LoadImageFromFile'), dict( type='MultiScaleFlipAug', img_scale=(300, 300), flip=False, transforms=[ dict(type='Resize', keep_ratio=False), dict(type='Normalize', **img_norm_cfg), dict(type='ImageToTensor', keys=['img']), dict(type='Collect', keys=['img']), ]) ] data = dict( samples_per_gpu=60, workers_per_gpu=2, train=dict( type='RepeatDataset', times=2, dataset=dict( type=dataset_type, ann_file=data_root + 'train.txt', img_prefix=data_root + 'WIDER_train/', min_size=17, pipeline=train_pipeline)), val=dict( type=dataset_type, ann_file=data_root + 'val.txt', img_prefix=data_root + 'WIDER_val/', pipeline=test_pipeline), test=dict( type=dataset_type, ann_file=data_root + 'val.txt', img_prefix=data_root + 'WIDER_val/', pipeline=test_pipeline)) ================================================ FILE: configs/_base_/default_runtime.py ================================================ default_scope = 'mmdet' default_hooks = dict( timer=dict(type='IterTimerHook'), logger=dict(type='LoggerHook', interval=50), param_scheduler=dict(type='ParamSchedulerHook'), checkpoint=dict(type='CheckpointHook', interval=1), sampler_seed=dict(type='DistSamplerSeedHook'), visualization=dict(type='DetVisualizationHook')) env_cfg = dict( cudnn_benchmark=False, mp_cfg=dict(mp_start_method='fork', opencv_num_threads=0), dist_cfg=dict(backend='nccl'), ) vis_backends = [dict(type='LocalVisBackend')] visualizer = dict( type='DetLocalVisualizer', vis_backends=vis_backends, name='visualizer') log_processor = dict(type='LogProcessor', window_size=50, by_epoch=True) log_level = 'INFO' load_from = None resume = False ================================================ FILE: configs/_base_/models/cascade-mask-rcnn_r50_fpn.py ================================================ # model settings model = dict( type='CascadeRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_mask=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5), rpn_head=dict( type='RPNHead', in_channels=256, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', scales=[8], ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)), roi_head=dict( type='CascadeRoIHead', num_stages=3, stage_loss_weights=[1, 0.5, 0.25], bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), out_channels=256, featmap_strides=[4, 8, 16, 32]), bbox_head=[ dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=True, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)), dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.05, 0.05, 0.1, 0.1]), reg_class_agnostic=True, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)), dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.033, 0.033, 0.067, 0.067]), reg_class_agnostic=True, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)) ], mask_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), out_channels=256, featmap_strides=[4, 8, 16, 32]), mask_head=dict( type='FCNMaskHead', num_convs=4, in_channels=256, conv_out_channels=256, num_classes=80, loss_mask=dict( type='CrossEntropyLoss', use_mask=True, loss_weight=1.0))), # model training and testing settings train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=0, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=2000, max_per_img=2000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=[ dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), mask_size=28, pos_weight=-1, debug=False), dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.6, neg_iou_thr=0.6, min_pos_iou=0.6, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), mask_size=28, pos_weight=-1, debug=False), dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.7, min_pos_iou=0.7, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), mask_size=28, pos_weight=-1, debug=False) ]), test_cfg=dict( rpn=dict( nms_pre=1000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100, mask_thr_binary=0.5))) ================================================ FILE: configs/_base_/models/cascade-rcnn_r50_fpn.py ================================================ # model settings model = dict( type='CascadeRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5), rpn_head=dict( type='RPNHead', in_channels=256, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', scales=[8], ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0)), roi_head=dict( type='CascadeRoIHead', num_stages=3, stage_loss_weights=[1, 0.5, 0.25], bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), out_channels=256, featmap_strides=[4, 8, 16, 32]), bbox_head=[ dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=True, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)), dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.05, 0.05, 0.1, 0.1]), reg_class_agnostic=True, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)), dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.033, 0.033, 0.067, 0.067]), reg_class_agnostic=True, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)) ]), # model training and testing settings train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=0, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=2000, max_per_img=2000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=[ dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False), dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.6, neg_iou_thr=0.6, min_pos_iou=0.6, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False), dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.7, min_pos_iou=0.7, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False) ]), test_cfg=dict( rpn=dict( nms_pre=1000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100))) ================================================ FILE: configs/_base_/models/fast-rcnn_r50_fpn.py ================================================ # model settings model = dict( type='FastRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5), roi_head=dict( type='StandardRoIHead', bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), out_channels=256, featmap_strides=[4, 8, 16, 32]), bbox_head=dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0))), # model training and testing settings train_cfg=dict( rcnn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False)), test_cfg=dict( rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100))) ================================================ FILE: configs/_base_/models/faster-rcnn_r50-caffe-c4.py ================================================ # model settings norm_cfg = dict(type='BN', requires_grad=False) model = dict( type='FasterRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=3, strides=(1, 2, 2), dilations=(1, 1, 1), out_indices=(2, ), frozen_stages=1, norm_cfg=norm_cfg, norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), rpn_head=dict( type='RPNHead', in_channels=1024, feat_channels=1024, anchor_generator=dict( type='AnchorGenerator', scales=[2, 4, 8, 16, 32], ratios=[0.5, 1.0, 2.0], strides=[16]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), roi_head=dict( type='StandardRoIHead', shared_head=dict( type='ResLayer', depth=50, stage=3, stride=2, dilation=1, style='caffe', norm_cfg=norm_cfg, norm_eval=True, init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), out_channels=1024, featmap_strides=[16]), bbox_head=dict( type='BBoxHead', with_avg_pool=True, roi_feat_size=7, in_channels=2048, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0))), # model training and testing settings train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=-1, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=12000, max_per_img=2000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False)), test_cfg=dict( rpn=dict( nms_pre=6000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100))) ================================================ FILE: configs/_base_/models/faster-rcnn_r50-caffe-dc5.py ================================================ # model settings norm_cfg = dict(type='BN', requires_grad=False) model = dict( type='FasterRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, strides=(1, 2, 2, 1), dilations=(1, 1, 1, 2), out_indices=(3, ), frozen_stages=1, norm_cfg=norm_cfg, norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), rpn_head=dict( type='RPNHead', in_channels=2048, feat_channels=2048, anchor_generator=dict( type='AnchorGenerator', scales=[2, 4, 8, 16, 32], ratios=[0.5, 1.0, 2.0], strides=[16]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), roi_head=dict( type='StandardRoIHead', bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), out_channels=2048, featmap_strides=[16]), bbox_head=dict( type='Shared2FCBBoxHead', in_channels=2048, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0))), # model training and testing settings train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=0, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=12000, max_per_img=2000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False)), test_cfg=dict( rpn=dict( nms=dict(type='nms', iou_threshold=0.7), nms_pre=6000, max_per_img=1000, min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100))) ================================================ FILE: configs/_base_/models/faster-rcnn_r50_fpn.py ================================================ # model settings model = dict( type='FasterRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5), rpn_head=dict( type='RPNHead', in_channels=256, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', scales=[8], ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), roi_head=dict( type='StandardRoIHead', bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), out_channels=256, featmap_strides=[4, 8, 16, 32]), bbox_head=dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0))), # model training and testing settings train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=-1, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=2000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), pos_weight=-1, debug=False)), test_cfg=dict( rpn=dict( nms_pre=1000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100) # soft-nms is also supported for rcnn testing # e.g., nms=dict(type='soft_nms', iou_threshold=0.5, min_score=0.05) )) ================================================ FILE: configs/_base_/models/mask-rcnn_r50-caffe-c4.py ================================================ # model settings norm_cfg = dict(type='BN', requires_grad=False) model = dict( type='MaskRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_mask=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=3, strides=(1, 2, 2), dilations=(1, 1, 1), out_indices=(2, ), frozen_stages=1, norm_cfg=norm_cfg, norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), rpn_head=dict( type='RPNHead', in_channels=1024, feat_channels=1024, anchor_generator=dict( type='AnchorGenerator', scales=[2, 4, 8, 16, 32], ratios=[0.5, 1.0, 2.0], strides=[16]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), roi_head=dict( type='StandardRoIHead', shared_head=dict( type='ResLayer', depth=50, stage=3, stride=2, dilation=1, style='caffe', norm_cfg=norm_cfg, norm_eval=True), bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), out_channels=1024, featmap_strides=[16]), bbox_head=dict( type='BBoxHead', with_avg_pool=True, roi_feat_size=7, in_channels=2048, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), mask_roi_extractor=None, mask_head=dict( type='FCNMaskHead', num_convs=0, in_channels=2048, conv_out_channels=256, num_classes=80, loss_mask=dict( type='CrossEntropyLoss', use_mask=True, loss_weight=1.0))), # model training and testing settings train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=0, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=12000, max_per_img=2000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), mask_size=14, pos_weight=-1, debug=False)), test_cfg=dict( rpn=dict( nms_pre=6000, nms=dict(type='nms', iou_threshold=0.7), max_per_img=1000, min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100, mask_thr_binary=0.5))) ================================================ FILE: configs/_base_/models/mask-rcnn_r50_fpn.py ================================================ # model settings model = dict( type='MaskRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_mask=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5), rpn_head=dict( type='RPNHead', in_channels=256, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', scales=[8], ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), roi_head=dict( type='StandardRoIHead', bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=0), out_channels=256, featmap_strides=[4, 8, 16, 32]), bbox_head=dict( type='Shared2FCBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=80, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), mask_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), out_channels=256, featmap_strides=[4, 8, 16, 32]), mask_head=dict( type='FCNMaskHead', num_convs=4, in_channels=256, conv_out_channels=256, num_classes=80, loss_mask=dict( type='CrossEntropyLoss', use_mask=True, loss_weight=1.0))), # model training and testing settings train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=-1, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=2000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), mask_size=28, pos_weight=-1, debug=False)), test_cfg=dict( rpn=dict( nms_pre=1000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100, mask_thr_binary=0.5))) ================================================ FILE: configs/_base_/models/retinanet_r50_fpn.py ================================================ # model settings model = dict( type='RetinaNet', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_input', num_outs=5), bbox_head=dict( type='RetinaHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), # model training and testing settings train_cfg=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0, ignore_iof_thr=-1), sampler=dict( type='PseudoSampler'), # Focal loss should use PseudoSampler allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100)) ================================================ FILE: configs/_base_/models/rpn_r50-caffe-c4.py ================================================ # model settings model = dict( type='RPN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=3, strides=(1, 2, 2), dilations=(1, 1, 1), out_indices=(2, ), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), neck=None, rpn_head=dict( type='RPNHead', in_channels=1024, feat_channels=1024, anchor_generator=dict( type='AnchorGenerator', scales=[2, 4, 8, 16, 32], ratios=[0.5, 1.0, 2.0], strides=[16]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), # model training and testing settings train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=-1, pos_weight=-1, debug=False)), test_cfg=dict( rpn=dict( nms_pre=12000, max_per_img=2000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0))) ================================================ FILE: configs/_base_/models/rpn_r50_fpn.py ================================================ # model settings model = dict( type='RPN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=5), rpn_head=dict( type='RPNHead', in_channels=256, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', scales=[8], ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), # model training and testing settings train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=-1, pos_weight=-1, debug=False)), test_cfg=dict( rpn=dict( nms_pre=2000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0))) ================================================ FILE: configs/_base_/models/ssd300.py ================================================ # model settings input_size = 300 model = dict( type='SingleStageDetector', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[1, 1, 1], bgr_to_rgb=True, pad_size_divisor=1), backbone=dict( type='SSDVGG', depth=16, with_last_pool=False, ceil_mode=True, out_indices=(3, 4), out_feature_indices=(22, 34), init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://vgg16_caffe')), neck=dict( type='SSDNeck', in_channels=(512, 1024), out_channels=(512, 1024, 512, 256, 256, 256), level_strides=(2, 2, 1, 1), level_paddings=(1, 1, 0, 0), l2_norm_scale=20), bbox_head=dict( type='SSDHead', in_channels=(512, 1024, 512, 256, 256, 256), num_classes=80, anchor_generator=dict( type='SSDAnchorGenerator', scale_major=False, input_size=input_size, basesize_ratio_range=(0.15, 0.9), strides=[8, 16, 32, 64, 100, 300], ratios=[[2], [2, 3], [2, 3], [2, 3], [2], [2]]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[0.1, 0.1, 0.2, 0.2])), # model training and testing settings train_cfg=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0., ignore_iof_thr=-1, gt_max_assign_all=False), sampler=dict(type='PseudoSampler'), smoothl1_beta=1., allowed_border=-1, pos_weight=-1, neg_pos_ratio=3, debug=False), test_cfg=dict( nms_pre=1000, nms=dict(type='nms', iou_threshold=0.45), min_bbox_size=0, score_thr=0.02, max_per_img=200)) cudnn_benchmark = True ================================================ FILE: configs/_base_/schedules/schedule_1x.py ================================================ # training schedule for 1x train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=12, val_interval=1) val_cfg = dict(type='ValLoop') test_cfg = dict(type='TestLoop') # learning rate param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=12, by_epoch=True, milestones=[8, 11], gamma=0.1) ] # optimizer optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001)) # Default setting for scaling LR automatically # - `enable` means enable scaling LR automatically # or not by default. # - `base_batch_size` = (8 GPUs) x (2 samples per GPU). auto_scale_lr = dict(enable=False, base_batch_size=16) ================================================ FILE: configs/_base_/schedules/schedule_20e.py ================================================ # training schedule for 20e train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=20, val_interval=1) val_cfg = dict(type='ValLoop') test_cfg = dict(type='TestLoop') # learning rate param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=20, by_epoch=True, milestones=[16, 19], gamma=0.1) ] # optimizer optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001)) # Default setting for scaling LR automatically # - `enable` means enable scaling LR automatically # or not by default. # - `base_batch_size` = (8 GPUs) x (2 samples per GPU). auto_scale_lr = dict(enable=False, base_batch_size=16) ================================================ FILE: configs/_base_/schedules/schedule_2x.py ================================================ # training schedule for 2x train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=24, val_interval=1) val_cfg = dict(type='ValLoop') test_cfg = dict(type='TestLoop') # learning rate param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=24, by_epoch=True, milestones=[16, 22], gamma=0.1) ] # optimizer optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.02, momentum=0.9, weight_decay=0.0001)) # Default setting for scaling LR automatically # - `enable` means enable scaling LR automatically # or not by default. # - `base_batch_size` = (8 GPUs) x (2 samples per GPU). auto_scale_lr = dict(enable=False, base_batch_size=16) ================================================ FILE: configs/atss/README.md ================================================ # ATSS > [Bridging the Gap Between Anchor-based and Anchor-free Detection via Adaptive Training Sample Selection](https://arxiv.org/abs/1912.02424) ## Abstract Object detection has been dominated by anchor-based detectors for several years. Recently, anchor-free detectors have become popular due to the proposal of FPN and Focal Loss. In this paper, we first point out that the essential difference between anchor-based and anchor-free detection is actually how to define positive and negative training samples, which leads to the performance gap between them. If they adopt the same definition of positive and negative samples during training, there is no obvious difference in the final performance, no matter regressing from a box or a point. This shows that how to select positive and negative training samples is important for current object detectors. Then, we propose an Adaptive Training Sample Selection (ATSS) to automatically select positive and negative samples according to statistical characteristics of object. It significantly improves the performance of anchor-based and anchor-free detectors and bridges the gap between them. Finally, we discuss the necessity of tiling multiple anchors per location on the image to detect objects. Extensive experiments conducted on MS COCO support our aforementioned analysis and conclusions. With the newly introduced ATSS, we improve state-of-the-art detectors by a large margin to 50.7% AP without introducing any overhead.
## Results and Models | Backbone | Style | Lr schd | Mem (GB) | Inf time (fps) | box AP | Config | Download | | :------: | :-----: | :-----: | :------: | :------------: | :----: | :----------------------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | R-50 | pytorch | 1x | 3.7 | 19.7 | 39.4 | [config](./atss_r50_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r50_fpn_1x_coco/atss_r50_fpn_1x_coco_20200209-985f7bd0.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r50_fpn_1x_coco/atss_r50_fpn_1x_coco_20200209_102539.log.json) | | R-101 | pytorch | 1x | 5.6 | 12.3 | 41.5 | [config](./atss_r101_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r101_fpn_1x_coco/atss_r101_fpn_1x_20200825-dfcadd6f.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r101_fpn_1x_coco/atss_r101_fpn_1x_20200825-dfcadd6f.log.json) | ## Citation ```latex @article{zhang2019bridging, title = {Bridging the Gap Between Anchor-based and Anchor-free Detection via Adaptive Training Sample Selection}, author = {Zhang, Shifeng and Chi, Cheng and Yao, Yongqiang and Lei, Zhen and Li, Stan Z.}, journal = {arXiv preprint arXiv:1912.02424}, year = {2019} } ``` ================================================ FILE: configs/atss/atss_r101_fpn_1x_coco.py ================================================ _base_ = './atss_r50_fpn_1x_coco.py' model = dict( backbone=dict( depth=101, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101'))) ================================================ FILE: configs/atss/atss_r101_fpn_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = './atss_r50_fpn_8xb8-amp-lsj-200e_coco.py' model = dict( backbone=dict( depth=101, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101'))) ================================================ FILE: configs/atss/atss_r18_fpn_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = './atss_r50_fpn_8xb8-amp-lsj-200e_coco.py' model = dict( backbone=dict( depth=18, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), neck=dict(in_channels=[64, 128, 256, 512])) ================================================ FILE: configs/atss/atss_r50_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] # model settings model = dict( type='ATSS', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='ATSSHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[0.1, 0.1, 0.2, 0.2]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)), # training and testing settings train_cfg=dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) # optimizer optim_wrapper = dict( optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) ================================================ FILE: configs/atss/atss_r50_fpn_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = '../common/lsj-200e_coco-detection.py' image_size = (1024, 1024) batch_augments = [dict(type='BatchFixedSizePad', size=image_size)] model = dict( type='ATSS', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32, batch_augments=batch_augments), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='ATSSHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[0.1, 0.1, 0.2, 0.2]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)), # training and testing settings train_cfg=dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) train_dataloader = dict(batch_size=8, num_workers=4) # Enable automatic-mixed-precision training with AmpOptimWrapper. optim_wrapper = dict( type='AmpOptimWrapper', optimizer=dict( type='SGD', lr=0.01 * 4, momentum=0.9, weight_decay=0.00004)) # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (8 GPUs) x (8 samples per GPU) auto_scale_lr = dict(base_batch_size=64) ================================================ FILE: configs/atss/metafile.yml ================================================ Collections: - Name: ATSS Metadata: Training Data: COCO Training Techniques: - SGD with Momentum - Weight Decay Training Resources: 8x V100 GPUs Architecture: - ATSS - FPN - ResNet Paper: URL: https://arxiv.org/abs/1912.02424 Title: 'Bridging the Gap Between Anchor-based and Anchor-free Detection via Adaptive Training Sample Selection' README: configs/atss/README.md Code: URL: https://github.com/open-mmlab/mmdetection/blob/v2.0.0/mmdet/models/detectors/atss.py#L6 Version: v2.0.0 Models: - Name: atss_r50_fpn_1x_coco In Collection: ATSS Config: configs/atss/atss_r50_fpn_1x_coco.py Metadata: Training Memory (GB): 3.7 inference time (ms/im): - value: 50.76 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 39.4 Weights: https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r50_fpn_1x_coco/atss_r50_fpn_1x_coco_20200209-985f7bd0.pth - Name: atss_r101_fpn_1x_coco In Collection: ATSS Config: configs/atss/atss_r101_fpn_1x_coco.py Metadata: Training Memory (GB): 5.6 inference time (ms/im): - value: 81.3 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 41.5 Weights: https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r101_fpn_1x_coco/atss_r101_fpn_1x_20200825-dfcadd6f.pth ================================================ FILE: configs/centernet/README.md ================================================ # CenterNet > [Objects as Points](https://arxiv.org/abs/1904.07850) ## Abstract Detection identifies objects as axis-aligned boxes in an image. Most successful object detectors enumerate a nearly exhaustive list of potential object locations and classify each. This is wasteful, inefficient, and requires additional post-processing. In this paper, we take a different approach. We model an object as a single point --- the center point of its bounding box. Our detector uses keypoint estimation to find center points and regresses to all other object properties, such as size, 3D location, orientation, and even pose. Our center point based approach, CenterNet, is end-to-end differentiable, simpler, faster, and more accurate than corresponding bounding box based detectors. CenterNet achieves the best speed-accuracy trade-off on the MS COCO dataset, with 28.1% AP at 142 FPS, 37.4% AP at 52 FPS, and 45.1% AP with multi-scale testing at 1.4 FPS. We use the same approach to estimate 3D bounding box in the KITTI benchmark and human pose on the COCO keypoint dataset. Our method performs competitively with sophisticated multi-stage methods and runs in real-time.
## Results and Models | Backbone | DCN | Mem (GB) | Box AP | Flip box AP | Config | Download | | :-------: | :-: | :------: | :----: | :---------: | :--------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | ResNet-18 | N | 3.45 | 25.9 | 27.3 | [config](./centernet_r18_8xb16-crop512-140e_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/centernet/centernet_resnet18_140e_coco/centernet_resnet18_140e_coco_20210705_093630-bb5b3bf7.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/centernet/centernet_resnet18_140e_coco/centernet_resnet18_140e_coco_20210705_093630.log.json) | | ResNet-18 | Y | 3.47 | 29.5 | 30.9 | [config](./centernet_r18-dcnv2_8xb16-crop512-140e_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/centernet/centernet_resnet18_dcnv2_140e_coco/centernet_resnet18_dcnv2_140e_coco_20210702_155131-c8cd631f.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/centernet/centernet_resnet18_dcnv2_140e_coco/centernet_resnet18_dcnv2_140e_coco_20210702_155131.log.json) | Note: - Flip box AP setting is single-scale and `flip=True`. - Due to complex data enhancement, we find that the performance is unstable and may fluctuate by about 0.4 mAP. mAP 29.4 ~ 29.8 is acceptable in ResNet-18-DCNv2. - Compared to the source code, we refer to [CenterNet-Better](https://github.com/FateScript/CenterNet-better), and make the following changes - fix wrong image mean and variance in image normalization to be compatible with the pre-trained backbone. - Use SGD rather than ADAM optimizer and add warmup and grad clip. - Use DistributedDataParallel as other models in MMDetection rather than using DataParallel. ## CenterNet Update | Backbone | Style | Lr schd | MS train | Mem (GB) | Box AP | Config | Download | | :-------: | :---: | :-----: | :------: | :------: | :----: | :------------------------------------------------------: | :----------------------: | | ResNet-50 | caffe | 1x | True | 3.3 | 40.2 | [config](./centernet-update_r50-caffe_fpn_ms-1x_coco.py) | [model](<>) \| [log](<>) | CenterNet Update from the paper of [Probabilistic two-stage detection](https://arxiv.org/abs/2103.07461). The author has updated CenterNet to greatly improve performance and convergence speed. The [Details](https://github.com/xingyizhou/CenterNet2/blob/master/docs/MODEL_ZOO.md) are as follows: - Using top-left-right-bottom box encoding and GIoU Loss - Adding regression loss to the center 3x3 region - Adding more positive pixels for the heatmap loss whose regression loss is small and is within the center3x3 region - Using RetinaNet-style optimizer (SGD), learning rate rule (0.01 for each batch size 16), and schedule (12 epochs) - Added FPN neck layers, and assigns objects to FPN levels based on a fixed size range. - Using standard NMS instead of max pooling ## Citation ```latex @article{zhou2019objects, title={Objects as Points}, author={Zhou, Xingyi and Wang, Dequan and Kr{\"a}henb{\"u}hl, Philipp}, booktitle={arXiv preprint arXiv:1904.07850}, year={2019} } ``` ================================================ FILE: configs/centernet/centernet-update_r101_fpn_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = './centernet-update_r50_fpn_8xb8-amp-lsj-200e_coco.py' model = dict( backbone=dict( depth=101, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101'))) ================================================ FILE: configs/centernet/centernet-update_r18_fpn_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = './centernet-update_r50_fpn_8xb8-amp-lsj-200e_coco.py' model = dict( backbone=dict( depth=18, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), neck=dict(in_channels=[64, 128, 256, 512])) ================================================ FILE: configs/centernet/centernet-update_r50-caffe_fpn_ms-1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] model = dict( type='CenterNet', # use caffe img_norm data_preprocessor=dict( type='DetDataPreprocessor', mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5, # There is a chance to get 40.3 after switching init_cfg, # otherwise it is about 39.9~40.1 init_cfg=dict(type='Caffe2Xavier', layer='Conv2d'), relu_before_extra_convs=True), bbox_head=dict( type='CenterNetUpdateHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, strides=[8, 16, 32, 64, 128], hm_min_radius=4, hm_min_overlap=0.8, more_pos_thresh=0.2, more_pos_topk=9, soft_weight_on_reg=False, loss_cls=dict( type='GaussianFocalLoss', pos_weight=0.25, neg_weight=0.75, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), ), train_cfg=None, test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) # single-scale training is about 39.3 train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomChoiceResize', scales=[(1333, 640), (1333, 672), (1333, 704), (1333, 736), (1333, 768), (1333, 800)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] train_dataloader = dict(dataset=dict(pipeline=train_pipeline)) # learning rate param_scheduler = [ dict( type='LinearLR', start_factor=0.00025, by_epoch=False, begin=0, end=4000), dict( type='MultiStepLR', begin=0, end=12, by_epoch=True, milestones=[8, 11], gamma=0.1) ] optim_wrapper = dict( optimizer=dict(lr=0.01), # Experiments show that there is no need to turn on clip_grad. paramwise_cfg=dict(norm_decay_mult=0.)) # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (8 GPUs) x (2 samples per GPU) auto_scale_lr = dict(base_batch_size=16) ================================================ FILE: configs/centernet/centernet-update_r50_fpn_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = '../common/lsj-200e_coco-detection.py' image_size = (1024, 1024) batch_augments = [dict(type='BatchFixedSizePad', size=image_size)] model = dict( type='CenterNet', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32, batch_augments=batch_augments), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5, init_cfg=dict(type='Caffe2Xavier', layer='Conv2d'), relu_before_extra_convs=True), bbox_head=dict( type='CenterNetUpdateHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, strides=[8, 16, 32, 64, 128], loss_cls=dict( type='GaussianFocalLoss', pos_weight=0.25, neg_weight=0.75, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), ), train_cfg=None, test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) train_dataloader = dict(batch_size=8, num_workers=4) # Enable automatic-mixed-precision training with AmpOptimWrapper. optim_wrapper = dict( type='AmpOptimWrapper', optimizer=dict( type='SGD', lr=0.01 * 4, momentum=0.9, weight_decay=0.00004), paramwise_cfg=dict(norm_decay_mult=0.)) param_scheduler = [ dict( type='LinearLR', start_factor=0.00025, by_epoch=False, begin=0, end=4000), dict( type='MultiStepLR', begin=0, end=25, by_epoch=True, milestones=[22, 24], gamma=0.1) ] # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (8 GPUs) x (8 samples per GPU) auto_scale_lr = dict(base_batch_size=64) ================================================ FILE: configs/centernet/centernet_r18-dcnv2_8xb16-crop512-140e_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py', './centernet_tta.py' ] dataset_type = 'CocoDataset' data_root = 'data/coco/' # model settings model = dict( type='CenterNet', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True), backbone=dict( type='ResNet', depth=18, norm_eval=False, norm_cfg=dict(type='BN'), init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), neck=dict( type='CTResNetNeck', in_channels=512, num_deconv_filters=(256, 128, 64), num_deconv_kernels=(4, 4, 4), use_dcn=True), bbox_head=dict( type='CenterNetHead', num_classes=80, in_channels=64, feat_channels=64, loss_center_heatmap=dict(type='GaussianFocalLoss', loss_weight=1.0), loss_wh=dict(type='L1Loss', loss_weight=0.1), loss_offset=dict(type='L1Loss', loss_weight=1.0)), train_cfg=None, test_cfg=dict(topk=100, local_maximum_kernel=3, max_per_img=100)) train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='LoadAnnotations', with_bbox=True), dict( type='PhotoMetricDistortion', brightness_delta=32, contrast_range=(0.5, 1.5), saturation_range=(0.5, 1.5), hue_delta=18), dict( type='RandomCenterCropPad', # The cropped images are padded into squares during training, # but may be less than crop_size. crop_size=(512, 512), ratios=(0.6, 0.7, 0.8, 0.9, 1.0, 1.1, 1.2, 1.3), mean=[0, 0, 0], std=[1, 1, 1], to_rgb=True, test_pad_mode=None), # Make sure the output is always crop_size. dict(type='Resize', scale=(512, 512), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict( type='LoadImageFromFile', to_float32=True, file_client_args={{_base_.file_client_args}}), # don't need Resize dict( type='RandomCenterCropPad', ratios=None, border=None, mean=[0, 0, 0], std=[1, 1, 1], to_rgb=True, test_mode=True, test_pad_mode=['logical_or', 31], test_pad_add_pix=1), dict(type='LoadAnnotations', with_bbox=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'border')) ] # Use RepeatDataset to speed up training train_dataloader = dict( batch_size=16, num_workers=4, persistent_workers=True, sampler=dict(type='DefaultSampler', shuffle=True), dataset=dict( _delete_=True, type='RepeatDataset', times=5, dataset=dict( type=dataset_type, data_root=data_root, ann_file='annotations/instances_train2017.json', data_prefix=dict(img='train2017/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=train_pipeline))) val_dataloader = dict(dataset=dict(pipeline=test_pipeline)) test_dataloader = val_dataloader # optimizer # Based on the default settings of modern detectors, the SGD effect is better # than the Adam in the source code, so we use SGD default settings and # if you use adam+lr5e-4, the map is 29.1. optim_wrapper = dict(clip_grad=dict(max_norm=35, norm_type=2)) max_epochs = 28 # learning policy # Based on the default settings of modern detectors, we added warmup settings. param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=1000), dict( type='MultiStepLR', begin=0, end=max_epochs, by_epoch=True, milestones=[18, 24], # the real step is [18*5, 24*5] gamma=0.1) ] train_cfg = dict(max_epochs=max_epochs) # the real epoch is 28*5=140 # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (8 GPUs) x (16 samples per GPU) auto_scale_lr = dict(base_batch_size=128) ================================================ FILE: configs/centernet/centernet_r18_8xb16-crop512-140e_coco.py ================================================ _base_ = './centernet_r18-dcnv2_8xb16-crop512-140e_coco.py' model = dict(neck=dict(use_dcn=False)) ================================================ FILE: configs/centernet/centernet_tta.py ================================================ # This is different from the TTA of official CenterNet. tta_model = dict( type='DetTTAModel', tta_cfg=dict(nms=dict(type='nms', iou_threshold=0.5), max_per_img=100)) tta_pipeline = [ dict( type='LoadImageFromFile', to_float32=True, file_client_args=dict(backend='disk')), dict( type='TestTimeAug', transforms=[ [ # ``RandomFlip`` must be placed before ``RandomCenterCropPad``, # otherwise bounding box coordinates after flipping cannot be # recovered correctly. dict(type='RandomFlip', prob=1.), dict(type='RandomFlip', prob=0.) ], [ dict( type='RandomCenterCropPad', ratios=None, border=None, mean=[0, 0, 0], std=[1, 1, 1], to_rgb=True, test_mode=True, test_pad_mode=['logical_or', 31], test_pad_add_pix=1), ], [dict(type='LoadAnnotations', with_bbox=True)], [ dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'flip', 'flip_direction', 'border')) ] ]) ] ================================================ FILE: configs/centernet/metafile.yml ================================================ Collections: - Name: CenterNet Metadata: Training Data: COCO Training Techniques: - SGD with Momentum - Weight Decay Training Resources: 8x TITANXP GPUs Architecture: - ResNet Paper: URL: https://arxiv.org/abs/1904.07850 Title: 'Objects as Points' README: configs/centernet/README.md Code: URL: https://github.com/open-mmlab/mmdetection/blob/v2.13.0/mmdet/models/detectors/centernet.py#L10 Version: v2.13.0 Models: - Name: centernet_r18-dcnv2_8xb16-crop512-140e_coco In Collection: CenterNet Config: configs/centernet/centernet_r18-dcnv2_8xb16-crop512-140e_coco.py Metadata: Batch Size: 128 Training Memory (GB): 3.47 Epochs: 140 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 29.5 Weights: https://download.openmmlab.com/mmdetection/v2.0/centernet/centernet_resnet18_dcnv2_140e_coco/centernet_resnet18_dcnv2_140e_coco_20210702_155131-c8cd631f.pth - Name: centernet_r18_8xb16-crop512-140e_coco In Collection: CenterNet Config: configs/centernet/centernet_r18_8xb16-crop512-140e_coco.py Metadata: Batch Size: 128 Training Memory (GB): 3.45 Epochs: 140 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 25.9 Weights: https://download.openmmlab.com/mmdetection/v2.0/centernet/centernet_resnet18_140e_coco/centernet_resnet18_140e_coco_20210705_093630-bb5b3bf7.pth ================================================ FILE: configs/crosskd/crosskd_r18_gflv1_r50_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r50_fpn_1x_coco/gfl_r50_fpn_1x_coco_20200629_121244-25944287.pth' # noqa model = dict( type='CrossKDGFL', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), teacher_config='configs/gfl/gfl_r50_fpn_1x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=18, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), neck=dict( type='FPN', in_channels=[64, 128, 256, 512], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='GFLHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), loss_cls=dict( type='QualityFocalLoss', use_sigmoid=True, beta=2.0, loss_weight=1.0), loss_dfl=dict(type='DistributionFocalLoss', loss_weight=0.25), reg_max=16, loss_bbox=dict(type='GIoULoss', loss_weight=2.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=1.0), loss_reg_kd=dict( type='KnowledgeDistillationKLDivLoss', class_reduction='sum', T=1, loss_weight=4.0), reused_teacher_head_idx=3), # model training and testing settings train_cfg=dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) # optimizer optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=12, val_interval=1) default_hooks = dict(checkpoint=dict(type='CheckpointHook', interval=12)) train_dataloader = dict(batch_size=2, num_workers=4) auto_scale_lr = dict(enable=True, base_batch_size=16) ================================================ FILE: configs/crosskd/crosskd_r18_retinanet_r50_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_1x_coco/retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth' # noqa model = dict( type='CrossKDRetinaNet', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), teacher_config='configs/retinanet/retinanet_r50_fpn_1x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=18, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), neck=dict( type='FPN', in_channels=[64, 128, 256, 512], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='RetinaHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=1.0), loss_reg_kd=dict(type='GIoULoss', loss_weight=1.0), reused_teacher_head_idx=3), # model training and testing settings train_cfg=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0, ignore_iof_thr=-1), sampler=dict( type='PseudoSampler'), # Focal loss should use PseudoSampler allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100)) optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=12, val_interval=1) train_dataloader = dict(batch_size=2, num_workers=4) auto_scale_lr = dict(enable=True, base_batch_size=16) default_hooks = dict(checkpoint=dict(type='CheckpointHook', interval=12)) ================================================ FILE: configs/crosskd/crosskd_r50_atss_r101_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r101_fpn_1x_coco/atss_r101_fpn_1x_20200825-dfcadd6f.pth' data_preprocessor = dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32 ) model = dict( type='CrossKDATSS', data_preprocessor=data_preprocessor, teacher_config='configs/atss/atss_r101_fpn_1x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='ATSSHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0.0, 0.0, 0.0, 0.0], target_stds=[0.1, 0.1, 0.2, 0.2]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=1.0), loss_reg_kd=dict(type='GIoULoss', loss_weight=1.0), loss_center_kd=dict(type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), reused_teacher_head_idx=3), train_cfg=dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100) ) optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001) ) train_dataloader = dict(batch_size=8, num_workers=4) auto_scale_lr = dict(enable=True, base_batch_size=16) ================================================ FILE: configs/crosskd/crosskd_r50_fcos_r101-2x-ms_caffe_fpn_gn-head_2x_ms_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] data_preprocessor = dict(type='DetDataPreprocessor', mean=[102.9801, 115.9465, 122.7717], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32) teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco-511424d6.pth' model = dict( type='CrossKDFCOS', data_preprocessor=data_preprocessor, teacher_config='configs/fcos/fcos_r101-caffe_fpn_gn-head_ms-640-800-2x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron/resnet50_caffe')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5, relu_before_extra_convs=True), bbox_head=dict( type='FCOSHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, strides=[8, 16, 32, 64, 128], loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='IoULoss', loss_weight=1.0), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=0.4), loss_reg_kd=dict(type='IoULoss', loss_weight=0.75), reused_teacher_head_idx=2), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100)) # dataset settings train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomChoiceResize', scales=[(1333, 640), (1333, 800)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] train_dataloader = dict(dataset=dict(pipeline=train_pipeline), batch_size=2, num_workers=4) # optimizer optim_wrapper = dict( optimizer=dict(lr=0.01), paramwise_cfg=dict(bias_lr_mult=2., bias_decay_mult=0.), clip_grad=dict(max_norm=35, norm_type=2)) # training schedule for 2x max_epochs = 24 train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=max_epochs, val_interval=1) default_hooks = dict(checkpoint=dict(type='CheckpointHook', interval=2)) auto_scale_lr = dict(enable=True, base_batch_size=16) # learning rate param_scheduler = [ dict( type='ConstantLR', factor=0.3333333333333333, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=24, by_epoch=True, milestones=[16, 22], gamma=0.1) ] ================================================ FILE: configs/crosskd/crosskd_r50_gflv1_r101-2x-ms_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_mstrain_2x_coco/gfl_r101_fpn_mstrain_2x_coco_20200629_200126-dd12f847.pth' # noqa model = dict( type='CrossKDGFL', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), teacher_config='configs/gfl/gfl_r101_fpn_ms-2x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='GFLHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), loss_cls=dict( type='QualityFocalLoss', use_sigmoid=True, beta=2.0, loss_weight=1.0), loss_dfl=dict(type='DistributionFocalLoss', loss_weight=0.25), reg_max=16, loss_bbox=dict(type='GIoULoss', loss_weight=2.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=1.0), loss_reg_kd=dict( type='KnowledgeDistillationKLDivLoss', class_reduction='sum', T=1, loss_weight=4.0), reused_teacher_head_idx=3), # model training and testing settings train_cfg=dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) # optimizer optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=12, val_interval=1) default_hooks = dict(checkpoint=dict(type='CheckpointHook', interval=12)) train_dataloader = dict(batch_size=2, num_workers=4) auto_scale_lr = dict(enable=True, base_batch_size=16) ================================================ FILE: configs/crosskd/crosskd_r50_retinanet_r101_fpn_2x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_2x.py', '../_base_/default_runtime.py' ] teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_2x_coco/retinanet_r101_fpn_2x_coco_20200131-5560aee8.pth' # noqa model = dict( type='CrossKDRetinaNet', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), teacher_config='configs/retinanet/retinanet_r101_fpn_2x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='RetinaHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=1.0), loss_reg_kd=dict(type='GIoULoss', loss_weight=1.0), reused_teacher_head_idx=3), # model training and testing settings train_cfg=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0, ignore_iof_thr=-1), sampler=dict( type='PseudoSampler'), # Focal loss should use PseudoSampler allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100) ) optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=24, val_interval=1) train_dataloader = dict(batch_size=2, num_workers=4) auto_scale_lr = dict(enable=True, base_batch_size=16) default_hooks = dict(checkpoint=dict(type='CheckpointHook', interval=12)) ================================================ FILE: configs/crosskd/crosskd_r50_retinanet_swint_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] teacher_ckpt = 'retinanet_swin-t-p4-w7_fpn_1x_coco.pth' model = dict( type='CrossKDRetinaNet', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), teacher_config='configs/retinanet/retinanet_swin-t-p4-w7_fpn_1x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='RetinaHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=1.0), loss_reg_kd=dict(type='GIoULoss', loss_weight=1.0), reused_teacher_head_idx=3), # model training and testing settings train_cfg=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0, ignore_iof_thr=-1), sampler=dict( type='PseudoSampler'), # Focal loss should use PseudoSampler allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100) ) optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) train_dataloader = dict(batch_size=2, num_workers=4) auto_scale_lr = dict(enable=True, base_batch_size=16) default_hooks = dict(checkpoint=dict(type='CheckpointHook', interval=12)) ================================================ FILE: configs/crosskd+pkd/crosskd+pkd_r50_atss_r101_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/atss/atss_r101_fpn_1x_coco/atss_r101_fpn_1x_20200825-dfcadd6f.pth' data_preprocessor = dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32) model = dict( type='CrossKDATSS', data_preprocessor=data_preprocessor, teacher_config='configs/atss/atss_r101_fpn_1x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='ATSSHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0.0, 0.0, 0.0, 0.0], target_stds=[0.1, 0.1, 0.2, 0.2]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=1.0), loss_reg_kd=dict(type='GIoULoss', loss_weight=1.0), loss_center_kd=dict(type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_feat_kd=dict(type='PKDLoss', loss_weight=1), reused_teacher_head_idx=3), train_cfg=dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) train_dataloader = dict(batch_size=8, num_workers=4) auto_scale_lr = dict(enable=True, base_batch_size=16) ================================================ FILE: configs/crosskd+pkd/crosskd+pkd_r50_fcos_r101-2x-ms_caffe_fpn_gn-head_2x_ms_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] data_preprocessor = dict(type='DetDataPreprocessor', mean=[102.9801, 115.9465, 122.7717], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32) teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco-511424d6.pth' model = dict( type='CrossKDFCOS', data_preprocessor=data_preprocessor, teacher_config='configs/fcos/fcos_r101-caffe_fpn_gn-head_ms-640-800-2x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron/resnet50_caffe')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5, relu_before_extra_convs=True), bbox_head=dict( type='FCOSHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, strides=[8, 16, 32, 64, 128], loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='IoULoss', loss_weight=1.0), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=0.4), loss_reg_kd=dict(type='IoULoss', loss_weight=0.75), loss_feat_kd=dict(type='PKDLoss', loss_weight=2), reused_teacher_head_idx=2), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100)) # dataset settings train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomChoiceResize', scales=[(1333, 640), (1333, 800)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] train_dataloader = dict(dataset=dict(pipeline=train_pipeline), batch_size=2, num_workers=4) # optimizer optim_wrapper = dict( optimizer=dict(lr=0.01), paramwise_cfg=dict(bias_lr_mult=2., bias_decay_mult=0.), clip_grad=dict(max_norm=35, norm_type=2)) # training schedule for 2x max_epochs = 24 train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=max_epochs, val_interval=1) default_hooks = dict(checkpoint=dict(type='CheckpointHook', interval=2)) auto_scale_lr = dict(enable=True, base_batch_size=16) # learning rate param_scheduler = [ dict( type='ConstantLR', factor=0.3333333333333333, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=24, by_epoch=True, milestones=[16, 22], gamma=0.1) ] ================================================ FILE: configs/crosskd+pkd/crosskd+pkd_r50_gflv1_r101-2x-ms_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_mstrain_2x_coco/gfl_r101_fpn_mstrain_2x_coco_20200629_200126-dd12f847.pth' # noqa model = dict( type='CrossKDGFL', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), teacher_config='configs/gfl/gfl_r101_fpn_ms-2x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='GFLHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), loss_cls=dict( type='QualityFocalLoss', use_sigmoid=True, beta=2.0, loss_weight=1.0), loss_dfl=dict(type='DistributionFocalLoss', loss_weight=0.25), reg_max=16, loss_bbox=dict(type='GIoULoss', loss_weight=2.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=1.0), loss_reg_kd=dict( type='KnowledgeDistillationKLDivLoss', class_reduction='sum', T=1, loss_weight=4.0), loss_feat_kd=dict(type='PKDLoss', loss_weight=6.0), reused_teacher_head_idx=3), # model training and testing settings train_cfg=dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) # optimizer optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=12, val_interval=1) default_hooks = dict(checkpoint=dict(type='CheckpointHook', interval=12)) train_dataloader = dict(batch_size=2, num_workers=4) auto_scale_lr = dict(enable=True, base_batch_size=16) ================================================ FILE: configs/crosskd+pkd/crosskd+pkd_r50_retinanet_r101_fpn_2x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_2x.py', '../_base_/default_runtime.py' ] teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_2x_coco/retinanet_r101_fpn_2x_coco_20200131-5560aee8.pth' # noqa model = dict( type='CrossKDRetinaNet', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), teacher_config='configs/retinanet/retinanet_r101_fpn_2x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='RetinaHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='L1Loss', loss_weight=1.0)), kd_cfg=dict( loss_cls_kd=dict(type='KDQualityFocalLoss', beta=1, loss_weight=1.0), loss_reg_kd=dict(type='GIoULoss', loss_weight=1.0), loss_feat_kd=dict(type='PKDLoss', loss_weight=2.0), reused_teacher_head_idx=3), # model training and testing settings train_cfg=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0, ignore_iof_thr=-1), sampler=dict( type='PseudoSampler'), # Focal loss should use PseudoSampler allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100) ) optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) train_cfg = dict(type='EpochBasedTrainLoop', max_epochs=24, val_interval=1) train_dataloader = dict(batch_size=2, num_workers=4) auto_scale_lr = dict(enable=True, base_batch_size=16) default_hooks = dict(checkpoint=dict(type='CheckpointHook', interval=12)) ================================================ FILE: configs/fcos/README.md ================================================ # FCOS > [FCOS: Fully Convolutional One-Stage Object Detection](https://arxiv.org/abs/1904.01355) ## Abstract We propose a fully convolutional one-stage object detector (FCOS) to solve object detection in a per-pixel prediction fashion, analogue to semantic segmentation. Almost all state-of-the-art object detectors such as RetinaNet, SSD, YOLOv3, and Faster R-CNN rely on pre-defined anchor boxes. In contrast, our proposed detector FCOS is anchor box free, as well as proposal free. By eliminating the predefined set of anchor boxes, FCOS completely avoids the complicated computation related to anchor boxes such as calculating overlapping during training. More importantly, we also avoid all hyper-parameters related to anchor boxes, which are often very sensitive to the final detection performance. With the only post-processing non-maximum suppression (NMS), FCOS with ResNeXt-64x4d-101 achieves 44.7% in AP with single-model and single-scale testing, surpassing previous one-stage detectors with the advantage of being much simpler. For the first time, we demonstrate a much simpler and flexible detection framework achieving improved detection accuracy. We hope that the proposed FCOS framework can serve as a simple and strong alternative for many other instance-level tasks.
## Results and Models | Backbone | Style | GN | MS train | Tricks | DCN | Lr schd | Mem (GB) | Inf time (fps) | box AP | Config | Download | | :------: | :---: | :-: | :------: | :----: | :-: | :-----: | :------: | :------------: | :----: | :------------------------------------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | R-50 | caffe | Y | N | N | N | 1x | 3.6 | 22.7 | 36.6 | [config](./fcos_r50-caffe_fpn_gn-head_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r50_caffe_fpn_gn-head_1x_coco/fcos_r50_caffe_fpn_gn-head_1x_coco-821213aa.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r50_caffe_fpn_gn-head_1x_coco/20201227_180009.log.json) | | R-50 | caffe | Y | N | Y | N | 1x | 3.7 | - | 38.7 | [config](./fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_1x_coco/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_1x_coco-0a0d75a8.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_1x_coco/20210105_135818.log.json) | | R-50 | caffe | Y | N | Y | Y | 1x | 3.8 | - | 42.3 | [config](./fcos_r50-dcn-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_dcn_1x_coco/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_dcn_1x_coco-ae4d8b3d.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_dcn_1x_coco/20210105_224556.log.json) | | R-101 | caffe | Y | N | N | N | 1x | 5.5 | 17.3 | 39.1 | [config](./fcos_r101-caffe_fpn_gn-head-1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r101_caffe_fpn_gn-head_1x_coco/fcos_r101_caffe_fpn_gn-head_1x_coco-0e37b982.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r101_caffe_fpn_gn-head_1x_coco/20210103_155046.log.json) | | Backbone | Style | GN | MS train | Lr schd | Mem (GB) | Inf time (fps) | box AP | Config | Download | | :------: | :-----: | :-: | :------: | :-----: | :------: | :------------: | :----: | :-----------------------------------------------------------: | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | R-50 | caffe | Y | Y | 2x | 2.6 | 22.9 | 38.5 | [config](./fcos_r50-caffe_fpn_gn-head_ms-640-800-2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r50_caffe_fpn_gn-head_mstrain_640-800_2x_coco/fcos_r50_caffe_fpn_gn-head_mstrain_640-800_2x_coco-d92ceeea.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r50_caffe_fpn_gn-head_mstrain_640-800_2x_coco/20201227_161900.log.json) | | R-101 | caffe | Y | Y | 2x | 5.5 | 17.3 | 40.8 | [config](./fcos_r101-caffe_fpn_gn-head_ms-640-800-2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco-511424d6.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco/20210103_155046.log.json) | | X-101 | pytorch | Y | Y | 2x | 10.0 | 9.7 | 42.6 | [config](./fcos_x101-64x4d_fpn_gn-head_ms-640-800-2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_x101_64x4d_fpn_gn-head_mstrain_640-800_2x_coco/fcos_x101_64x4d_fpn_gn-head_mstrain_640-800_2x_coco-ede514a8.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_x101_64x4d_fpn_gn-head_mstrain_640-800_2x_coco/20210114_133041.log.json) | **Notes:** - The X-101 backbone is X-101-64x4d. - Tricks means setting `norm_on_bbox`, `centerness_on_reg`, `center_sampling` as `True`. - DCN means using `DCNv2` in both backbone and head. ## Citation ```latex @article{tian2019fcos, title={FCOS: Fully Convolutional One-Stage Object Detection}, author={Tian, Zhi and Shen, Chunhua and Chen, Hao and He, Tong}, journal={arXiv preprint arXiv:1904.01355}, year={2019} } ``` ================================================ FILE: configs/fcos/fcos_r101-caffe_fpn_gn-head-1x_coco.py ================================================ _base_ = './fcos_r50-caffe_fpn_gn-head_1x_coco.py' # model settings model = dict( backbone=dict( depth=101, init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron/resnet101_caffe'))) ================================================ FILE: configs/fcos/fcos_r101-caffe_fpn_gn-head_ms-640-800-2x_coco.py ================================================ _base_ = './fcos_r50-caffe_fpn_gn-head_1x_coco.py' # model settings model = dict( backbone=dict( depth=101, init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron/resnet101_caffe'))) # dataset settings train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomChoiceResize', scale=[(1333, 640), (1333, 800)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] train_dataloader = dict(dataset=dict(pipeline=train_pipeline)) # training schedule for 2x max_epochs = 24 train_cfg = dict(max_epochs=max_epochs) # learning rate param_scheduler = [ dict(type='ConstantLR', factor=1.0 / 3, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=max_epochs, by_epoch=True, milestones=[16, 22], gamma=0.1) ] ================================================ FILE: configs/fcos/fcos_r101_fpn_gn-head-center-normbbox-centeronreg-giou_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = './fcos_r50_fpn_gn-head-center-normbbox-centeronreg-giou_8xb8-amp-lsj-200e_coco.py' # noqa model = dict( backbone=dict( depth=101, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101'))) ================================================ FILE: configs/fcos/fcos_r18_fpn_gn-head-center-normbbox-centeronreg-giou_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = './fcos_r50_fpn_gn-head-center-normbbox-centeronreg-giou_8xb8-amp-lsj-200e_coco.py' # noqa model = dict( backbone=dict( depth=18, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), neck=dict(in_channels=[64, 128, 256, 512])) ================================================ FILE: configs/fcos/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py ================================================ _base_ = 'fcos_r50-caffe_fpn_gn-head_1x_coco.py' # model setting model = dict( data_preprocessor=dict( type='DetDataPreprocessor', mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32), backbone=dict( init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), bbox_head=dict( norm_on_bbox=True, centerness_on_reg=True, dcn_on_last_conv=False, center_sampling=True, conv_bias=True, loss_bbox=dict(type='GIoULoss', loss_weight=1.0)), # training and testing settings test_cfg=dict(nms=dict(type='nms', iou_threshold=0.6))) # learning rate param_scheduler = [ dict( type='LinearLR', start_factor=1.0 / 3.0, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=12, by_epoch=True, milestones=[8, 11], gamma=0.1) ] # optimizer optim_wrapper = dict(clip_grad=None) ================================================ FILE: configs/fcos/fcos_r50-caffe_fpn_gn-head-center_1x_coco.py ================================================ _base_ = './fcos_r50-caffe_fpn_gn-head_1x_coco.py' # model settings model = dict(bbox_head=dict(center_sampling=True, center_sample_radius=1.5)) ================================================ FILE: configs/fcos/fcos_r50-caffe_fpn_gn-head_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] # model settings model = dict( type='FCOS', data_preprocessor=dict( type='DetDataPreprocessor', mean=[102.9801, 115.9465, 122.7717], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron/resnet50_caffe')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', # use P5 num_outs=5, relu_before_extra_convs=True), bbox_head=dict( type='FCOSHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, strides=[8, 16, 32, 64, 128], loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='IoULoss', loss_weight=1.0), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)), # testing settings test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100)) # learning rate param_scheduler = [ dict(type='ConstantLR', factor=1.0 / 3, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=12, by_epoch=True, milestones=[8, 11], gamma=0.1) ] # optimizer optim_wrapper = dict( optimizer=dict(lr=0.01), paramwise_cfg=dict(bias_lr_mult=2., bias_decay_mult=0.), clip_grad=dict(max_norm=35, norm_type=2)) ================================================ FILE: configs/fcos/fcos_r50-caffe_fpn_gn-head_4xb4-1x_coco.py ================================================ # TODO: Remove this config after benchmarking all related configs _base_ = 'fcos_r50-caffe_fpn_gn-head_1x_coco.py' # dataset settings train_dataloader = dict(batch_size=4, num_workers=4) ================================================ FILE: configs/fcos/fcos_r50-caffe_fpn_gn-head_ms-640-800-2x_coco.py ================================================ _base_ = './fcos_r50-caffe_fpn_gn-head_1x_coco.py' # dataset settings train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomChoiceResize', scale=[(1333, 640), (1333, 800)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] train_dataloader = dict(dataset=dict(pipeline=train_pipeline)) # training schedule for 2x max_epochs = 24 train_cfg = dict(max_epochs=max_epochs) # learning rate param_scheduler = [ dict(type='ConstantLR', factor=1.0 / 3, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=max_epochs, by_epoch=True, milestones=[16, 22], gamma=0.1) ] ================================================ FILE: configs/fcos/fcos_r50-dcn-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py ================================================ _base_ = 'fcos_r50-caffe_fpn_gn-head_1x_coco.py' # model settings model = dict( data_preprocessor=dict( type='DetDataPreprocessor', mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32), backbone=dict( dcn=dict(type='DCNv2', deform_groups=1, fallback_on_stride=False), stage_with_dcn=(False, True, True, True), init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe')), bbox_head=dict( norm_on_bbox=True, centerness_on_reg=True, dcn_on_last_conv=True, center_sampling=True, conv_bias=True, loss_bbox=dict(type='GIoULoss', loss_weight=1.0)), # training and testing settings test_cfg=dict(nms=dict(type='nms', iou_threshold=0.6))) # learning rate param_scheduler = [ dict( type='LinearLR', start_factor=1.0 / 3.0, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=12, by_epoch=True, milestones=[8, 11], gamma=0.1) ] # optimizer optim_wrapper = dict(clip_grad=None) ================================================ FILE: configs/fcos/fcos_r50_fpn_gn-head-center-normbbox-centeronreg-giou_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = '../common/lsj-200e_coco-detection.py' image_size = (1024, 1024) batch_augments = [dict(type='BatchFixedSizePad', size=image_size)] # model settings model = dict( type='FCOS', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32, batch_augments=batch_augments), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', # use P5 num_outs=5, relu_before_extra_convs=True), bbox_head=dict( type='FCOSHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, strides=[8, 16, 32, 64, 128], norm_on_bbox=True, centerness_on_reg=True, dcn_on_last_conv=False, center_sampling=True, conv_bias=True, loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=1.0), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)), # testing settings test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) train_dataloader = dict(batch_size=8, num_workers=4) # Enable automatic-mixed-precision training with AmpOptimWrapper. optim_wrapper = dict( type='AmpOptimWrapper', optimizer=dict( type='SGD', lr=0.01 * 4, momentum=0.9, weight_decay=0.00004), paramwise_cfg=dict(bias_lr_mult=2., bias_decay_mult=0.), clip_grad=dict(max_norm=35, norm_type=2)) # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (8 GPUs) x (8 samples per GPU) auto_scale_lr = dict(base_batch_size=64) ================================================ FILE: configs/fcos/fcos_x101-64x4d_fpn_gn-head_ms-640-800-2x_coco.py ================================================ _base_ = './fcos_r50-caffe_fpn_gn-head_1x_coco.py' # model settings model = dict( data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), backbone=dict( type='ResNeXt', depth=101, groups=64, base_width=4, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://resnext101_64x4d'))) # dataset settings train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomChoiceResize', scale=[(1333, 640), (1333, 800)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] train_dataloader = dict(dataset=dict(pipeline=train_pipeline)) # training schedule for 2x max_epochs = 24 train_cfg = dict(max_epochs=max_epochs) # learning rate param_scheduler = [ dict(type='ConstantLR', factor=1.0 / 3, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=max_epochs, by_epoch=True, milestones=[16, 22], gamma=0.1) ] ================================================ FILE: configs/fcos/metafile.yml ================================================ Collections: - Name: FCOS Metadata: Training Data: COCO Training Techniques: - SGD with Momentum - Weight Decay Training Resources: 8x V100 GPUs Architecture: - FPN - Group Normalization - ResNet Paper: URL: https://arxiv.org/abs/1904.01355 Title: 'FCOS: Fully Convolutional One-Stage Object Detection' README: configs/fcos/README.md Code: URL: https://github.com/open-mmlab/mmdetection/blob/v2.0.0/mmdet/models/detectors/fcos.py#L6 Version: v2.0.0 Models: - Name: fcos_r50-caffe_fpn_gn-head_1x_coco In Collection: FCOS Config: configs/fcos/fcos_r50-caffe_fpn_gn-head_1x_coco.py Metadata: Training Memory (GB): 3.6 inference time (ms/im): - value: 44.05 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 36.6 Weights: https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r50_caffe_fpn_gn-head_1x_coco/fcos_r50_caffe_fpn_gn-head_1x_coco-821213aa.pth - Name: fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco In Collection: FCOS Config: configs/fcos/fcos_r50-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py Metadata: Training Memory (GB): 3.7 Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 38.7 Weights: https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_1x_coco/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_1x_coco-0a0d75a8.pth - Name: fcos_r50-dcn-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco In Collection: FCOS Config: configs/fcos/fcos_r50-dcn-caffe_fpn_gn-head-center-normbbox-centeronreg-giou_1x_coco.py Metadata: Training Memory (GB): 3.8 Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 42.3 Weights: https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_dcn_1x_coco/fcos_center-normbbox-centeronreg-giou_r50_caffe_fpn_gn-head_dcn_1x_coco-ae4d8b3d.pth - Name: fcos_r101-caffe_fpn_gn-head-1x_coco In Collection: FCOS Config: configs/fcos/fcos_r101-caffe_fpn_gn-head-1x_coco.py Metadata: Training Memory (GB): 5.5 inference time (ms/im): - value: 57.8 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 39.1 Weights: https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r101_caffe_fpn_gn-head_1x_coco/fcos_r101_caffe_fpn_gn-head_1x_coco-0e37b982.pth - Name: fcos_r50-caffe_fpn_gn-head_ms-640-800-2x_coco In Collection: FCOS Config: configs/fcos/fcos_r50-caffe_fpn_gn-head_ms-640-800-2x_coco.py Metadata: Training Memory (GB): 2.6 inference time (ms/im): - value: 43.67 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 38.5 Weights: https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r50_caffe_fpn_gn-head_mstrain_640-800_2x_coco/fcos_r50_caffe_fpn_gn-head_mstrain_640-800_2x_coco-d92ceeea.pth - Name: fcos_r101-caffe_fpn_gn-head_ms-640-800-2x_coco In Collection: FCOS Config: configs/fcos/fcos_r101-caffe_fpn_gn-head_ms-640-800-2x_coco.py Metadata: Training Memory (GB): 5.5 inference time (ms/im): - value: 57.8 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 40.8 Weights: https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco/fcos_r101_caffe_fpn_gn-head_mstrain_640-800_2x_coco-511424d6.pth - Name: fcos_x101-64x4d_fpn_gn-head_ms-640-800-2x_coco In Collection: FCOS Config: configs/fcos/fcos_x101-64x4d_fpn_gn-head_ms-640-800-2x_coco.py Metadata: Training Memory (GB): 10.0 inference time (ms/im): - value: 103.09 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 42.6 Weights: https://download.openmmlab.com/mmdetection/v2.0/fcos/fcos_x101_64x4d_fpn_gn-head_mstrain_640-800_2x_coco/fcos_x101_64x4d_fpn_gn-head_mstrain_640-800_2x_coco-ede514a8.pth ================================================ FILE: configs/gfl/README.md ================================================ # GFL > [Generalized Focal Loss: Learning Qualified and Distributed Bounding Boxes for Dense Object Detection](https://arxiv.org/abs/2006.04388) ## Abstract One-stage detector basically formulates object detection as dense classification and localization. The classification is usually optimized by Focal Loss and the box location is commonly learned under Dirac delta distribution. A recent trend for one-stage detectors is to introduce an individual prediction branch to estimate the quality of localization, where the predicted quality facilitates the classification to improve detection performance. This paper delves into the representations of the above three fundamental elements: quality estimation, classification and localization. Two problems are discovered in existing practices, including (1) the inconsistent usage of the quality estimation and classification between training and inference and (2) the inflexible Dirac delta distribution for localization when there is ambiguity and uncertainty in complex scenes. To address the problems, we design new representations for these elements. Specifically, we merge the quality estimation into the class prediction vector to form a joint representation of localization quality and classification, and use a vector to represent arbitrary distribution of box locations. The improved representations eliminate the inconsistency risk and accurately depict the flexible distribution in real data, but contain continuous labels, which is beyond the scope of Focal Loss. We then propose Generalized Focal Loss (GFL) that generalizes Focal Loss from its discrete form to the continuous version for successful optimization. On COCO test-dev, GFL achieves 45.0% AP using ResNet-101 backbone, surpassing state-of-the-art SAPD (43.5%) and ATSS (43.6%) with higher or comparable inference speed, under the same backbone and training settings. Notably, our best model can achieve a single-model single-scale AP of 48.2%, at 10 FPS on a single 2080Ti GPU.
## Results and Models | Backbone | Style | Lr schd | Multi-scale Training | Inf time (fps) | box AP | Config | Download | | :---------------: | :-----: | :-----: | :------------------: | :------------: | :----: | :------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | R-50 | pytorch | 1x | No | 19.5 | 40.2 | [config](./gfl_r50_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r50_fpn_1x_coco/gfl_r50_fpn_1x_coco_20200629_121244-25944287.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r50_fpn_1x_coco/gfl_r50_fpn_1x_coco_20200629_121244.log.json) | | R-50 | pytorch | 2x | Yes | 19.5 | 42.9 | [config](./gfl_r50_fpn_ms-2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r50_fpn_mstrain_2x_coco/gfl_r50_fpn_mstrain_2x_coco_20200629_213802-37bb1edc.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r50_fpn_mstrain_2x_coco/gfl_r50_fpn_mstrain_2x_coco_20200629_213802.log.json) | | R-101 | pytorch | 2x | Yes | 14.7 | 44.7 | [config](./gfl_r101_fpn_ms-2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_mstrain_2x_coco/gfl_r101_fpn_mstrain_2x_coco_20200629_200126-dd12f847.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_mstrain_2x_coco/gfl_r101_fpn_mstrain_2x_coco_20200629_200126.log.json) | | R-101-dcnv2 | pytorch | 2x | Yes | 12.9 | 47.1 | [config](./gfl_r101-dconv-c3-c5_fpn_ms-2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_dconv_c3-c5_mstrain_2x_coco/gfl_r101_fpn_dconv_c3-c5_mstrain_2x_coco_20200630_102002-134b07df.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_dconv_c3-c5_mstrain_2x_coco/gfl_r101_fpn_dconv_c3-c5_mstrain_2x_coco_20200630_102002.log.json) | | X-101-32x4d | pytorch | 2x | Yes | 12.1 | 45.9 | [config](./gfl_x101-32x4d_fpn_ms-2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_x101_32x4d_fpn_mstrain_2x_coco/gfl_x101_32x4d_fpn_mstrain_2x_coco_20200630_102002-50c1ffdb.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_x101_32x4d_fpn_mstrain_2x_coco/gfl_x101_32x4d_fpn_mstrain_2x_coco_20200630_102002.log.json) | | X-101-32x4d-dcnv2 | pytorch | 2x | Yes | 10.7 | 48.1 | [config](./gfl_x101-32x4d-dconv-c4-c5_fpn_ms-2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_x101_32x4d_fpn_dconv_c4-c5_mstrain_2x_coco/gfl_x101_32x4d_fpn_dconv_c4-c5_mstrain_2x_coco_20200630_102002-14a2bf25.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_x101_32x4d_fpn_dconv_c4-c5_mstrain_2x_coco/gfl_x101_32x4d_fpn_dconv_c4-c5_mstrain_2x_coco_20200630_102002.log.json) | \[1\] *1x and 2x mean the model is trained for 90K and 180K iterations, respectively.* \ \[2\] *All results are obtained with a single model and without any test time data augmentation such as multi-scale, flipping and etc..* \ \[3\] *`dcnv2` denotes deformable convolutional networks v2.* \ \[4\] *FPS is tested with a single GeForce RTX 2080Ti GPU, using a batch size of 1.* ## Citation We provide config files to reproduce the object detection results in the paper [Generalized Focal Loss: Learning Qualified and Distributed Bounding Boxes for Dense Object Detection](https://arxiv.org/abs/2006.04388) ```latex @article{li2020generalized, title={Generalized Focal Loss: Learning Qualified and Distributed Bounding Boxes for Dense Object Detection}, author={Li, Xiang and Wang, Wenhai and Wu, Lijun and Chen, Shuo and Hu, Xiaolin and Li, Jun and Tang, Jinhui and Yang, Jian}, journal={arXiv preprint arXiv:2006.04388}, year={2020} } ``` ================================================ FILE: configs/gfl/gfl_r101-dconv-c3-c5_fpn_ms-2x_coco.py ================================================ _base_ = './gfl_r50_fpn_ms-2x_coco.py' model = dict( backbone=dict( type='ResNet', depth=101, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), dcn=dict(type='DCN', deform_groups=1, fallback_on_stride=False), stage_with_dcn=(False, True, True, True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101'))) ================================================ FILE: configs/gfl/gfl_r101_fpn_ms-2x_coco.py ================================================ _base_ = './gfl_r50_fpn_ms-2x_coco.py' model = dict( backbone=dict( type='ResNet', depth=101, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101'))) ================================================ FILE: configs/gfl/gfl_r50_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] model = dict( type='GFL', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='GFLHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), loss_cls=dict( type='QualityFocalLoss', use_sigmoid=True, beta=2.0, loss_weight=1.0), loss_dfl=dict(type='DistributionFocalLoss', loss_weight=0.25), reg_max=16, loss_bbox=dict(type='GIoULoss', loss_weight=2.0)), # training and testing settings train_cfg=dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) # optimizer optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) ================================================ FILE: configs/gfl/gfl_r50_fpn_ms-2x_coco.py ================================================ _base_ = './gfl_r50_fpn_1x_coco.py' max_epochs = 24 # learning policy param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=max_epochs, by_epoch=True, milestones=[16, 22], gamma=0.1) ] train_cfg = dict(max_epochs=max_epochs) # multi-scale training train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomResize', scale=[(1333, 480), (1333, 800)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] train_dataloader = dict(dataset=dict(pipeline=train_pipeline)) ================================================ FILE: configs/gfl/gfl_x101-32x4d-dconv-c4-c5_fpn_ms-2x_coco.py ================================================ _base_ = './gfl_r50_fpn_ms-2x_coco.py' model = dict( type='GFL', backbone=dict( type='ResNeXt', depth=101, groups=32, base_width=4, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), dcn=dict(type='DCN', deform_groups=1, fallback_on_stride=False), stage_with_dcn=(False, False, True, True), norm_eval=True, style='pytorch', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://resnext101_32x4d'))) ================================================ FILE: configs/gfl/gfl_x101-32x4d_fpn_ms-2x_coco.py ================================================ _base_ = './gfl_r50_fpn_ms-2x_coco.py' model = dict( type='GFL', backbone=dict( type='ResNeXt', depth=101, groups=32, base_width=4, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://resnext101_32x4d'))) ================================================ FILE: configs/gfl/metafile.yml ================================================ Collections: - Name: Generalized Focal Loss Metadata: Training Data: COCO Training Techniques: - SGD with Momentum - Weight Decay Training Resources: 8x V100 GPUs Architecture: - Generalized Focal Loss - FPN - ResNet Paper: URL: https://arxiv.org/abs/2006.04388 Title: 'Generalized Focal Loss: Learning Qualified and Distributed Bounding Boxes for Dense Object Detection' README: configs/gfl/README.md Code: URL: https://github.com/open-mmlab/mmdetection/blob/v2.2.0/mmdet/models/detectors/gfl.py#L6 Version: v2.2.0 Models: - Name: gfl_r50_fpn_1x_coco In Collection: Generalized Focal Loss Config: configs/gfl/gfl_r50_fpn_1x_coco.py Metadata: inference time (ms/im): - value: 51.28 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 40.2 Weights: https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r50_fpn_1x_coco/gfl_r50_fpn_1x_coco_20200629_121244-25944287.pth - Name: gfl_r50_fpn_ms-2x_coco In Collection: Generalized Focal Loss Config: configs/gfl/gfl_r50_fpn_ms-2x_coco.py Metadata: inference time (ms/im): - value: 51.28 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 42.9 Weights: https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r50_fpn_mstrain_2x_coco/gfl_r50_fpn_mstrain_2x_coco_20200629_213802-37bb1edc.pth - Name: gfl_r101_fpn_ms-2x_coco In Collection: Generalized Focal Loss Config: configs/gfl/gfl_r101_fpn_ms-2x_coco.py Metadata: inference time (ms/im): - value: 68.03 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 44.7 Weights: https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_mstrain_2x_coco/gfl_r101_fpn_mstrain_2x_coco_20200629_200126-dd12f847.pth - Name: gfl_r101-dconv-c3-c5_fpn_ms-2x_coco In Collection: Generalized Focal Loss Config: configs/gfl/gfl_r101-dconv-c3-c5_fpn_ms-2x_coco.py Metadata: inference time (ms/im): - value: 77.52 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 47.1 Weights: https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_dconv_c3-c5_mstrain_2x_coco/gfl_r101_fpn_dconv_c3-c5_mstrain_2x_coco_20200630_102002-134b07df.pth - Name: gfl_x101-32x4d_fpn_ms-2x_coco In Collection: Generalized Focal Loss Config: configs/gfl/gfl_x101-32x4d_fpn_ms-2x_coco.py Metadata: inference time (ms/im): - value: 82.64 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 45.9 Weights: https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_x101_32x4d_fpn_mstrain_2x_coco/gfl_x101_32x4d_fpn_mstrain_2x_coco_20200630_102002-50c1ffdb.pth - Name: gfl_x101-32x4d-dconv-c4-c5_fpn_ms-2x_coco In Collection: Generalized Focal Loss Config: configs/gfl/gfl_x101-32x4d-dconv-c4-c5_fpn_ms-2x_coco.py Metadata: inference time (ms/im): - value: 93.46 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 48.1 Weights: https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_x101_32x4d_fpn_dconv_c4-c5_mstrain_2x_coco/gfl_x101_32x4d_fpn_dconv_c4-c5_mstrain_2x_coco_20200630_102002-14a2bf25.pth ================================================ FILE: configs/ld/README.md ================================================ # LD > [Localization Distillation for Dense Object Detection](https://arxiv.org/abs/2102.12252) ## Abstract Knowledge distillation (KD) has witnessed its powerful capability in learning compact models in object detection. Previous KD methods for object detection mostly focus on imitating deep features within the imitation regions instead of mimicking classification logits due to its inefficiency in distilling localization information. In this paper, by reformulating the knowledge distillation process on localization, we present a novel localization distillation (LD) method which can efficiently transfer the localization knowledge from the teacher to the student. Moreover, we also heuristically introduce the concept of valuable localization region that can aid to selectively distill the semantic and localization knowledge for a certain region. Combining these two new components, for the first time, we show that logit mimicking can outperform feature imitation and localization knowledge distillation is more important and efficient than semantic knowledge for distilling object detectors. Our distillation scheme is simple as well as effective and can be easily applied to different dense object detectors. Experiments show that our LD can boost the AP score of GFocal-ResNet-50 with a single-scale 1× training schedule from 40.1 to 42.1 on the COCO benchmark without any sacrifice on the inference speed.
## Results and Models ### GFocalV1 with LD | Teacher | Student | Training schedule | Mini-batch size | AP (val) | Config | Download | | :-------: | :-----: | :---------------: | :-------------: | :------: | :-----------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | -- | R-18 | 1x | 6 | 35.8 | | | | R-101 | R-18 | 1x | 6 | 36.5 | [config](./ld_r18-gflv1-r101_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r18_gflv1_r101_fpn_coco_1x/ld_r18_gflv1_r101_fpn_coco_1x_20220702_062206-330e6332.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r18_gflv1_r101_fpn_coco_1x/ld_r18_gflv1_r101_fpn_coco_1x_20220702_062206.log.json) | | -- | R-34 | 1x | 6 | 38.9 | | | | R-101 | R-34 | 1x | 6 | 39.9 | [config](./ld_r34-gflv1-r101_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r34_gflv1_r101_fpn_coco_1x/ld_r34_gflv1_r101_fpn_coco_1x_20220630_134007-9bc69413.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r34_gflv1_r101_fpn_coco_1x/ld_r34_gflv1_r101_fpn_coco_1x_20220630_134007.log.json) | | -- | R-50 | 1x | 6 | 40.1 | | | | R-101 | R-50 | 1x | 6 | 41.0 | [config](./ld_r50-gflv1-r101_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r50_gflv1_r101_fpn_coco_1x/ld_r50_gflv1_r101_fpn_coco_1x_20220629_145355-8dc5bad8.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r50_gflv1_r101_fpn_coco_1x/ld_r50_gflv1_r101_fpn_coco_1x_20220629_145355.log.json) | | -- | R-101 | 2x | 6 | 44.6 | | | | R-101-DCN | R-101 | 2x | 6 | 45.5 | [config](./ld_r101-gflv1-r101-dcn_fpn_2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r101_gflv1_r101dcn_fpn_coco_2x/ld_r101_gflv1_r101dcn_fpn_coco_2x_20220629_185920-9e658426.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r101_gflv1_r101dcn_fpn_coco_2x/ld_r101_gflv1_r101dcn_fpn_coco_2x_20220629_185920.log.json) | ## Note - Meaning of Config name: ld_r18(student model)\_gflv1(based on gflv1)\_r101(teacher model)\_fpn(neck)\_coco(dataset)\_1x(12 epoch).py ## Citation ```latex @Inproceedings{zheng2022LD, title={Localization Distillation for Dense Object Detection}, author= {Zheng, Zhaohui and Ye, Rongguang and Wang, Ping and Ren, Dongwei and Zuo, Wangmeng and Hou, Qibin and Cheng, Mingming}, booktitle={CVPR}, year={2022} } ``` ================================================ FILE: configs/ld/ld_r101-gflv1-r101-dcn_fpn_2x_coco.py ================================================ _base_ = ['./ld_r18-gflv1-r101_fpn_1x_coco.py'] teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_dconv_c3-c5_mstrain_2x_coco/gfl_r101_fpn_dconv_c3-c5_mstrain_2x_coco_20200630_102002-134b07df.pth' # noqa model = dict( teacher_config='configs/gfl/gfl_r101-dconv-c3-c5_fpn_ms-2x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=101, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5)) max_epochs = 24 param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=max_epochs, by_epoch=True, milestones=[16, 22], gamma=0.1) ] train_cfg = dict(max_epochs=max_epochs) # multi-scale training train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomResize', scale=[(1333, 480), (1333, 800)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] train_dataloader = dict(dataset=dict(pipeline=train_pipeline)) ================================================ FILE: configs/ld/ld_r18-gflv1-r101_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] teacher_ckpt = 'https://download.openmmlab.com/mmdetection/v2.0/gfl/gfl_r101_fpn_mstrain_2x_coco/gfl_r101_fpn_mstrain_2x_coco_20200629_200126-dd12f847.pth' # noqa model = dict( type='KnowledgeDistillationSingleStageDetector', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), teacher_config='configs/gfl/gfl_r101_fpn_ms-2x_coco.py', teacher_ckpt=teacher_ckpt, backbone=dict( type='ResNet', depth=18, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), neck=dict( type='FPN', in_channels=[64, 128, 256, 512], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5), bbox_head=dict( type='LDHead', num_classes=80, in_channels=256, stacked_convs=4, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), loss_cls=dict( type='QualityFocalLoss', use_sigmoid=True, beta=2.0, loss_weight=1.0), loss_dfl=dict(type='DistributionFocalLoss', loss_weight=0.25), loss_ld=dict( type='KnowledgeDistillationKLDivLoss', loss_weight=0.25, T=10), reg_max=16, loss_bbox=dict(type='GIoULoss', loss_weight=2.0)), # training and testing settings train_cfg=dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) optim_wrapper = dict( type='OptimWrapper', optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) ================================================ FILE: configs/ld/ld_r34-gflv1-r101_fpn_1x_coco.py ================================================ _base_ = ['./ld_r18-gflv1-r101_fpn_1x_coco.py'] model = dict( backbone=dict( type='ResNet', depth=34, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet34')), neck=dict( type='FPN', in_channels=[64, 128, 256, 512], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5)) ================================================ FILE: configs/ld/ld_r50-gflv1-r101_fpn_1x_coco.py ================================================ _base_ = ['./ld_r18-gflv1-r101_fpn_1x_coco.py'] model = dict( backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, start_level=1, add_extra_convs='on_output', num_outs=5)) ================================================ FILE: configs/ld/metafile.yml ================================================ Collections: - Name: Localization Distillation Metadata: Training Data: COCO Training Techniques: - Localization Distillation - SGD with Momentum - Weight Decay Training Resources: 8x V100 GPUs Architecture: - FPN - ResNet Paper: URL: https://arxiv.org/abs/2102.12252 Title: 'Localization Distillation for Dense Object Detection' README: configs/ld/README.md Code: URL: https://github.com/open-mmlab/mmdetection/blob/v2.11.0/mmdet/models/dense_heads/ld_head.py#L11 Version: v2.11.0 Models: - Name: ld_r18-gflv1-r101_fpn_1x_coco In Collection: Localization Distillation Config: configs/ld/ld_r18-gflv1-r101_fpn_1x_coco.py Metadata: Training Memory (GB): 1.8 Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 36.5 Weights: https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r18_gflv1_r101_fpn_coco_1x/ld_r18_gflv1_r101_fpn_coco_1x_20220702_062206-330e6332.pth - Name: ld_r34-gflv1-r101_fpn_1x_coco In Collection: Localization Distillation Config: configs/ld/ld_r34-gflv1-r101_fpn_1x_coco.py Metadata: Training Memory (GB): 2.2 Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 39.9 Weights: https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r34_gflv1_r101_fpn_coco_1x/ld_r34_gflv1_r101_fpn_coco_1x_20220630_134007-9bc69413.pth - Name: ld_r50-gflv1-r101_fpn_1x_coco In Collection: Localization Distillation Config: configs/ld/ld_r50-gflv1-r101_fpn_1x_coco.py Metadata: Training Memory (GB): 3.6 Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 41.0 Weights: https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r50_gflv1_r101_fpn_coco_1x/ld_r50_gflv1_r101_fpn_coco_1x_20220629_145355-8dc5bad8.pth - Name: ld_r101-gflv1-r101-dcn_fpn_2x_coco In Collection: Localization Distillation Config: configs/ld/ld_r101-gflv1-r101-dcn_fpn_2x_coco.py Metadata: Training Memory (GB): 5.5 Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 45.5 Weights: https://download.openmmlab.com/mmdetection/v2.0/ld/ld_r101_gflv1_r101dcn_fpn_coco_2x/ld_r101_gflv1_r101dcn_fpn_coco_2x_20220629_185920-9e658426.pth ================================================ FILE: configs/retinanet/README.md ================================================ # RetinaNet > [Focal Loss for Dense Object Detection](https://arxiv.org/abs/1708.02002) ## Abstract The highest accuracy object detectors to date are based on a two-stage approach popularized by R-CNN, where a classifier is applied to a sparse set of candidate object locations. In contrast, one-stage detectors that are applied over a regular, dense sampling of possible object locations have the potential to be faster and simpler, but have trailed the accuracy of two-stage detectors thus far. In this paper, we investigate why this is the case. We discover that the extreme foreground-background class imbalance encountered during training of dense detectors is the central cause. We propose to address this class imbalance by reshaping the standard cross entropy loss such that it down-weights the loss assigned to well-classified examples. Our novel Focal Loss focuses training on a sparse set of hard examples and prevents the vast number of easy negatives from overwhelming the detector during training. To evaluate the effectiveness of our loss, we design and train a simple dense detector we call RetinaNet. Our results show that when trained with the focal loss, RetinaNet is able to match the speed of previous one-stage detectors while surpassing the accuracy of all existing state-of-the-art two-stage detectors.
## Results and Models | Backbone | Style | Lr schd | Mem (GB) | Inf time (fps) | box AP | Config | Download | | :-------------: | :-----: | :----------: | :------: | :------------: | :----: | :---------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | R-18-FPN | pytorch | 1x | 1.7 | | 31.7 | [config](./retinanet_r18_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r18_fpn_1x_coco/retinanet_r18_fpn_1x_coco_20220407_171055-614fd399.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r18_fpn_1x_coco/retinanet_r18_fpn_1x_coco_20220407_171055.log.json) | | R-18-FPN | pytorch | 1x(1 x 8 BS) | 5.0 | | 31.7 | [config](./retinanet_r18_fpn_1xb8-1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r18_fpn_1x8_1x_coco/retinanet_r18_fpn_1x8_1x_coco_20220407_171255-4ea310d7.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r18_fpn_1x8_1x_coco/retinanet_r18_fpn_1x8_1x_coco_20220407_171255.log.json) | | R-50-FPN | caffe | 1x | 3.5 | 18.6 | 36.3 | [config](./retinanet_r50-caffe_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_caffe_fpn_1x_coco/retinanet_r50_caffe_fpn_1x_coco_20200531-f11027c5.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_caffe_fpn_1x_coco/retinanet_r50_caffe_fpn_1x_coco_20200531_012518.log.json) | | R-50-FPN | pytorch | 1x | 3.8 | 19.0 | 36.5 | [config](./retinanet_r50_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_1x_coco/retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_1x_coco/retinanet_r50_fpn_1x_coco_20200130_002941.log.json) | | R-50-FPN (FP16) | pytorch | 1x | 2.8 | 31.6 | 36.4 | [config](./retinanet_r50_fpn_amp-1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/fp16/retinanet_r50_fpn_fp16_1x_coco/retinanet_r50_fpn_fp16_1x_coco_20200702-0dbfb212.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/fp16/retinanet_r50_fpn_fp16_1x_coco/retinanet_r50_fpn_fp16_1x_coco_20200702_020127.log.json) | | R-50-FPN | pytorch | 2x | - | - | 37.4 | [config](./retinanet_r50_fpn_2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_2x_coco/retinanet_r50_fpn_2x_coco_20200131-fdb43119.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_2x_coco/retinanet_r50_fpn_2x_coco_20200131_114738.log.json) | | R-101-FPN | caffe | 1x | 5.5 | 14.7 | 38.5 | [config](./retinanet_r101-caffe_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_caffe_fpn_1x_coco/retinanet_r101_caffe_fpn_1x_coco_20200531-b428fa0f.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_caffe_fpn_1x_coco/retinanet_r101_caffe_fpn_1x_coco_20200531_012536.log.json) | | R-101-FPN | pytorch | 1x | 5.7 | 15.0 | 38.5 | [config](./retinanet_r101_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_1x_coco/retinanet_r101_fpn_1x_coco_20200130-7a93545f.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_1x_coco/retinanet_r101_fpn_1x_coco_20200130_003055.log.json) | | R-101-FPN | pytorch | 2x | - | - | 38.9 | [config](./retinanet_r101_fpn_2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_2x_coco/retinanet_r101_fpn_2x_coco_20200131-5560aee8.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_2x_coco/retinanet_r101_fpn_2x_coco_20200131_114859.log.json) | | X-101-32x4d-FPN | pytorch | 1x | 7.0 | 12.1 | 39.9 | [config](./retinanet_x101-32x4d_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_32x4d_fpn_1x_coco/retinanet_x101_32x4d_fpn_1x_coco_20200130-5c8b7ec4.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_32x4d_fpn_1x_coco/retinanet_x101_32x4d_fpn_1x_coco_20200130_003004.log.json) | | X-101-32x4d-FPN | pytorch | 2x | - | - | 40.1 | [config](./retinanet_x101-32x4d_fpn_2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_32x4d_fpn_2x_coco/retinanet_x101_32x4d_fpn_2x_coco_20200131-237fc5e1.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_32x4d_fpn_2x_coco/retinanet_x101_32x4d_fpn_2x_coco_20200131_114812.log.json) | | X-101-64x4d-FPN | pytorch | 1x | 10.0 | 8.7 | 41.0 | [config](./retinanet_x101-64x4d_fpn_1x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_1x_coco/retinanet_x101_64x4d_fpn_1x_coco_20200130-366f5af1.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_1x_coco/retinanet_x101_64x4d_fpn_1x_coco_20200130_003008.log.json) | | X-101-64x4d-FPN | pytorch | 2x | - | - | 40.8 | [config](./retinanet_x101-64x4d_fpn_2x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_2x_coco/retinanet_x101_64x4d_fpn_2x_coco_20200131-bca068ab.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_2x_coco/retinanet_x101_64x4d_fpn_2x_coco_20200131_114833.log.json) | ## Pre-trained Models We also train some models with longer schedules and multi-scale training. The users could finetune them for downstream tasks. | Backbone | Style | Lr schd | Mem (GB) | box AP | Config | Download | | :-------------: | :-----: | :-----: | :------: | :----: | :--------------------------------------------------------: | :-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | R-50-FPN | pytorch | 3x | 3.5 | 39.5 | [config](./retinanet_r50_fpn_ms-640-800-3x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_mstrain_3x_coco/retinanet_r50_fpn_mstrain_3x_coco_20210718_220633-88476508.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_mstrain_3x_coco/retinanet_r50_fpn_mstrain_3x_coco_20210718_220633-88476508.log.json) | | R-101-FPN | caffe | 3x | 5.4 | 40.7 | [config](./retinanet_r101-caffe_fpn_ms-3x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_caffe_fpn_mstrain_3x_coco/retinanet_r101_caffe_fpn_mstrain_3x_coco_20210721_063439-88a8a944.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_caffe_fpn_mstrain_3x_coco/retinanet_r101_caffe_fpn_mstrain_3x_coco_20210721_063439-88a8a944.log.json) | | R-101-FPN | pytorch | 3x | 5.4 | 41 | [config](./retinanet_r101_fpn_ms-640-800-3x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_mstrain_3x_coco/retinanet_r101_fpn_mstrain_3x_coco_20210720_214650-7ee888e0.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_mstrain_3x_coco/retinanet_r101_fpn_mstrain_3x_coco_20210720_214650-7ee888e0.log.json) | | X-101-64x4d-FPN | pytorch | 3x | 9.8 | 41.6 | [config](./retinanet_x101-64x4d_fpn_ms-640-800-3x_coco.py) | [model](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_mstrain_3x_coco/retinanet_x101_64x4d_fpn_mstrain_3x_coco_20210719_051838-022c2187.pth) \| [log](https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_mstrain_3x_coco/retinanet_x101_64x4d_fpn_mstrain_3x_coco_20210719_051838-022c2187.log.json) | ## Citation ```latex @inproceedings{lin2017focal, title={Focal loss for dense object detection}, author={Lin, Tsung-Yi and Goyal, Priya and Girshick, Ross and He, Kaiming and Doll{\'a}r, Piotr}, booktitle={Proceedings of the IEEE international conference on computer vision}, year={2017} } ``` ================================================ FILE: configs/retinanet/metafile.yml ================================================ Collections: - Name: RetinaNet Metadata: Training Data: COCO Training Techniques: - SGD with Momentum - Weight Decay Training Resources: 8x V100 GPUs Architecture: - Focal Loss - FPN - ResNet Paper: URL: https://arxiv.org/abs/1708.02002 Title: "Focal Loss for Dense Object Detection" README: configs/retinanet/README.md Code: URL: https://github.com/open-mmlab/mmdetection/blob/v2.0.0/mmdet/models/detectors/retinanet.py#L6 Version: v2.0.0 Models: - Name: retinanet_r18_fpn_1x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r18_fpn_1x_coco.py Metadata: Training Memory (GB): 1.7 Training Resources: 8x V100 GPUs Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 31.7 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r18_fpn_1x_coco/retinanet_r18_fpn_1x_coco_20220407_171055-614fd399.pth - Name: retinanet_r18_fpn_1xb8-1x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r18_fpn_1xb8-1x_coco.py Metadata: Training Memory (GB): 5.0 Training Resources: 1x V100 GPUs Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 31.7 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r18_fpn_1x8_1x_coco/retinanet_r18_fpn_1x8_1x_coco_20220407_171255-4ea310d7.pth - Name: retinanet_r50-caffe_fpn_1x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r50-caffe_fpn_1x_coco.py Metadata: Training Memory (GB): 3.5 inference time (ms/im): - value: 53.76 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 36.3 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_caffe_fpn_1x_coco/retinanet_r50_caffe_fpn_1x_coco_20200531-f11027c5.pth - Name: retinanet_r50_fpn_1x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r50_fpn_1x_coco.py Metadata: Training Memory (GB): 3.8 inference time (ms/im): - value: 52.63 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 36.5 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_1x_coco/retinanet_r50_fpn_1x_coco_20200130-c2398f9e.pth - Name: retinanet_r50_fpn_amp-1x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r50_fpn_amp-1x_coco.py Metadata: Training Memory (GB): 2.8 Training Techniques: - SGD with Momentum - Weight Decay - Mixed Precision Training inference time (ms/im): - value: 31.65 hardware: V100 backend: PyTorch batch size: 1 mode: FP16 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 36.4 Weights: https://download.openmmlab.com/mmdetection/v2.0/fp16/retinanet_r50_fpn_fp16_1x_coco/retinanet_r50_fpn_fp16_1x_coco_20200702-0dbfb212.pth - Name: retinanet_r50_fpn_2x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r50_fpn_2x_coco.py Metadata: Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 37.4 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_2x_coco/retinanet_r50_fpn_2x_coco_20200131-fdb43119.pth - Name: retinanet_r50_fpn_ms-640-800-3x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r50_fpn_ms-640-800-3x_coco.py Metadata: Epochs: 36 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 39.5 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r50_fpn_mstrain_3x_coco/retinanet_r50_fpn_mstrain_3x_coco_20210718_220633-88476508.pth - Name: retinanet_r101-caffe_fpn_1x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r101-caffe_fpn_1x_coco.py Metadata: Training Memory (GB): 5.5 inference time (ms/im): - value: 68.03 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 38.5 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_caffe_fpn_1x_coco/retinanet_r101_caffe_fpn_1x_coco_20200531-b428fa0f.pth - Name: retinanet_r101-caffe_fpn_ms-3x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r101-caffe_fpn_ms-3x_coco.py Metadata: Epochs: 36 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 40.7 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_caffe_fpn_mstrain_3x_coco/retinanet_r101_caffe_fpn_mstrain_3x_coco_20210721_063439-88a8a944.pth - Name: retinanet_r101_fpn_1x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r101_fpn_1x_coco.py Metadata: Training Memory (GB): 5.7 inference time (ms/im): - value: 66.67 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 38.5 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_1x_coco/retinanet_r101_fpn_1x_coco_20200130-7a93545f.pth - Name: retinanet_r101_fpn_2x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r101_fpn_2x_coco.py Metadata: Training Memory (GB): 5.7 inference time (ms/im): - value: 66.67 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 38.9 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_2x_coco/retinanet_r101_fpn_2x_coco_20200131-5560aee8.pth - Name: retinanet_r101_fpn_ms-640-800-3x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_r101_fpn_ms-640-800-3x_coco.py Metadata: Epochs: 36 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 41 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_r101_fpn_mstrain_3x_coco/retinanet_r101_fpn_mstrain_3x_coco_20210720_214650-7ee888e0.pth - Name: retinanet_x101-32x4d_fpn_1x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_x101-32x4d_fpn_1x_coco.py Metadata: Training Memory (GB): 7.0 inference time (ms/im): - value: 82.64 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 39.9 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_32x4d_fpn_1x_coco/retinanet_x101_32x4d_fpn_1x_coco_20200130-5c8b7ec4.pth - Name: retinanet_x101-32x4d_fpn_2x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_x101-32x4d_fpn_2x_coco.py Metadata: Training Memory (GB): 7.0 inference time (ms/im): - value: 82.64 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 40.1 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_32x4d_fpn_2x_coco/retinanet_x101_32x4d_fpn_2x_coco_20200131-237fc5e1.pth - Name: retinanet_x101-64x4d_fpn_1x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_x101-64x4d_fpn_1x_coco.py Metadata: Training Memory (GB): 10.0 inference time (ms/im): - value: 114.94 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 12 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 41.0 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_1x_coco/retinanet_x101_64x4d_fpn_1x_coco_20200130-366f5af1.pth - Name: retinanet_x101-64x4d_fpn_2x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_x101-64x4d_fpn_2x_coco.py Metadata: Training Memory (GB): 10.0 inference time (ms/im): - value: 114.94 hardware: V100 backend: PyTorch batch size: 1 mode: FP32 resolution: (800, 1333) Epochs: 24 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 40.8 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_2x_coco/retinanet_x101_64x4d_fpn_2x_coco_20200131-bca068ab.pth - Name: retinanet_x101-64x4d_fpn_ms-640-800-3x_coco In Collection: RetinaNet Config: configs/retinanet/retinanet_x101-64x4d_fpn_ms-640-800-3x_coco.py Metadata: Epochs: 36 Results: - Task: Object Detection Dataset: COCO Metrics: box AP: 41.6 Weights: https://download.openmmlab.com/mmdetection/v2.0/retinanet/retinanet_x101_64x4d_fpn_mstrain_3x_coco/retinanet_x101_64x4d_fpn_mstrain_3x_coco_20210719_051838-022c2187.pth ================================================ FILE: configs/retinanet/retinanet_r101-caffe_fpn_1x_coco.py ================================================ _base_ = './retinanet_r50-caffe_fpn_1x_coco.py' model = dict( backbone=dict( depth=101, init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet101_caffe'))) ================================================ FILE: configs/retinanet/retinanet_r101-caffe_fpn_ms-3x_coco.py ================================================ _base_ = './retinanet_r50-caffe_fpn_ms-3x_coco.py' # learning policy model = dict( backbone=dict( depth=101, init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet101_caffe'))) ================================================ FILE: configs/retinanet/retinanet_r101_fpn_1x_coco.py ================================================ _base_ = './retinanet_r50_fpn_1x_coco.py' model = dict( backbone=dict( depth=101, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101'))) ================================================ FILE: configs/retinanet/retinanet_r101_fpn_2x_coco.py ================================================ _base_ = './retinanet_r50_fpn_2x_coco.py' model = dict( backbone=dict( depth=101, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101'))) ================================================ FILE: configs/retinanet/retinanet_r101_fpn_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = './retinanet_r50_fpn_8xb8-amp-lsj-200e_coco.py' model = dict( backbone=dict( depth=101, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101'))) ================================================ FILE: configs/retinanet/retinanet_r101_fpn_ms-640-800-3x_coco.py ================================================ _base_ = ['../_base_/models/retinanet_r50_fpn.py', '../common/ms_3x_coco.py'] # optimizer model = dict( backbone=dict( depth=101, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet101'))) optim_wrapper = dict( optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) ================================================ FILE: configs/retinanet/retinanet_r18_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/models/retinanet_r50_fpn.py', '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] # model model = dict( backbone=dict( depth=18, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), neck=dict(in_channels=[64, 128, 256, 512])) optim_wrapper = dict( optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) # TODO: support auto scaling lr # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (8 GPUs) x (2 samples per GPU) # auto_scale_lr = dict(base_batch_size=16) ================================================ FILE: configs/retinanet/retinanet_r18_fpn_1xb8-1x_coco.py ================================================ _base_ = [ '../_base_/models/retinanet_r50_fpn.py', '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] # data train_dataloader = dict(batch_size=8) # model model = dict( backbone=dict( depth=18, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), neck=dict(in_channels=[64, 128, 256, 512])) # Note: If the learning rate is set to 0.0025, the mAP will be 32.4. optim_wrapper = dict( optimizer=dict(type='SGD', lr=0.005, momentum=0.9, weight_decay=0.0001)) # TODO: support auto scaling lr # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (1 GPUs) x (8 samples per GPU) # auto_scale_lr = dict(base_batch_size=8) ================================================ FILE: configs/retinanet/retinanet_r18_fpn_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = './retinanet_r50_fpn_8xb8-amp-lsj-200e_coco.py' model = dict( backbone=dict( depth=18, init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet18')), neck=dict(in_channels=[64, 128, 256, 512])) ================================================ FILE: configs/retinanet/retinanet_r50-caffe_fpn_1x_coco.py ================================================ _base_ = './retinanet_r50_fpn_1x_coco.py' model = dict( data_preprocessor=dict( type='DetDataPreprocessor', # use caffe img_norm mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], bgr_to_rgb=False, pad_size_divisor=32), backbone=dict( norm_cfg=dict(requires_grad=False), norm_eval=True, style='caffe', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://detectron2/resnet50_caffe'))) ================================================ FILE: configs/retinanet/retinanet_r50-caffe_fpn_ms-1x_coco.py ================================================ _base_ = './retinanet_r50-caffe_fpn_1x_coco.py' train_pipeline = [ dict(type='LoadImageFromFile'), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomResize', scale=[(1333, 640), (1333, 672), (1333, 704), (1333, 736), (1333, 768), (1333, 800)], keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] train_dataloader = dict(dataset=dict(pipeline=train_pipeline)) ================================================ FILE: configs/retinanet/retinanet_r50-caffe_fpn_ms-2x_coco.py ================================================ _base_ = './retinanet_r50-caffe_fpn_ms-1x_coco.py' # training schedule for 2x train_cfg = dict(max_epochs=24) # learning rate policy param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=24, by_epoch=True, milestones=[16, 22], gamma=0.1) ] ================================================ FILE: configs/retinanet/retinanet_r50-caffe_fpn_ms-3x_coco.py ================================================ _base_ = './retinanet_r50-caffe_fpn_ms-1x_coco.py' # training schedule for 2x train_cfg = dict(max_epochs=36) # learning rate policy param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=36, by_epoch=True, milestones=[28, 34], gamma=0.1) ] ================================================ FILE: configs/retinanet/retinanet_r50_fpn_1x_coco.py ================================================ _base_ = [ '../_base_/models/retinanet_r50_fpn.py', '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py', './retinanet_tta.py' ] # optimizer optim_wrapper = dict( optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) ================================================ FILE: configs/retinanet/retinanet_r50_fpn_2x_coco.py ================================================ _base_ = [ '../_base_/models/retinanet_r50_fpn.py', '../_base_/datasets/coco_detection.py', '../_base_/schedules/schedule_1x.py', '../_base_/default_runtime.py' ] # training schedule for 2x train_cfg = dict(max_epochs=24) # learning rate policy param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=24, by_epoch=True, milestones=[16, 22], gamma=0.1) ] # optimizer optim_wrapper = dict( optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) ================================================ FILE: configs/retinanet/retinanet_r50_fpn_8xb8-amp-lsj-200e_coco.py ================================================ _base_ = [ '../_base_/models/retinanet_r50_fpn.py', '../common/lsj-200e_coco-detection.py' ] image_size = (1024, 1024) batch_augments = [dict(type='BatchFixedSizePad', size=image_size)] model = dict(data_preprocessor=dict(batch_augments=batch_augments)) train_dataloader = dict(batch_size=8, num_workers=4) # Enable automatic-mixed-precision training with AmpOptimWrapper. optim_wrapper = dict( type='AmpOptimWrapper', optimizer=dict( type='SGD', lr=0.01 * 4, momentum=0.9, weight_decay=0.00004)) # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (8 GPUs) x (8 samples per GPU) auto_scale_lr = dict(base_batch_size=64) ================================================ FILE: configs/retinanet/retinanet_r50_fpn_90k_coco.py ================================================ _base_ = 'retinanet_r50_fpn_1x_coco.py' # training schedule for 90k train_cfg = dict( _delete_=True, type='IterBasedTrainLoop', max_iters=90000, val_interval=10000) # learning rate policy param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=500), dict( type='MultiStepLR', begin=0, end=90000, by_epoch=False, milestones=[60000, 80000], gamma=0.1) ] train_dataloader = dict(sampler=dict(type='InfiniteSampler')) default_hooks = dict(checkpoint=dict(by_epoch=False, interval=10000)) log_processor = dict(by_epoch=False) ================================================ FILE: configs/retinanet/retinanet_r50_fpn_amp-1x_coco.py ================================================ _base_ = './retinanet_r50_fpn_1x_coco.py' # fp16 settings fp16 = dict(loss_scale=512.) ================================================ FILE: configs/retinanet/retinanet_r50_fpn_ms-640-800-3x_coco.py ================================================ _base_ = ['../_base_/models/retinanet_r50_fpn.py', '../common/ms_3x_coco.py'] # optimizer optim_wrapper = dict( optimizer=dict(type='SGD', lr=0.01, momentum=0.9, weight_decay=0.0001)) ================================================ FILE: configs/retinanet/retinanet_swin-t-p4-w7_fpn_1x_coco.py ================================================ _base_ = './retinanet_r50_fpn_1x_coco.py' model = dict( backbone=dict( _delete_=True, type='SwinTransformer', embed_dims=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24], window_size=7, mlp_ratio=4, qkv_bias=True, qk_scale=None, drop_rate=0.0, attn_drop_rate=0.0, drop_path_rate=0.2, patch_norm=True, out_indices=(1, 2, 3), with_cp=False, convert_weights=True, init_cfg=dict( type='Pretrained', checkpoint= 'https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_tiny_patch4_window7_224.pth' )), neck=dict( type='FPN', in_channels=[192, 384, 768], out_channels=256, start_level=0, add_extra_convs='on_input', num_outs=5) ) ================================================ FILE: configs/retinanet/retinanet_tta.py ================================================ tta_model = dict( type='DetTTAModel', tta_cfg=dict(nms=dict(type='nms', iou_threshold=0.5), max_per_img=100)) img_scales = [(1333, 800), (666, 400), (2000, 1200)] tta_pipeline = [ dict(type='LoadImageFromFile', file_client_args=dict(backend='disk')), dict( type='TestTimeAug', transforms=[[ dict(type='Resize', scale=s, keep_ratio=True) for s in img_scales ], [ dict(type='RandomFlip', prob=1.), dict(type='RandomFlip', prob=0.) ], [dict(type='LoadAnnotations', with_bbox=True)], [ dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction')) ]]) ] ================================================ FILE: configs/retinanet/retinanet_x101-32x4d_fpn_1x_coco.py ================================================ _base_ = './retinanet_r50_fpn_1x_coco.py' model = dict( backbone=dict( type='ResNeXt', depth=101, groups=32, base_width=4, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), style='pytorch', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://resnext101_32x4d'))) ================================================ FILE: configs/retinanet/retinanet_x101-32x4d_fpn_2x_coco.py ================================================ _base_ = './retinanet_r50_fpn_2x_coco.py' model = dict( backbone=dict( type='ResNeXt', depth=101, groups=32, base_width=4, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), style='pytorch', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://resnext101_32x4d'))) ================================================ FILE: configs/retinanet/retinanet_x101-64x4d_fpn_1x_coco.py ================================================ _base_ = './retinanet_r50_fpn_1x_coco.py' model = dict( backbone=dict( type='ResNeXt', depth=101, groups=64, base_width=4, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), style='pytorch', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://resnext101_64x4d'))) ================================================ FILE: configs/retinanet/retinanet_x101-64x4d_fpn_2x_coco.py ================================================ _base_ = './retinanet_r50_fpn_2x_coco.py' model = dict( backbone=dict( type='ResNeXt', depth=101, groups=64, base_width=4, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), style='pytorch', init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://resnext101_64x4d'))) ================================================ FILE: configs/retinanet/retinanet_x101-64x4d_fpn_ms-640-800-3x_coco.py ================================================ _base_ = ['../_base_/models/retinanet_r50_fpn.py', '../common/ms_3x_coco.py'] # optimizer model = dict( backbone=dict( type='ResNeXt', depth=101, groups=64, base_width=4, init_cfg=dict( type='Pretrained', checkpoint='open-mmlab://resnext101_64x4d'))) optim_wrapper = dict(optimizer=dict(type='SGD', lr=0.01)) ================================================ FILE: demo/MMDet_InstanceSeg_Tutorial.ipynb ================================================ [File too large to display: 12.1 MB] ================================================ FILE: demo/MMDet_Tutorial.ipynb ================================================ [File too large to display: 12.3 MB] ================================================ FILE: demo/create_result_gif.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os import os.path as osp import matplotlib.patches as mpatches import matplotlib.pyplot as plt import mmcv import numpy as np from mmengine.utils import scandir try: import imageio except ImportError: imageio = None # TODO verify after refactoring analyze_results.py def parse_args(): parser = argparse.ArgumentParser(description='Create GIF for demo') parser.add_argument( 'image_dir', help='directory where result ' 'images save path generated by ‘analyze_results.py’') parser.add_argument( '--out', type=str, default='result.gif', help='gif path where will be saved') args = parser.parse_args() return args def _generate_batch_data(sampler, batch_size): batch = [] for idx in sampler: batch.append(idx) if len(batch) == batch_size: yield batch batch = [] if len(batch) > 0: yield batch def create_gif(frames, gif_name, duration=2): """Create gif through imageio. Args: frames (list[ndarray]): Image frames gif_name (str): Saved gif name duration (int): Display interval (s), Default: 2 """ if imageio is None: raise RuntimeError('imageio is not installed,' 'Please use “pip install imageio” to install') imageio.mimsave(gif_name, frames, 'GIF', duration=duration) def create_frame_by_matplotlib(image_dir, nrows=1, fig_size=(300, 300), font_size=15): """Create gif frame image through matplotlib. Args: image_dir (str): Root directory of result images nrows (int): Number of rows displayed, Default: 1 fig_size (tuple): Figure size of the pyplot figure. Default: (300, 300) font_size (int): Font size of texts. Default: 15 Returns: list[ndarray]: image frames """ result_dir_names = os.listdir(image_dir) assert len(result_dir_names) == 2 # Longer length has higher priority result_dir_names.reverse() images_list = [] for dir_names in result_dir_names: images_list.append(scandir(osp.join(image_dir, dir_names))) frames = [] for paths in _generate_batch_data(zip(*images_list), nrows): fig, axes = plt.subplots(nrows=nrows, ncols=2) fig.suptitle('Good/bad case selected according ' 'to the COCO mAP of the single image') det_patch = mpatches.Patch(color='salmon', label='prediction') gt_patch = mpatches.Patch(color='royalblue', label='ground truth') # bbox_to_anchor may need to be finetuned plt.legend( handles=[det_patch, gt_patch], bbox_to_anchor=(1, -0.18), loc='lower right', borderaxespad=0.) if nrows == 1: axes = [axes] dpi = fig.get_dpi() # set fig size and margin fig.set_size_inches( (fig_size[0] * 2 + fig_size[0] // 20) / dpi, (fig_size[1] * nrows + fig_size[1] // 3) / dpi, ) fig.tight_layout() # set subplot margin plt.subplots_adjust( hspace=.05, wspace=0.05, left=0.02, right=0.98, bottom=0.02, top=0.98) for i, (path_tuple, ax_tuple) in enumerate(zip(paths, axes)): image_path_left = osp.join( osp.join(image_dir, result_dir_names[0], path_tuple[0])) image_path_right = osp.join( osp.join(image_dir, result_dir_names[1], path_tuple[1])) image_left = mmcv.imread(image_path_left) image_left = mmcv.rgb2bgr(image_left) image_right = mmcv.imread(image_path_right) image_right = mmcv.rgb2bgr(image_right) if i == 0: ax_tuple[0].set_title( result_dir_names[0], fontdict={'size': font_size}) ax_tuple[1].set_title( result_dir_names[1], fontdict={'size': font_size}) ax_tuple[0].imshow( image_left, extent=(0, *fig_size, 0), interpolation='bilinear') ax_tuple[0].axis('off') ax_tuple[1].imshow( image_right, extent=(0, *fig_size, 0), interpolation='bilinear') ax_tuple[1].axis('off') canvas = fig.canvas s, (width, height) = canvas.print_to_buffer() buffer = np.frombuffer(s, dtype='uint8') img_rgba = buffer.reshape(height, width, 4) rgb, alpha = np.split(img_rgba, [3], axis=2) img = rgb.astype('uint8') frames.append(img) return frames def main(): args = parse_args() frames = create_frame_by_matplotlib(args.image_dir) create_gif(frames, args.out) if __name__ == '__main__': main() ================================================ FILE: demo/image_demo.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """Image Demo. This script adopts a new infenence class, currently supports image path, np.array and folder input formats, and will support video and webcam in the future. Example: Save visualizations and predictions results:: python demo/image_demo.py demo/demo.jpg rtmdet-s python demo/image_demo.py demo/demo.jpg \ configs/rtmdet/rtmdet_s_8xb32-300e_coco.py \ --weights rtmdet_s_8xb32-300e_coco_20220905_161602-387a891e.pth Visualize prediction results:: python demo/image_demo.py demo/demo.jpg rtmdet-ins-s --show python demo/image_demo.py demo/demo.jpg rtmdet-ins_s_8xb32-300e_coco \ --show """ from argparse import ArgumentParser from mmengine.logging import print_log from mmdet.apis import DetInferencer def parse_args(): parser = ArgumentParser() parser.add_argument( 'inputs', type=str, help='Input image file or folder path.') parser.add_argument( 'model', type=str, help='Config or checkpoint .pth file or the model name ' 'and alias defined in metafile. The model configuration ' 'file will try to read from .pth if the parameter is ' 'a .pth weights file.') parser.add_argument('--weights', default=None, help='Checkpoint file') parser.add_argument( '--out-dir', type=str, default='outputs', help='Output directory of images or prediction results.') parser.add_argument( '--device', default='cuda:0', help='Device used for inference') parser.add_argument( '--pred-score-thr', type=float, default=0.3, help='bbox score threshold') parser.add_argument( '--batch-size', type=int, default=1, help='Inference batch size.') parser.add_argument( '--show', action='store_true', help='Display the image in a popup window.') parser.add_argument( '--no-save-vis', action='store_true', help='Do not save detection vis results') parser.add_argument( '--no-save-pred', action='store_true', help='Do not save detection json results') parser.add_argument( '--print-result', action='store_true', help='Whether to print the results.') parser.add_argument( '--palette', default='none', choices=['coco', 'voc', 'citys', 'random', 'none'], help='Color palette used for visualization') call_args = vars(parser.parse_args()) if call_args['no_save_vis'] and call_args['no_save_pred']: call_args['out_dir'] = '' if call_args['model'].endswith('.pth'): print_log('The model is a weight file, automatically ' 'assign the model to --weights') call_args['weights'] = call_args['model'] call_args['model'] = None init_kws = ['model', 'weights', 'device', 'palette'] init_args = {} for init_kw in init_kws: init_args[init_kw] = call_args.pop(init_kw) return init_args, call_args def main(): init_args, call_args = parse_args() # TODO: Video and Webcam are currently not supported and # may consume too much memory if your input folder has a lot of images. # We will be optimized later. inferencer = DetInferencer(**init_args) inferencer(**call_args) if call_args['out_dir'] != '' and not (call_args['no_save_vis'] and call_args['no_save_pred']): print_log(f'results have been saved at {call_args["out_dir"]}') if __name__ == '__main__': main() ================================================ FILE: demo/inference_demo.ipynb ================================================ { "cells": [ { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "from mmdet.apis import init_detector, inference_detector\n", "from mmdet.utils import register_all_modules\n", "from mmdet.registry import VISUALIZERS\n", "import mmcv" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "processing rtmdet_tiny_8xb32-300e_coco...\n", "rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth exists in e:\\mmdetection\\demo\\checkpoints\n", "Successfully dumped rtmdet_tiny_8xb32-300e_coco.py to e:\\mmdetection\\demo\\checkpoints\n" ] } ], "source": [ "# download the checkpoint demo\n", "!mim download mmdet --config rtmdet_tiny_8xb32-300e_coco --dest ./checkpoints\n", "config_file = './checkpoints/rtmdet_tiny_8xb32-300e_coco.py'\n", "checkpoint_file = './checkpoints/rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth'" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loads checkpoint by local backend from path: ./checkpoints/rtmdet_tiny_8xb32-300e_coco_20220902_112414-78e30dcc.pth\n", "The model and loaded state dict do not match exactly\n", "\n", "unexpected key in source state_dict: data_preprocessor.mean, data_preprocessor.std\n", "\n" ] } ], "source": [ "#Register all modules in mmdet into the registries\n", "register_all_modules()\n", "# build the model from a config file and a checkpoint file\n", "model = init_detector(config_file, checkpoint_file, device='cuda:0') # or device='cpu'" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "d:\\anaconda3\\envs\\mmdet\\lib\\site-packages\\torch\\functional.py:445: UserWarning: torch.meshgrid: in an upcoming release, it will be required to pass the indexing argument. (Triggered internally at ..\\aten\\src\\ATen\\native\\TensorShape.cpp:2157.)\n", " return _VF.meshgrid(tensors, **kwargs) # type: ignore[attr-defined]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n", " ignored_instances: \n", " pred_instances: \n", ") at 0x237adee4970>\n" ] } ], "source": [ "# test a single image\n", "img = mmcv.imread( 'demo.jpg', channel_order='rgb')\n", "result = inference_detector(model, img)\n", "print(result)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "d:\\anaconda3\\envs\\mmdet\\lib\\site-packages\\mmengine\\visualization\\visualizer.py:163: UserWarning: `Visualizer` backend is not initialized because save_dir is None.\n", " warnings.warn('`Visualizer` backend is not initialized '\n" ] } ], "source": [ "# init the visualizer(execute this block only once)\n", "visualizer = VISUALIZERS.build(model.cfg.visualizer)\n", "# the dataset_meta is loaded from the checkpoint and\n", "# then pass to the model in init_detector\n", "visualizer.dataset_meta = model.dataset_meta" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAApQAAAG/CAYAAADmTEdUAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOz9Z5BmyX3dDf4y8/rHl69q3+N7/GAMZgaeAA0IgATorUjKkaKkV1opSK1eSdRLiZQoyjLoRFKg6AmQoAFAEB4EMBgMgMG4HtveVXXZx1+fZj88Pa13v2GXitjYiDoRFR3RXdHdVXVv5snzP+ekcM6xj33sYx/72Mc+9rGPffx/C/n/6//APvaxj33sYx/72Mc+/v8b+4RyH/vYxz72sY997GMffy3sE8p97GMf+9jHPvaxj338tbBPKPexj33sYx/72Mc+9vHXwj6h3Mc+9rGPfexjH/vYx18L+4RyH/vYxz72sY997GMffy3sE8p97GMf+9jHPvaxj338tbBPKPexj33sYx/72Mc+9vHXwj6h3Mc+9rGPfexjH/vYx18L3tf6iYfubbu406EUjkJrhJZ4siAOQvJiiJU1dW2wTmCNoCgsURRgTA3CgZFI5aE1eCLED0OSKAJbkVVTEudTNlJMFuBrx21vXGO8NebsYzvcc/s9zB16DaefeZF+dh6BIfP7/ODfuIH3/do6Jx6a46Uzl8nzHjfd2CW/WnDqqSGv/aaIzprHaKTQ0TZnn4zpLXoY49Mf7NLtxWxfyZFRSC1KmkELiaXXq5kMKoRskqZT+hfhn/8//iXv/La7+fCHP8nG9h6fe+LjWAxSTQmIUVXJuLbUfsA4q5DCwzoQOHwMpVPo2tAlQhSG3lIXE+ygNTTbPkVR42mPnUsCPMfqDQFRo2K43mTrQkrjiEUmBq9Q+D3BxGhEFeACjRxaMgNx4tFIPKwtOHLoBGVVsb7dx487BHEXaofQFiEtpa3QxiGFQ9oaJUNqa0A6mo0um0+dQ4xTogjGsSNQMYEISG1JGQgOH0pY6B1lNClId/YobUWQjRB1SagEC4eXWB/0qVPBZFwhdnrs5QNWD64ymUw4dEMbuaJJ0xwna5yQRJlgd8ey2Ftk96UNZBAilxWtpYjN3W28skVnpUNV5NTFmHA+ZHd9islD4kAQtxzTVFBXjvnFitiXpGOFlB61KrHbFo8IO19jFBQDjdCgvRCnSoSLSFSJFRHpOY8jJyKSo3uk0wBtSxTQ6iSML3YZXJjiiXr2ckQeuqqRIqbXa1EWjuF4nSD0kCLAaIk2BUEQAA7nHNZanDPEUUSe5wTS0Y0UvnRUUrGXVWSVIfQ8fKlwaOIoIS8y2u02Vvt0O6so6XH2wissLqyh9ZQ0TSmLmk63xcrqAcbjKRtXz6PiiG/+um/ko5/+OFEs2NsbMT/fY6Hhce7SAJxPsyuJvC7ZxHLLbYd557vexSsvn+eP/ui3UTJEKIlQkiLL8aSiNpqiLPimd38L62fXefGlZ4kiD60tQiiktMQNAaKiKi1VEeB5AcYWxMkc73jHO/irv/pzNjd2+ZM//xQvPvcMP/GT/5RH73+Uf/CPfpDv/t4f593veBd70ws8+dxXueu2++jvTNnYOUV7LuLA8l1Mp2M8X7C8eCv96qvsXcxwWtHPSpYXGnQbIecvrDPOSlqJIgm7VDqjNAbwOHJoCVfFJPMR7/rm7+PXfv1XcWKbqrTY2mKNIo5j8kJSmyEq+F83i8VhSTVSiKDNVA9oBzAtPZaXD7C21uIrj79Eq9VA64qy8nAYbF3RaCZ4yqc0GdaVJEGLqnSz97LQSOVoNkKyIqfT6GCdYGdrl8XFJYywlGXJ8uoaFy5cQgiBws3WGd8nywuEUCz35pmWKZN0QidMiKKYTFcUeYV1NVVVIz3J0aNHUcphjOPc2Yu0um3qskQ4QAqQEgtordFVTSNqYGyBsD5ZVuAHoLWl1ewShILRcErciCirAmstvhfg+z5lWZI0YkzlsK7GIUAJJtOS173pYc5fPok1LYIERqMpzbCFQJNPSqx1rBzqEsYVtucjRz5Xr+zRbCb0dwcsrCyxMNfg0rPbdNcOcHn9FOPBBF8JGvES1o45cHCJMCrZ2Blxy21tNjdGDLc9hKxQMgFZUBaWMm0TL1VUdU3DKVxYoU0PX08IwiZlYdCmoKoFt916B6dOn8QPFEUp6HR6GHeJ0Z7Ckx0aLY9OO2Iyrhj0Rxw6eJD+aJM0GxGGHsaE5JklChWOAj+QONtE6wrtLMYY/DDAiwPipqLTaZAjYOMyHi2O33QHX/jclwiSmIPLXbqNReaOLvPMM8+xsX4Zz1PUtabdbmGMBaEwxuGUh6MkUC1EAJ0VgS4Ue+cLRFAgncRTwewhFxatCw4cXOH8+fP4KkAq79ozpPFVE6lACkMUdMmrKVVVEYYhMPsapO9hraWua5w21JXG8xVJ0kRrjbWWKIqQUjKZTGgmDbj2zMVRgDYGY2uSJCHLMowxgKPdbpPnBdM0Jwx9hARnJUnYwbqKqs4wAprtJnGzwXg8oS5KhBBYramLCqUUAHVtZl+vkiwtLVGWJWWW0+m0mEwmGKvxPA9Tz37VWqONxfM8gtBnOhnR6STUdc1oWOOLRTpzjrIekqcenW5EVRrqtMQGDiMkQkuUJ7DCEqgAnTvKIKchfTIcDRHjGZ/SQBDV/PAPvIc/+ehneOPf6LKzXfGet/4Tnnn8JB/6wh+zU6/jmwW8sMuCKsnSkGq0yfJNXXw9YX0wpBrGtNseg+EIZ3yC2CE82NvRhJ7EGUnQiwmURTuPUHaJFmB5dcyXfqcvvhaeKL7WqxdXbhdORhF+s0VRWQIV4UuNQYM0uKpACEdR1VTaYYyiLDVx5GGsRhhwoYd0EAgPv9HFFIbdyzsIX+AEtJSgsRIy3MnJh5J204c5jagNdx4PoAzx5SrNgx2Gg1cIQsGls5ppEFGXU0rtuOlQB70esbVzmfaBmDiRjDYqRmGNMD6+b1g72GMysGyvj/CaAl0ZhC8JgphABUReQa83z+mzGySNmLaY5+HXvYkf/uFv5T/8u1/j6Rc+y9KBkCxL8Z3EFBWNQFL6PnuZYTypcQ68wEM4i3LgnCATPoErCNKI0XbNoZs8nKfprIXktcNUNZ6n2b4K2iaIMCMUIDTInYD5WxUuqqkLga5qNKBrEBXEcQOUpigqVlbWePjBb+Cjn/wIlRA0O23q2mFNRhh4CBeRFSVR4qMkmEpiRYGhIjKOkRQkIiTd2EakJb2gxVjXyEyyvBZypYDBeESvEXDknntxwjIY7dLxfQI7pT/dZjSu8G2bOPEIozGiUOSFYTg1JCqmtTzHNLZkWzs0vIBSSeIgpiF6FCYn8ytcmpFfEmhS1GJMPdX4nTa6KKmKXWSkMLnBZSHSWTrLHllRkGeOZgydZkCW1uB8nGdQhWUyVMwdjRlkE+oxCAlGNPD9nMpYPA1Bs4XbSTl4U0QeZZh6Dk+WSFehMYRBm/nGPNUkYnR1xO75FBMawjCk0YjQ2jAajQj8BCfAuoqyqmg1mywvL3P+/Dl830frCmstYRiSZhMOdxsIY6hkSAqkaYqSkk67R7vtUdeGfr/PiRMnqCu4uj4kbjQZTXbptVcIQok1kksXL4IoiKKEZnOe/mCXNB+zOr/M5t5VoshHqgSnKjyRMdgLOLy8TOUbGu1F9tZ3SRLJNE3Z2d6j224ThJAXBRaHRCCtY3l5lbOXL9JaWEDVgmk6QCmH53mUVUYch1jj024uU9ZDynoIxscLBEKFRGEPo8fU9YjFlQf49d9+H8OqZP3yi1T9gn/4t7+X7/72b+Ezn/ssXjQ7qPaWSrIsJy88rCtZW76Z1eWbKdji1JmvcsMNd7O35di9coGFhSVqa6jqEW9961t57DNfZGv9CkHsQSioaksx1sShh9/usdi5gavbr+AHGdZpTCEJQo90AkIYvDDCmub1NdEPpiReTCUNNjHEYY3SJVsbJa+5/26ee+Y0tvRx3hRrIwpd4BsP3/fxgghrNcsr82xubqLwKcscITWlrjlxw62849vfw3/46Z/hkTe9gZNPP41xmiBKqKqaRqPF7k6fOPSp6xrfE2RZxute/xbe/j3fym/8u//K/KGDTMmodvqceeEcyzce4fve8128731/xJve9AaeO/kVBqMhUvjMzy9y7twZVOBRFQV5ms2IoNE4IfA8D08FRL4kn9b4vuK7vvs7+LM//QiDwQ5CajwvRKCQnsA6g5SSKIxJ0xQhFIuL8wz2hpQ6p9FoUGnNgYPHmBZjuvMhV9Y3OXT0GIPBgOFOThRrTAVCtwibhm4vxuv6SAyehfnOrVR6l8WlmPXLQy68vEdRj/F8y/z8IpcvXiEJOkymu7z5zV/H4489ThB5ZPmApYUVEIbBaAefZW659QSvnP0CYQxVqWkvSKJpgygekfmKy2dCVpdCtvYGKNlBSB+nNFmWISVICVqDEhqBRxT28JTGDwSeiKnrmsuX92g3QtYOJQwnA0ZD8LwmYWiwTiNFSKUr4jhmOp3iKR+DwPN9wkZIVeUEecx0uMud9zzE3/oHP8rv/fK/oRpeRXQOEAWr7A2ucPXqVTq9iDRNGY8tGoVxFcoPcK6mu9DCCwt2NzXWNsAHYX1MMUC7kkbSJVCz/7OQmrwYU9eWTifBZ7a/GQtRw6OqKgJvDk8FOMbUWmDtjGgJIXButh4YYyiKAilBCIExM94RBAFK+nieRxiGKE+Q5zmhH2CtpSpKqrpAKYVUirIsCSMfaw1SKoq8mh2qlMI6jbYlzgha8RJh4rO5fZW41aTSE+KwjacEaTpFCAHWzg72FpTycc4RJU2MrRFCEAQBVVVgqpkoJoTDkwrnBLU2hGGMpyKiMOSOO07wuc/9Fffedwf3vuZmPvznj/GWt7yVvcEVzp07x8bGZaZjC6ZGRgpRQRA1SW1G6AR1XhEshSShYHAxQK1aIisIfYEj59/83K+yuXme//yLv8Tf/LvfyhOPv8I7f/g7efqxP+Ozjz9GVfosde9g/sZDrDZfZGvjIhuvhLgwp8xi0lpjdzv4YZ+4GaKtQ8uMvK6xOsK6El8qVLOJKwuac4d4y6Nv5sOf/30OLnmc/PDgayKUX7NCaSKQMThPYkqLwWGtj/QV+IrYc5iqZLHTYjLKSZo9eotLnLp4GikMFIppqTl6YI2VxWXOXbyK1gZfCZA+nqgYVQ69DibyaS7VWFmSmJAqsjz4yAnaS47zF3f40Mc3WIosbRXSikOyag/nZi90Nm1w9FiHje3LaK1ZOwgH2/DKOdiONKrwmYynRN0FFlJLWYzwPYlTPrm2CKXwZIt7736UUj/FzmCDw3etsHRM8Ou/+UFeOXeaqFWQlzk4iecrdObInCEtHIOxRaHw5eyAb5BoK4l9TVYXOAdKCpKGRxAq8BxJAnunCoTykC2JsBbKHE97GGsII4mYq9g4Cwu9Fl6cITxJVVjC0McYS29ZoI2HHpYcPn6QvNAEMkDVJXZQ0573mOQThtOUVnKQJErwpKQ2FqtCMA7je+RA6HuEQqNXIgLXQtGi1+8zaIcMfOhZqFpNqqDi5b96kdsffogjC0uc2z2LnzmMtgROs3duzKRqccPKPK49gRB6xyqyS2NkcZRm5BPMxwBIJ/CDBsY4kiQiTTOEJ8jrMVGjgfMbdHqawhra7RVUW7G5vYO0Mc5zOCnJ64Ko5dPs+tRlRoXFKIFSjqCMkc2KZqMiLwtE4aGUo9IG4ee4wpI0E6wqKN2UziK4wOKAIB4jQ4MnPEQF1o4oqNE9n/lDirUbW5x8fAo2Z3dnF8/zmOussLM3IYh8tNGEYcx0ktHvv0Sn1aYo8tkLGASUZU0QNOlnFd1mwiStyWtNHMWAZZrlOKtot9vccdud5FnGU8+8SBx2yDbWUYFDV44bb7yRqhTEcUxtUsp6gBlVdDpdmkmD/mQHzwl0KfGjijIvqUSLtZUGoag4eOg4T7/8Eul4SjoN8SPHwSPzmEJiXcHi0jxb29vA7FSfpikSQTmd4gUlSkmk9NCmJAg8dC0QQjCe7oCrCaImRpYoGZGXBdPRFZz1SbqSSxuP873f80Pcefe3EzU2ePxTH0V6mrvvv4f2Uszv/84f0W436fdHRFGEzn2K0qNse1y6dI44sqy0buELnzjLwsIiunKcOXWaO+55DXGjxcriQZQMyMoKL4gwRYlQjk6ng3SO8XSXMp2QNCKETaiKPtYotAZHhWci6mKE9e31NbEuU0Rh8cOYbK9AeZq63WSaF1y4tEtn3mPjXEq7KdGuJpGOylSUZYUfKZzVbGxsoDywJgcscZgQR5KiqHn8C19GBTFeFPMN3/xO3vcn72ep4SHETNXxfZ+6NkRRRF2XxHHMqdMD2p+6wNmzu5y6fAFRCe56+AF+4md/hP/xC/+dN7/n++nrkEBZSvEsF9c38JwkS6cIUWOtQkmP60KDcWhr8KQP0uHLkCO3Heb0qQucOXWFqp7SaDTIsgKlACcxRoOYKfHGGKT0UEqRZQXNTkJQeaTFBN+LyfIRaZbSbCcsr7a4cOEC7dYcKjY0exHFpGZvawiqy0hqWrqgtxLRHEms22Mae/i6ohYhzfk23mRGTvIsx/NDbrztOA+/9kf44hN/xaSe0PFjPJHgezFllTPXPkKlS65snaa96HP33S2+9Nkxy2uO+mrF8sISG1nG/FLO3ja0u1209jBOM8kmCBmRZSVJIvF9ga0iPD+kNkOq0kMVAd2uJWkqXv/GNTBTssxnb8cnjiW1nmKJQCg8T6AtTCYjjDEEQYB0Al0WpEWOMQbtBnRWDrAznfC3fvTHONYJOboyx8vrG6wd7fLoa9/Mhz/yPjrNHkXmiJMK6zwK7RFEAidT0r2KMPIJQijLglg5ymqEH7bQLsDWdiYUuYo3vuFRnn/+OZaX17h4YZ1m6NFslIRxg42r24TBjKRa5xDSI44DhBBUVUVRzIig7/uEYYgQgrou0dqitSbwI5qNNtZaJpPJTJFMQqy1TKdTgiBAKIktoK4Lms0mnhcgkERhNPv82MdaiKKARqNBUU8p8gmhZ0kaOfNLkBcly4tzOKcYj9LZAUwplBKAxAkHWLQ2eFLi+wnGGJyd/bn0PYR1SCWIo4TJZIIQiihKmOstEYYhr7x8FmslOl+imPZYWOzwmc9+mO2tIVHsAYIkaTKqJ/jCgTKk1Rjp+UwLSTLf4si9ktOfGyG6LQ7f2aHXESzfUfKWE99POulTzHn82Z/+Np954gnq9qd5+vnP8YlPf5G40+Nbv+6tDDcdK2vzfOLTjzEeRiwfl1z4igTR4t47buWV0TNY0aSa1tR1hUqufW9tjZco4lZzJgoaQejFnHz+cVphQCJbXytN/P+AUBoQlaXKpiB8mm1JllWU2YhGo4UyoIsK7TRYg65KQt+n12gwyQps4BPVjt2dAelUU1cGP/BoziX4vqD0BX5uSHdLXO6gBUZD6Tw8Ab/+m8/TbsNNDyacuCOmf7mgmlREB2Pqiz6erZHWZ3d8mduO38bf/rHXMbVP8tRXv8RNx0L6ox7jaU6SWJwNKacjjq8tcO4Vy423JJzc2qKyAWEsybTmA3/5ERYX54lbglcuPsszT3wZnTXodGsC5RH6oJ1HrXOUgloKxqnFCRDCgbM4Aw4Pg6CuYlp+yvRSSG4K5g8LalvSjpqI0iGlj3aWQd8iPYhshFIlRSnIC0PLD1FSsH5xwoF7fZSV5OdK9IrFNQ3DrRI/USh81q9sYRcbKJUzHvapsh2S5jHmOstkOxvkRUojDBA2xvMkQSDxRIA2FXnpCKeSQAhMow27Nbtbe0RiRLXiMxqGHG4tIHSFpwTK5Zx67PPc965HaCU+01FNOtYEtcdC0mAynVLkMdlOTSESlu4C59Vc2j7PctLEqi55Zmk22oQdjXKCbHcXb7vk3LOb9BbniQ62EV4bs1vRaPYxaclgkNHqNJhOM1ygsKaizCRCW4xK8UOJdZooVjgdokxIpqd4TY98p8ZVHtILkV6NFBJnHP2tjJXFFnk9hcTHeRZjIJaK4bkGxJaFwxm+JxBVSRxbSuGzs5USBB55OaHZaIMw1HXJgYOLjCcpi52j9Pd28H2L78nrI56iKGajRykByTjX4FuEHxApH6c1VjhqWzOpNNsbF5l7aInJuKDT6FAbRa83zyTtk6YTLl26xGiYEQYCFdaEvmRrs48VBXHUJul0me7too3BMxB7EXld0GuN6cyFPPnkY+R1RKOp6LYa5HlNXWm63UU2t4YsJku0Gm12dvYQQpFVg5kSYSxGewRBgHMOpwPqyl0bJ9VYpymcQ5SCuXaby+tbPPjGt/Lub/9urpw7zflTT/OVp06zdeVTtPQmP/5j/4QnPzqi02zSnjvExc89RToxhGHNfOMeRvk5jt7QZf1ixe7gHG9484PsbkjOXf4q3/9d38bu8CWCzq0Mh2NuO3KCUTbkZ3765/nP//Xn+fITX+L9738/vbk2ntLk+ZBxH5pz4FzOdFSgJIR+j5qcIi9Ikia5mRB7Ad61ERmAK0OK6QCZaFzbo3Q+jC0RIXubGcdv6TBuGRwaaw0OHykrnBBYXSKEQimF0wVSgbWOJG6RlxWTacbVq+vUtuaxxx7j7rtfg++3SdOUIIhmqo+t0VXFDTcco9Vq8dJLL3DbzYsstKfcctthOgcXaeK45d43Mt0W7G1e5ru/7TtYW+1x7uwpytGETqeLNQXT6RTfi3C6ukYAIoy11NqysraMqWqSOAYc585expiaz37+YwR+EyEqoligDZi6IowDpJopVePxGCFmtgFr9YwgE6CnGt+Dza0rhEnMhfPrvP7NJ1Bqg+3tnKPHl7l8aZu51jzZZJO8zLn3oTvYO/00yJhzFy7T6I2JDh5h6/mMUT5k/eoOC81Vdnc3ueXEIfJMs3V1zJ//6Se4/6Eb2Nh6BWeGXDoDnsq46ZYV1tcHTMttTOGY81p84eNjtExYPz9H7IbMtWKGg10WVgR1pVCyS2536Q+ndDvzqEAw12uTTium0ylxVKGNJQoDrKiJI5/hYMzKWgsRFJisie+rme2j0aYZ+ISxx872mCwXOAdBEGGtpcwzoijCuRKlFHO9HtOiYm35MGl2mQVvTCM5xsqJB3n+kx8niVucPPcyu5MawpJKSLKqpNX20JlGlxblt8EIXK2IkogiyzCFw5MOB2A1QRjTajQYDksee+wLhGHAcDhka+sqeSPi6OETtOdiDBXrl3exzkcFBmcNVZYhpbyuTEopqeuaPM/RWl8jcbCyskIYxIxGI/J8drgW0jEajXDu2tpRVQAo3yPyY5wz17iIAwy+H+B7MVEUoZSi1WoxnnQopuuEDQ/pbxPGBiUblKnFuhqJo91s4pyjLEuccwhma7KUkGVT5pcWGY+mTNMx9913H5cuXWKwuzMbb/sC5xQqUPRHfYqiIMumOGdYXVlmc/QYX3rvR2g1G7Q7MY4aXIyzAm1SIi/BcxO0t4Dy9nBjww13rvDgewI+9HtnWDzSY/VYh72qxe0PBDz22Re5d/EhNkZ/Tpou8z/WP8MTn/oN5g8t89nPf4jv/tZv5Z7bX88ffehzfMePfjsf+pXf4fSLfRaXBVcvztNteZT+Ls9+9cvEfoARNaPBiLWVRSb5Hs5KnJAoL+TgkRs59/xJpAzIqsvUtaTZhG77a6aJX/vI+8jDTZelBc5JGt0eRqQIG3BsrUvazxhspYzSKXNLMXlpcS7GaocSJc4V5H6ArTVWSpTyaSYz1lvpEl+AlzjKvKB0FpU5dOkIUdhGgBKSwk5wBfTaEYdXjnPfnbfwzBdeIm9XjHb7DKohtQYpfLrxYf7rf/wlPv6xz/DJz/934tigHLz4VUlzuSTu+DSVz52ry1x8sU8ZW6YNRVGPKSbmmlJYgg4QdQBJTpjN4ew2VjtarRbGVVinQRuEcdQo+rlFC4VzNdLOSLgVIJTAEwlKp4xOB9x86zyyN8RQ4vkSPJ9a1EzHinRa4isfbWqwHmGi8ZVgui3wIotXQrAwT22meHlJLmBuLsELK6xw9PuGu07cz2i3xpJy9sxlikwiw4KlAy2SboJ1IUq1CDxFXkxYW1kmT2Fa9Xn5wmV6oo033qVzuEt+ZcywMCyvxowGPkFlCRYURRGzd3abI3e0WbxNsbMLib+IiHzOXjqLqww9zyDyEiFD8B1BEGBUjjOWyo+JKZF1i+1nfDpLHhyvoDCYLKPqGPJzAWoSsPDQccbrewyu5Bw94XPlxRTnStrHPCb5FAkIIZleFXz/d30H2u7xF5/8BPPLTap6CjWEdHGNCaUTlLlkOq6QVhIEAWVeETUU+dgjrHsUcZ+DKzVxN6E0Ds+EnPvMFOMr7nxrm6o1IFYSISusiznzIVBhRFXvEXktPN9QV5LOXIeVtUNE8SK+rHnu2afJ8xRPCmYc0mMwGuJ5Ps4YhOeBkiTBzL+U5wWtVgtPga4tzhgCT+H5kjyrqWuFdhona5QMqMqctdXDeL5EeRWPPPoQj33+i2ztXiWyPpmW1LXBUxEyKEgLjZSSN7yhw3CcUowXOX1qGxFIhHXgDFJ4xPE8ab7NsSPHGQ2mBHHCax55GM9TrC4u8PP/7meJGguEvkVIh9GzUReioqoqFhdW2RrtEqDxXEhlclzYYK57jD/8vfdy6w1HuenYCrIZE8gm01JQphmtpGY4mRCGi8x1PSbTgt/+3T/lfX/0P3n/H/1PjhxZodlJ2BuM2dnLiETFz/3s7zAsz/HeX/4jvESxduAIsi5Ip9t83/f/Q1q9mB/6/u9gcb7J7s6U7/+B7yVIWrzv938L30vwfENZj2iEBzl0bIWzF54jiHyKUcbC/AppVVxfE/PCMN/sMRwOkXrE0W6TOoC9KeyNLSqQCOkQRoJnyPIaz81+Bo1mROCHlKUhCB21ztFVSLfr0d+bIv0EjCRsRuTpFC9QxFHAdJoxPz9PmeVUVXVdCfpX/+pf8973vpdHHnktC/fczvBMnxMP3MWFbc27vvld/Mjrb2bxtqPoSUVVpgz29hDCmxHUyKPWBikDgmD2Tkgp0VqTFjmL83OMB306zRaDcY51Nb43U4iMrWekB0lV5wReQFlV1HVNs9nEOUFZllirCaOAqqpmo1xj8DyPoi6ZX5pjeXWVpBHQ6Tb55Kc/z6HjB/BkwN7mLtYYsqnh3gfvQogcooThK2cIAkX3+CLnn58y6e/grMDvRJRZwTu++T184mOfYzjcQPkVURThqQbf9Pa38/LLT/HCcy8RhS3yYkwQWRqtLtNJTkXC29/4dWxOrrJx6XlMXlGHNVLGWOvI9qCqUxpJF2skyBKtC5K4B87D6ZyyHhL4MZ7v8P2Q6aREKchSg6RBs5UwGY1xzrC41COKfc5fvEicgLOzQ0Zd1zjnaDXaWCcwBhpJBz0cMzQTfD9kKYqIOw2mlcSvfUbFiGkxQLgIocC62bqAMAg0jaRLXVfUtcbh4XkBeT7B6IogbCCEorYpvvKxRlIUFUEQEIY+CM3NN9/ET/7Tn+ELT3yU//gf/yvLCytM88FsfCx8wjimyLJrxHH2dUgprymTNcYYhBD4vk8SN6nrmizLAPD82efNrAMz9b3T6TDY6wOz0fh0Or0+RhdCYOzM6jU/v4AfhrP3oSjY3b3EgdU1LBlrhxbY3BiweXUPIR1ShbRaLfI8x1kwxmCtvTbenr1LBw4fYjweM5nOyHHg+9R1STNOmKSzPVtKZodn51AIlCdxbrbuOzIECXHUYG6+ybu/5T384i/+IkJalNMoD8alxPM8bK1pri6zcnOTVrPg4Td6/NF/usRdj76D7/kHt/Kr/9cHKOY1rnMOtXMLh5YNyZFdrjxj2L4y4fhd72a+0+aTn3mct37Tt/CxP/5Fkm6bdlCydTkntA1Ey5B4BeNRxWgQEoUdsnyADDUqDBnlBasHl7njjjv5qw9+kiO33MJb3nEvzz75HNPBNpdP9RldNv97PZRL9wlXlxAlTWyQICOLZxow2UPv1ExqSBoSbXKE8JA0cAKkqvHimqmyeMYgAonG4duQRrNDLRSekYh6wsZWwZFD83hRSl0IdoZTlDZQQ3NuDqVyVhodFo/N0XK30lgYcurKSdZPjtjzJKouUDrAOsVdD6+SNI5Q5hmXLj/D0VbA+ZcMA5FCCJ0FxWragtNDxkckK3MHoXZ0W6uc295g2shYWFrmyvltcjEm7Ws6rYBAKqw1LCzFqHqe4eAsvvUZj2syzyNHEyUQqYBsaqmNxQ8stfWJ/RpvsMLSSos8OE0QekzGAtHTuFJRTjSyjtC6wPozQtpsCJoh5LlHIWoCEeOlksFWiuoolm8SKM9SSMnWFU2v2aHpt3n+ixvc/ZoVdvvr6LKH6qY0OxqLZZwmSNVjcalHogRzyRKe12Fz7yX2drdw+My1fC5sX6ERdtgbpiTSp9FpMO3vooYdwl5ErQp6nTXsfAeZnWV3c4dgsU3DxezsjqiUYUlAHIfUpqDAQ4QFgQKqNmJTMDWQjjN8E9E8GKLWCtK6pusrJucl06FP7Vvyq2NWDrbpHAx4+Ss7HL7RZ+w5lIwh1RShoBpk/PC3/X1uufkE/+o//T3WbhLoUiDrBntXJzRakua8YncoGE0q5ho98qEjiqfkUuM7RThJ0KHEszB3NMZrjtg77zHpQ7qVcuf98zRuKvH8ACcK5o5a8ufn+OSf9FlaVUxGORJNEvXQlGgnCYIljhzssX7lEliDc4bpdEy73cU4wXA8YqHborSSoijA1igvQAYRnWYLWRdYT1NmBXEYUpYloNBaEsQz/wsuZHfvKocOHiGJO9fGTT6PPPIIv/sH76XTaSFFg718j8RJlC8ZVwa/DPFEQbK0wHK3zSunXsRPmlgTQG0IAkfY8JhOSo4dOsapV84zt7JCIQX/9Cf+CS8++zS/9xv/g9e89g7OvHwRnIcfCIpyMttENBw6eIzdSxeh0+TOhx7mbW98E3/6e79KnSqcf4T/9Gv/jX/0o+/i8uU9WrLBXrrBDbecYHfjCl7gUw4G2NAHz6fbvQUnMjrdkka8wkunn+P4CUU9zjl33nHvHW+n07Z0u006TZ8//ZOP8FM/+1OsLHT5sw8+w3f94DfwU//yJ3j5mZfRVc69D7yBT33+s/zE//G3+eVf/A3mFppUVTWzXZSalaOCznzFuVM1epxwz113XF8Tn37yBRqhR2k1j7z2Vs589ascO3GcG269h7Dr8cnPfJHh3jrj7RCpHM7NiBRSYF1NGIZ4KkabCWWV44sOws9pRG12RmN0VrF29CAUNbVw2NJhnL2u1Pi+QmtNvz/i6PEb6PXmWT5+K2/9vtfz3FOP0Y1u5tlnTtG7eYFXPvRZrChoRILdrQF7OztIUbMwN8+FizusrMyhXcl4VNBoNBBI0iKfbbRFSieOCZWkn2s8pa4FLwKqqkZIiTY1jaSJrmtqrSnLkihKiKKELJuidUUQ+uB8rCkR+CjPoY1g7dAKN9xykC8+dhIlJK2lAOV3mEz7hMqjGGuyekC7vULYitFO0woSYg8m5R7nzl5lpX2UyutTZhpPdjl28ARnzn6Fw4d7HDo0x9lTOwx2UxpLB3nk0Tt48flnuXT2IkHocFpRuwmB16HRc3h6lcrfIwprSpeSZwY9DKBhmO7VCELiKKAsc6SYKWRVPaLTS0gHGb7XYjwe0UjaNJtdNq9u05trIf0xe4Oa4zeuUkxy9rYnMzJupyg/YjKyKH/2vUuSiCiKSNMJQRSTZrP3+eiBG+jLEeWlPke6ixRRgAyb/I0f+C6efOIxvvCZzxDEkrw2THODNiBlMPMkuhwlHKOpImgYYuVhjY8fz54jjGY8yWd2FVORJAlCCJYWVyjLmptvvpmrOxnptM+Vy6eZ780x7I8Q0hAlMVLGFGVGXdczsmQtSimazSbGmJknXKkZcRQzL7EQswOHVBCGIWWZX1vf5HWl0xiDJ9U1a0dNVRV4nker0yXParSdhRwrXTM3FxGpmCoH63Lysk9VSeKggyVFejPCOgsI6+sWq7qu0Vrj+QLlhwAUVTXzVQazMXwjihlOxsTxLEBUFCVFUREGMb6aeT+tMEhlCFUHPwBPxRRpjfTHCCKq6RTV8REuIuhoujd4eMKSTlL8xTm+5R0/zDuO3c9HPvMkrdftsnnu8wynFcuLEYcW7qV/6Qq//4GXOLC6QFWNKdMmdlCQdnKW/IQzz20iVjycE0S1YnC5xJM+vQWB38vZvgK9zgEqnTEs+2gnqY1hZWWVPDMU/QG91SWIPXptn2K4zmSnYOsV+7+XUD7wpshtVYKJSOg2fQKjyfDI8hFqWCIjR0BIOREoClpzimyi0Naga4drgWtaQtGhnoyopEfcSAhdEz+s2LmSkqY57XaCoeDm227l8qV1imxCpxUgphGdFcXe9h5x5LGxaVk+2OXw6s1cunyS3VFO2BHIyuLLkPtfcxfjccUwm5JPzrF6cJGNLc2Fs3sEoaK7YmhKj4aDurbkynEwP8rBXpcRhq9sXyBaComsxUSanfWSWNW0PYXJKyrroeIGxW6BN9DojiWLJDIxNHyJcI5xETCZljQCgQhAKoUrHYm19LoeybLDBYbpOKYio5gKxNQnqysCJahxKANBGONsjhf4LC6vceXyDtiM0G8xt5pRa4+9K4rCZshWhwYlk3VB7/Y2rGcEsU8ZaSqtcM5S5im1DpBhxH333EwnjJjsCQ4c6GFJefzJJ2nOr+KHAeP+DlKDzAq265RcC+ajJhRTKuHTbCwT+B59u0vsUqzVSBWQTjIcPs5Y5nyBihRpVoAvUIFAoagnksm2IxvWNP02RZzSiwPiZcN0Lqe8FFG+ACJssnigRXBzyuSqZvdMzuKthmlRE5qATNSEAvw4RuxVdHsHqNsX8fHIygpX+Tx696NUpDz14lcQXoPSpDSigGp0lCqbMHBXacc+cRFwecPglY47Hm6TL++w9VeLhEGD9a0LLK0s8dDXSz798T1uuXuFm+70Of2C5qkPXWJ+voNSinSag5uZuGGmzGqT0Wo3SUcZ3W5AECTESYKMNGfP9AmDBCkExszGQkVRIJilpXVtQSliD2IVEsc+u+MxQdyl3WnQ3xlT6QFKhbMNRMzGocZoynJm8vdUQBI1Zql6aiwa52ZqvDWKhbl56rpmMNzEUy3quiaINa3mHJPpGCVnG4QuFQiNdTlhsEJZaIzY5siRu9ncuILnCbJ8ShyH5GWNcwIhFK7OZuQCyde99bv5t//ix+nvbDA3v8xv//qnuHT1Ih/88C9w4s67CNs9trfOcOexu7l4+WX6U02Z1ZRVRqtrOHzgHibpFfJSMxzu0uhEREuaH3n3v+DAgZt4/3v/mOZSi1cuvMRP/OOf5JZbbmNbTPjs7/wBx+94mNCW/MgPfh9rB+dY39jmn//Cb8HVF/jwBz7IS1fP040Uus6oxCKB1fhzBcOdgkcefIBk3v9fh+zeUT78B39ClQnuevAOXnz+eRZXb+AH/u6Psrl3lQ9+4H8QyozBjiSvZqb3sqwRws18X7FPI24wGmiMzfB8R547FubnGed9lCxIGitMc4PnxiivSxzBaDgmCH2UDHDSorUjz6d4nk/QOsLBI7dS25RxfwdRjKmyHarSMRpPuPOO21lebfHJjz/NkWPLfPv3PMJv/saf0u2tcfOJHk/81SuIa+pVUU5nPuu6pttr4wcwHs8CC2mawrWwTl3X+L6Puhaa0FrPPHNyltR1dkYWpJTkVYEnfISDyhQE0Wy86wcKIQx5GtBuJ1hRM51kzC90Cf0u6+tnCIMGt959nPWrV4iCDgvzc3zliS+xMr/CwkKT9StXcfjkVYmxr3pdPe655x4efvhOnnrqSc6efplRFrC4OM9D99/OX/7FBzEO8lTS63Torfa4cv4iS4sC7WqMjXDCgihxRlLlAXme46kYpSROW5aW1lg7MM/u7i5XN7YRckb2q6qi1WqxsrLCYLBHWeU0g5Ag7HJ1c5dmy9BoTFieb7OzWXPlUkl+LU3sBT7Wasqynk3EjOHG4zeSa4FLMzaGe7SB1twanfmEt73pHVjf8Wv/8V8w327Tz2BUpxxaO8DGzhWsiKGyNFs9lFLs7m7TarXQ1ywOxhiUUqwsdrm82afMDKpZ47uYoMxoLC5y6eougoKGH86Cd8Zw4NABirRie2uIURVKWKQVWAJKY8GURNKDwKMAOkGM7/t0u13quqYoCkbjAc6Z620A0bVnoihmwcKiKIiCkMXFRWpt6Q92CYKAxcVFpPDoj4YMh2PiOEaXFUuL8zhnSLMRVVVizMxWNPOlxuR5ThQmVFWFEzP105iZIvyqomqsxjhHs9nCOcd0ms2eb1viCYXQgHUYW2GtI4wSrAXpA56PyyYkjTbWpoz3arSCwBMsri2xcNsCfjPi0uYGzfkhelriobh4UXDj/Gt423d+K0dvThhc/hVeOJ1y2623Mtw0fMM3/RB/8ru/wO/8+ldZeU2Xu255kB/8xu+gtRjzH37xp9jZvswrJ3PCOYUShq5YwnM1raWc0gqytEZYQRLFVLVlXDj8IGY6HOGkQ2Pw44Buu4cXzTG3lHL+S5dQzQ57L/xvDuWMTYgf+yyqmHKiGRcRc/NtIq8i9Qs8D1whEJToymO458DTqNiS5wFiENBrGAZXK47dvMrlnT2K6ZggktSipjMvsDZimmZEEbz07CmCcDZ6mw419WjEpIwIujHDtGC+LdGuz8uXX+DQvE/SdlROsbdVUCrJSxfPkdCkMLtESUxVplRpQOB79BYMpm5R6JyLL0sWD/m0Fku2x+fZ3FjGa1oW5xUTOyXzW0hd0kgUgfGQrsCPoNta5PT5IXFQUqsEmdTkqaUdK5IwJogDJlt92k1ouxbWL6mlxgUhjUiS51PWmosUao80zahTga58okAg7Mwn4oTE9xRO5MRNRVYY+sM+2mQ0Yp8oSSlry2RsGG0r2ksBWo6x1qOd1PSSLldcQRQYmkEb2ZBUxYSotcw4BT9WrJ+7xME7HyZZ9nn49a/nmede4uhh2Nm6yHQnp53MMyw26Kcph4+soitLMZXYZofxpI/UU5xT1BND3Ao4fOwQgR9z+oWXGGUlUaOBF4eYvKTZShhMJhQGgkAT9TyW5xJGe9C/UhDTZDSAsZkw12rgdUt6d0dcvDSmN1/jm5BhWlKXOa4KaLQsBoEdG/JSMRfNQSyp/R2arZCqdKhaMl43PJXvsXYsIgkTLpwOuePBJrXdQ0bbXD07R6xiplmOyjvEsqCQAlG1cFuO3Z0Ri/MVcx2Y7pY89fEG9V7Es58ccP5pj8XlNq1WPDORo/C8miAIiEJ/FlyRgtjvkE0cC4sdJuMCXSv2hjuUlSZKQuqqAidxTmCkAOdTm4per8Ntt9zKZx/7PAEtClMSRg2ECkGW7O6m+IGj3YjJ0pokac4CarWichVJHKGUj3U5WdUH5VFk5bVxXISpfUxVUOspSkX4fkgjichzR1lpxqMKhMRiKasCHAjn02i2qc0Ag0aJDhfPn0EpgbEO5wxlWaKEoDKGIPCoVYjGwXiIzDZpH7sFceQYsQv45//tbs585XOcfP79vPzsWQ6uHGF3sMkTu2MG/Zrl1TZhkFEWNccO3YlwkmNH7qEoS07mT3L77bfyyY9/nsP/5CE82eUbvvXb6B6e58inP82Wq/jgL/4O977xDbz7R36MT/zpJ3njN72J93znG/nIX3yKZgSnvvBp/vOv/gpv/sbX8v3f8qNkboCvmjStYeSGyEmTZq/J+uYGN80fub4mPvLGR9lYv8CFsxcYZ9t8yzvfgO12mV9o8vGPfYW9KyPuuutGJvlzeJMGpspn1TBOIcwsFFCWJa35IVoDukcQDbnprhYJKzzx1FNol7E8b3BBiElLdnczAj9EKYE1GlxN6MUErQbptIDJJU5+4TxLh5d4+PV3cOaVHSKxyKULe6yuLHN1+zJOHOGWE3Psbm/wynOb/P0f+x7Onb3A0185hzEWpSqEhE67x3Q6pahS0qxgKV5Dm42ZanQtFPTq2M8Yc70O69VUr0IR+DN1J0mSa++Cj9YlptZEcZssLfBDSztusrszIvQsRTlFBiVhPPPuHbtR0ZiE5BPD1nafpYVlqCQvPvcsYeTTWuyysbNBpc21piM3I+eF5ujxBSbZOr/0C8/Q6TaJGj6Ly10efM3b+MD7fx+HZW5unvvuO8ZocAl0TLeXkOkAR0GV5izMdZjkhslQUOmc0A/JpjULSx1kOLNAbW+NGAyGpGlKGAUEgUej0QAkW1s7ZNmUyWTCLoqiuMrcfI9xv8DlAj3ss7a8ROMYPHfRzRRsa9HWoXxvpnxZx9nz59jrj+iEMSNVIoImvrRsnHqJ/laODByZCnFVxfa4wI/bXL48wvcaOBkQN9Q14mRoNpu0Wg12djKyLKPRaJCmKZNIkZZD0I6umONqv8+//Efv4Suffpxz52ua7YgaSza1SOXY2tihqh2OmkjFeMoxzTTaTDAOhPLRrkLmJb6SVMwUyslkglKKaTq+Ftziug3i1SqhV1VO3/epdM3Gxga9+TmMMWRZxmAwmK0zSpFEHvW11ozpdEoUBZRlie971xXTVw/nr1Yivap4+sq7rvqXZUmz2cRXIaPR6JrPvQJmRNMjBCdBgjE5QoHnSbQpcdKB9aHOZ4c87Thw0wJH7pzjyI2HmYzPc7WQuDghm1yh0Uyppzn5Ro+b77yJ1xxpESyn1JMP42dvIJ+ucMgb8sRTp4kGNzD3XTdT5yuIFiwtr/LQI1/H8Xvvwk1y7JaiMCXtHnixj9YWxAS0YzoSxB1ot3yKzFGUU5wLmZ9bw4oaq4dU2iJoMt/rEARHaC0WZOMNisxxw4mFr5Umfu3F5luDlNJoTAGj7SFRo8aLC4Ry5CZgOvRxFqIQ6tIRBgYlQGofQcXSItiBxK8s1RTmWqv0WglVNcQaCBJFGCuiSCKFQEkQoqLZUFitUd0I6Ve4aY4fO8quoUzb/PrPvZdH3vhNHL3hBu6+8zZavsRH02pNEfICMsjJjI+uFeNpH19JtHZUZYbFELQrtHMUO5bX/s2I7usLroQ7rJspaSnpb44YX8jJpwXNlmDct/S3JTvbQ7TNySuPGx9YImp3CMImwnQY7sH2Zkrkx3i1ZLJuqdYD5MQSK4fF0IiafOWjI17+1ALVwMfDoagp85IghLl5QasdkE41cQTSlwhpGQ2nKB9qA8K3DPYkg52QZtOjyiuUbuB5Acvzi9TDAq8h2N0xXLi0yfkLGxSVI4g9xtM9xtOCaWoQfkhuUj712ec4eeYC63vrLC3ezBse+UbClmBjuI3yK3yvTRQuIf2AIIhYmz8we/ljOLDWIOn47A6HnD19htWlDssrDYQQ9MdQ+AGZjfGSRRqtBZQXkeU1g/GIRgcWbsmpM0u70aQeS8xVQejVFM0JR+4BFQkCN6AcK+Y6LTAew7MJ/XVJO+xReQKbZuT5gHIo2XlBEoYV0vOhXGA4vMoTn3+a7XMB0+0BF14ocWmLSLZZXmrSCkPiUJIPC3yvxCjDxbObrD87JQgFUdjC1V2kmjDo7+LLilgKqkHKmacvUVYV0zRlMBjg+z7z8/PMLy4gPYVzjrqehVNGY0OjtUBlNLW2NBoNbB1gbIYfWoJrPVHG1igl6Q+2kL7mrrtuZml1nkPHjnL20hWq2tIfjChLTT4pqEoBLsA5iMIE6/TMz2hLHCVKdKi1pjbltUXQo0hrAt+n1/U4dvwQg+HeLKGpJ7OuQQBXY02BsRqcQChHqyMoqim6dijlgyhJGgGLS/MzUmlqrNXUdYl1miydYMvZRhkGLS6dX8fUJYOspk/GRmY59sjD/NDf/Kd89M//gF/7rZ/hlrvuYjAuaLcjRuM+VRaCsUR+wvMvPMOZM6d44st/hVSW6dRy7MAt/PzP/HvWy5ynTp3io3/4ZQ689jC6mmCC0wwufZFf+E+/xcvnzvH4k8/zzu/8SVwKYTfiI3/+AX7td/6Qiybge/7OdzDZNhAkaFICL0ZaRVN0OfvyOnPNu65/ZAN48gtfYWXxVv6Pf/lvcYdXqZzPpStXufHYGmWWk08dSTzrDfW9eKaU1AG+L6gLQZFr7n3gIHUpSZKYRqNFlQ6onWS+18VD01kK0OWYMIIobMw2C+uoypLA98nSKaPBECUlb37He7jz0SO0lxM++/lTXFgX7OWK3JbgQ1bVbGzvMBw7eouH6S0s8+RTl3j22T1eOXuFqlII0WA0LvmO7/p21g6tsLq2hrGOwqQ4J8jzEs+btROAxFpm/rt6lkx+1Sf3qg9Ta810Op2plbrGkx5J0kDrgij2OHb0ZkK/TRQmSFVjLeisTeDHZLng2PHbOXzsMKUtmExzVhcPsLW+QT7NuenWm7DCMZiMkUGANrODjK4VjcRjobeErSXdBYfVFWXh0d8tefLpx2gtFsRNR6kneKFkeekEL774PKaOKKoCh8TzG/iJxROCxbl5XnPfPUhVI5TF2Flo5NzZC1y6fJaynBEXgKrS9PtDhoMRo+H4etJ9cfEIh48dY5yOqIWjcAmZnuPynuD8Vo1xFqSgrGd/t+/7zC8sXvf9rSwv0JxLWI4XuP2WW7h64RwyN5w6d5LNjUssrK6B1+Wb3/l2jBlT1SUy8DG2oM70taoyTZpOuHz5MlLKWX9pNgufpbnGlI5mEjPcmXDk2M1EsU/a36bVWqQVRCQi4aYDHm+6tc1hpTkYSQ51I1Q6oawCJAW4AA9JaGukl6CTBCf9WRer0+zubTOZjq77JgGknIVjjDFUVXXd3/jq7wklGQ77aF2hlGAyGhJ4CufMLPAlDJ43+8uKokDXlrrWgMBahzGzijbPm+1fYRgThuE1m5GgLAqSJEFrzWg0uq6YvuoFtdaSxCFxGNBo+ghPEYQ9gjhCO4PVPrUuQfvE3Qa3PRLhOopxwxIuzHHglgfIrCOMU44f9jh+ICHbXaQ7P8d9972d7/ub72Kzv8GRO15Plnuo5aNc2BIU5ipnNl7h/OQ8C82Yg0sBnYbg5dNP8Lt/9ml+7rd+nbTzCsJaWh2BND1ajR4Lq4ZGzycdtagziedClK8JggilQqbpmHRc0W0dQYoGqwfnSfxFKuOwrsDoKW99+1vozXf/9xPKQ61DNJxB5AU3rR6k6fnktUPFEVFgCFo1XqxptASmNPQvhYgC2qHHQjzP5sWSdJhjTQmuIgoK2skxOvMhyipQFcqvEELh+Y64ZbFOop2ktaAwuqCsoQ4TcuNhRz75OON//N6vMhoE1LsLvOHet3Pzkdu54eaQ+16/iFU9evNghyVnnoMwCtCuos7ADwSdxYg7HgXhS9R8QOEvMx15rDSXCHOQ0wLfOYxQSAnptECqhDRzDHZryhwGA83Z9XWuro9ZmFuk3V6mLGNGwxJTOupUUBQ12W5AVPVwQ4dvC7JRiaebpNt9Lp8UuGHC1osN6klIq+0x6c+MxwdvViAF2VQSRRJbB4AkSjTWCUZDO9vEKUk6MeQVtk7ZXJfUoxDhaQIfaiMpS8X29pThKKc2lqLUGCepbcVLFy5z6fJJeqElHw345Of/kq+cOcfr3vyDLLZuYTCwnHzmJZ5/+sucefElWg0Pv2Ep6xyTBzjfY1ppigoOrh3Fj3w63YikIYkiizFTlDQ4W+BsNju1Bh5e4pEx82qpcEA63SBQhvHGFMoGTgrqymMqJ5SuRdIKacYNTBlyNDTE2ymB0cRNTToak6UF/as5O+crfDNP/4olzypGg4p2s4WyHp12g8lVxcXnIi6/7Djz4iVsbolEiHCGWs82jEBFUAXELY/CFRQWXABhHBLGCicrrKhZXp4njGYJ57gRgbCk2YRTp05R1xWtTpvSTLFOzUj0aA8vnI356kpS64xG0iMKmzO1ujKEQUwjaXPk8A2cO3uJu+96kHe/+9088OAbeMNbXk8Qe7RaHeJE0m53CYM2VWWoK8NgMKAoSxAC6c1qMWYmdA9PhQjPR3qKznyLvcEub3nb1/PjP/ZPmU4qPDULETgn8H1vNiWQDoGHkLN0urEz8uCcwhoQUjMaDej3d2k2E9rtNlWlZ8+pH7C8tITUFVYbXDPihdNf4Q9+6Ve582DCK09d4fGPfYX1DY9/8M//IQ+9453MLR3j7OlLNDoCq+pZ2XtVUOSap57+Cq22ZXvzEkcOHSX0FJ6bxw8E99+Y4K6e5OWzL/GJv/hl/u3f+z/xhhV3HLqZKh9STJ/m0FzFWivkscee4fgtx8mNhariYx98L5/7xJ9z9I6j3HHHUepyF+vHRNLj5lvupL+5yQ23HaG5lFz/eOq5c1RVgK5GnHn+NNtXRghVECcBl9e3cV7IhSunqVIFjJESPNVA+hOE8LHUPPymZQ4dOsS477O9tUd/MOXC2SlBcJC1tYPc/5pHGezNs7cesbMlCCOPW2+7ieW1ecJIUlZjwkiwsjqPo+JP/vjDXHx5i+G5XVYCzZvvvY2rp84zHeQMdqZ4GPJik/F4SG9ulWdfOstH/vLTnF4/Qzw3jxfUFGWGFB4rK2s88vDrOXHH/STNBS5duQrMDkF1XdNutzHGXCuxhl6vRxBE18eG+pqXUqmZR8+5WfWKFB5J3KTRjGi2YtY3Nri6tYnyHHHcwNYahyUvhjTbio9//JMUZYofKFpxh5Mnn2dzd4cbbzvGJB1zdWODKIjRlbnmuzPoKmNh7gjDQY4nPBa6yzjZR+sxx2/usHqkRIY5SVcQNppcvFjy9HMvsHC0yerBJRbaTVreMZrdkL1hivKbNFuCne3ZoeuGW2cTiWlqMHI8S8hT4CSsHjhAu92cBUiY+Qh7vR5K+aT5LkbPFMHaGgqrGZuSC9tb9ItyFi7RGuX7aOvACYbDIVle4vkh/cEEKUKkk1w6fxGpS247cQt/64e/F0RNuT6ingbcf//beNfbvxFMgbEFKIPwa7SpqOqCIPTwfHldWRZipooKGdDw2mRFgYs1rSjhF37hLzg/8OjO+TSd5aYjh/i6r/sBlo69laB3gLe97W2EdcBP/J2/wzfde4C60HieIQrbVKJD7ipiKTEY8jylKLJZgKUukZLZvyvEdc+kUuqalWKmGr76569Slk6nw3xv7pqPsaDMC+IwwtSasspRvrw+vs7SAmvAGonRAs+b1Qs559jb28MYdz2U02y0Aajr2YUVcTz7GWutZ+tPUTAZ9al1dt3yYVxFXQtwHp7n0W40KcqKB75+BdksKXQD50uefvolnnrmLO0wohMalg7GnHp2k3bPcvNDB7lw6Qt89umn+aHv+HE2Lme8dOk5ZN3CNSt8Jzhw9FbWwjbtg4sMRpaLL23zgV/+IB993+9y/srj0PBZuiUiGwh0lqKLHCETiqqm1hacjzECTzYIQoEfaExVMx2W5FMDNuTEiUd49E2vI4hqinwX359j6ehhVg71vmZC+TWPvH/u536eP/jj3+ajn/pLMlPS6LQQgYcMuyDGVDk4qbnxroiDRySf/cspk1QQtxSNtsKfKqKmxfN8tten3PyaFpUxiOwA5XiT6UQQhx7puKTdSVg9sEZVWtJiA6VCwpZi58IQl9WEbY/KFvhtePa5r3L3XRuMM41TGVEwz6Q+y/oVQbBUcvQ2n4UGfPFTNVZp8KHTEbTmAlKTIYqQA8cEF07VfOSXt/CCkvkDbbx6diLJ0TTCAOMrpkUOOLwmGKORkyaezNjedAhjGA2G9BZbzC83yNMBdV3QW1LUDcF4UFLpBk3VQqc+65dLPG1wngMrufoSCJvjex5lMfP/TCY57fkIGWgagcHUlrk5iZXgh47pBAQg0Hhtj/4459DCrKLgbL/PciBZWG6RliNiExM3FUkU4+qAlfmD9NMhzchjd/Mqx+Y6rB2f5/lXzjO3sEgQVFx46ZP85tmT1NQzL6lf40ceUtdsntnk6O03IjhNWW6xvR3i/JT5tk9V5TTb8wipOT7nUUxHXLyoSfMx1hmEiHBaItTs6/RkA1TJyk0hepKwc3VAo5NQaYkXOuLQUVdz7Jk9RNgiMF1Ey+OAP2SBgOd2HJ0ln7SoIezMyLYuOfNcidJN/ECweqBJmvV53UPfwsbVCzzxxBdnpeZDn+nY4qkGrXYbZIERNZHXwoqQtSMd1revkKbTWVmzjajtjFx5UUxVaLJ6DEiU54GYjf+GwwHtdgshZmRIuBDlp3hiFnARokWSNMiyjCRuzcqQo4jV1SXyPCdNc2IvYHNzk7oq+NAH/4KV1UXOnL7I7XfcTBj6+F5EOp2QZuCFGdIDYyuMm3kvrZXXlKQSPyyptaWqLKUumZtbmhnxheXU6U0u/9J7Z6qSiFhbWWCvv4OSYK3BV01qneGsZHH+KFtbW3jKoBQo5ZFlxSyYUFVsb+fceOONtFodtra2CIKIO++8k8986pMIa3HaEQYtfuW//U/OXYq45a7D/MLP/J/ccu+7eOgNb5wt9OunkBIawSJZMWFSBrz1rfcyGWnSccWDD93D8688xisv9MGreOH5T9LfzTh24wnGX36MsNrimZNf5l3f+Ca6xw6xEj7Mn37gv9CXE44+8na6ty1TffnDfON3fA//7T/8DPPdiJe+8BJff/ubePLx89xw951cPL1OMO/QxmOU9Zlb82h2pvzxb/3e9TVxd3uPIDK88PIZ7nt4i294/Zs4c+YKn/jox3j5zPM8/JaHuP/uu/iNX/o1Or0G1rQoyhG6CjFiyoGDS7zp9d8FQnPP/SPe8nVv4td+9bcobcgXH/8Ih44dJW4GbJw/T9SL8K3CWI9HHn0zo/Eun/zYJ8hyS6UFLq2w1mO5abHO56bbb+NK/wJ9J7jnDW9ivmcYbU4489JlurFmb7fmycefQoY13V4PsOTZEOkEYSRx1uPJL7/A3nDAU09/icl0gKdmm+trXnM3X/3qV6+FJ0ApNSujDkN836coiutjxlfH31LKa+qdh7MlZTVFG8jGU+6+7w6iKOJLj38VF8UI4QiSMeORI27CgYNLSCnpdluo2rKxtcnDb34DhZ7QP38BWVsCz8MoSX1NvYoTj3S6y+rqAXZ39igKTeTNMxzvYcqEl5+/QhwvE8US52LKvMBvKFYP30A9tYz7Jb35s3QXehy64QgvPn8Rp/psXQrRZmbpyHOH58Vo59DWIoRBKoUfKMaTAq1rwjAiLzIuXrzI8vIiiJpBv8/K0gH6/T66LmY3ZnkB1s4Uyaqa+fJeLdfOsgLnZl7sBx54gM2N81ze2OOGG9c4uHCIOkw4/fJJFudW8YtNUrXLT//sT6BEQNJuIrXBFwkmqghtSJZlCCHodrvo2hLHMe12l36/TxI1qOsBVeGYmzvEztXzBI2ScaVxmyMe/q738A//7g9SjHb44Ps+TqRCsmKXQydW2fNjdi5dZa0JVyufqc1o+AZjBZNsQiICTBBcCx0lhGGIMeZ6+rssZ1VaeT57rl61VLzqyW02m1hrybIMqx1JkhAEAUEQsL29izEWbWbj8lcPO7OD76ukVF2/aUfKmSXB92e2JK1n3tXRcEQQBHhCosvqWqUb134WIYqZ0un7bcbjHR58+C7GQ8OlC+tIf4jWMVGjQ7IQUw1vpR1LimzK2tI8u6M9In+ZYvoSpzeG3P66w9xzV5etjR6yLTg8fxe11+U1Cxn9zr1s5C8xFi/S6vS44RgErsnei2Mml0rsZIoqKlx9luJ8g+aiz81fP8e5v9pCiIJsWLFpDQuLHS5fGJJkMV5jgq59TFmBCBDXbtiajPaQcY+tTc2hGxMCNWL7ygCv2WO7X7K4MPc1E8qvOZSzeGPobr/xDkz7Ehd3xnSSo9TVkEA1KUd9tidjyszSbUasHNZMJ4qyDBj2a+ZWCgIr2duSrB7XDPfAuAbH7jyM1QlmNODky+cIr1UHzC82WVg4zOlTZ3nwtTdw/vQunWaHwbjPlUt7dBYlfmKpNh2J1+Mbvu+NPPXk01Cv8Pr7b2K3+gh7VU77oEez2WTjqynPPTFkCkgbEYiCuQOKSQ47Fz3CoMRrgWcloVBYX+OcosotVlqyKQhP0ogtVepjtcb3HekOCOnTWnaY1DIpoDGvkLKm3fBI4tltHsOBwFQNbP8o3YVL+IHk4mmHc0N8FVDXFZ4LCOKAI3cHjPw+uu/jIkvYMaiyiRAFo75HI5m9fE4ZRiNDlUHS8EkLjRCOQ90O+bRgs1+TeBFHbm9z+eUdwuWQpBUReg0GOxlW+lin8BAcP7BMtDDHpZdy2g04fGvIld09Xjh5Cc/mNAOHZ2EzM5RGEk4t1aBm4dASupFS2pRWnCBURYBEKB+ZzBS2JIqRQcV0ErC3s4k1NbaucNbQjELytCBQHrZZEHgSkwuwUBiDESFKBXjW0GhoChQ90WDvdMr5zZLXH4ZWaPnEpYiVxYjRznhWuVF4hKGPQ2JsTSNp4fmS5aUDPHT/o3zsY5/AMmZ7MyNq1hhb024coBEGnL1wls7yAlIHCKkxqqK/M0HamWnd4tAmJwpjgtDHuQlKJijpUxTVdWXm1Y3WmJpOpwOiJktrqrzJbXf0WN84Tzpu0Wz5rB1YZv3KJlWdE8WSqioBgUTgBx5lPsEqSxQ2sK5kZ0dzww0LZFlGOhY4JxEo5hfnuHzlEr4XoNRsY5+bm53kt65u0ul1KYoMhMP3ApJkdpXjoD+mGUesrs0xHBWsrhxiZ2eP8bg/e7amGkeBUi2qquLYsWP88A/9KP/m3/4LtJ1gjT/zSV6rBgniiLIsZ7dnOEdVFCSholQh0kEc1BjVIk3HtMKDNCLHeDQiyyukl3LsyOs5fqPPM08/RpXXFKnPPQ88ivIsL7z4NItLXcLmBr3WXbxw8hzalFjfg3GELfdYvPEwP/SP/4DKvsDWc09y+MFDfOF9n6G/NeBy33LotmNsbJ/im9/5bfzF+3+RbGdAJznCe9/3e7zvE3/J2kKTX/6//j1OGLTS6AxwhtXjHquLh6+viVuXRyysLHHvfXfwvvd/kkb3Zn74e9/NY1/5JCdffpa5uUM8eOLr+MAf/jyNlgA/xJgRnoox2nHk4H102j0cmpMnn+en/83/yZef+QQf+uDHOXos4uSzQ44cvou5pZqnnnqe+eYSGp/V5RVefuU5krCJsQVJ3GGSjQiDxswuUdUIoUDUSBEyGnu85zu/hbOXnuT44SP8nb/99/iJf/bjXLm8RzYG35uV3HtiHiNGsxCVVTO1ejoiagbce98dvPTyC2RpSRiGMxXRudlVmsaQ57PbbzxvVorebrcZDPauk8lX+wg9L5iFkpwhiCLml+aYW5zj5DNPYy14TiF9ENKiy4RSpzSaMTfcchhjDBee3yR1Gfc89CCXr5yjHE/R01m/oHMzr6+uDUnS5ODheZxVXLx4GSU1SbhKUaccvauBnyj2rhpGewFJR4AaIKqEuvbZ291mabHLbbcGnHlhG08t0bvxIucvaXZPCcJIgY0JYztT4m1IbabEUZtaZ/j+zCeYJAmj/uT6mjAL3EkOHzxElhUYYxhnKX4YkGWzzsl0Mr1+y0zozchSce2KwDAMOXJ4lc3BLuun1nnbN7+VG4+s8eu//Nt0OjGLnYOceOBhuu2KD/zhh5k4n6TbRVUjykmOiwzddnd2rWBZzuwXetYXGUWzUW+Vj8kRdKMewjqiYA+jDe35gxxbvIHXvu07+Yf/4Fv5+F9+jgdfezs//7P/mj//3c/QXWqxcGgBO52wXfpc3r6IqyzNVgffN6TTgkxblKuuK4J1XVPXmiAIr4+UPS+4HvR6VWVsNpvX64Imkwmm1iRJMlu/8pz5+XnG4zFRErO3twdAp9X+f+u0nN3SIwmiWaAojmOKLL9OZJMkwhhDWVcYM0t8/99VU9SMmwhncddqkCwQRx2ktHQ6MZNxynAn57Xv7JBHHocPPEISwoHlm7j1lgN87otfwFrN+pUv0lw7hItSHr35NvT0MLe95h7saI/f/9BX+am/8//kox//T3zs3G9y/NZFpKkodQO/5/Pyh87zxGckSzdI8t0Wi/M9vOgqAYZUROidkI0LQzoLEas3ziaS6W5C0lAsHq5xJkOXIGTM1c2U2OvRTBbwmk1cMyTsKuqNlK3NZxC9BV57/ztYWhjxWz/9J19TKOdrHnl31wynd55iYjRJZ5nCaowbs71zjsorabUc3XmBto6rm5ppbqgpiJqauobCBNxxx1FkvYAXRcSdlK29l+gPr7CxdYWVJZDW4UuDElPWL7xMPal46jPn2DmzzYXzZ5DWEMSCOq1hbPDnBMu3Kz7465+h6O+wvfkCD97/TTSDH0KIBmLS4Nwz22yvW1rzHj4+raQkCAOyzBICh4+UOCXQ4xgpLRaHbx1Gz6p8vKlHEIQkgUfgtagMEEimU2i2PKJQMDcXEHZn441m4rG81AFlyKqavHa0el1uuu0wx+5tYP01rl4OaCYB1kBdK9qNg3hBTFUGJHMhXsOjtjXdpdniqk1Nc07gqLBSg/QZj2aX4Ck/oK4kZSVpRR796ZTKlcwtNDC1R1pWCNUkaSVs7fSZVCWTasooHzBKU0aTDGs1p547S5CMuO/RoywdOMSBgzdww80HOXjsIIUNKcM28wttVjoxcRIQLyr69TZlWZOIWWJP2gTlNWd3t5oMpjmjy7tkk5p0J8U3LTzbpZksEzeWKGwbF85RezE6naMsJZqErIqpap/ClNRugh84DAZhal5+Yczi2u0szB/i1E6LTQmxb6jrglarzVyS0Ah9qqrE90OCRpO8GqErj/WNi/zyf/8FvKRPXhicGlMWlnzqsb2zwdbeVebXFpF2VvRblGN2t7cJ1ax3zPMh9CMC1cQZZqp8nVCkFmNmicAkmfWiKeVd8+04QJI0IoIgBqaUU0cjahA3UpyRbF7dm20wGra3RqRTgxQxde0IPUUUSoTt0N+xlFnCQm+Bq+sjGvEcVW2xzqG8iG53garU1wuE67rkdY++lnd889dT65nPqNVsMtfukQ7HPPLAvdx/9wluOj7PynIHIWfeykuXLlDpnIMHD6M8QbsT46suWqdIP2NjY4Of+lf/FusqEBYp/tcGYAXXi4qruiCMfLzQw5gEKS3Sy7FG4eyUTrNN0t4jNZs0V2D+aJOgI1k7aHjg4R/ikW//Ae79xnfSW23z9MnPcPK5p7jxxjUW5pYY7S6glMRLMpZXjrPaXeMnf+VfcuKh+zHG8NN//1He/7vvZW75EB/8yJe4MtnllQuvMEhPcunpT6H2Rvz5L/8O3eYBwo7HcDrgtz7yecKmz7nnNhlPK/AdUoQIqzl0eImHXv867nv47dc/UiE4fNOdvPDcZQYbm2xfeIpvec87aYdt8sEee5uX+K3f/i+cuGue1QNd0rTEuQZFpclLmFaXubT1LOc3voyIdvl3P/df+PSnv8hkB86frwk9nzAQvPTKBfQE7rrvNqbTTc5ffJFGIsmLIb25xqwOxtNMs72Z50xl5DpH2A7//Zd/iQMrgg++//2MNg1xdIgf/qF/xuaGYDCasrCSoK1AyiZW7WBsOVPUpUH5Fe1uRCNqcencFmjF4cOHqKqSvCqveSRnXZjtdpuiKEjT9FqIYXagetX/9mqKWEiNlB51ZVleWaTda/PkV57BDxs0mhG1NVgqjJasHmzwwEPHkdLi4ZONc0aTEcvLi1w8fRaT1zPfuJylojWOwA/x/ZDFxXka0QqTyQjI0bViccUnbCmCoMNgTyKEotU12ApMEXP5yiV2dzcQQrB2OODppzbQ1RxXt65y9WLAa193iG6vTeB18LwIoyVhGM6CQc05lIJbbj1BVc1uxpoFK2e3q8RxTBTFMyJTFGxvbzMaT7BK0lvq0l72UY0Ji/M9lBD41753syta9fXanRdfOMml8wPe+tCtnHzyOf7i/R/gvpsXecPr3sbexjlEM6E/1Dxwxy3ceaTLaHcDLWq8wCGNz2g4YTrJECiMdteJ3Gg0uGZVaBEFMcIryfIxeyNDbTpMRjkvnDtN7GX8z//5cV7/jW/ij//iMT74iSeQSyPGZcn3f8/fI1xchnqX+aRDaaDUhkqHWCWIPX297Nw5RxiGxHF0LdglWF5eJQ6j60Tu1ZGzc+56CEcJyerqMlpXODerYDtz7iyTdDor5/cVeZ6ys7dNWeYY82qrgkZcqyZKkgTf90EK2t0OSTJLaM/NzV2rLqoJgwB1zbrhnCMOwhmxvOYLdhgCzyefGqoyZ2f3KsPdMa97d483f2fMqD/h+We3WVm7m/seuINTW49xdfxpdHaBEzc9zL23Pgr9g9xy9B08cP/DHFu6mVfO7LLoJJ945aOcH/4ZR46HXDi5R3/d4gUVrnGVxUMhQeEzzwIdoxicucpwJ8bKNqMXS65uZEg5U4F7Cx51BVIK1lZXyfuL2LqN8j2kcphaMBlU7O3usDfYwIiCdFoz2dkmHwgiX+DqIZ/+1Be+Vpr4tSuUN97Xdr2bNKNpTTFukzTajPZ28GRNriukkcRxMPNPRAZjZtUwvqdRJmLqah648Xbe/s0/wO/+2Qe5vP0YNY7dXUc39mmHwawTy8tI04put0Hoa/pXDK4wrJ5ocmlrQp4LbB6yeFCzt21pJwHdTpOjK/fhN3Luu/tW/vCPHueFk2f4ez/67eyUL/DS2ZO0uw3c5ACR8vncl57n8C09dD/jzvslX/d9R/nJd7/EyoGYsl3i0NSjBBNkuFpSOUfbm5mGpxnUGCIEeq+Jkhmq5eG1NcXIEIWShZUE2cjIUgsGdC3odE6wsHYDQaB55eQn6PgB6dhnaz0lHzuEgzA23PWNijwICTNLqXImo4RGkNFdbHHh9ATfKYrSYpxDeooqny2I2sFas8fmdEynZdBTRSMSqJWQzbOzFF3QtCwtrnLh3BaVqfC8iEArvvdbv5X77znEen/Ec6/s8cGPfI611R7NlmJxbRHrF2yubyOpuXzuCrqUeNLHofBC6CwK8ixge3uHVjMgakIraDI5VZKPDCt3H2CsBzMvnvQQhERRhO+BNgUKQZ3OCpNtkDIZgTQ+LjJY4XGoGzHWOREzs3Cxu4CXLXN28wqdOUdcahodn2w8ouXPMbQjCu1RpiFeomHP4He8WR1KYAgbGlcvMBrtzUbw1kdJS3ehR7u3RITi/JnzFKTUpQRjkaFByQhHRhg2UMpR5jO1GONjXHV9xNfp9BgO+1RVRaMZMx6PSRotojhHGEs69PCCmkYH6jpmd7ciTuT1q8CKYjYSwmSsrjaZZntkRYckbrK3myHQKMVM4VnskZeW4XDIaNTHDxSeCjDVjOSK2RrIsduWuHj2Mvkw4OjBg6weEJw7e4YwmMPzArZ2RyTNNq975Bt46tkvMc2mtFvz9PuXMLaaqVJGkJcFRZGhvJmZf1bOC7WZdbbJa6nK2VnVXu+iM74mKGebh/IifL/GSo+8mNWQ+KFH7SoEHvm4oq5LwqTHwvJBlEqZ7K0TRrMKj/neAbzQsD08i3UBznqETvK6t30nf/T77+XwgYN807vewqXNDT70+39JAsiODy7jxIl7SK0h3a1I7QidFwRVzLC4yr2Pvok777iXl5/5Mi8/f4aiHiMDnzLPuefB13H4+BzTzf919eKttx1jZAMe/+JnSPfWwYb48QLD9W2q+iLKb+J5HsvLAaPpLotLJ1hfX6c2Q+YXE/p7E5Q9RBCVfPu3fQ9//Ce/y5WzAz740ffzK//tN/n8lz5Br7uEIGWvXxCrDirQs3q1VoOHX/tGPvwXf8Liwiora4u8fPoMophgaNNcCMH5+K7F7t46QRKD76GKMcpLSPMxcQN6vcOEvkK7PufObKNEkzjxmE6HWAvz8z3SbEpdadqtBe667za+8IUvEAQBtjbXS6xfDeRI6SGu2T6CYHbXsxDi+niz1tW15wIa7R5WWibpiLXVZfKsZLgz5viNS1y60KdINTfcfJB2TzHYUVy4eIrXv+VRLpw7w5WLm/Q6s2v7sqKk2UnwvZDpZEQjCnnDG97A1atXefGF0yjhUZspc70FUlcSd5bJ6zFznYgkkOzs7ZEOBLYQVDJlca7BZNzHaEVZBzz6ujfx4gsvMJlcxlYeCEsUtjCmunbDisLJmiRJWFxY5dy5c/S6bdavbBJFEdJxreNQUVk7u95PSIypieKAaZrjy5hGmOA1BP1+H0/6VNWsjgmhrl+FWaY17W4L5TS74zFvfeRmylpS7GwTxws88cJZwnaPB265ndgM2NgbcnEwpLKCXE+J1OzGolcT1K8Wel8fgbsCWXoMigE4hVQGIRPqvOCf/av/ANUVfubf/xb/5B//a37vd/4N5XSDVncVjcBXPcpyj3ll8Oo+ca9LKlpc2RoxGE7xk2AWyvK8695Eay3OCbSe+SSTaHaPeavTJkkSBoPBtV7YWSemxBH9v1j7r2jdsrM6F37GGDPPL64cds6Vs7JKKEtIIAESEmCQAAHOJvrYB44zv0kmGgzYBoxEMAhJCCRAoSQVVVLluFPt2jmsvNYXZ55jjv9ifnvJp7VzwQW7td3qolrVSnON+Y7+9v50z6PX69VnJGA5Nrbt0hsMcCw5ScVXWFLVdZVlXeDgBT4oSZHWMH7XdXfJBEWWMzU1RZrGjMfjultdSuI4xvU8AIpKY9khmAJTKSqTU5kExw4ZjyLue9V+/sXPH+W3fuMC508mdPYucvjYXXS6PZ548bO888H386qj95LLNk+98Bhq2+P93/d9zE63+djP/wZxs80r3zrP5z//K6xdHRPeHWHGOTeeiCiLLvd/OOTsX23Te/FWjt81xx//wZeZnlNY1jRpv0+WSEpRELgpaSx4wzfPc+NahkxmUcLhhdMv0GjaTE03cVywLZ/VG5vsbCYQBhy97wHcIGPt6efobSnahx0Cy6MfDxmdHv/9KpRpmaOMhTAK4QkyhqRFwfZGhVfZ+FZFVaR4gaEsA5AVWIqoMIxNhrBKnj15jl//L7/I+ukblLHHeKDQI4nIKja2MrJiTBoLpGxTGElpZew/4dKcUaTDEQstWJ4BR4EvPfYtSTruHFdODnGtkA997z/kxIPfhgpn2L/XZ5xm3HLofdxz66uQsuTMU9t84AMf4b3vezvDnYTCChkKl8DZw4MfOMAgLrAjUCWoRoIowcbFFxZ5AaUGy9H4NgglsaYjvGkbS2W4tsSUYGG4+HxOemOGmdBCVC7SkqytnOXUFx7nhYefpRcVDP2IqeMVx2/xamU3Lpld6DLeaXHhyzkXHnFolo0aim4USa5RNjVKQkiqTKKMxrdDbNOhFXrkKsVGYzIbLSSxsVBli/aMh3Edmp1ZeuN+fQGQEt+GRrvDte1t7Pnb+OLXYg7tW2bJhcvnznH2/Is8+pWHufHyFYokxVdd5qbmkUIjLYEWEXk1QhcOIklZ6MwQ2jZu1cT1NdMHCyrfEGV9XBmCrtBlQqljknREEg3I4yFp3KMQ22idEQqLrq9RIkEWBW6lSXTGfKtJ2AnotOdxD23j7jnPfCdisJ7TdgPaXsBwDKujlDILaEibxa7EKwSlLevUYFVQZBb9DYv+zjaWtLGEU7/gBJjCY7yzw+z8HPuPHMKSFp12iLJl/T3XGa4dorCxRABGUhYVQlW0Gm1c22M8Tun1Bvh+yMLCEhjJwsICvnIxeYc4dVC+QKk2VR5iqRzPFTiWA6IGgVt27RfKckmeW1Slz9LSEoKA6aUMjaGSJds7fUzlsL2+ynAnIfQC8riA0sbzHJRV0Z6eYv+hWwimmtx77738o3/6XQzSVcaRS5kGzC1FbA4i9uxfZhzlPPzVzzEY9NG64OqNkzRaTWwnZDROKKuCoiq4/a47aHe7tSpZ1ZVpnu9iTwYIrWszvZQ1vNiUGkqH0lgox4AcERUZ2hQszM/gN2yKWJJHCToFt+HQ2ePQaPTprz3PzuYKltUgTmOyIqGXXGZUXMBxJba0me7M0Nq3zPu/6R0cnlrinnveiHDm+Mwf/hX33XOUow/cQiEdFg7cwhve9kbuvO12Ll1+maookVbBoNrC8RUXTr9Mb5jxLd/3vbzvw+8jMwm2P0JSMBwO2Vwf01ic3f37wsnL/K/f/k1eOnuW7f6IJB4zWDtHdyblxJ1zzC8ZXMdme7uiLB3WV6+ixxEms9nZqNh3rEEVbLM1HvAXn/ss3YU2J+4+zL/6iZ/h2eefoSpztrcG9Hs2tu0wu9Di+PF5mlOKrV7EP/nX/4of+Uf/kBs3bnBhbcR0u0lhORQiJk9zovGQ1a1zGJUxHm4i8gFRGeM3G7iBQ5rB+tYN/JZPb0dhNLSnTF0NWNbd8VIZtK6HpTgZ8syTz2MJG7SY/Hz93S5lZUFVlbueONf1aTbbOI5HGmfoskQKhakkgd9k1OuTRxkNu0kRadpBG8v2uXRxhSDw2LN/gRtr1ym0S0LM27/1XeSe5t3vfw+uF9QKFSm27ZKMM+LhgFazS7M9w9bOiF5vhKEkK0f4QYPN7R0ajQZTDYvp0MNRFju9ITubg5quoHJC0SAZSfp9H10JhC548isPY9Nn79whsiilylyk8OpyjAOG0TgmGTqI3OfS2YvIsmLUSwkbHlJ4COWydDDE9mxsUzde6cogLZvuzELNarUMg2TAeJTg2D5BI6TZbqJshaGcNLNoVNtCJxOgdqvFs6e2MYOU/bcdotdb4fbbp2i5ki8/8RjnhxXLJ44hZYLRJb7XpqT+uNp8fe18s2+7KAqKpKI37tNtTtVBl1ww251iYWGBU08/zK/+yu9xZP80v/NbP4VlWeQmZGOnz9Zmj35/Bc92MNondgV70pir5y6T5grpKgJRgpKURlNS4AcWnq8IQhtjCmxV2yI8z6EsEqJxH1PVLUdT3RlsNyRoz6C8JjMLexnGGeMow7Z8Ws0Onh0gqDhy+CDLC4soKbEmAR+APM1IoxioEKYiTxNcS5EnMY6t2NpcJ00S1MQ3mRcFUimysqA0k4txachSTV6M0VWCFC5FGhC2Pd79ra/g1//9OZ795ArHHrgbvWnIkxf44uc/Q7Lm8eKXNuh0TnDjyhp5X7F45BD7puc59bWTbAQ3OHinx+XNP2LYy9i5Nsu0ZRCZy33vXGT53ohn/3CDZDXgmaef4RN/8DAd36foO2xdv8o4zcnlCMcNkV4HqwWXzgluXI3YHu5w4cqFWgBwfHb6hp2Bh1Qd9u09hG1LSErSG+sUyQYmEVgixfVgfbNHOSz/rmPi312hnD4hzPL+Nl4HhsMA21Fcu3KdeFuyuFjhu10qUZKkIyw7nMjCOZ5nqEqBNC5KpPRWIBlDJcFy2kwvCrywZHUlRhcVTd+h1WlRkqBUhGXAqQQaF+Xm5BWMY5vx0OLgbYbA7zLbvRWr8nn2ySfQwQJ3n9hD07MYph53HNvHY4+/zEr5ECe/MuDWO6d51etu49L5M2zuRFy5UZGWhtvulPRXIBqkYCkKXeL7IXmmKUyKqixsT6PL2mROVWFbiuluwGhYgnIYRgMkHp5tsXYeDhyWLBwt2R6kuBK2tgVu2WRwoyIfaewmTB8N6e4rsceGZ748oEpbOFZGZRyOvWWACT2KUUmUlFBMdJ8KlLQodYmoGsRxRmdaEw8rTGkReBZ5nuE2ApzAJSvH5NkUyq6Ik9Gkd1liWxZSNyl1gnQXuX59hfe/87UsT4X80n/9I2b2G6SsyIYumTRYEvbvnyeJU3Z2UoSEykT4gUJig8pp+A5CKISCwJUk44zNkaHpztUwbanR2tRmaZ1gWSWOo1B2SVWVlBW4gYNlQRE7mGyWqhgzNj0C2SS0LES7D75NyzVcfTohWm/QadZIFJTEVDm+ZzM3PcPq2oCdXowQdYBESZuaeVxNenLN7oqu0w3Z3Bjjui579s5x5vTL+IGN7/u7t1UhBLbl7iZYG5Ne2Gg85u577mNra4vhcMjU1BRCCAaDXq3W2Ir1zQ1sV7A0f4R4PKIsBjRa0+z0d2quYCXwvTZRvIVBo6SL1mMcGZBmNp0ZSWOqz40rEqNrr9FwNMKxmkzN+AyGG9x66228dOYyiPr3WkmPRqOBP9XmTa99kHMvfY2XTq/ygz/w/YSux6/92i/jdy0G/R1GfcWdd9/Kyy+/zHgMQegghMZQwaRCr9VqceKW45w+fZrhcIgUahe9YYyhNLWCV1WTmjVn0nCRxEhLEI8Nge9Q5hVz3b380D/6MP/pP/8n/sUP/zCnXvwqTz73BFLb9FaHVI6PFXhYhcTYObYDRVnheAFZWqHsHF04/Oov/08+99d/zbv+wYe57xVHefTPP833fu/3052V3Hv0lZzvnSYea6KhRejPk+URcbaCZZXce8/9XLm8QjzaoShs/O40H/nBn+D5x57ksSc/gR9YpD0wekBuhThWuHsm9gdrWAXcdcsJRDPn8rkVciNxPZiZb5MmFUkkQGh6vW3KQrF3X4M0T8iykHAmYTSMSGKg8JifPUC/t8FosEMY+LWaayvKQhBHOfsPLPLTP/sz/MJ//fc8/9hLfPAf/Bgf/sB7+aF//AE2NtbIc420nLoffBJmuAmtLrOcQ4cPcue9S/z5nz1KuzWF43g8+KZXcvLUs5x+9gqz0ws0Ow6XL11HKMnCwgw7OztU2sLzHbIsmqxz7d3fG8uqVcmbYOqbK26ogdH79u3jpZdewnHql/pNcHWe18qmkWKiIIHjTODoac0kXFj2uPv+Yzz11HlcZ5406+N02nz423+MX/r5/wfH6SMJwFhIq6DS7CaEpztTDIZ9dna2sKw6gDEex7Sm2zUrMnDRpiQMA0I/ZGN1izwpsVTK9EKFdEpuXPQRWhK2NM3WIe6671Y++acfx/VCDh3ey8bWDp7bZHNjQGm2oPAo45yZuS7CLllb20Epn0InHDoyQ1UVOFWLS5ev0Wq1J0ilfBfb43u1J7DRaNBohlRVxdZW/flnac709DRlbhhlPcrcpumGHPA1sttmZnE/Xig4/eI5Ll1bpdVpkMQVTd/CC2AUgeW2SJPBrmew5i+aSRpd4/s+WxvbeIG76wOvdP1znJ6e3Q1h2cqi3Qy5vrqCEIayqn/2w/4AZcFte46z1j/H3Y0Gzw1yVqOKqkxphSFaCbIkwbYVt9xyC5cvXybPS6qyAlMTK2xbMRgMGEfDmhkpa5pEo91hemYW27ZIk4TNzXXQN7mT9ap8caH2lldVRZbGGCRZlqF1/Tkq26rV8qIkTVMspRiPIxqNEN93cV2f/mBAXqQg6krFcgLJl7IuBlHIyZpeIYVFWQ1pdDwGvYIqc9l7ZIYrG2ucOBxy+J0SNe7iWy3e/+6foLnH8PCX/pLZ2bt4zWtew5XHh/z8r/8j3vpPu1w5NWDvUY8v/nKP82sOr/9en7n5TUoESzNdfvb7r1KWDpaqcC2PdrPDTm8dQ4bt2uRFfYkbR308z6Y7PcVoMKIsKyxLkiQxU1NTGGwsx8NUCelwQBqXpIkGITn6wDLrJzdJs4L2EYtolOIIRe9c8ffblNM8IIztwzgWHDo2j1ExW2sjLGmghGbXZ9xPkBa4jkVZgjQenl8fbNrUfMUqCelvCu6760G++X13c+KuBf7Lz32W5y88ShqlpHGJ40j8pkQJ8GxBMywpxhXChjxTlAiGI5vZPRZZNWbP4gnagcVXH3mR/d37+PYPP8iXH7rG4nRIlMacPHmK9ehlxhsFd71iika7wdMPrWDLChNUKF9B6ZEUCcVkVa8sQ7NtKKsStI3QBb4vkJZLr5dhe4rWdI0Ect0p3KrL2tYFihxcCTKaZbRVIIOII3c4JDLCsRxsmWOmILrRZe1vK4xjiEzF/AHFfDvk9HM7KKsizuHV3+ITmyHZNuiixjrIykJzUwnw8F2fLI9RoiCLQRmP/k5M0IDmdIgT+ERZjs6bKKVA1U0Xlqco8hKdOhRljKvHqHZIFPm86TVv4InHP8+1qxtMz7XJrT5W2cS1PXZ2Nul2OyRpNWlayevOdumQ5DFL000sk2Acn0pplJOQFyHjnQrHaqIrRWUKSj3GcRVQd6IqR2FbKVJUpEmN36mqkiz2GI1iSBWZZbCx6bZT7OkSPwC7P81zXxwzNW0hpCROkwm+oa58A0muS4SxMKZukXFdF9upX7ph0GRnp0+cDGm2HLKkPoAct1ZS8iLFshSO4zIajXdBzbWR3WN5eZnBYMC1q9dY3rNMo9GapCZbXLx4kSiqKwi/8V3v4K477+G3/8d/R0pJb3uIkhVCaQw2UuaYSuE6AYUe1522hSKJM6an58nSAW5o6pDXdl0dppRNmkXISTJyZrbNG7/hLXzucw9hqFfTRV4zAm+96zbSAZy/+Dx79x7kW9/7HTjK4md+7t+zuGeR0Wi9XuNUYlJrZhgME9C1N0xZ9ZDg+z75xNCfTQbxm6srlARqX1ZR1od4GPrMzc2wOd4kG+bMd2dZ39hCOSFSeFR5QTbuceSW1/GOb34bq2tfZbgZIYzHEye/QJ4qsrSi5TXqFia7Is9sLE/gN+tDPlC38rO/8d+YPzBPZmaYb455/quP8Kd/9Nsc2LOX3/3jTxIP1snzEinrGrjFpWkeuPdVnD93jstXT+K5NmWuwQ3IswZlPKLVAeXY5HGEJRqEymE76e+eiUeO3kqjHXDr7a/CNS4vXPk0zz56huW9HXa2a+W+2VZoGdMIu1y/tk5V+rS6dZgjGSt8r1MPOK7NOO6xuDhHnmREwxwQGFGHCm679W5OPn+GV7zuXTx/+m/ZuHaJvUeP8G0f+CDPPvUoptfn5NXLDLeHNdppwvC76QErs5zjx2+h1Q55/LGnaDVD9u7bhx9YXLh0miIGzwoZjEe4k1CVPfkdqHTNULTdiuWlfaytrZDlKVmaY9suRVYyPz9PFEV1V/1knWo59u7nUV+YqkljSb1mBAjDkCRPsKw6ONFpz6KrAozkxK23ISU8+eTTtUraEGjV5bbDb2N17QkGo3MouhMvb0EWGxaXZkjiDNdyKMqcXq/HzMwUw+GQqqptIkmSYETdH724WPv2Vq+vkqUlR+4LMAy5cSVntFkrXp3uLFu9MRl9vMrGdUP27Oty8eJlyszFdgTTCyWrV0v8pZyZ1n4sVdJpV+zckGQjRTS6Wp+fUTphdapJqEnW5zL1qlYKNUl2x7sraSEEWmvm5+cZpxnb/QFNkXL3VBv2HcHp7uWhv/gb3v/t34U263zqz/8S168tKr4bUOQpluuRVQkmF/X5L2ufYhRFNULIsnaH8ZtNSJUxpGnKvffeT5kXnD59mumZLt32FBuraxgpan8iBjO5RCTpmLuO3MN2tIbYWWeAxRgLz7KI4gSpDDMzM6yuruM4DlQGKevucoHC90Mcx5l4X8ELfDwvQJcGvxHi2D7Xb1zDmFpJN8ZQpF+nDZhqst72HKJ4VA85QpBnJdWktUxrTegHdatTUWBZcoK+MnSnZun3+5RVQZIkOG5t6xmPI2y7tnOYUu1ilowxuJ4kzQts2cVrg0orolHM0dct8d5/fi/Dl1xa++7jlttDnn7qD3AXJWpzkeun9/G1hx7h+D1DFh5M6YYVxbWDfPS3n2ft8jYis7j/g1O89psWGb004vd+ZYtOO2A42qHKwfUUyXiyhpcRtmrgeEUdiHQ9GmGHoijp9XZoNJ26EtJ1yXKD4wXE4wHtMGBrrUdZiDrYuSdkuJJSVILWsk0UDwhCxfap7O955R359NahTC3WbmxhMp94CJay8EIPbRKUC9Wk7cMPIM9TqqJBWdiowMbyBevrNh/6we/lx//dD7Kw7xU8+xycO7+KEGMsVQ9yRakZ7mgGPUORG0aDitIoHNclaEhsqQnthN5lQ0t5dPyY1715gXe8523obIfPfPI5PN/n4N5ZDhxewgk9bNXB93xOPT3m0b+5ipO7MO5QZXUwZhxF5JUAoZAip+EUYDTIGiA+rhxWdgwrWylIUQPS4wKZO2xc2aHpBbTdI6QJSMdCzW6y996Epm9z/bECBvNUlmRQWPSvd+lfl7gtH0iZVh5bL2WcP7lJGDRBw3TQ4dqTLfykhRIuWVKbmSsUQpXYHmhd1Em20pCnasLqKpEWOH5AmpfkBejCxbIljUabZrO9C7bd2t5ks7eBkAUmDBmVBaU95NTp53n3W99OFRm21yPQ87gyxxQJ7YZHmkWUOkZXaY10cDyErAh9G1uOmG+7hHaBMTE6beHoKVzHw1Bg2xm2nREEDcrcxRAglI0uBWXuQ+bTcDxkWRHvFLhCcXjfHHOLDu2Oi+2MsWSTtjxMPvSppvuceJVDr5cQJxW6qofCnZ0caKG1jWV5SFnDaf2gBrpmWUoQ+FRVRafTQSmQhLi+xgvqF54f1EOjoE4Fuo6H4zi7iuXs7Cybm5tcu36NsOnT7/d56aUz+L5PlmWEYVg311QVp09epLcTMRzEDIZbeL5Em5KiqrlsWiukVIzGOxS5pipd+r2YY8eP8Iu//PO0ZqZozyY0p2NsJ8QPLJIkImxWKGUzt+jR7oT8yZ98gjwzFHmFMYpG0MZ3A86+cJJz555j774pHEvheQFnXjoN0uKVr3yA7/quHyIa14PBieO3k6c2YdhAqozZuTaWZRHHMUwg7XEcU5Ylw+EIIwWaCiHYDWC4rovruuR5zvXrK1iZIB1IXvGqezl2YhmdxphiSJr2ELbD7JTiLz7xu3zhL5+g3TrCzkggc5tbDvjcf99+kqiq2ygqiR94JGObIpfMzu3jez7yrfzWz/57wuYcT5x6nr/88kUe/sqjPP3QJf70k59juJ5B6WIpl3bHx3EkqyubvPD8S+xsRQTONBK/HoSyAiG3mZpysa0AYRmEEuR2xSAXyErv/v2hf/wRfuxf/lt+61d+mYe++FHuPd7GWAW9QcooThknY1ZXUwK/SW7W8ULozkn8hsC1Wxw9eII07qEQVFWfPUsHiaOMtZXBhBnqMR4lfMMb3sof//EfEjQkX/ib36e/uUK31Wa4PcBxWjzw+rdx7MS9HDt+/+6g8vVBrn7um802Z8+e5dlnTjI90yEvY65fX+HcS5eYnZnnfR94G69+8CB33DvL8nKHZrOBFJpDR6dpdjOyYoguYX19EyUs8jTHsS3yLMEYzXA4pCzL3aS/67oYXdWDgjC7SlgNVa4/J6UU/X6PwwcO1p7MqkYXjcdjfD/kySe/xiOPfBkpU6bbDaaa04gy58qVxxmP1ikLQWVyLFswHsYszC+RpjHRqIZSp3HNJ6zXqN6kUcWr8S9CgZakUUZ/u48uKySCiyf7XDkZkGz6WKZAmYje+hUawjCtHIQuKdOIC6fXCNQUnhtT5QXb1zWOFLjbHpefucLwWsTK5T7r2yO2kgH3vv0VTN+yzP79+8myYhcoXpbVLsrGVPwfyu/XBzyow3Kbm5sUcYXKDU2vy/XSYX0rYTwY8y3f9C5e+8Z7CcIQXYEftvA8j7KsKIra8lPpDFtZhH6AMaYePGzJ3MwsnVab0WBAnud0u926tUYplJRUuuDChQt1rWwUsbJyHb8R4vs+apJEt6yJt9Szubp6g/4woycc8kqhEAhd7vJKNzc364uOrofJPM9xHAfXqxtxeoM+UZzURI2yhpGnWb0dGvS2EJWmFQZ0u1327dvHgQMH6HSmsG0Xz6/LNJIkwXFsXLd+rrSp/z83Az/j8bgeWD1ncrkxGDRh2GRhaXm3MlYgydIcy6pDUoIaCg7WpPe7pMgtfHsKZI8kSsn9nKTMuWPP3fzrb/kf/F8f+nEajef4whM/zd989BwLByS/91//jD/4Xx/l8vo5rl3KWd47i4rm+PRvXmA42qbTDjh4yxSXXhzw+d+HJ77coFlClm7hWx0cx6coU8KGhaU0wkBZDkmitA69JZpoGNNstHfVcMuuKRuWZdWiiePUlIbSYGEhKsPm9S3iPEZahnwMVeJTJvb/x0T4//3n76xQqjllTKWw3RKjDK5Ty+Kl0Xg+SFEXveeJRlmGTtcniiJCf45Op8PO+GVs2yYZuxw7vpdo6JOkGYeP7+Whh75G2OgjJSQxCCy0LrGkhaUqFhYtdJ4jBQgBjvTROmP9usfMQn3Tnd43z9zcDFsXLQbbDne94ghvfuP9/PVnH+bPvvBXmNEQS0E4b+FPGVxdsXnJwusUxJGilJIoKfFdCystOHxoL41Zj5duXMDybLbXQMi6XkqJAiUNeVrQmVLEsUa5LTAN+oMVXAtsRxG0wZMW0XXB6EbJ4n0OrWWbiw8n6M0Qf8aiMorBcANLVmijoDK4ysKzBP1eh8U7C9rHcnbWcmzbIIwHaoQuOiRRShCA65TEUUmlBUpYSEtQVZJcW1ieR1nlSOESeNNUJIzGO1RVSRrXhmLLNrheiNQZWWRj+4pbb72V0yefY/tGn6UZh2LSS+0GkkxXjCJZe3GqFCVqT2kYdOh6KeNVjfA9vJmY/jakkUd7NqyxDHmMZddsMK1dKiSIEllZ2MEWQoDR9aDtCJvBDUHSc+jucZhqL9Ndyrm00efw/vs5vH+ex574PEV/i+unPPJiDJaZvKAhcBskWQrcVM1LqkpTlnUq1RjD5uaQdqsJoiKOUpb3NfE8h/W1PiARxkNrQZoNaYQ126zRaLC1tUUYhrsDVDkx6Du2R6vVIo5rJaLdbjPsD0izGgBuqD+fRtNmHEdYskVZjqm0RAqbO++6ndWVbTbWBiwstfnYH/82P/3TP82Tp7/IwROabGRx+axFPrZY2ufSnlKcerbH1LQijgqisUWlHaSq8D0LyxIYkTLVdElii2anzamT13jFA6+nM+Ow0++ztM/l6qURp0+/yDve9jaOHbmLX/7V/8zMnEs8FrhOi7JM6fcHgKDdbDGKhrWx3ujd4UBKufsydN26uuwmxy3wPOKoIGw4uH5Os7XEv/hn/4qf+jc/xmAr4Z//8x/lTa99DSs7l/jN3/t9Tj9zkmNH9rOycpWpTof/59/+R/7gj36fr371a3Tac0TpFkmqeOUr3kqvp7D8Ea/5ptfwp7/1SY7P7OfalRfY6W+ipiVlIZCVTaFN/axZkiLLkVZJK/DQhUWcF7TDBtrkVFKzNHMMYdmkZovL567jNlzKQvCut79390xszrdQKN76+rvBXudXfuV/8+zjV2jP5Bw6vEiWwskXztNut7DdGtPUnh2xuTbgxLHDXDy3w3AY4dpTXL22wnd/9z8lbBV8+s8/CWXNzEuLlNDby9vf+s184lP/Dd+zyBKB5WpyXfBN7/4efvw//Ed+6p9/Pw89/Flcq04R32T4VVX9IpWmBjY/8Oq7yYuEk8+fZGZ+juMnDrK1vcp73/VNXL20wfWVM1y9vM2NG3WITtkS2/IxlUNZVCwuLHDu3FksGxYWFjDGsHJjDWDy/NuUE1SQbdukeb0xcFybJMtQyF3W4k3f3vLycl3NGI/p97foDVLCoIk2JZUucG0LJernSJsSIw0CBzBYUhJHCY6rOLD/KOcvn2Hf8j7iOCUeR5OAWD1QdDpTtU9Qa5SqKw3zvO4eD32/HuhKB6lKmu0GmY7RumB2ukNVJvQ3S4xnsG13kmivPXjKEuw5CEFDMBXeyWhY8dRTz2C7PnleUJY2rakGt9/ncuaJPuNxjG3V9X/VpPLPVEzadL7OY3Qch5mZGaamplhZWWF9fZ2G5SMcUKrJsNTIfIfv+8B38typl3nlW97E1o0b/MEf/D4zs12yJEdZgrIs0LpCyhApNEVR0GgENcbJrpFOZhJeMYaJcvr1ysIsy/B9vwaNS4Ot6iFEiDolXVMBDKPBgMqUKG1TWRLLlVhakRY5vq1IKSYXnQljUhsqzS4CaDgcEiUplmWxuLi4W414+fLl3aE6CDwsy6Lf77N3/0GKQiONZDQaT9LaBVka141AptgNMyHkJDBWD5RMOJ+mqiYhspLKlHS6y3USP4vZ3NxE6xpyfhOLVl9OGhS5qc/YQFJpRRynSCujIWx0Lugsz/HXX/ozPP8Ej138DB//4k9y8mN9omKab/vRKX7pXz/B/N6Q9SsR737/a3nfD+/hX773LxjlhunWDCgPq+NQjYf0d1JinWEGQyzVpqhGGFNvkoTMsW0HjEWSRthWfTGOk5Rms4nneWxsb7GwMEsUj/AcwSjK2bd/P+PRgK31LYS2qHJw7BpzV9mGVtulqgRl6TBOxujt5O935d04JIyUUJUWWJosM0jpYLslGAtETlmC50PQoMZtoGu1bWoOnZYkeoTXLsjykiqFt7/tXTz3whqPP/c0U1O1VyhPS6QUVFrh2Jpmy+B5CmIbITSWU2A7Nepga9XGch268xH9TYVSitGG4N3v+BD3veYAtxy7ixdPr/JTP/MTKKuHyFuUzhDlwXRDkCcGpwVXr4IcCG69v0ORCy6+FDE/73HoxF4uXlxnMNzElpI8q0DUOAPXDyi0pjIZC8uzmMTiytpVkrFiflpSZRr8FCkEDSdg8FJAlkR0bjF0Fi2yJCferNg5a1BOSJLWCWJlJSg6ZHobW3XITcEtb8wYjEqUNEgFSsHmikIIm+lpnyzpU1U1K8tSAVor8kJSUR9QwtIIZSFMgJIlaTpGSCjyMUrW61+FItsYsnW1xGn6VJVFe86hOd1DJwIlHISU5DrBKMk4gUazTZwM6/7VSjIeV8x1MpqehbE8LD+F0mVj3cN2NEqGJEmC7aZYboofepS5SxIZBJpGuyRJIoyQNYuuMpB3uHiy4tbFAFWu0lmcQ6iYsj3PwXtezc6piC//+ReRvkYYPUFEAEhsZX99NTsBA5dlga4y2u0m43GMqRTDQUKz2UaoEVpXdKcatSdQBwR+k6LQjKM+tm3juXWyME1rmPdNT5iuCtbXt2k1G5OVzaiGOSvF/n0HsV2L1ZXr2ErSG27T6gZI4TIe5CTpiMBvgCiYX1wgjgrSfIfv+OD38K3v/SG+7dveg7uwwvxcl2iQ0mpIFNPcf/+tfOUrX+HIgW/gmSdfYKe3XntTBUihoFIEocPy3ibj7ZiVtYTXv/613HH7EX75v/4mt99zP3/4sS/yMz/3I3zs93+H7jIcObTI6tWUne0IaSQSH+HkyKpOTE7NzrG9vU0Y+pNVSkCa1m0UxhiEMXUXruuys9P7ejrVaAJbEQ9y3vu+93Bp5SrvfNe3Y+Pw8f/9ewx6JY888ih/++Wv8ou/9MPcdustfOkrJ4nyNSwqbjvxWr7/Bz7Ij//oT5Dlw7p9KMuJY8Wb3/5qHnzFm8HP2Rlm+MWIP/ofv0ew6HHy+SG+72JUPewa6oRmGEzheyUVCTqXRGVC22tRak1aCo4eOMTr3vAObLfBr/3qT+KEM8iiz/2vfdvumfgNr38jh44cZuXGiK3rF/irh/6UrNyktzXGd6fx3IBLV84wNeNRVR7t5gKXLp6rFYS09kQGgceZ01dxWxpHzpElFv/kn32QG1ev8Gcf/yxT8x4Lyw6XLq5TJAHCilEqJMtqz2pTLqLas2yvnWZuRjHKJge7EORptgsUV6J+cY/TlLl5l9nZGS68fINKt7n1xG3ccecCcbLCjavbXLu2QTTO6fWGCOExM71Mv7/DOOrVSW4FRZHRCH0OHjjMyZMn8X0fXRVUZT3E2pMUr+XUFw178kLOs2LX43nTb+n7IZ7n0Ww22d7uEY0TsErChlcPjqWkyNM6javH+F5IWTooq0AAlbboTnn0hwMOHjyI1oa1ldXJ8DJJEgsotNlVT6enpxkMBpSmHpriKK23GFpQiToY15xqMLPUYWenz3AnIXA8wpZLd9pjdt7n+efOMhrk7Nm7wGCwwup5zcLiMst75nnxxRewrQbIBERBVXp4nk0xYUA69gRDI8yEB1nVnv28vgwEQWNSaSk4dOgQUTTi7NmzmKpA2DayqhBGML20hF+UnL1yg07jAMrpoay6hnVraw3PnSSqKz1R12oLxU0YvZBmcukzUBmUbZOntV2l0+ngui5bW1u74O9S1/+9RCFUzXWMolF9Yc9zLMumSDPswEXYNj4uoyhCTqoqy8m4IUVtd7KkvQtw7/V6WFatGM4vLuD7NSEjSSb4HwNxMsaya8XMSEUWZ/heWCOH/IDNjdXaGiMhy+siEiEklTE0m21c12U4HDIeRuzdu5csjRmPhwgBlSlJs/rZPHL0EFevXp30gavdZ0moCiE9LFVf2uemF9ja2iJNx1i2QkmHoR6xd89R3vm6d/Ann/1L0myAHg2Z3X8ApyzAhhv9LVpeSZRI3vCtJ7j47Gkun5HMHlri2C1zXL824trZK1R5SUlFq6HQKiTeiTAyBgNCuChl14UDJkapmvFKVavP0pK1Z3KysXADn2i8RZoUzMzMAVV96cAlHsVUOqMoNKUEy8lxLI9SG7I4x8TV3+9AefT1TZPnOeM4JxqBkBZSCvKywHHBDyx0aSi1pjsNOrfrFYDwUcamtDIqNcJvKIQwWITE42k2tlYpRIbSYNt1PZIxhmZoY6qCVrM+OPLSoHUtURpKggZkicXq5YDDtyXYqkAVe7nr1qMcP/o6trZ8zlx+ku2R5OEvfhy/C1YFOYAb4DoxNgKNwbZCRE/Tas6y98BedrJtcm0x6G3gWRnJ0GLEoH6IrIxoWDE15bM0P8tLL66ytLCHuZbgyvgKvbGm4YPOHGw/R6QNei9XJBVMhT7jSDN1JGXqRIokxFyf46VHr9FoVeSZX3vpLIfEKEIbROYwfWxA40CBJUKKtKC/ZWMmfbnCGlNVBbqUGAGO00ITYoRNmo6RRuI4PsZKKLM6+WsmKcw43ahxEe1ZikyjkhFF6YArsMSAaMubmO0LKkuSJhWWDZYjwHKxHIVtK7LU4DopUewiRILnG+IYptpT5ElOon3KbIw2NR/Rsg3trkORj+rbfqLr7lelUaJCKQut2xiZ4rhw46JPJwu55cEZLu/0ed+hO3nqk39CtCwoGk0unhFs9wc0/AYYNUFu5HieQ5JG9dpDC4oyw/NsdJXh+x7jOGL/viPYlsfLZ1+mM2WBUfSHtbJWlS5CGhzHxnFciqJgNIx2O14XFhbI87xmcE4YZxvrW7s3fGMMRZ6zuLiIkCWOskijijjdRipFWdlMdV0kAidM6Pc1vZ2KPIPl/dN8xwc+zN985hHWd15gaiGk3x9x+NAc1y5vkOVDfHeaB990H+fObHL2hSs0W3U69Mq1K0gJSjQwWiHVpPM5y/m1X/slttb6/Pj/9WO84RvfyR23fyOPPvszKDvn6vkdpqe6DLcyttdKPM9hOBxgWTatVouqgsoYFpaXWN9YJRmPabcCbKnY3OjjuPaut6iqqkl7SO3ZsgWUJscISdOfYmamwZVLaxw6eB/bowtkgx1KschMx2F5VlFg4YdTXHj5LDgpQXMPb3zDm0iiAU888Qg3Vi4R+rPs9Mbcc98rUU7I2efOcezOu7j3RIfnn/tjLkYN4kTQGMJmtobRYCkPaRRzcwtAxYXzF7n7njvYHo9Zu3INz+9SISiKAXnicvDw/YjqPNfXVvEbNuON/OsD5ds/wm2vupvZfYZHvvZVyu0hX33oz7nzrlvZuJ5y6cI1pmZt0txQFBbL+1pEacTC7AJHjhzhM5/+G8IwQOuKIFREUQZSUMQewmg++J3v5unnH+PaygUWFjqkQ4trN4aUlWGq3QJHEm1u4zYclO2QjARZOaDZbDE/P8/5ly/iuTZal3iOhZSgq4C8GOE7HnmVoUvJvr1HeMc7XokuE4bRVV4+u83TT1xgYWmed737TTz++ONcubxCEmsQdSinVvuhLCuazSZlXiuBGEOS1ipTENYXLmX/vwdI1/F21Z66U9mtgxlVxVu/8Xa+9LmzSOEQNAyDwaD2pXsWiAxpHIocLNvHUKDLnGbYIs1jjAX7lvcxGAxIopiiyEDUH7PV7rLTH6FNjTcCCEIP1/cJmw2UssmLirLYYHt9k+GGS9jw6cwE9PslcbqB7wua3cPcdp9iq7/GxpqhzDO2rma84c2vIc/7vPTSyzQbHXY2BiRRji7BC8B2gKr+XYnjFInY9WIja1ZtWdQ1iJ7n0Wi0UMpmPIong1FtITNC4VjQbjXpZwnusCKxNNo2yDTHb3gYrchTm9e+9h5urF7j8qVV/KAe4AR1iG48HqMsseuzrTFPNoZawSsmBQbj8ZjhcEiWZZMLmUSYetBVSlHoOoVtJpsfKW2qPMcKPAI/JE9SXN9jezTAKivMTVA4NYbPdRwG/RGu607S5u7k/1tOKg9zlKjVSdtRte9xcikoKlMTJLipOCparWC3khFR7X6P0zSl0WzWl7lCk6cZ7XabwPOJkzE7O1sYU+EHTYbDIbPzCxO4eQ1TN2bCUrUqpAjJ8xhERbMxTRh0GA62KMoYoSQNqSgM7Ixi3MDBVRZWZUgdm7YHWWVRWTlOYYg9H7upiM4N0Q3NnXft4eiJ2/nMp08RbV9hfq/NeGTo2l20kKxurBCEHlUFaVZiOx5h2ydoOGz1dsiHMVLW2wlLiUlYyZBkKY7jUZYplnSJRlE9lEpwnUZdpdrwyfOcuFCYfFQjpaoCVVmUcf53Gij/ztWLVZUTJxqjXXQJRmU4NvgT8GqWgHA0FZBENqFn46qEaKtCxxZJB7KR4Pgt0Jm2uLY+ZnuQEDRL5vxpUDnb2wlCgDCatMywBPSHKa123XtbolBWCaUgix0aTY0rC2ZbCywcaVBcbfGvf/Kn+Ccf/N/c9XqPnrzO408+xfS0TSQ0Jq8IpE2pY6xUIpVCy4IkTenMajx7k4c/s0ZncY6ZfYKl6WXGaYbuaOx4QJ4nVJnED+oHaqd/jUPHHbZXL3PJFTS6DZSwKftDWq05+oXG9lM6e1y+9f7Xs9Hf4W8/+zUGLxV02k2S1ghrrk/3iMX2FYHtaPK8IvAEjoSySJB2SWd5ivFgm6gHgdvk6pmMI3eCsEuyssBUEtsWaOOghYUTQpoOUI7EEi5GGwQSy1ZkWYrnNepbqVYEYYiwHZJiTCgFRg+J4/rCQCsjSSSqsjGpBDSWVcOrdQ5JmhM2LILAxdMS4WZkecVwoFB2QH9jiGu5GGuMskoEmiwr8fwmlYaw4aOLEkGFcWNKDbmGpu+y/nKfatwhlBl+kZCpIedeLGDG4eVVw9LBDqP5ZR7fimnMDVhfB7vpMOj3sW2LMPQpy9pHYyqJwOA5LnE8pt0JULacvOCKehVWRmRJl3bXxcsaFLkFIqlXcw7kudj1iN30UHpeQLc7PblZ16s8zxvverbyPEc4DltbW7Q7AZXRKDy64R52ohWydIQ3N0MQDIhzQ9JvYVlj3vWe19Fu7OdXf+W/cc+9R5m1Q9IkxpFDtq41yUaa7uw8h44tkY46PProV/iB7/sWdrZS3vDgu/j13/xVbqyeARVhWy6dboDj+hS9HitrMf/zv/83TtxxJ5kZ8Ad/9c+455UugzWXSkiuXNnCs228hmLUG7J/3wm63Sanz7xAZQSlqViyFHlRUJmS0WiEMBrXbVCWOUKyG8DwHEmS5zi2jbY8Gr6NqhRZvsUgi3BaNqfOPkoz8GjPegw31vC7HqPA5sbLhn2LLjMLs1y4+BLTi01W17botgIc28NRsxgDrabDqZNPoIXAtxXPPfJxnv6qz96DDoG9RlbMsJnEVLpJZVIcW+HaNa5kPATbDgmaIb04RZclWm8jXEE7bBA5OdfXHkJmFsLSlLlHa6m1eyZ+6fO/y8NfanLbvbfzmnvv4q8f/yoLe6HVarBSDlG2ZmnpINdurJEmGWs3Ig4c28/yvhk2t1KE9EizGF0aikzU7SshCDtj2I/467/8W8ZxhOM1iQeaPBvTDFyiLGdxtos9G3Jqu4djuvTjIaUT4UqHPM/Y3NycvOjrtK7jWBRlhu/VF4Us1djKIITh8pVzLC1/I1/84l+AcUhjw959B/jQhz7ETm+DF184TxjW69JKaJTlglBUWuC5HhhB2GgxGPY4fvQIWmtefPHk7gtdCNCmTpqCoihzhKhtEnVoSKCkjW1J/vahs7Xi5QGJRbPRIUljBqMxprDodBQHDs9z+dpVKEI6nTZHj+7lhRcvEucxqytru800UhiKIsf1A2ZmZtjaGeA365dnEPgs7Vliu7dDnCYgS6Jxwj1v2Mf+VLLYCfnr/32R4WZJEJZ0wmUABuMNXni6YG5xiqUlwfScy+PRJa5fvkKejRgPLe68e5Fb7w5YvwqXXh5gTESR2RQlFIyxrDqMUodxDIYaCcTEVjQejylLwHzdQqKrmklZiQqhBcM4xpcw8jKEcnDzCit0KXOXsGnx6lfdxzgaMBwOcbz642SJRFkFBo1UIJXCsm2KPCeOk3qA0BWHD+ynv73DS+fOIYWoK2mFwJlAv42p/YbGaFCCkgIxCeMZYUDVXuoizSirEpEnFKbCs1wqwWTIqy0Mpa7T5fVFtP4+jMcJnuftboAs28agKYoKU9VJ6zwvkbaFoZoglaAoEoypffFZljE/P8/29vbuBTdLU8pJ0YQSFtvbPRKv9q96k7BWnIwnRINk8nzWpRZlWavJZeZQMUYpm6KAJB2CKSgnFyJtBIkjyMuEsNnBFDnjQqBMhRVnRFMO8UiwZ2GKzBojDRzYF3F2J6fTbDMwI56/9CW64QgdBwxykLnNdrpNYkn8sEmh64CzpS2kN0L5Mf3RNGGrjW9bpFEKKMTEvlCZgtmZKba3tylLDUphWQolDLbrMx6OmZ6dRSpNnJUYXeIEClNUtRocqb/rmPh3D+XEuU2mNdkYKpER2haVAVNVKAml0DilxBW1ZymNY8aJxAlc8qFBrKQ0LJ8zLyokTWYbNnMtgSsAE7E0HbK00MXzDRpFUYHlWjRaNQZDuSGOqm/ZxjJYtiAelBw6cAgdH+T+5ffwi7/zW3zt+Yj56Rbf8K2vI7pgoKoImhJloAogkRXChdQD5YNbQqNVYbtwx6vvoNG0IV3n9iMak1xmbjrg8NIMs629ND0XTwl0HiGEYmtLsblTMXfIQuQVV05lCBwa88ukIiZwXJaWKpypmK985Wuce+kSOhA43YqXnxnRGLXoiJKpWwoO3OGgcXC8BlmeY2KFMgrflbz8/BY3nmgTXZ7m+tmUxTkb104p0xF2JVG5QOUVShsoErL+OlZU0qhKyBMKAzoTKC+gnNSgudKmEy5jyiZ5UmGPBmS6olCSSkM6hmSokFVtqMdkWI4iThWjWCMthTSaKqowScz2ikXad3dv4baocFtTVEGTvLAQ0kU4ggpwRMjepb0kWYLjudjCpeGDi40WEKkxU0ttQmlDJSlsG7KSeGWF2YGPsHqw/3U8te1iQp9eaVjYM4+mx8JCm/m5KfqDOthgDCiVIawCKk2rPUNpMmRZYSub65dXee7pJ3DULEVRUWYp3WYDQY4SAikVaaaJkoi8jGk0XYwRZKnBcWrs0NbWDlmmuXbtxi7sO01zGkFz0hAiqMoUqpQkT7j/za9ENhS+H7C1tcXp8yN6owC7UbDnEJx/+Qyf/5svYEzC6Rcuc/n8Jo4Vc8tdbcKZhPnFBk0vJNnRXLz0Nd7zzW/h9jtegbRC/vjjv8dLZ09hSpciccmLmN7OiO3ta3Qbgn//H36UC9dWOXTsKDo1WHEbe3wrzz00pOw7+KJBldVKred5jKMddra2UFZt4K7ygpbVYKm7QJFolHHqQJisCBsevidwHYnvWSRJTtPv1jzMZky7bVPKCOVYFJnBQhP4Dsq2SeOM7swyqzcG+PlRfuJHfhjHcWiEXcZDwfkXL/KXn/hzPv2pT+I6Dq4HRRmjTYnrKlxVo6iCmWkOHFtifSfGUvvIx0NElWFTEDoSNPQHGT/wQz/CO971Xj78/R/i8UceR6c5XsOn0BZNt40lUhqepukFBF2fbneaqpIIne/+7XRy3vQN92B5OQNh869+7Yc5dnCJR77wIkW5wW23L7G1tcEbXv8a7rnrFiyRceP8JZ565ElePv04jl3VyrWvcEKF9HNaoUsrnOX//skfo99fw1Ixs9MzbGxoNnqKoOUyP99ka7DNtbMrWLZPyghHVvg6RGBhdMWg30cKgZIuYVAntKUUSGVIk4rlfYs88OrXkKQ5VVXwiY9/npdOJTz3zArf+Q++i+/63gf50kNf5YXnLiBlhesIGk235rZWQAWVzsnzIaWOKcqEdrfLR37o+6jQfPeHvou5+XmOHD1OURoUtbcsL8vJsFf76ApVsXBwD1pW7GxvcvvtR7jrzluII40lmyRp3eS1b2mJ228/gqkEBw9N8YoHDhOPI37kR34UKUN2ej26jTZawiipK1hNYSErt4Zz2yXdqQUW987RmZFordnY2KIqHNJxSrS9hVONePIvz9Lvh9wYJgxisGzDOIVeskE0ivFtRbGluHJ6zKEjD7DTz3ngdXvpLlcMM80dr2iSmytorXn5zDa2VeFIH3SFo3K0UbhWG0sEZEldBiCkRiqNEnJXoayqnFKnIEpKneL7daBFGYOyBTrLGCcpqpKU4wStBLmWlFWEpWwefuRRHvnbx4nGeb0CTQosVSuDphIUuUYJi9BvTC7Abu0pzUqS8cTDabl02lOMRyNcx4HJcFJUORW1smkrhSVq4oQRsn4ujEQJhRf49fmZFnjGIitrD6UxtapY5DmWgunZKZTlETZaJGXO4v5ZlKNJkhGu7WA5ik63S6kNusq4+QBKU2EphajELhYoSSPG0RBEtfvPsixxbI+yqMBobAtsS+OonCTZRqmYSke0GnUXfVGW7Ozs0Ov1dhvAoC7DVXY1ScIbbFtQaVM3ldkSqWyqrJgwVyW+q5CWwHfqn3FpaXQkCeyU7e1toqHGMjlFLplfnGNpzxyibONYbew9LvuOL+PqObQocW0HqUGTYQeCtDBQGJbnuuzfF7LYnkHaDmgJJsdRBtd2EJSUBfT7CZbn0O22KcoMpEE6LnmpsTyH7e1NDBLLFSByKqPQ1IgoEei/80D5d1552/stY6sKpKEy4FoWKrNIswxtS7zWJGmkArAqfJFSZmA7HlNN2D6X8fpvnuOWuzv89194iakpn8acpjeS5GVKkNuE01BiyMuSZCSQpQJjcG1DEE4jwjF5ZqiKCi1yrMChGHZ4673fza2vPsZ4sMUXH34C11Tc/vrb+Pgf/yFleBUTeYwjQcNPWT1nsNqgQghzj31LHZzFjPG4xdrjI1a3djiwp8n7f+goX370LFtPluyZP4g5vpfVlfNExQp5WftGtclxbQgDcFWXlWsZ01NtXLdNroek5Yip+YxOt2DlqmHrHFiJi9ESrW2klXHwbhfRTqgsSe9Fh5XzJW4zxLaGVFmLQlSkSUR32qrbY6oW4eIW3nSMAarcr1ODqkQbsG2FFB6ljoAWFT6VVNgCMpFTVBo/cPHcFmhBFW/gVYJp3+Xy9VVyAcKFKIXKgLJAF4ARCGyMLNG6IvRdyqKg3VL4QcH2NQhaNklRYrshlu0BkrxMKKqSTnuWeLSDbTRH9h9j9eoKdiMnVkOSymIqtMnzhHGswFa07TZbL0T4ls+wikFV2IXD7J5DOKHirobH5849jeNMs7m2Q7PlEfWH+LaH57bZGa4QNOo2gGbDYbRpkzd7dBwJBIyGkiqCuOzjN5pkUX1rdp0KU9l1QKwYU2QKg6RCYyuvXoWbulliemq2XsmVKZZdgbFIE02rNUUcx9i2qENTPowHAtsZoYucVmORVsfmpdMbCFWjqJJYM7/okSQFhhQ/DCizAGRCUaTsOdJksBlx5VyF65e87Z3v5ud+/td4+dxl/vP/78d48umnaDZazM4doDOzw9mT1ynTFrY/wnWbDAcZnZmQo8dPsLXTZzweYiqLK1duEIQWlqzX9INBD8/zCIIG40FUqwLG8I73vJPQ6/LHH/09PvKRH+eNb3od3/WB9zK71CVJRgS+qqF1SDyni5CaXI9pdxZY3rPAy2fXGYz6tfl+1Ieqqm+zkwuL61dUGgQe971qD/fc/k62trZw3DrtG+WXa1XbPki/l3DbXTM8//yzJImNskrKvEbVJHnB4uI8y3v38PjXHqPd7mJKTaEHOFYLz1fs9COyNOCBV9xB6Id86aG/ptkJSdIxtmphK83hIzOsrKxgTEjYmGN5/hBPPP1Zjh7/epf37JEBsjrAjYtDzpy6wrFbj7NwYI0Xnx5w9Ohx/EbG7cfvZ7w5zyc+9VG67ZBUDwmbkvEAsrRCizGeM0WS9TDaQeOSFymve8138bXHP8X0LAy2dibPXkW73WV2dprz5y8yHmV0p2ZwvFqRG/RH2MpmfmGGG9fXKAqNJaDVatW4lKpEFzmFLlG2O0GIRJPOZAtLWDi2z2sfvI0oXecvPv4cnc4sFRF79+yjLGOuXV2j0/Upy4JoXOI4DmGjZruORpq773wtZ86cotEIyIuERtNja2uNOBnVrEmvWTdj5TkGGOU5R44cQ5QVF146z7333U5ZFpy/eGHSyR3UHlBZd2Pnec7SniaV3IRiietXRvQGqzQ7DvFYMtWwSAtq1Uxq+jsZb/vG+/gHH3kjP/3Tv8OhpXuZW8wJ3AU2dzZ5+dxl8thw/coqDd/GKIth1ef+V97NhRfW2Li2TrMbMIzH+JPecstz8TwHY7f55z/5vZx54Wk+9ht/ze1vOMCHvv07OH3uc3z+Ly+DKSmzOrxjiYA9e+c5f/48odOt1Tud1ZsgWStkRS5JkqhGc+U5Wt9UMdnFB8XjZLdhR028fTdT9LWtoPYnGl1bcIoy211rCxSWLSfr7RpqfrN6EWpVkYm3sapK3Ali5yYPtU7hp/8vvNDNSs2b/30wIVwEQd0NnqZpPbza1sRqUBHHOWHQREpJp9NBSEMcjxmPUnSV4Lo1x7oswHZrP2QUpUhhoSQ4rlu31giLJIp3N0ZlWZLltdJ6swXnpv0oibPdJrM4jjFVhW1LWo2g5h+XFUYo8rLuqC91zUtut9sYo3etTcCuracOLBV16YQQbG1tIanX8mGzwczMzG63uFKK8XhM4NtYwmeUrCPVPPtvs9kZjInHCiU0+450MO0hGxdzFuePMBhuc+mZNTqzXZTfY1zYeLrBKB1x/PaAo4cf5JZ7D/LYV57ka489TJE0aLcc8jSjiEsw9c+nSGuvf2u6DpaORmOUqANXiBrb1Ol0mJrqcOnSFYRQKGmBqVFf8Vb09+uh7N5hGRMbCmVBBXmR80Pf/D3c8+rb+af/8l/iNgVFLrCaFVgeOi5oNizQFouLgs6SZH2jYH6+yY0zPZIeSGVj+wVSFYy3fPYeN4xGITk7SB1S5DHKVDiOTZkXTM/Z7GwaVFgiQ4usVCzNKsL01fzId/8jDt1zhN/9jT/iwtnnWHxlmz/+1Jc4eKRHPKzoxwIz1Gy/7GI1LSIrwnNhrqNAdviG+9/ER37gvfzBn/4un3vqC9xyaD95NiKrOmw+V1LKELeZsL0zBKcirepqMssIFD5pnqGsCpcDLC4vMBxlJMWIrd4Njtwmsadz+s84XHkiwfN8hBcTeAFl0WbvnRnOUg8RCc48rBC6hZEZfqMg9Nqsrw+obJuG8AkbFu0jG4xyB2kq8iwBWfd1SlSd5qwM2BVatym1i60EhBqTB+gipVRDdGXjWpKWW5Btu5SbPTJjcDoemawYpTVyRJkSoyUoKEsQFDQaIUIlpHHFvr0NRuMxsrSRtiTNMxzbQdk+o1FMXhiYpH5lNUZHGhPPMRvs5cixFi9t/i2mqyijojaNG01JzszcNOsvFqTX24j2Dllp8G3NoKy47cRB7nNsPnHuJRynQ1nEjMca31aMBxqtS5b3hfR3asU4bJT0+4q5qYBMj0H6LO45wumXTiELjWNKHL9Lmo1QMsd32sxMLzCO+txY2cC2FZWqTe0CNWFUloxGGe32NEWuseyKNE13b8p5nlPqnG63RZKOkMLnxH1HOP3Yc7z17tuxnIIvPbdFpG10toYlLL7h7bdx5tRl+jspx04c45WveAO333GcX/q1/8D29jayXAJTsLmzyi13Hmdx4QhPf/UcYWcVZJNOuMxOb4uy2gFgYTnk+tVt7r/7G5hfPMbv/v5/x3VtgsBDWRB4AaNRRJknu8qR49W1mDVWQ+y2Q3Rn64+9vrLCHbffy5GDt/EXn/0ooRdiuyVFKggCF6kqilxi2QYjKlqtvczOz3Hm9AuISfJXSkkYBGxvbzLY6dUvoSJGiTYHjle84S1H+NQfXmVxaRYnAEs2cL2I4bbPuXMnCRsOgTtHlGziBCXxyCOJUxrNkLzUuK5PUUiiUcRtt9/KYLDNxQvXaDYkRZ6wd99hbrvrCF/50lNs72zSDHyGw5TFfU3KxGFx/hAf/OB38ou/+AsYMSQvEqQwzE55ZMnX1z+9EXzb+7+RUyc/y/kzCXEG99zvUuY283tneOaJNbSxmF0MWF9fJ3DmCZs+vcFanajVtR8QYGo+Ixq6pGMw1ZiyKPAbbYSjKIqMQHp0gg4vX79Ao+niecGun3dheZaNjQ0ENtFA8i9+9EN89KO/i2936PV6dKcaJElUs0FzkKpCKpc4zfECwVR3lt7OGF3lSCPxA5iZbQOK8y+toSzJ0tISUsLVK6t0plyWFhY5dfJllLJoNG2iKKEqXbIinfjr7MlgUtWBHT1pQDEV8WiMqSpa7S5pWdTA6tYUZ06eQgDd6RZRPKybeoQ9SWOnmKrAcRssLnZptHOk6bJyY4fBoA+qYKo7R64l48EWrrBJ05T5fTPM7gl59WvuZ9DLacy7HD/4ShaXW/z27/w6vjuD54Y89rcvkud9yl7CKE7odA6Q5at4jkPYlAjVYX2jR1VVjHtDvNDD2A6ve92DPPXMV8BJuOve29i84nHh/GmUEtiWwvFjHE8z3A4JGy6UKcOdDMeyGY3GWE6dXE6zMbqwdtPxWn99SLw5wBVFwf4DS1iWxcmTZ2k2fcqyqlWySYBHoHeHKK01QeBNAPcueV7s4pu01pjJcHQTIVN7BCVBUCfAsyRF2dYuy1TK2ud5kztrT5i8ctJ5XaOAKhqNBuPxeJf+UJYlQlk0m82ae+lYjOOadrG0PDehv2SsrFynETokqaHTnqU9o9hcH2NKgdYVrpcSRzlSWSR5QeA38F2Xhbn5mo1a5Oii5lAivs5gNdXEx23bwORzygsQFXme0mo1EUIyGkUYUX89UkrSrA5slWW+G0pyHRulrF0vJ0harRaW5TAej8nzfHeYjaKI2dlZoF49jwZDgqBBGPogSorCQoVjkqyg2eqysDdifk9AWs4zTmLOPjtgdmEKo9e4cjqikhrbCOKdlAfeup8oX2Hreou5vdOcfPIyTU/idhVB4NMfbOE6DaJxrQoXeYkwDmWaM73QJkvS3crNPK9wPQdjBJ7nMBiMcBwP3wuIk4gyL6jG+u/XQ1nmFY2GQVUTJEso2Njq8ZYHP8zhvR/j7PoLNFwXURgkKW7QReuIThix/jJcfRGOvRLWrxQUVcnUosvGegG5wbUVSqZIZRiNUgQtijKl2wHLcjh4FKKxT7CQMHu0xclHh9xz6+2cvHoZt9tiyj/Hb37p91j86quZnfb4wE/9Q37nf/6v+hZZNaj8ESaTFKGmcyIj3jTIQpFbmmtbhrbT54775nl2Y8Tc8X3cvr2fLMpItqEwmpl7myx0XBrBHv7mc08yGEXYQV0BGe2ALHNyCe2OjeWscObUDn5TIh2HVsvC80d4riAOcqTt4bUd4sKgTZ0YvvAMHIjnOfEqyeqeda6+EGPZJbblUDAg9C02RxXtsACt2LhhM3tYMNw0WFVA5cRQGpRw618kK6EyIVoIpGvjBQ20FoxEihd6qJFmHPUY+TAc2VhpH9sXVMpibHKy/OYhAbrUKKHIixzHDkmTAmM0nZaHkjFZAVoHQEFZFDi2Q1Fo0nSEKeuh1HYUusyIIo1JbVQ15srGi6ysuLiBz6LwSKciNjc1Hb9CebBltmgeapAO+hgkNkOygaC52OG6TjGphbd3H+PRiPnZLvELKwzHFhgbITNWriVgHLwgQ8l5tDVCVC579iySpEN6az2UA3NTx7l+5QVQNQvNcxWDnRHT7VnQFaFrYWRFKQK0LhGy/gXMsphWx8d1qWHMpeTI4S6nzjyNkCW2AzPtefJME/pz9PprHF88yMG33cozzz3MgQOHMeUZZAbvee+3cvbic5w+dYODB25h2I156skXWZy9h7Ubz5MXMZ1Wl6k9PcaRy40nXZAJLzz/ENPLDhcvROzbu8Rb3vI6Tj3/HEYcY25fhy986a8oioK7b30lX3niYUyRoXxJUURUlWI7ihGVRRLlGKsiCGo+XRrXidebB6FlS5JhxHg0pNl0eOnUSS6cPUPLCyhLjbID0rKuwNNmzIFDC1y7voIwPnfe8QCPPvZFpBWT5gXjuGJ6ap71zTXuv/cBijTjq199hOXlfeR6g2FP8pcfP8few7PIqovrOHzlS08QhooyH9Pu2jg+7D/Q5dTpLdLIq4eVQBClI1wnrHl5cUSr7XPlygWGoz6tjkdZjMA4uK7m5HMX6PW3CUOPvCy54+4T7D8QsjB3hDd/w3uZmZnj7e98kI//2ceYmvJJypxxIdGit3smZqniq18+z1vf/c0I72FOv7DJE4+P2LN/D9rb4cCJaQJvloe/8hgLcwsYnTIYFGSJYGbBYtDrUxY2YDHcKcni+hwsMxfLNpSJZqE7x6Ur1xibbTINnhdQFprEJLUfCrh04TK269BsNJg64PD0k2fx7AZl1efAgX3ESYSyE1y3y/Wrq9iOR56l+G592dzc3CaJy0l9Ys6wLxGVpNlNmZ6ZYjgYce3aFbQ2OH4LqZrEmaQ7NY0wku3tbZRysSwLv9HEUvYuVsf3XYTQ2I4k0ymBE1J5HkWe1y1SolbMbCSNZoDRMDMzw/jKoH4JR32SJGV+fo5oNCRJh5w/P6IZtrCdGD+wUdJBKbcO/GVjhBYYryQpSnyvtmqcPKs5evsreOiFh3j5+nM0xCYvPvYc29cFSIEb2DhOizQ1TE912NxcZWpBkMSa/rURnmdIdcb99yyjKsmTT/QJrJCHPvV59h7cz8HbpvjSXzwHpExNd1F2TjSECo0xEtSAN7z+exBVxKc+/hfkVb1mNgIsV9JyQ6IoxZLeLhfxphJ2E8klpWRhfpmZmRlefvkCRVYrxEbW5SHSkvhuSJqmuK6L4yhKnZPlBZax6HTabPd2EAiEEmRJ9n8MWnVIxhhDv9+vFVGpdoNCjXaL0WgEml3W6U2s0E2Qfpqm+G6w61msqoo4TvDDul0qSRIafgfljRlGOd3pKYQzYn1jhNJdPNfCdxbResD2YIWNDYl0Ja1GF8GAsjQoyybJC9qdGTqtLr1eD2U5CGVhmQpb1YO3bXsTnFu5y2S1bRvH8XZV2SRJqUyFNobQDxiOkokfs2aBGlOh9SRAJATWZDi++UcJC2VbRFGCMfHuzyjLvk5XGA6HqIlfWGuNLk0N/1cBjbbD9o5HoUvWekOWlxpsXC+4cmWdYTUkcG0KnVEWHvsO+MwcsDj35AaVBqujOTZ3L2V6ic3eOfYeCdjpx3itBp1ul0Y3ZDwaEEcpjuWgsxJZweze+f+jQ71+xsLQr8sAfI8kyWrBMJ3g9lx31+/6d/nzd8cGHRFmcU9AbyNGOtBthswtdjg8/3bOvPQMZ1bPYUwKhSDPNX7TBS05sMclH+Uc7LyVd37gAX7xN/4zyk7xGobBqF5x5VlMt+Vi+xn9gU2cFOQZuI5g/+GAeOgx1fZJ3YwHHox56hMOo7UZDtzncvbiZR583WFeuniKtfNdPvDBb+W3PvpXHDtoaC6NCZw2L1+9zPaaoNCCqG/wKoNtFBoLYWeY2KE9K1jcf4JIrrG9klKWOXcf8FkoA3a8nJUrQ97ylveytOcw/+m//CJ+WyCdhPHAQGGDdMjziMXZNkVmsb01xm/lBO48yt3m0O3TLC/4fO2zG5x7rCRs5TiyxeLSDEUZs7o6ZGmxwZ0PuuwMp3j8c1fQWYoSDVqtFHyLUEB/S5IGYw7drVk9V+G6Fo25Eo82g3FMhsbxDWXVRkof5YaUGCwiikqQ5ymeEIh8hFQ5hVbkuiQva2OzApQlajh2AZbFZL1vaq9KAVKWhA2FQRONwJI+ggQUeL6kEdR+nKqCopQ0W1021rcpS4HCJfQ1gV+RjCDfsWhIl+4xD29hyMbVgqBjUTU0w5dmMddLwpmc8SBn9qBicwSZXzE7PUM28MhkHzvVNEWb69fWELIgjwW/8F9+mr/6/Md48YVz2GKB9eEAb8Yj39rk0N5pTGYQyZidWNKLE9D1jdvojMBtMx72mFtoEEdlfSu3HPIixfdCGo02jUZIZXI2N3oszB2kMjlpNmY0GtbDRpYxNTWF69Z1dL3ekK4X0mx3udq7RDRweP0r21y5fJHjR9/HZnaVQS/l+pUtulOaVqvFypWI3mCFE7ctok2AbN5g0HMYD+D+B+b5ymeusWfPFJaXkSYJve2MX/i5X+NjH/0zzl9+njjKCP0Gw9EOUZTRnWpRGU3YbJIktZI07PW55ZbbGI+H3Lixiu/7JHk2aSYJybJ6ZSYBy3JBFHQbC1Q6I02H2JaHqBTGqnbrVmdnFun1B/hhl+Xlw5w881VcxwKheOMb38SLL5zi/Pnz7N+3j9tOHOdLX34Iz5cEgUuRWwgkO4MeybhietGlLOHYrXUlXzc4zPWrQ3Z2diiKDD9wELL+WoocKmyKQuM4FrZdKwmj0Zh2Z4p0lOH6MXlm8EMXpEsS5zh2g3d/0ztYWz3P7NRRDuy7nRdOPUejJfnkJz9GO6yVoGboY6lg90xc2t/ijd/wNt72tnfw/R/+MVAZ07NtnnvhNFKVUIW8+a2v48rl69y4uopQEf1RQZEWeIHEkiBERZH5SFnRnXbpDzOyyaqzyBJEGnD7LYeQjstzp56m0exS5DFFkaPUTa6eQAiJweKOu/YwM9vm2ccusrke4/gpyqpoNjrMznVIk4gb1zdqQoQQGKxJe40mzQ1797SJooQ08th/oEuWVqyurmI5FnlW4DS7eLaD49Trxng83g1BTHVnWF9f57bbbsMYw+kzJwkCjyDwyIuM0pQEbq1+5Xmt+pgJX/ImcLko6md/PB4jZIXrKhA131VrwdKeBqsrW4x7No5X1MON9pia8RkNY/yuRTYyjEc7dGbmeOWb7uDqxilcM8tMq8PDjzzH0aNHefCNd/PHf/SnYCyyeIAqIbCm0IFDEfVwnRab/S08p02oPAozojk3zZE3TOE1M849ts36uQzbEYiyor85QtoBmR4QhLUShsnIU4G0JZ4vUdUeKrONhUscjUmSGMtxkDa4dh38qEp7MnyYXfWv5nXWw19a5BRpgbLAd71dlUmpOr091a3LFvI8x7IlnmfvthNpXeEFHnEcT2wEdfq5LCeD6UQRBXY7vrXWZKXe/RiBW4dXiqKYDBvOzZbXXTh9XXgw5K677uDOu+/io7//MZqtGrSeJxW52UIqH9f3WdobEg0L1q8lhKFAV4IKydzCIp/6zB/xMz/963zs9/8HgRdSZBEgKStoNLvs2bOXyxcvcWDfXpIkqptxqnqdPx6Pa8SQZWEqMSGgOOR57W2vE+Qpli13a3mz9OtczjiOqSbfDzlJ/dSKq5x8v1w8N2AcJ18fFrUGqt2wGUCZF7v2ACEEnu+QJBVpNqbValGWJdPTs5SFIAxDjt/t8dSFFylTn0IYdno99s/s4657jnHyxdM02i26sy7Nzn72Lezhq099mv4goUzGLN/uc/UJ6G1pZufalNWY4SDBFgF5mhA2fCpdD7pFnpJlGUmSYlu1RcD1A8oyJ4q+/jVJWV80ovW/55X39G2umV7K6ThtTr804OjRBj/5G3fxVx9/AkfuJd5weOHURe666yhBsMPHfnsV14FoaPHm9y5gT/k89Nd9ulMDQiVJopRCewiZEw0qAm2z55hiODJsbeW0Ojbbmzm33tWg2bbp9RuMxjlFPsQzHr3rOb2dnL23drj3wQXi8Q2+errP8UNL7Fzt44mE1G4z3fBYubLOdqypthVvefsrOXdlnbMvXWCqK0FWlCU0vYDpY1OIapqFOY2OS+JszOZOxKI4zlRgePlcTBw3GIpzWJ1tNA7bmzm+44BdoShJx7B3zxJV5bGyepF2pwkVJFLz1re9jgN7Ha480+f0E6dZmt/Hnj238qm/+DSJySk3NTNLDV797SdwvQP8zR9+GSJDWhYEQcpokNDqBOy/u2JzIyfbajO7d8g411SRC56mtAHZokLjurN4gQ8SxkmCa7kMt6/RcR16F3Okzpg5pOnrEq1tdKko8wwJJKnBCLBdmzipcB1DUVZYou4SzxNwPSiL2h8iKxssjVAF3WkbS1ZksSbNwHFbNbhda6o8Rxe1D6lyCjzhMLhS+2333duhuygx5SJG5Ky+fJHNsw7tOcNoXHHiLftYX+kxXs2ZWVikTCMuDde4deEIh2b389nPfJFXvOpWThy/ja8+/DRzy5pRtM76ik3LSpja3+bCOcG+owWDNQ/PylldjTBCk2cCJaER2FSFxHYUy3tm2N4a0O8P8Dseo2FKmpS8+c1vJggavPD8GRYXlzl79iRb2+sEQUCz2WQ4iHabORAVU1NthqOY/jDl3rtuQcfbXLx4GafR4PCJAzzx6Mscv/NBHKfkicceIQgMnh2SJXld/5kHtNpTFMUOlS44dnw/2/1NnnnqMntnl3DtlO/80Hfxyc/8DdPzJzhz5gl6a2t197tj4XdtPBqMowIDTM9NMxiMcD3F9s4aD9z/KtZurLG2toZSNRMuLfIa2WHX3sR6QWbQZcm999yFkg7jaBXbTDMcxOzsXK/DBVJS5opWN2Aw7pHnFq12A6o6NCOk3O3+bjVDtjc3aDZDtDZIYWNIkVIyHCTs2X8A5JigM2ZrtSJJMhZmj7GxNmA0WqXd8SmKgrBhURaSIgddWhghGI17IAzdzhTRuF7pKVliSkWajTHC4LjUa0Yro9s5ThJvMRz0yIuKhX1dXvPaV9LfiHnkS19jasEn2sw4cc+B3TPxn/7kv+MVD7yD5x/5At/x/g/ys7/2M+zttvie7/phFvZUVEJw41qO54cYMcJyXYRpcPSOnGQcsXrFp8hhZl5w4k6fl84OWL9sY7TAcku0Knj/d36YV7/6Fdxx7Ag/8D0/wMp2H0yBMRVFUeHZDsNoOFnzuUx3p9i//yAXzj9DZ9oiTwXbm3UzVXuqzZ7lLpvrQy5fuUDg1w1ViBLbVUTjnKPHDjM9X5GOfIa9imvXrqFUTfJQNmg8kAlK1N9vCfiBZGtrB11KvLAGjjeCupdaCIHn+vWZkZeUOtrFw/h+3S510xt5c9ApC41tuyA0rquQigkn0uHAwUWycoMbl1KCoDmp4JMcOLjMiy+cZXl+ge1+j7w0hK0mh44f5Y1vfxMvnH2OQ4eP8OVHP8W73/yDXL94jo/+zu8xM9/Ga2i2t8a89z0f4vq1izz7+JNgFO3ODI6nsSoYjfq0ppcYD0ckZY5nSRpOi7wsEG49/I1GPUTlEEUxApfAs8hSU5eBaIfS7IBU2NLHVpI0iykKwcz8LONhn6qEoix3h7ebKp8QYtfvaPsW0jCBcwuKogQjsSynXtGqep3darXIspozaUw1gV9L8jKb8HjL3YvvzbKGJEn4evK+/pryvERY9XBhOw6mKHfX8LUK6lGVeuL51BNLC0RRxLETx9nZ2amh9VFEFEW7K3Bzs3rSrus3TWlqEUJLUj2g5b2C933LD/Pyyh/yxS98mtCdJUt3aiuFrEs1mo02w8GAPIk5cHAf4/GYNI7wA3eXN1mWJXLC3PQ8b1eVUxP25s3B2BjIsxJd1SzNPM8Rk4vWza/XdiwsOakI7XTx3IDBYIA2tee0Vkbrf28M2JbCUs7uxcDzvBo5l4M2Y4oc8izm4IFD2J5gayNlGPXoHggxUrG12UP5Ga+571U4skust5mab3Hj6gYzrQW21l9ifRgjbJ+G9BhxA9lzGA0rhsMhra5HJSRZAlMz06xcvVrb7MqSRqOB0dUuDkrZzgTflZLn5e5zoHVBu91k+0rv73egnL1HGMdyuf2uBoduafNbP3uRd73/CO/57ikuXxtziB/kk3/xpzz+led41T2v4P/+T+9j5cYqF89t8Ief/ARr6RZ+6OE7FfEGtKcLLl80TM9Kbr2lxRNf7rO4f4rlg7N86QsvsXzIJR5XjMeGxT0uoSMJp2YYjzTbV7bpdnzsJmxsx3zzN72aVKzwwhevUroFiVuy8aLLnuOGG+sp+QAWpj1aleHbP/BPGNltfvnX/x2usPA8aHcC4p5hZvkQWbLN4uGSq6s9Nk+leA40j89iZbOcuGWBzc01Hn3kNM2uhzAFUU/guYbK0yhAKpieaRK6dzCKthj0NhCqT3e6xdqNiAfue4ADe/bwh//t44SihZQViS4wpotgB6vy8BemOXzfHC8/0UP3E0zYo4w0uJqFwy5WMGLlHPjCZ3afYWeUkseKqTmPSCcUVQfl2YR+jS2gqnAaXVZH25SDhKUK4s0euu1gGhllWlEaqLQ9ObjAMDF8K4EuJVIYGs3632dpzRkFDbo2J1cmR9oWZaUBQxAIQt/B6IooLtCFhW3BwkK9Vr2+MsJrtAkDh2wcU+1IRknG0n7JvsPTLC+9gs31IS8++xjjLYUrS3oa/GYD4eckkWRprgluRHxulvd8+wP8ySf+CN+bxbFDLr10gUbbYXGvQlPS62ec2LufSCl2+n26foPVKwlHF1yaUvP8Wo7v2bheRW9zzIEDhyjKhKtXVihNibI80jSrPTFCsLa2UXtOfMHCUpPbT7yaL3zh81RVSbPZBqNwHI8777yTRx99lIWFBlGUkETQaM1y6OgcZ1+4ztKeBpevn2actZifXkKbDYpszHCnPpxrXEaObRcszc2imOXD3/sd/Pwv/hrdaYeqLPmT3/8bzl6+zEf+8fcRzvYIg2l+7t/8NqfOPMLv/s4n2B6s0PQa6FIyHA9odht0O3P1sCBdbNWit32lbs8os4mvK2AURZSlxnLqlhkhBEYkCFzmZ1v4rsfa6g5VIfAcnyQZ4bgWGAfHE2wPtpHSByqUNLsvKs+rVZIwDKnKfLeXt9IuStoYkVKZGixfiQJlCZIsxnN80jTGGIGtGgSeRxSNam5fS5LEdbmCkhZ6csALJGlR4jiKO269nWeefpp3f/M3IqXFk0+dBAbcuDJAqwxZ2TQCnzgveP93v4NDR+b5xB9+iZ31HUo9RqQlzXln90zsHjwGeho5SHjTGw9xeQU2Rzu88MJTzHSaXL50nv/47/4tZ09f5fzFU3S7B/jzT/8p+w83CMOC8UBy/XLCA6++l2F+CpMt09uJ2LqxTbMZUnoV7//ef8a+PW/mG996lPe/6/Wsr20jKAFDVdSrzn0H92KM4dKlqzQbXWbnuhw+tMCZM6fJU6v2yZJw9NgBDDYKi9UbN7CVJMsTqgmSpqwqdCm4/Y69NMI2zz99Ea0NZSHxQwetCzItQSRYEiwVogRkRR8joCptoqxiYW6WLB1Sljm6rMM3vh8SRQllmWOMoNPp0O122Vjf3F2/1U1aKYFfr0g9z0MqwXDYB6AoSvYsH6Awa4wGJZQhusq47a45hn1NvxcRq4RiaAikwQsM+w6/ksWDe7hy4wlOv3CDdrvNvuWDvHT2OcKGQ1qWFKaiNQeiOMCUN4VOI44cXqQ/yHj2uacQEw9eoTWqcpGqxNiKLMlxAFwf6Xk0VEklYDwe4vshoRcyGG5SJArb9pmeCbh6/RpB0CAaD7AsRVnKWh2kwndC0jzZbTq6qXoppTDUwRlT1aSTZtig1PnEs+dQ6JJms8me5YOcP38ex3JpNkM2N9cpdT6BpQcMJkPWrrdx0hxjjCEMQ/I8rzvF5dfXnNUkkV9VFZaQlGU9IDmOQyMIaxj2ZI2qVL1CT9OURitkdXV14s+sfZ1h6BNF2eR33gYjEdKQZQmYuuI2zQdIFRJFtUf4lmPHefnli7huThKXYNfYsqWlJbY2N8mTlNnZOiDZbjXqgpPRcLdT3oh6AE+SpFbmqVfelS45fPggaZpy48YqUtY4Ha0rqslArVRdDSmlRCqxGyJstzt1crwsJ5zK+vnVZtK/br5eQ+v7/qSFyGBZHoVOqZBIYyGNh+dbIEuanSY7gw1UUKDCJh1fcPRuh34/5uWnY77/Rz7Cpz/zec69+DK33HYI3ynwWi2urqyydmnM8Vv3s7y0QJYVnHr2JXqbfXzPQxcljh2QZQV5lWJJNamWFCRRgrK/7tG9ubYHiTGasixotgIG1/+eFcrl1wjTDVxWtzJ+8j+9hr/61Ao3Xtzge//tLOdXxywGx+itNfjyJ7/E8X238Qu//jO4HGRn/TqXr67xQ//mIzSmLaJRRRbl7D1scEOHbsNC5pInPj+mPSfpj0pKbbHvqMOVSxGVVghLU2SgsPEbilQbCm2zd5+mKDIWl7vc+8Dr+PzvnEGr6xx7W8ypZ2B5rktv6//P2n9HW5bd953YZ++Tz7nx5VC5qrur0RENNBpANzIIEgQJBoEUFamRlmxJtrWsZI080ow0Hq8lc2RbsiVrFE3FoUxKIgmJJAQQOTfQOVRXd1eul9+78eSz9/Yf+95b4PxjeC2+te7qrqqX7kn7u7+/bzC89tqYT743YXPrFF/81phpXbIsB6S3ImRSE7Q7VCdHFG3DOx9N2FwuuH4ARsDwQHI3r3FHNgQ0bnuMs5qigrJikbunKoPvu3SXFOtnBa4bk443GR4fkI4VZlDR6TgUjc/WhcfodI546+s7VFPIVIqvXaKepqhjfBHR78LB6ADDErHb0OgKfyNl7aLDaLfDjRfGtFsNnfUQERWUuUuvJ8mqCoIOwl3DC0ua8QiZSeTKKqL2SNM9qnKKlBXdXshkUpMXatZ0IlEYlDY4josnjA2sLQEpiSIP6dZkU2G1c/M+Vu3jeBUN4Ie+1X/VNWsrbeqyoigqtEqo6inCgfZSl8ALme7n1JVkeX0N5WdkuxPS3YZkLWPtwQ7t1QuM9k8Y3hhSTypaS+c5Hg/IiglGQdVzaTsOHzvzKNf3p1y7/Sp+JMkzTSuU1GpKf3mNsmqoJxlFU7HSkwyHEHcijJ4iaRFHPe7c3WNleQ1jSlwnwvMcghDSCdy6vYfnC/xAWGOJ28FzY5QumGb7+IHhvksP8eabb9JptZlOU4q8YWlplY997GP823/9bzh9apmihr2dA9pJyKWLD7B5+hy//fn/yNZmj8OTHNf08P0az/NpJUsMhjuWJWgU2aTiRz/9DHd3Drl+7Q4PXn6En/3pP87Va2/jxkN++Zf/OZ4RLC8ZTo4dfuZn/zw7Bzf47rd+E5FHJH3F44++n73DI27evMngZEIQCvIiY7m7RJHVPProw7z51hWmU9tEM51mqNmD0T5crdhdEuE7Ie2uRDUVUlTUjaDV6jAeDzHao2pKtLC6zJVNh+uv5bi+T5yETKdjHnnkYa5du4bn2AqxaTqkqTzb7CRsj65mjG5ahKFL0Io4PDwkCB1cKWgaayp3JTiOhx/YbL2yqK3WSgYIbEWb57tsrG+ys7OHEA4f+uBHGEyOefW1F7h49hJ7ezdYXX+ARx96hMOjt/j857/Bu555is2tJZymwyc/8ST/x//2b2HqHCHv9do+/pEur7+yy//0t/8TL7zwTf67v/l/Zfuiz2Avor+cMR4UfOiZn+Tu3tscDg5QyqNSB4yOXAwlRgvbb+xWtJccyjxCFjVGGJR2abTh3Lkz3Hj7hH7vAY7S7+BoSeDbsRUKjo+P2djs44UBhwcDwpbk1OY5qrzg+GQH6WjQEVku8HyHtVNL7O/ewRUh6yvreK7m5s2bGO0hZG11x/4KfpAznY5ZWd6kKjyG0x3iqEeWZQRBgNYNvu+imoqyLq2juK7xwi3qumAyOWRlqcVkPKKuS5QB1/Fw3Jmr2IhZnIs9nnXTzOJyoChTlpaWSKcZKyu2mamqc86dOwVCMxoU5OWAMjO0Wyt8+BPn+NLnXyQJzzCc3EDXM5OIJ6m0wo08trY65OmUtPF47KF3cv3GGwzHeyhZIglZ6nvEYcTOtYLI7xIFkpPDKXWjmUwHnLtwP1U9YXh0jNcLKfIUr4loKCnrmlBGeE6DcgOUavBdnyRMLOMlBVk+pZX0SbMC3RiEbHBcQ1WBdARSaoySVE1NXTWL8PUwDEGYhS7PdqNb84ueZVkaLfDDCMdxaLVi9vYO8NwAz3VtlZ6x0UGe56E0VE2N697T9f1g97tC482ajOYaTmUMnU7HyhTKemFuKcuSVpws/mxNifbfk3a8GMenqU0SEIBBglC0kjZ5lgGKqlEIY9lD6djGoOmkIAx9ksS25gVBxGR8bF3YjjV/grGVnFojhWV0u602vaUeBwcHCyd7mme0Wi3rOq8q4sC6wvMspd/vznrSj4nC1sx579gN9WSyAN5WU8oChOtG47nBYjNksMePGfPaitsLYD5nOKuqAjQ4DdrY+8XFAlc/jIhaEdrkTBrB/Q8ITp1pMxxqEj/mxq0RWeXSOHv03PMMmoZe1+P81ip7R2P2Dq9w4dyDZBMXx8m59vpdJieaMHaoVYpqoNNdoqonNks5iKlrNRtrz5lYGyWllCKIwlk9qvVAjO9mv7+A8oFPCNNUHlI2TIcRf/zPXiKvRqyfjxilGUloKEqH2BtSpQmJ/hTHL7d55gOPMPV7/A+/9JfJzTWyEw/HaO5/zGfpQkgtB9z4nKZIXVrdhiuvQBglbJ7PwdUc33XpLTcMp1BlIToocAXUU0F/2yHseiDadJMYIQS3X7nOOx7ucDDQTIY1H/3FNb7+/9hHhA4yalMWBb21MdNxGzGRDPcNeTLm7FYMTsHTn1zjvZ+4j3Gu+K1fuUV5XLIWr7M7uMvh0ZhSS9KysbE6Chu34giUCiidkqByaG82hJFHuB/SeqhFeuySHhkm1+7g+gGsJVx+4gLnznQZH2maYpcXvnWV4lDQeCE+LVQ9xesEDA+mtBOX5WWB7EuizYib36pwmgmjRtFtdeitZwivIXPhTHKa63lJP/IoqxPMRBEmK2S9dZzpgJEakQ8HCA1rp5YYjE4oj8Dz3NnNYkEyswxHGzXi0tSNdfFKSZblSGl3a0pZRggcvKCy4deNZm3do5UYTAVZqigxSAVNLUAYkrjH6lqX8XSH4UChTUJ10uA1hqaEpZWIpftyJrImzX0i2vjpJnsnB6wt+QxOMjbv26QeHvK+Rzf47ot3eevGlKff83H2jwZcf/sbbLTWaa82HN2ZcDJteOC9LYY3FMnpCf1ej9svlIT9Fs0oY29P0O1FpNmYbKpwnchquEJBWda4sg2ipqqnnDlzhiLXDEdT0jS1D2dpiJIYISsm44pON2IyGfGH/+Cf4PP/5eu0e6uoZsjDD1/iNz/7OdZXV2hKF9cvKQpQTUXcCRjsDQk6PmVTU001cS/keGfM+555H+1OyJe/8rv0+23yrKJIQ4I4orea4aqAwYldALW2WYPQEMU+0nVo6tl97kiSJCGfaacEdvfuaGlrK01Jf9UuPuMTl6ywYzDdgHFclFRojAVEFSx3Wzz8WMyLL445c6ZDnsLJ0RjpeNSVpGoaVCNBpqhKcPmBRyiqijDxePPNq3bhTdrk44xGpQhp3ZRx1EVIj0JVdJfbnOzvozUW4DYGEIsFy/M8fNfDc0tcD3TtUBYCAzbHEAgcF2MUrh/QaS9z+84tlleXOH36NPsHO6gqYXtzjYsXTvHq66/gt9vsHRwSRy267R4Ht15BCU02LRbPxHe+axtlQs5f+hF+7d/9cwJdsrK9xsc/8Ql+9Vd/lV7vFNfevMrSehtMRacXzBayiqqEsqhwHIeiLFlbXeX4eIDWPgKNJx1bj1rXrK4u27Bro2fRJTG4DY5oULVPVaY4rkKpkNW+Iu43HOx22FhfIQocpuOCxjh0V9vs7+xSFCXtpIXruvT7fU4Gh5RZSlFmuNIuoIGfEHghFy5d4u23rtNutzk8PrJ1uli9Y5pOiOOYsi6YTHPilo/UwSx2pZ4F/YcMBoN74Ah+j2avqdXCJKG1RrohgmrmzpUIqa2O25GcPn2aq6+/QRL3LesS1DSVzyPvPMPbbx7QmCl1rSlz+NGfPs1rL+1x/j7J26+1OTg4IAw63P/OFY4OU04ODGtrMePxkOFJSRBavePS0gpFNiHPKqKww2h8wvapNeJWm8ODESeTEW4jUXUDnqCuFY6pEcIlqyq6rZZNBZBQlz5ra2sMhzs4jk+RN+BZ6YjUEUKWeK5DEIScDE7wAx9hNGHQxkhFVmZEYYcoCtjd26XXXcLBMJ1mNvfRGKIwWRQMWCmBZTZtpE2+OO5zwGiUxHEbNAqBzY+cd14bLWfgol5ME6wr2m7yXM+G2QdBMKsk9JhORgDEsZWfSFy0Mba0Qt8bqc5NMXMHdJK07WZvxnzOHeHTabaISKqqiiiyZqtypvnz3BDHi6jKEa4UGB1SqwrcGq1dotAncD3SNEVgUKrG9b2ZaclHMmNbseeurmsC17lXgymcBYM7Zxjj2Lbv5EW2OMZzFhljI7CkcBeB+vMkC8exWb52BG4oCuuanwPMOWs5n9rEcYwrPeK2S6ZOCBLoL/cplWL/aEqnv8yPfexJnv/2LZ599lnOXexw68aI9nJAXpVcOP9OnnrPOb74hS+x3Ovz6os3WV5pkaYpvdUuB/sTqlRQFCme66C1RNWWwDBaoox9trmOJYiqQuAHlq3Mj364ppwfGlBe/JBrotgjzws2T3u8/8Nb9FYc0ibFj3q0lqd43hpZIegGIS/8huGL/5+7/Hd/88/zla98ns9+6XOcubzE0eSEsopQA5+kM6W9Ldh/U+A7NXHk4vqSspZM0xLHszsMoQ0ePie5hromDl0aPNotTWvZo5T2whnlgvJE8tgD8PM/8RD/5O9c4dN/I+TVb7T57i/fYPmiwVt1UUIwuuswvevTNCU/+6fXKI3hjdfvcOniA1x/q0RGKXeuVZhCsrUF7/7og/zHX/smpQbpgnQisrTAcw2+FzI+KHBiWytY+NBFkr4JqxdjmriNSg/QmUd/2aWUNV64ztbpLVJVkAcZj5/RvPD1A248l7HS67M/OObn/sjP8Pyzr3Pr6lVcH05dXiFlyMELin5HkGpBuw2d1iqVP0AnDfpA0Dq9yc1Xd4k6sLoi0P4qqfRp6ZhpekxbTinUCKEjwnbJzqGmUSClO2MNIpTOLbBsfNtsMcvAT1ON50obmSQdhLQ5aAhj9ZSuh+tJvKBidc1BoimnIbnKoPaQjkCbCq1sZubSUkxZ5qixpB4HDOsUP27TNhOMCFi5tIR07nIycbh9JSZNK9bPrLF9+jxxuyCSMafcFnfr17n7Vo/N7hJf+N3fRXiay5c7LJ0WPP/VipVzPucvtnjrBUPcytk6JRgfaurKakrGQ9v1PU2HBIFLUZQURUWv16bb9xmND0G3cUTLZjgaQ7e3jDGG8eQY03jUeowjAuoKWq0WYdjife9/nJdffpX+0jpX3niRf/HL/5pvf/WbfO53foud/QErGz3evnETnIrTp5cZ3RpxMipYXl9jND7CeA1bS+f5sR/7KX75X/5DwshFm5wkiSgLCEI7ypAiwPEqpnlK01hAm6U1TePbGKPU5seVTX0v4mP2YFNKgVa4IiaMBL1lqCtNNgmpSoMb5uSFi+drkDlVKTAqJIgaglZBXUl07qJNTRTFeK4gSZbZ2z3GDQV16TOtxyTSwXdccHwG6QlRWxA4EpVrSiUIAgdlajwh8ZyAItcEfozGILyGulZoYzWeeW4ffgtHrLESi3YnpKlqJuOCOOnYcZwqyacldQOdXkBR24XaiugNG+tnEabkeDhgMBqyffYcCA9XOuzevEnkuWRFivHtvTH/aHmS5e4ar13dYf10n//0m7/JZ3/r3/Lyi29x9fpXONoNKMoJjQLXE3TaXYQoKIqaPG2I4w5Zllnmp9YodW/RVVWNNpY9OnPmDNN8OuuHB+kElE2NJx2ksOySsHY6XDRRu4XSHr1lmB44GBrcaIRwu9y6cZs4DvGkhyPtiPEeQEyp6wlgNx1NrZlOM5aWlnnooYf4wue/yNJyF7Bd9o5jmRxlGnq9DkVdcfv6DusbazOXd4rv+5SlZTDnTVOu684ctz5SOEwmk0UjTBjGtiJSKcvcCWtQkRIalROGLRqV4Yk2WT5B1y73v2OTO7ePKOoRQkhQXX7ypz/A0fAqRwcpH/3AH+Gf/LO/y5lLHSZFzAMPbvP6a1dRxqfIHHrLmv7GgIPdEr2/Bc6YplGMh9YI02q1GI1SPE/YZpgyRQCNkKjG0A67TLMhcSeejccNTS144IH7SNOcnZ0dfN/mHyol8DxBEEQkrZCdnR08zyNKWkzGOa6wOmMpJY5vK17nOaHGKIQ2GGPX9iCI6PV6pGlKlmVsb29y587OTIMpFjrJOcgEqMsGIW22o+Nas5kxtW3pmbHGnu9ajaQfU1WzZqUyRwhn4Qj3PDtN0DOGM8vseHp+f4zH41mlpDXDTCaTxQjaxnrVC3A2Z73txoLFiLnf7xPHIVk2tVKIOKYcpaR6gidDjJEUdYpwXLRyaKGx6cwsNilyxio6jgWRniORwplFJYU0RqPrBnfGnBdFhevOJgDMW33cRc6l1nrBVtZ1je9ZJrgqm9/D9jmOQxRFeJ7HYDBYMMGNViTJPbPj3MhkjGF7e5vxeEpRZPSWe+BoJpMR3aUuTuBSNQ3HE03gCuoyQzImGwr6S21On2vTCi9gEoPvuOztHPDGK1eI45Dl1Zi4FXLz7h2oJY4MQYVUVYNSJQiDaqykwnMdwsiC3qps0Kai1Vrh5PbhDwUof2g/uHQVhorAdRkf1zz/VU0SrdPvreIGmpe/HfJb/26Xf/nfXOXv/Zm3eP7Vm3S7bd7/+AP8/J/8Wc5uuyh3TG9JIkYr1KMx62suu3caglBT1HDrtiboNMiwZFoYjkfaupCF4WhaolWN57rkssHoHKkcRpXBJ6YqBb4qWWoX+B2X8+9d4Ze++BRx0uX5b6b4p7s0XZ9B2ZAeQXVc0OuX/O1/8R7+3N/4BNOJ4eZtn6v7b9A9c8j4oOLiqZCl1QGf+iMXePXq2/gth/5SF+n4KFURJx7KQFkrPvbjj9Dr9ShH4OMQuponPyXY3PY5vVWSZiHDvED7OVErQApDcVLiDjKWp5Jvfa1g+4HTPPz4Fll5Ak3IV3/tK5zcPqJ7CoJ+iJeMaSYQhxpXONRFTa+1xd7dY1Tj4giJ7xnuvHZIcVSxknjkE8k4zzFGYiJNEVSYpI/qx3htzejQIS0FyjhoYTVAjuOQtB0cCWhBFIEjBUJqen0XPwBj9CLQtm4McdgnigVeWNPUmmziMZ0qcMDxFQ6CTs/FSPt3XgRGeIyHikBGOMuCzoMBbq+FClLGrZATFfLG94eEZoPzp2LO3+cSSBvCerB/yHe/8Qpf/+bz/MxP/1l+5P1/nelI8/1nv4cjXFq9AOU13Lw7JVmSTIcZ3/3cIcNsShRF3L2ZczT1qIsJQRCRFyOyfIDWtR2d9Fq02m0GoxG1atjYXmJtY3k2cnIJgoiyyJlOxggzY1KER+CVhJ5me/U8505fsI7VbMTtG7s0Gv7IL/xpXvzuy9x5+waNSnnHIw/RVFN8CVdeOOL9H/gE/6v/zR9lODpkfXmJ1eU2Wms+9/nfIEkipkN7Tta3Eqo6RStFkadsbq0hhY/n+nS7fapCIrBC6+2tM1x+8AGEK/F9DykFWis8R9JUJZ4j8b3A7lBFxd7dkqaKiNsN0i0oc0XDlGk6hcbDFQ1rKx6TcYFwIs5c3CJJEtbXTnHx3P0IYRiODkGUlGVKlp/QjX3a6yHhSoBwHHrRKqFKaHIPKbvE3YSiMWxunqO3skVWKYyrmVYDlEwxRix0W0VRLhitMAxwpYOUgklaMR5WSNel3bHnp6oqkiThyafewzvf9Q7qxqUuQ4q8mTUeaZSZ8vCT7+bSgw/Q6bTZv3ubfiugHbsoXdDtxfyBP/xp3vX4OynGxeIlZchRuUd7wzLx5/v3c7gn+Q+/9jX+5l//+7zrvRtkU007WsUYw93bJ9y5NaJIXaYTq/nLMwsmpQPtTmwDruuaxtj8xk6nw+7+PqPhxNZnNqBJOXO+RxBZp6pjXKqiQJkJTmRomJJWRzz5nsepdUqepxSp4sZbt9la24bGIUtzyrJkmk6Ik4BGlYSRZ6vphGAymVBWOdLRZNmE73zn27Q7idVQlyVRFFOWFePpxHYjN5BlBb1+m7LMmUxGeJ6zCIOeM1TzMa2Nbclp6go/8Ehm0TIIGygdBBFGaTzHp6lZmES0bkiShCyf8MhjFwgil+OjMU1TWpmO5+FHGV/50vf51pf3ufFmzj/+Z/+YsOUxGYEoKz73q1/n1pVD6uk+qtrneKfixmsu5ahLXkyYTkrSSYMjIwLfZ5qOCCMHpSuEMpjGoc4THAxJFCEdxdkzZyiyjFbbslNRFFCVsHPXms7qpsL1FVEQkk5zNtZX+fAHPkFTgXBcWkmP3oqPkP4sC7KFNC5xHBEHIYEboOpq0bw11+UdHR1RVRVh6HNyckJZ5rOg7gatbRe1EFaO5fs+rseMQXMXTuR5x/rcUW5mG5mqahDMR7USrSzr1uv1CAKP8XhMlueLDYkQgizLaJpm1pIz3yDIBcALw3hxTShlGUK7+Q4XRkartbTxX8PhkCAIWF+z3dppU9kINxlSIHHcCEc7NAiGeOimQTcKjJiBQol0fYyQmMbQVPWCMTTGIPQ9V7sF6d6CIQUWrGNRFAvjUVVVZGmOarQNSQcaVSEdG7MzB92TyYTj42PbKz4DuXMpg/3eZqFbdGbGIc/z8NyILNWMDirII4Z3U8Z3J9TDEj/fJ9/fI9sdQ95C54JPfuoz/NQf/HlqT/LmtZfZObjJG2+8ThA7VKXDZKS5ce2ITtzBdUMCvwfCQTgKxysJ44owtiyxQZDlJUVZg3SoS5hOsx8WJv7wDOXljzrGhhUbwsBB4NNbifjkH97iaDBF0uGX/18v8Uf/5Aq//Y9dXv76Ho89cYHPfu03+KW///d4681/zeFEMnjL5cb3FGcvBGw/OeXKjYpERTRuTuwluJ7i9u2CRoP0LN3qBwLZaKYV+BVEq4qVpQ71kUNyaoXpdELcwfZC62Oykx6U8NRnVljW6/zKP3yezrqGZYWThTAQaEexfh6C1hKv3prQaQ/xPcPZ+31WthqOd87z2MXP8Ju/8puY5jb+cog2OVXls3NnjOeD8BRNDXUp+fRPP8Jbu3vc/d4JJwPDo4+u896PrDORJ/RDhy/+TsH+gUaoEXEroHBd1sOzdFxYunia7Qtdrr7yKm9+7QbT4ylVIzBNiRHw8Z/Y4vXbd3BFwO5Vl8AU6MYjJycMEqpUsnnJw++XFAc+d24KErfi0uMVkxzq3gpGWUlA4QlCXzCpDknSnLsvKrzTNWVmiBJwhI+gJmmDQ5uj3Ql+IHG8gKLMiFqCpobp2CC01YEIqXGcmDBpaFRFGISoBoq8YH0tJkoysqnADx2qBlQJwtOoqsNqfAmn2aMIRyTSo6xcDg8d2u02veUed27ewG3GXHiXNfMMdrrkZcju/h7xkk+jPN559lGe/9ZzHAz2We46dNYioo5gOpqg6RC5mqYQFNOSaVDhlSH95ZDdI8NKYKjLPpPpTVzXs+dVOKSpIWl5KJ0ReD0mkwme2yHPSuKWg+dIWq0uR0eHSKfGcyKkgbVNze6dCa7TZmXlDBcvn+Xu7h2uvPYSrW6PtreEVNdxI4/rOxCHqywlAyYnGa2tTZ75wE8yzAb87m//BxLTYf30Iwg8TiYvUTcpn/7JP8j6+ir//d/8H1ld3cTxJ3z8Yz/Dl774Wfb3Rwgh6HQ6jIYFYRzQNAUbG6d457sf5Tvf+Y5t9ilLy1bOxla6UQjHLiZNY53Q7Y5Pb0VTFobRoKSuLcsR+R6dngvS0O70CFtL/KFf+Et8+fO/wmd/8z9x5vQ5vNAwnhyiGpe6kbiepE6n/NQv/hR7x/t89wvP0nXaDNJjGqmIopi15RVu3rzN0lKP+x+8n2s3rpOmVqivGkk6HViHZF3PFqaQMAyoqoqqKHF9nyKt8XyHXicgTgLqsqapYTSa8J73P0NZKF5+5XkuXjrDnVvHZPmQTqfD4GSCchtcxyH0fFxjtWrSEQihiGKfCxee4fhkl8OD1xfPRB+HOGpRqCFFHeOK89z/SJc49KnrMdffvMH+3gmrmz12dk+4/8EH2Ll5e5ZVV1FV98Zjc7YCrEPT9/0ZALOGBy/wKYoC1wNHBlx++BLZ1G5UTKPwfIemcvCjkP5qzHSkyfMRuskJvIiyauj2l3nkwcvc2bnLjRs3EELQ7XZ59NHHeeH5F61u1ZRk2dSCgsCbLay2jrXT6dDUNYdHJ7TbbYqixPN9sjxFOI5t4PJcG/kjnBn4sYvrD15vURRZtq5uZnpMjZoBaGMMRa4IPJ+mmY1KfYgTnzTNUaZB6hhkyf/6f/sp/tU//wKD4xIvUjS1ptc6RVbdQcqQ1bUeg8GYos5pdbq8933vJBYVt2/fYTKOuHXzLrXOWVtfZzSdUFcSVU8xxkGKueSgREhrkCsLhes3dLsdWu2AG9cnbG32OT7aJxt5fPij72J/P+fOnVsk/ZzBUW2bVuSEgz3r3HYdRRR2mIwtyGj3M/xAsL9npxqT4YjtUxs40rfMZuBQlbWtPVU21DsIAhzHWxzPsixnZsmKIPRnIKjCmekkm9qOwH3fR9UaxxUY06A1IOy1Zx3K3uzvFK7j4TotoihgND60uZOVw+pam8lk8nuMVHOgZMP6IYwtMzeZTFBKsby8vBh912W1AI5Wo2dH90FgCxVGkzF1bY93r9eZmYSsBlMbRafXZ+/WHUzjEHsBXqI4zkpiN6bMC/xIopTGIJGei9bNbANtwaPn2n5uMTOe/OB4ez6GnrcPzd32cy0p8HvyN23oeUNZNiSxT7fbJcsLiqJgTufNQ+TBAvcwiAlCa26dj82lcBeMreuGBJFlLasix5mBTbvOSrzOBJolPvCRd/Hcc89x8+4hFy7exyjd4/AwZet8n3NnznPr+m2O9vZxjGWMPV8SJDVloShz+/3CmNnEA4rUUKQBQlZI42NEie/F+IEgLyqqkx9u5P3DM5TGx6gQYRzqWtnWjDs5u1ddIkdxPN4hKAI++89L0nHN/WccsumQP/rzf4YrX/kmolfhOYZbr9gFeVSU6FoSufZAO8Kl3S+pypJe18X3IQwErtOga5uH1O4YQk+h8oS7N0tUNaTbKlhb2kYUCVEvJNiQOIHm7ANTWv2Kz/2Xb7J+ycMkAYqKdDzG1RpVCgbHOddf2yeRAwLfCuErrckrDx28zee/+XfwV24yFQUnR8dMRhnT8YAkAtMIHCSeA0IaXn7+GnXmcP+7N3nivStsnjvDpTOfphX/HBXv5MzZNvVBxdLyGVSnoee6vPKNq8hkmTtHA579yjXC5iLv+ciHUcZFGp+Jpyhd+Orn9lDpEjrr4ZIReJY6D7wuk7SgVjXDvYyDtzwODwYodcLypsPyqmd1i2kFVUM9PKHrR3hG0PeWacYOrW5DK5JsrXTox20cDL2+Sxh4oDRB6CF1jBEFcQuMNgShh5DYh7uU1hlHTZ4rWx/XgBAly90WTabxWaLfjZiOBcLMdHs6oNVqoauI/RtThjdKmtJH+iNWt2LcyGF1e5OnPv5T6FMrvH0jJ89TuheOKMUhlXLoba4gRMz3Xv0uDz+ecvYihIkknWi6fsT7n9iiFzdQ9jmeNqw/0OLCuRXqvGByMiQJBSJxmZS3cGQy07c1BF4PaPB9j35vncl0ONOOTnE9aLVCtk+f4vL9j+O7S6wsnUPVLsgpWVqzurLN6fN2xDUcTuy4RUlUlTE4OOKDT/8E3d4lkm6fpjjmdNel1wq4tP0eXvj+C3z5K79DO9nE87ucudCnEYd0e8u8//0f56/87/8ON99OUdrwqU9/jHe/5538h//4q7z7XR/il37pb5MkEUVRIN2corRB3nfu3OLzn/8Ch4dHNGWNI200S+AFeI5HEIR4QQg0uI6HkIqyykFH9HodotjHUYZ2S3D+Uo9Ot8/xyYSNjQ3e/+SP8Jf/d3+V3/3d36XX77N3sMvgZEhRlDOZhB2f1KLFl37rWV762hWmw5zD0QikzZh8/9NP4fmSX/iDP8foeEoxNFy+8CjjoylCafLxYBa0fo+lnO/0dWMXtLpSeKF1JuZ5jdYG6Si0KWm323zrW1/nu9/9BlLafubJZILrCZqmZmlpieVglbaXIB1oZEEjC4ypcR2HOm947lu/w+233kAKb/Ga1DXKqzACqAs8/yp3b9/ktdde4/nnv490FJce7LK0tML25kXbRqM0p06dnuna6oVOq2kaptMp0+l0sVDZl0JKh7puiKIYXUuqUvDyc9d46+oOQWBZTAcf14kp0ox0kOM60O0mLK938BLDu9/3EE5Qc3C0z97eDhsba/iRz2g64jvf+w5RK6FQJXG7RRBEi1G1zYwsFt3Itu3IIcsy2t02nm91mOfOn7Eb1nLOZugFU6OUoixt9ZuUkjRNFwC6aZoFgzX/WXZcW6GUIAxD/KgijC349p2IybjkzOlz3LmzM4vVsQy153kYOWB740E+83OfxJhZbmDZwXfbjE4Mr74sGE/6uF6flfUtwlbCcDKlaTR1M0U1LlpXKJMhZIUXSRotqWqJ4wt660vc99CjhMklNraXmBYjRgOPv/CX/ywf++gneeXl1+h2u/S6CZ1uRNVM6PVjPN/hzJlzhEnIODvAjQrCSCCFj2k8XOOhm5x2J6auNLu7d3FcyLOK+++/H89nwarleb4AdXF8Lxc1acUz4OjaiBvXjq6dWTC1vZ5sPA5IpKNxHQh8n9CPQFsZjOf6NI1lMhHaGqmMwvVgNBpRluUiymi+WZiP18M4wRjblGRrC43NdNR6McqeRwjZEbMFYFmWcXhoHf9JEiElHB4eks8YUCEEnutzsnfER3/m5/g//c//b973qR/h/R/9Ed77kx+lTgQ/8pM/QeDHGCwzqZq5VABcaaUyRoDj2nzasswXY+y5vncOAIVw7PeYjarvhbWD5wULEFrXDZ1uTBhHTNLpQtMdhPcC1OfA1I7K77UfeZ5nx+mO1bm2222khKrIqasURE2eT+n1Orieh8FlnAlWNs9z/dYh2jH8wT/8cVZWBY8+fJ6H7r9AWUiODic4DjgO+FGF40JVpyz1O3S7IdJRdLoRYRgxGRtCv8vq2qq93l1rmpq1vqNUTeD/8MHmP3RTjhfUIG0Woa58pHYoBvB3/+vX+Myf2OK/+m/fy91vfZdv/qtDepEgOgOD44wXf/u7nLlfcN/FDZytCe94P7z89QbfcxgPEurM0A4V47JhMkhwZIPrljYsVVtdXhhpMhxIJYHnMCWl7Uvy2vDq1dvEYcap1Rh0g6vPIVrXyUqXL/32IYHfIc80SoKfd3DdCcr4iGBKFQiWzpYkSUihDJO9krh22e77yLM+NzuaG6+lRFOJLiSNUvgOFEqxvtFjMi6odE071ozTCafcFXaOArYvOLzjAw9y5fbb3L074cylNd7xzgcZXq148/YOIla46Ygoa/jSv/8cP/eZT7Jr3uS//MpXWTlzFr+dMNzdo9fyaCpN4dakx2OSVoQroWw8ykrgyYIgiFB1QeAbqqYin4S4rmZaZLz2QkC/38KhZDIY0OQ+S+cb2+/qDDg+FnScFQI1ZHg0AeGxedbHyILjQ0WRVwhcBBM8Aa4rUQ22T33WF29r3DSqNPi+h9EVVeUhhaQUU4SBt640PPruDstrGkODLhvySoF/gONOWV3rYthkZa1LUy4xrW6yefEyr77yJvnBHdqbLbJsRDHq4PY1UbfktHea11+9jhk6RChaj7aIQofrN6Ys9yMiR5CfCMrMUJoBxsCbNxoSZ4mkW6CahgvnYnbujnHzNjIukVKwtrpFoxXbpzbwvTYnxxMunL+PvZ0JRqdcung/f+yP/SK/+mv/hmef+wK95S5Hh7v4nod0bSRGbioqk9LqnOH0qYs89/y3UFqile1HPR413Lx7QlEUXLrYYWWt4pU7Divlm0yO3uLS+kVeff0uf/7P/zn8Vsx3vvcdfMdn//A673rqMsPhmNPnlnno8tP8xq8PkJ7mmQ/9GO99+r089fRX+K3P/mfiXoBQoKiQs+YSIcRikVHK9gfneW5HP8pHaTs6cxA4MuD2jQmrG21cNyZuTblw33mE43L9jRu0egHPPvd9vvA7z9Jd0cTJMoKQdt/liSee4O237nDz5k38QKK1wbgZ6Yl9gHvtgChO6EcdjscnPPnRT3Dm4RFvvPwymWrwOh5u5LJ+ahOJJAwSjoe2Ezf0A8q6oigKy17NWkSEdNCmRggoqprhYMzqescupDX4AZw6vUFdubzw/BuEfhcpS5TSIKaopqJWFdKzOZkePlrDyUlKFIW4XRfPNOTje6acvIA8avA9F4RLpRzGhzsI0+LdzyQ8/60RN64pltanSNHn8GiX5d4aV65cpS4r4pZ1tjaNDaF2XYe6ulftOF+M5tqzpqpn+Yw1jhPRNNDrBExGU5oGpDclki0kDsIoHr78TgbjITdv7LHUvcBdN+Ptt26RdFocD0cz57C9r4eTI1zPJc9tF7Lr2nrMLMtwHDFjo2pOTk7Y3t5mNJqQpimdTodOp8Xuzl37OboBAVIGZFmxAI3zbue6rn9PrMpca1YUGY1uZot4TRRb/W5Zlpw708PzNeNBhlYBYQxlWfPdb9wi8LtojjDaMnBaOQxHR/zqr3yOPKs4feYCZXOd6djlG1//NkI4VHUGQhH5MVIElKXND6wqB9/xULrGkTFZUWFkTjArajh3tsvqykUODyGdpkzHOb5Y4d/+2n/Lj378x/gjv/An6PZ9GpVzuFfOcgcDpiNFuyVJ0x3Cdo4X+0wGijxLETLAc3wuXjzF3Z3rFEWGaiRBHGGMpqwqWq0WTz31JF/64teJw9CmZmgL0g8PDxdM5Xx86rr3xrbzYGoArQyOJ6mKmiD0cFxbu+jO2syoa6qyomkkGMeWNRQNRoPRIB2F47j3AN5sopHnOe12lyiKCMP4BxjPgE6nt4j2CsMQV9rzPQel8+BwOcunraqK4cnAfi9/bvCymw5lDMb1+Mbvfocvf/3reJMRK+tniM4vUxcFt966Q97k1KrGEa7VspYKz7egsm4qHM+z5psF2DMLJ/Zc//iD9x4zrnH++cYoytJmM1ptsL94L1JCVdvpjx94aKPQWi2Ov5RiFhVVL5hdrRW24lYQhB5h4DFNcyYTC6SDICbNCpAOQhj63imuvfEiymg+/Qd+kgfue5yT0TdZOVXx/BsvkU9DfCGAAq0EuJKy0Liez63rNq+2KjSOo3GMg+vYzF+jrblUG0MQO6A9fN9FGwX63pj+/9fHDz3yfuhjnjFOQxALilGb5aWGO68l1FlFb7nkxz/zFLeO3yIfNrz0WwMC2dD4AYUIOT4Y8PRHlwkfHLO1tMp/+HsDltfXaS9vsj++yVJcs/Fgyt1rkp3rFUYLpK9BGhrlIN2SxoXAD1HTgkYJvCAhrhuMrKk70I8ChvuK6YHLE5fOs3Eu4cXnv0cQJKikoPY7lNMG6RgaFZNW+/SXDN1OhOPkBFHIdJpz9n6XSw+3GB357O8NeeNlgUERaah1Q1VKHOnR6jic7JdkE83GVoRYKciva7bPn2J/GLJ6MeaRh84y2Rvyu19+lbOXFdunVnjzcwVXv7VLZy1Ei4rAtNhPJzzzo6dJ8oIXvrGHbCWkdY40PoHjkzcTet1VRumY7Q0PGSuuX5FEXk2uLMDfPueiioq7uzmdVo9kteDGWwUr3ZjVB1uc7B2A49I/1WJyt2H9ss/b365o9nM2L62RJAFvvnWbcw8mFHrMaGL1KXmR0umBFwRMpyWqsY0kSs8q01xLm+cFONIjigWNqsDYmyiJA5SyOxenlaNrl3bkIl3JoMo4v96iuXGKt/ducGr1NFlWQDRm+YLH4X7IaH9AM0pJNkL67fs5mJywsmEYHOW4nsf0OOXphz/AfeeX+ZXf/DXqosR1ApaWa5548N1899Xn2T59njtvl9weH9PtegROTjauOX+/j6oCJtcrdNtwcqgJIwdjGiYjgdGSpOXjyjbD0REIRRwlGG27rv/+P/gfORkc8lf/yt9geSXEkRFVNaYsNEtLHRw3IfR7fPwT7+Mf/oN/wwc/8jA3r9/AaJ+Dw2OE43Jqo0XHj3n1xk0u3NeCLOeBBz/I177zTU5tXwK5hu8OePWVK/SXYzY3tnn55ZdJWhFlDmVd0ekmeEHM3u4Jq6t9tGmsqc0xKFPgSB/0LJ7CcZGO3TGnafoDLkSF51oA0enZxWk60iyvrHDu/DZX33iVJN5CocmLExoyllfaPPPej/LFL3yHaTqmqTVxW4MOiIJl8mJMWZY47oxZdCRh0mKYZTz++OOcPXua9dOn+fZLL3P24gZ7N27w/a9/i/e++31879kXkE7AdDJiqdvhYH8Xz5sZiKR9yMdxjFG2maOsK1zpLNgSVxqSVmAf4tKhqCp049HuxCytSt6+UlAzBCSO7FHplFYUUxQV+diaX+67/xTnHzzN1rlVXvjKyyytbLN28Z4pZ/30Cv/5332Jk1vH+F7EKJV88MMPoLXkcPotPv2Jv8Sb19/gV//df8J1faIktNmFvs0SrKqGqtQoZd/LdDrF9+91J88B1zx8WSnF2mqb0XTMZGxNce984jTTyZjjowKla+pSUjcV/V6LLNNUdUoU9ghDl6QVzvRx9vyPx2PCuMV4PMaYmRtYV0hjjRFrK6sMBgOm6Zg4DlGqsZFcnTZFXs7Cs0vqpgBmDI6ZOVe1Y8enRswWdLNYeIUQeI47Y36aBUCxq64NzJZOQ1NL0mnFH//FT5GVe1y7esDVN3aR0vYn7+8NEY7G9RxUY5k0V7ogM1xPkaWKdjeinWxyd2cIoiLwJJ4b2vceBRTFFGEknheQF9MZgdEmr3KM37Cx3aJqcu67fB+T6Zgzy09w+9aEuzuvQqG4cPYyf/Gv/G1u3HqFv/Xf/yVaQYTrCYLA4/SZTQaDE8q65ujoCNUY/EQQJ4IoSpA64PiwYDrW9Ps9jo53MEaRdLozx/MUgQV7SRgwGAzs180MInOX8A9WNNr/b3BcSRjafu48L8GIhfGlzBsL2CJbDWh/BiRtSVlphicVrhOjdArCLNzgQWANUo4j7NcJ+z3zvCRJEnq9JaTrUs4C0l3XZTAYEAQBSZJwfHzM9uYWStWLv6/r2pYTJMnMFFYvphC2W1wstJlgHcjalHhOgOP6DIZHdFodHn7sCb781S+xst1heDzGwcOXLmVZ2EBzYa/pIIlnkUsa0+hFR3ddq8Xmba6bnGd0zsGlUvXCoLMYZTsSV4qFtlUpe53Pv4/vegu5h91cVQvwr5QiCCKqykbrJUlM4PloI2aRPjZto6oKNre3ODg+4PSpi2TlCVHS4wMffpJvfPO7XL/9Jpcf2mJvb8x4lOMIgZAGXRmaupplzCowjjV1iRrXt1mUZVkjTIgjghnDny9C2re3T7O5tcIrL7/B4Obw99fl/cSnImNEY3u4lzuM9xQ3XxK0PcmZUz1eeuUWj74vYfmRkNGdbd743ZcIQ6hDh+wwYeOyw9qDglvfh3IM8UrCYCTB81FG88jTBVvbki/95jFVEeDGA8rCw2/VGC1oUp/lSLN9rsdoWpIeZgzvNnTam3grhs65fXpOyLNfznn/05/i5//UX+Hv/o2/xo23X6F7LiXsezR1TH1iaHc93OUJIl+i0SMORcqpRKI8jQg8lFDoxkNQUk8C0CVSgBQeTe1QFhVCaHzhYmqXbFLgLnlIDF1fUHrb+FJyNLnG+fPbnLvwNN//1gvc95RiyT/Ll/719zi+M6azGpNnGjfTlI1g8+GQw1fHPPxQj1q3+f6Lt/DbPo5RdFs+k6LN9sYSg+yEYpLjlA06rHFFyLSqCIwhbLuUk5IzD/c4OXKZDg7w1zsop6RUDad6p7j9nZv4Z3zKQUBiJpRen45XcHyiqXTD9v0ReTMF18X1bT1amWqMcMhzhee6lLWNGErigKIoaWrrfo/jEGU0SlcY7EPB9RWBFlRGIoBuAp1Vgwlc1DjGGwrq8YRh4dFt+5QVmCXDqUuaYtAhr1ZZWmt47vm3cVTANC2g8OgvexwNR/yBj/00o1slz735n+l0WwxTw+bZAqf2CGKPjf4W03TId58bsbxWEIuIwZGh9jVlXbHebtMojXQU00lJEtk6TN8PGQ4sg9dqhUzGDaury/iB4c039rh83/sYjY8YTt+Y9brmOKKLdHPCoM1kXOHKFivrHkdHJ/i+S60a8kzhOS6okKI5otYObU+S1hV/46/+RX7jP3+JOzu3GBwdI2VMryPJ65LV/gVGowFrmyGj0YCi1CRxl2k6WHSmV6Vl9ZWxphOtjM1Wc+yDrakqmqbGdZ0FIxCGIVLM3JahaysOlz0u3LfE9bcGOHKNveM3iaKE0XDK449d5vUr13n80fexvr3Eb/7mb+IKSRBpXF/R75wF5XB8sk9ZKBwnw/VaFDrFFwGBaPO+H/kQk7BEacFHnvgE16+/wb//N/+Cjz/zFPk05ytf+zaubw0qhhqYLzI1jbbNHkLY6CopJVVVotR8ZGW1d0LD5tYKjU7BSAYnU7a3zvHU+x/i3/7Lz9PqS8qioW6syfXC5TO0+gGuozG1ophqbt864Kn3PM1TH3mQ73372xyflItn4vs/+KPcvvkq3/ny75CPKy4+cBGlcq68vsOTTz7D2uopvv3sd1k5dczxQcnhriTPMtrdaDE2xlj2BAx+YN2qVsdVL8KG2+02y70+OzvWwYsjiKKY0WjCudNnOHPmDC+//CIIRUOJLu2i6zoJlx+8yPPPvYDvQ+R3qbUdsfu+TzjT23m+1ZopZUeYllkMqQoLFCbTAUIYHEeS57Ypa27kaBobPB+HAUIapCsQ2MUqTXN8z3Z850U2A9KudbVXVrPXasULllxrjRfYc+75DeAwPFb88V/8Wdr9ilvX7/CFL3yfpd460qlYXl1FCMFbb72Fqt3ZIl5QFtBKuiQtfwZ8EqSbk+Y1rvAQTo7nhvbZ6xcYI2gqH20Kzpze5PCw4JGnluis1rzw/C5+2KbViTk5cOh7EQfDO/gy4UNPvZ/P/qfPEbfP8MBD76DdnvDbv/55lpY6SAc8p4fjCLLiYMZ8OTiBBTjZFOpKkU4bzp+7D+MWXHv7Jr7jE7cjqtLgeRFhJBkNrfHJ9TSBGyyA2FwqMXcNw9zRbCN+jFELQ40j7WSibjKkCXGdmHYnmY3OG6Sbc/b8MqNxzvW3hrRbPRqdonWDK9vW4e3bBiOl6gWgBWzRgBfQW1paZE62k4SDgwOSJGE0Gi00kkbpWWOOYN533TQW4EZRxMnxMXqGSYyx8qp5J3aSJFArpjpnNe5wN51ghGS7tcaZyw/wxosvcubhPq+/+iY+Hh4+aqaJDKOEsq7w45lON8txpcRYaeTCXW5mmZtzYLuI+1noLK3rP01TO20JA+q6Qhhj2UjXQxizuHc9x50ZCYsZU2m1p3MdsTXkWEY5TbOFrEGK2YZrFgF14eJF7u7uMElL1tbWWN5YoVC77O6dEEUBw3E+02MbwshF6sB2r2sH3CkYlzBI0HWDMjVGuxhcGp1iNASeHYWndcPW+lmQOZNRhXTs82JwbfT7Cyjf9VOuaYzG1RHFtOLayw5tfESj2X5ok2JyTOwU1MagonWUmXDw5gnduMuwHNNOOrz35xKef26HN7/o8fjjD7GnDjgaa2RTUR1pVs/6XHxHTFl6HA1vUjU1aS5xI03sO2Sv+IRxwJlHOlx7fg8HSS4kjz/6JO/9Sfjy57/H9Rd9svEJReaytLKGn6TUTOmtOAi3ZuduwJ/6cz/Pnf3Xef2V2wR+hyLPmKQneJFLXYyJag82GxwXfBfqzKAEBA5IFVJMCox2qBuB51tdisTBlYppZY/XaucUT37iMm/dfpGo7nFy0GaSX8fLfZ752JP8+r/6Eu5xSqZgogPawpC7IFXFn/5jn+bXf+MFrt3c5fJZn9VOyot3fdrtDe6/7xw3j25wcucAUzlkIifSIX7boS5rXM9mh1V5Q7cfEm/V7BcKtxQELZBNSHMAk6MpTigsdagFrU6CoyZkU0GyFKOXSjwFE6FRjcb3JPlIkXQMo1FMo20eqHANlTHIxu7afN9FyHlcQw3SamxcFRDKnHHosh3WhMsR0hEUY5dmxyPqVBzsSgih7Ubo0SH9iwl6bQmnPIfXKnj1uSt0usvQ1GSjCaPcgPFo1SXvfXyZuNXnu683qOEurVM5b77mcn5bsHa5pBootG4hnRLTLLF3cMJkVKGbkEZPufTIOdI05+TA5sxJKSm0IM8rPAVh5FmDgRFIIWZjnvaiMk5KA8LqLu095aAVXLx4kYPDPYLIY3g0YLm3zDQv0Ch6y+vcvvk2vSWDxtAULpfOfoDXXn8FzZj3vucj7B8eEMaK1658n1a8SplXaFUznZTEUYc0H9NfaaNmmqfRdLIQmlv24gdu9tmOWymNmWkP19bWODk+Ym2zz+7uLr1em62tLUajEceDKVHYZmVtjZOjY4Squb27z4/85E/w/PPfox+1uXb1Fn7iUTcFvucQhj55VtuQ/MZWqwlTEYd2RJSXOadOX6K/tsK1W28StNa4/OBjmDznwYeW2T+4zbU3dzg52md4qKiaEZ4bUDcGhHWFem5IFLUtIyNrhGwoUjuOAgO64dKlS1y9ehXX9en3+xgxpWkqppOa6dgabRptEMIhTCTZyKPWI6JOi43TW1y4fxUPj6/+5+9QT3M++jM/xmM//m6GL7+5OJ7f/uJVVs+c58VvfpbWSsiZ0+f54I88yZe+9qtsrV/mznXDc889hxCGs2cucf7CGb733LMc7Y8JAo+lXsLJyZAz25fIsoL9o9t4QTxzf1YzBii0WYezAPiyypASWknfNoSsbHOwf5dWF9AxeTWyRp5a4MiY8+fP06icK1eu0Ot08QPJ8dGAVqtlhf6uxvc924jkurSSzgwsGobDIcv9JTvWLFIA0mm10K/WdYUyerHwLi0tLVywc2nFvEkkDMMFA5SmY/rL60jXoaoG6LohCtqMBic4XgttajABvt/Q1ILLly9z7lLCKy+9zbVrB4scxKWlJVzXZW9vj3m1XVYUhL6/GKXPNalCCGtqcrkX5q0bjIEoClCNg+869FZW2T+6w4c//gFu7VynaDLaXcGNq0MwHpHwOTkesbWxRjZSCK3oryxzMtln82zI268fk04M/f4yaTbE9+UiF7KuQEsFChzpW7lSYzMbj44OqFSFJ6320D5XCnrLvYUDuCiy2QjV1vsFrnVr9/t9yrJkOBziuMw0kFYb6HpzU4i0WYONIGlFXLi/Dc6Yt66MMI3D5voFGj1kd+94UY1pjCHP5q09ZsbOgVJmAXxgfv6tTGYyHbGxsYHnOYvzMm+LCUOr9yyLGt/zFqzkZDJajHwDP+JkcLTQF7quS5bZ+Kk4jskK2yPe6/VQjZ5JeZxZFFWAlDDJJtRNwWOPPcLdu7cZHg9nWbqGRtnfv9VqWUCubGC8F1gpgZg9R+fAfG7ScVxhga641+89B8VW+9vMpgmzNUCz+J0nE/tMDkIfVTd0Op2FhlRKZ6HhVEotgOf8+Da1JowTHnjgAY4OT+j1XV568RqnLmzixYrDoyOCKLZVk4OM6SQnCH2CwPa/e75dh7TWlGVNlPiLe8HzIorMsqnG1DYOUCpqIrphmzAJKaYHHGVTzL75/TXlTD2D63k4CIZ3PVpuSYOhcEsme0c4jsPeUU5tYgbHnBcJMwABAABJREFUB5w/t0Z/rUfWaDw3Zu/2iLtXFVu9SyBqdvdukQQxSeDjuS7JpmCaerz2gktVBLSCHio3rLQ8tvs9xIkk0C75OGcwGLByRtBdVyxvptw+eImv/Poekz2FqcZsbsT02g2tYEzsTyiPmTnKApq64Lc/+zVe/v4uabHH5mmX9c0lWtKlHUSsbPgsXdJEgYRKkg1teKzvOwyOI/IUGiUIwjbSaVB1SOBHGF+hfMnqWosHL6zhmkO+89tvEsgzxBcmnD93g47fxqlbvPHqNTYu3UforXC67dHuS1RU47oVRS557e2c9ilFb7Om8SF3Vylzj2c+8gG0hk5nivELpCdxETSipCwawjDG99pMJzlJKyZLK9Jpg+NpKlMT+REqs07SpTUPzwtwRUw2VZRZg3BXiLdD1NKEoqmZZjWR1ESeJjCKGkNWJAhZ4CcG4WpkLfBLd5H5VRQl0NhgY1/iOgIpNMptuNMYROOyOwxRjUvS1ySnNceHBdfedNm5mTLda4jJeeABwfWXU6Z3hmydjum1L/HOh99POWpw3SXOXb7I0kpAnk8p/DE6foivfSfFCY74b37qj/Px0Taf+UNPoyuH/ds+edpibcswHRru3DjAFYCuKOsxWglM45GONNJx6K12kCagPswQ0xzX1ZSFZjAYUNUpSI3r+PS6K3iei+spkCWe55LnBa1WG9932Npe5dbtGwxOJhwfHnP61EXW1pcJPZ/JqOJw7wDfjanKhqa27MKNu99g+5zAaJfLD95Pp695/bW30GXC+558hnc+/ih/8A99hife8wjnL5/lZ//IL9Be3WJ1dfX3OIatEFwuxOD3woLrxcJujGE0GpEXJXlWsrqyznSak6YFw0GKUoaiKLl14zZ5ZhgMRjz4jktk2T4//qlnOD7ZIU58fDcg8F2W+is24wxBGPq2WUgbBD6TYkgjc8IwYjQ45tabbxIpiT7epdq5S7ed8crzr/H1L7zK0d4dDCcYY/D9HtITCGEz7qIootuPMdh6xqZSpCPb8jAXzrt+yO7uPsvLq0gpmUwmCCGI4xZJktBf6liwLTV1XbLUXeMdT2zhex7NJGNyZ8LNFyY8ePE+fuoPvZveaZ9nX3iFb/yHLxCF7cXr07/4MT76zHv4qZ/6Q4xViu/EfPW/HPDK92q++sUrfPvrz7K9vc25c3azcvP6Hr7TmoU7a/Z3LatQO0ccDY9Rxi5C83MIzDYGc5DmzP4sGI0mIEvG6S3CWNtga/deQLTjCKqq4M6dW9y9e5ckCm2moRa02xbczwFfmmZcuHCBVqvF0fHBIlRaSsnJydBuFL0QjKTTaeP7s7YgYZkvKRzCICLP85mhQSxY0HkW5XyMX1carQUePuU4RzSCwEsYj3Jcv43SBZ4vKIoptcpptT2uvnGD5559i1s3DvE8B993Ac1wOOTg4GDxnpumQcJCB5dl2WIcPNcQay1RjaSpBbrxWeptgnFxPU3V1Fy/cY12p8WNa/u0oiWmoz3qVNIK1xF1G5o+73/mAzzx5NPc3dtHxposLxkcTbny/AmTocBQMxgc4so2UgREUULTgJAaXWt81yYrSASXH7iPw/0Dut0uqrpnzhqPxxR5zvHxMaPR6AeyGZfodrvcq8czjMdjRqOR1bwpM2MOrSaxKGq2tk5xavs0ZVnhRyV16bJ7u2D9dI4jWjz6yLtA5hzMNjpJEiHErFt7dh26rmXX5qzoPKx8rg8UwkqdWq3WLC5nsIj/abfbRFFCEAS0ks6iPScIPFZWlmi1WhbUudbssry8DLD4Ob1ebyH9mDvCi6JYGL2CwJsdsyEHB3v4jkuvu8RwOGYySWebnAZlNI4LcRLMtKXMGNKKMs+IosD6BFSNkAptGoQ0IDRlXc6ylhVNPcuUbCxIE8LmsXquT+AHVnM6kyLMNxNCCFSjEUIyGAwB6zyfa5bFjKRwXWcRTwQspjK7u7u4rsvNaycsLa0yGec0tW0niwIfrUoalSJdW5SgMdSqoShrlNFkRY50bf6p1hohoaymCKehrvMZGHcxmcHXMK2OGQ4HFAZaYeeHhYk/PEN534+5RlUKiUPL+Nz+fkgjapRTs9RxCCOHWivSvMHUgt6yxPV67O2e4EtFMdZsXFylqhV7NwY4wkF5ku7GKnEcMUoFeXGCEB6ba20ccjbPSPZ3UpTqs/fmEa14xIWHfW4dNWhjWF0SmLomTQOMr1hZSxgfwWQ0xfc0rhPhBjAdSXrLtoat0qCNgxc6tJYqXN82t9QTKMYtkIrOqQpHKPBAanCkQ7hkuP6KoRUJfF+jCgfpKYZHEMc+rlsjXUOVxzhxxZ/8Sxf5zpf3efnbBYNjB39Dcv9yzO3bEacurHCxK/jWb7yOMYpJ2XCoNG1i6ibD6StaHRje9kiiFo0uWF7a4oHHnuCtl7/HtLhN0E64/nLO5raPnwh2b+cEvr2Bp9mUIDDkqaFzGupWjao9oqbN+LbEETm9nkOjCtK0QpVrCC+lvVXgJIppDe0Qllpd9u+OCDuQFYJBFoNICbVDWii8AJJEUGWGxtjRhXTAdW3JVhDaXlTPDcjKglhF5H6OHwgC4XD20hpukfHsfymQUhMZl8E456kPORwc9Ng8v0KoJMrzCM8+gJrWvH3lBUZpxoX77ufu7bdQZUZaKsa7gh/96BO0g5KXfvXbfPzcOW7dd47PfeEbbJ1PSPOCJAlZWTcU+YS7112QCrARFv6Sj2msGN+PQg7u7PDkQ09zfDTmhavfoxUkNqsvdNEajBIsLy8znhwjHXvzq8bg+3Yk1el0uH37NqoRuG6AHwiW+0vcf/4xvvu9r/FjP/4pXnr5Da5cfZY4bFPUBarx8TyHpOWyunyBw4MhZbNPGLTpLW9x6tQ2Rms++WM/zpU3XueXf/mfcurMNp7nsb9zmzTPcV0XpfTC9cpsgZ07DcuyWux+HcdBmllAb2CZkTzNSPPM6uOKiqJu6Pf7pNmQVhTi+glnLlzk+GSPN1+/SreV0O6HLPdPMRgMODkZUJaW1RDGkKU5ruvzyac/iJ8IvvzNZymaAj+JqPKa+y9dIgmWmWYpk3QPPzBcvXqTbr/N6ERgZIkX2I5p3w8Ig5jJdITExl1UpUJKO7KNomgxcptnVOrGBqJvbfdtEHNjSLMSITzb/bvUJU812p0tkpWht7zE1uk1Lr3jHEkSAZLJcJduJ+b41j1X7cd//Mf5zGd+nq/8l1/nb/3tP0FWRey9nfMP/qf/gX/+z/4ld2/lnDt7kZPJXa6+/ibg4bg+rS6kWQra4dEnTuPHFa+/fGwBcWWjgsQsFNpo+77nJowoAYGL0R5GW7Yy9DrESUjguUzSjKLKaWqNVpa5qeoc1xH4XowxalZxaF23URJydHzA5cuXqcqGnd07hEE8Mz2B6/qL41lVla3gK20WZZ6Xi3iVMApmES2WrVJKLQK156DDGIORBl1XCCNxHYdW28PzQybTkkmW4uIhHYPnxly4b4mTkxPuXE8xNASBQxgHMzNZOXMQz6612fU9H8POjUzz/1ZVNasptO1UQoARPmVhwUO37xNGgjJ3GQ7HNI2HGwoefWKTG29MkERcOHuZ2zuv00o2ufzYec7dr/mX//izTI5rRNOl3dGcvtjntVdu4YgQpS3LZa87geskNoYssLV2ZWENSjaCqaCoSrIsx/PcxabCC4MZGDazasXOwjXv+z6+4840sOqe7nDWouO6knbHArwiL2kaTZwEGGab36yh113GcSUnxyM8X9KQ47k+dW2o8hrXseH3jaqR0hYeGPODjTDMfpYPs47seYNV01g2bn5f2uilGtezm7wkinFdC4p7vR7D4dACME8uANny8jJZli6c4rWy9/nCNS6cmY4zX7jejTE4rmQ0GmGMIY5tgPk8TsnzAmjsNZq0YisXmBUmaKWo6+oeQJqBQmVm/ee1+T1RQ/PPmesq5/fKD8YN+b5PVVnG2MyeU8AClFv2NpwZnLLFz5Sz1ipHejiePzMvgR8LGq1oNASRZeOzLKNRFU3l2KBypXA8m3ox12EKIQgSG0eWTnNQhjC02m2MdYWLKsCLFY2jUSea3qkudaEY3Z38/jKUrcKh2/WIlxThZs6FDxnqYEpTCLSw7Q1KF7hODVowPjHs3D1ASIWRCjfS7N48ZP/WEZ7ngBQ0ueLkzgFSwdp6n/XNHlGi2bkz4mSo+NCHP4NiE+VqLr3rFJ1LMbtpjggkXj9CEQCCqikRWrBzY0qtRqxuOFRFDyNrXF/hx1OyvAST4HmSpKsQAlQVgXEJWtDalHjtKY6rSWtF4iboEiAhbXzu3jK4XsB4rHFcl8YokNBfdpCOosGgGkkrKqCGf/J3r3PzisNTT65z+kwPNQq4fvuY5a2SazdeZeRv88GfeQ+hbJhKl8CX1NMJulaYos3xtQRfJNRqxHCQ88ST7+bK6y9xdLyLND6bSxf4iU9/mKoQmAbbCFIKsnyCI2MEEikiyrymHSzhBxGmAlUO0Y3i8MAK7NdWQ5JWTe/+Fby1SyRxl/ayix9IpvWEJhEUTgCewTUFnpQYx+Mv/oU/xbnzWyjXJVqFppYYA1UJdelitENVKeIwwndafOSJj3C538fLPMjs5uOVrx/xypdtZaPXrpCyoLPmc/d2yOhgiEoLskJyq845Guzw9u5V3MhlYyPipRdfh8bj6fc9TJF69DZjRvslulzj0T/60/z76Q7Xb1/lwfc+wDAbonXD3u2CO9cMUbDE1vYK/c4qjjRIWSFTQZ9VvNSwHMQkUYdc1WT1AFeAoZplhgUI4eCHkml6hNY1GIkU3uxhosiyjOOjCVp5GGOdfFEUMBxlfP5Ln2P79DmuvPEmh4e7JHEfz0vAeCRJwGg05kc+8TF+8b/6WXr9hOODHN9rc+nB+3ECwXef/R5/4c//Ff7tv/l3BKHHnRvXOTzYhZkeaT7mtk0WzuLhBizYhvlIpa5r8qpEuPbBNTg+oSwroiBiOsntAhSHgKauNKNJTr+/zK3rt7j6+pt0Oz1UA0nQYW/vkN29uzhuQ68TI4yx45cgQJuKu0Jz5XBCYSpiP0amFRe2tui2VvnK177Kzs4untNnf2/IUn8JTEQUu8RJiJQuCE1R5AyGRwSzcVjTNLie7Vi37LAV7juON9N6acpasbG1wUc/8qPoBoJQEAS2gz4IPOrCaqlCNGudNXzPoyzGvPbCa9y8sk8sVkj3DXHY5mg8pLvpLF63r7/CV599gS9+/4tcOOMzmlT85E/8LB/9wC9yavNRVtdb3Lj1Om+9eYPtM5s8+b53cP8D5xmNUhzho4zC9/rcvqapSpvXN18Y5+YVuyGwYNf1JOgIpQSaFC8StJJlPLfFZFxwsG/ZRCvsb6ibAm0UcRQsWLr5uZ87g+cs3uBkNKt+1ORFOpOv2M2W5/qoRlOVNVlR4vjeLKjcodNt43qCpOURxd4iDmU+6pw7fufZfRiPWkmKUpF02qxtbhO12xR1RV0ZfD8mz+1I8+TYlhBEic/yyjLSM7ienGX/5QvT0vxa/kGTylwDV5U1Ta3wXB+BJI59fC+k1V5l61SXBx8PaC/B8FiRZ4o8HdNudfBcQS9pc3CzYTpsePq970TVI/buHDItrjManPDs147QTUy3vYxiwniSMZmMcF1BlEiCELQSlIUkiTtIqWm3OwRBQBxbDWPdlGRFTlGVs/F1j16vt8iNNEYRBPZ6DwKri8uyDEfaRiU/ChcVl/PrpdWyQK2qKsbjsd1cRQFrayt84hPP8KnPLLF5KkYry3Svn9I8/aOGpilm+ae21QgpZ7E7FiRl2T0d4A8CprnMYR6BNQf087HydDpd5E7OP9/3XYoiYzQeLH5PGw0kKTJ7bvv9Pvv7e4umGa01/X5/cW1JKVG6oapLXM/mhpZNBcIwHk1IomRWpaiRwgUpcJ3Q6hOljTaqSltBaTdOzu+ZDszrQY2AJGnTNPZ9B0GwmPJEUUQQBACLTM0fvAcs4LRf4/s+nhuQZyVNrRHc+/uyLKjrasZ8zgFrs4gzcl07Wk+6LhfuW0H4JWk+5eRkyOHhgMmoRM3ApJmZ4dACoyAOo1k8kZhl+uacPrNJd9lOehDYDN6ghXADzpzd4PzpU/zojz7D8a2CpPPDkY7w/wdD+dgnpRG+QLoCJRWtpYDhmz3e/tYR62tdZGCQMsNoxXTaYLRLHEbkZYUyDa700I0hL0qkK6lrjTQODS5rW9v4qyFG+7aiaixJszucO7+FxmH9rM/B4S0OTw7QqY8jDdN6TD92WY077BydIBqXnVuK+y4LNk+HVIVLUzmk2RSjJZWIKOshdelSayvE18YjjF02z2p65wK8YgyVh+7C4fUa44cU45qjQ0VTB6yuOjRFTpKAI0OEbMDUoCRp6mGo6IQWYCoC3n7OkA0cfvSPfZDKHLFz6zZ+z2NJtXj9m3usnF/CqJLdWzt0llbR44ppLlHOBB8X30uotaLf6fChT/wov/6r/4yNUx1uvD2gFa0SdSRHh0M21vrcvLPHuVNbDMb7COkjhEtVlKyf96mcGK9dMHgrxykEeaGJOy2mwzFJ0kaLjPaZDfJ6A7n7GqbXUI8NW+cD0mBCpNroVOM5FSNHk/uKB9ae5uTkhP3J6/S6sHNN0mhbQ4k0CFETBA5GgTQOP/2JT7D7UskL16/S1IeM8wndfp9qklNnEunUfPzRiLGOGZxohD6mdFy0u8Hztw7Y3jrDxUfWuPPmDkd3dym05tTZUwReyvGJxEUzfnuXBx96kp//Ax/nr/6f/w7vvPwQO7sj6moPxxNMhiVKKzY2WnS6EXmqaXUcxpMSo/qEfoBSKUpXTGvNqKzQpmS922J4PKXMagQSP/AIQoGZLWhK2VyNe80gEq0EruvPWMGUadrg+w5GNgR+m+GxFWCvri5TqSkYq9tptxMee+wxHn/0af7xP/17fPIn38MLz13l5KBimtvxnudF4NZUTYlqXKQTsdRpLcZAcxDZ7XbRWnNwcLAAm/Zh+XuDem0vu/0a3wtmUTGWzTx9ZhvX99nbO8TxbQyHMQZ0RaMlcWCbUaTENkfpChB4bsTK8jo3rl+zRi0tKIqGVivkZDTm8uXLxK0IR0Ycn+xydHBIkljJxt2da7aTW0mCILYmACVwvIYodNC6IUtLHCdAaBt9Ipx7OZUgZ41GFuALIVCVptX22NxqkWcVZe4jpMM0HVBXsL6xQlVntsddJBR1ao9hfxU/CHjsPe+kMtd54v3vXjwTj28NySuXUuwzuv0yV14M+eDTH+Hf/Kv/gOsXeGGKI2OM9smKgu3t02xcgJvXj9jbGeO7gg9/8JO8+NyrHJ68TacTUxbYKBExA5Xcc677gUueWYZleTXiYP+QJFqjlTik6TGDk5KVlS6gyMsC17FtKkZbx2pTg+vB2uoGJycnFKU9z9IVFHlFq9XB9aDI7c/QGjvia2yD1nxBr6pqZqwpabcjtKktC2caJAll+XvzNecRM5YhlEzSKf31JRpdsrTcI/A8bt+8SRK2qfKGtfU2ZTFjcExFOrHfQ0gbUea6Lum0QAi5CE6fA/DFaH0W1G2MsSHd8w2I08I4h0ymGkd0aPUKotglHTu2sUiUYDwqVWGMwDUJXphz36VzvHHlCsoI1tb73LmZkkQrnLvQ5sqVO6T5lJWNPtVUo3RmI1tMSKsdolRJPOvczotqwRLP77/xeDpjU+36N79Pm6bCDfzFhKGqKjw3Whg+oihCm4amrACrw5tOp/iBNzMSthZNM9Z0pdjePE3SzXjuO7u0OpZ9j9sVynhkaU0QeBRFRTFt2NzcwPMCjo4O7Mh7xnzVla0ynDNm9nhbZnQO9ObGoXlXN9gkgyiK2N/fZ319leOjAzY3NxmPR4wGQ5aXlxeu6zzPycuCKLJSCiPutfGEYThjyPMFILOAN1swo4508T0Px3XJSvv1QgjrGjf2d478iDyzhhu0QDgSOYsRshsSy1pmRY7RLK7p/2X/uOu6FKV1tqta3QNXs+uxqfWCwbRyFGdxD0VRhB+4KDVjQJtqdgzdmc7RjsMd6SEcydkL53GCgrev30TrgCD0rUs8L6nzAmYJHkLbNjsjrUZXuoK6rmjN2q7SNCcMIlzX1ltXZcbKyhLpcU3tjQjdLf763/pz/N//L/+IvdEe5c7vc7B599SjVFWbUmnCjsO1b8PuG/u0uw6TYQpkoB2aytLg0rHaA2EkniNRWtPoeiZYtWyWFgrXKcmnxyDaTNKcSpd4iSFqhdy6tcPe3lt8+XPfZufaCZEbIvUYlZZEchlQ3N0boVREVhi2Lxm6Gx6jqUK5Y/Abwp7B7whqPaTTjWh1Y3r9ANf1aKqafFLx1ksV0xs129vLrF5qWDtrWD7rICjZ2PLY3gJhGjzP8MR7zlJUBpwGLWr8MEB4mjjQ9HoC7TSMSwcRwv1PVmAy7r74FhtL5/D9kKPDETcGR7Tvy/D6+zz+gQBHBoxPCkzc4CaCVhzM+oux3cuy5nvPPkvoQVkVCBkxGAy4dWeIH7l4bsjycpdG1XaXIkuqEqJEQC24+/YBIjcEbhekpd/LYoITaAbDMWUh2b+2x91rrzA+SZnuOxwdaY52C3yTkB5OcHLD4LrErQO6LXjtpedp+yGbK0vQRCA1ujEURYURBW6gZou6Q1Xn7Awy2usbLAcpnoRlBeq4pBdDq9vwrvsEk5HisOxwm5JkawljIqRj6CUd7l65xu0rb+NSU6SaVhji+Ar8VZZWY971/j7P/PT9XHn9Zb76H/8h73u4TVndwHeus74EgRfhh4azZ7c4c2GVqq7JC8X7P/gA3WSNdihpd136nWXeed87eXBznYsrHTbay1RTC260sBoc33dxZIDrRJSldU17vpmJuDWedOi2Q7RJCSM7Dlzu9+kvx0RhQpbmtHsecSLIy8w6OF0bXXL27Hm+/o2v8Y/+0T+yDmAVceeGFZivri1T1RXSaWh1HFpJhCtCfOFZgDCr+JqzNnaR8qiaBsS9B6AxesZiusRhMMtytFVgURzatg0JcRJyMhwyHJ3Q6kgu3Xcax5eYWuEIlzgMaUxJFBn8YMZWNJK6MrMddYrnOaRpzmQ8xnMVdVXx6EOX0FXGs9/8Hg8/eIF3P/EEZ8/dzxNP98nrQ4IoxPU9hJDUjT0+dV1SFPmscxdarR6qUpRlzhPvepg4ThYLixXL2+Mwb+WI4xZ1rUizkjCysTRzYbrnS8bjKccnGTWKVJ+wtNGhMSVG5jhehfAUrmd48cW3Fq9rbx2AHrC+vsq1axnjwTV+4zf+NctrJUm7wnctC9Rf9ljb8lE6Z7m/RToW+L6HH7l8+SufY+t0yOkza2Rj67KeGxIWmtAZW1WWJZ4PdW0zOB999FGWVgJWtkLWNlZZX9/A911c31tU84HEmW0mFjFEzTw3Ty4yJsMwpmk0qjG0WlbnNhyO7O8wW0rsSM6aP4TR+L7EcYUNonaYgSDrFr/HRPkLRqeqKqra6rs7vYSzl84iPIfxdMRjj14miS3zJLDaS8vuGaQjaFSJ68GcALEVonLB6PzgWN72RQeLRV8btRiZTtID0okmDmI63ZAyTTi86+GIGOEUZKmaSUFcQi8kiDTdvub6javEiaDbDagqRTvpsLWxDU0LKTNW1iPe89SHWFnu4hDguQlxHFKWKdIRdiJX2zq+PC8Zj6ekRclwcs80ZJlXC5Tqurbd1bOGmbKsZ+fynlGkqgvKLF+8f7DVf1mWLZ4FcXzPRd/ptDg4OOCl7x8TJrPMRVEzHtWUhUaYkPEoQ5U2xshKSfSCEWxm0TrAIj5nXtc4D1Sfv+Y/+wfNWHNtqCMk+7t7ti++qSnyFD+QGFWidE2aTWhUZXMkZ6N1z/MW7L1dV8SiK3symSwYThtf1MNgdY9mBrR911tE/yhVImRDmo0worRRWR4EoYvAIc9mRh3Pp6obtDI2Aot7bWLzZ8ucnRXGTj3mx2NebzsHsfMczjlrWde1PTbS/B4gPN98zZ9fRVHN7s8QtKEVL3P7xgBhEpQyVKU9FtooHN8+3z1pn/HGGALPo0gzoiDEczzSsaIuDI6w8WKWAQ/wvYjxKKe/towbJERxwV/7P/wSk+E+UaJ/WJj4wwPK0xsfYG3zEZzQJULg5woy7ELiGMq8RCFABjZUVAeMpinC0Qhdg6kRs4eOURIaD4EgSUJoCo5v3yHQAtdIBAWB18KomHaywYULp2iHMWWWsr6xjRs14Iw5PjHUKI4OSlotl064zpXvC5ppgM4SkBM8tyEIGnpdadF/MyWKfNotWFuFxJWYwlC85nB0d4poBSx3eyx3AmLlkE0KWqsxrZZkdFRw8/oN4ti34bcuKF2iFaigpsgtgG4lLnUhcFsRy+ci7r59nf/5//br6Okq952+hKSBoOb2JOXZa0dsvbONLjPyoqGpc/LU1ubVpkDKKUWpuHHtDRwZououSuecvbzNo0/dTz5p2Nnfw3MD9vdO8NwESYTnN+hGc7A7oRV0cCYtsnHO0uoS3W6bJOwShAFhIkmSkNW4w7LnEq9ErPRqlrqaauRQ3HWpDl1r7ulucv78uxBZQH9N8sZruxzfaiOkw9pymzB0CQJv1r1rNW6WUbBu4jEl5dGUVDv4Pej5HqWJaPlLfP/2EjfVEi/efpOB0rx24vH6QLFbSGrZELUEo/0hYSBwQmhKxfR4xPHeIVXWcOPGkNv1kGRNknpLNG5NUxS40Qobl1oIOcaRIaPpEdfeOqYoXGoz5LvfuoM2Cr8VIxqX5XabpspwpU+347O06pKpBsMsiFZAUVSkaU6el5SFQkoPx4kZDDOMdhlPc3BcfD8kLwvKqiIMC1RljSCu0yaKDPddTkB5tJI2WTakacbcuHED1WiCqMFxDb/7ha/gBhlX3n6NrKjodDaZTKCpPHTp4IuAVuQuRk3WcWg1TEdHxwxGQ3q9Hr7vzyrX7EjGEYJ6tlhJKcGR1FoxHJ4w7wD2PAeENXxUpeHOzj6TSUYQ+ISBQ9PYTtyqamY77oYkaRN4IceHQ25cv4PjeDzwwGVWl1f4M3/mz/FX/tp/zY/95I/x4z/1KdCCJx59F8e39nntte9xtD+mzJtF1Z0xGq3AcQTtbkgrbnF4MCabavJJRVXaaruNtXU6nR6nT59eMK5VVduUgdmHEArVwGjQgHDxQm0rXYU1sNTTlHbo0437BCLg4csXUGVF7IU89OgKh7u7FIXP3tu3F69WJLn61j5f/k+v0Ew0fmwjsuaxH0pVNConzxoefvApLl28zPPfvcpocEzgN/h+QX9VUXEAzoTuckw6sRo71/FpavsgT9OM6TRF4KBKu4gPBgPe8cj99NcNTzz5MBvr72AwbGapCjNmpNFoZcjzkqpsOHPmDBsbG6RpSpZPF3mG84zLOWC0juKKKIxmpodgpkGUqKokDq1mTema6dSO3TAeTS0WdXoWxMeL620+8pZIsklBOszxtE+TQj3VNAVUuQUqw4HV3ebFdHFdh6GP40g2NzfxPRsY7vvhwgkMFmzGYYAwGt1YMJskCUmSICTkRcb9ly6ysb5NUWQIUsKwJAhqpuMTVGHfr1bOQsKS5WNct0WYOMRtH2UKolZOa8llkO1ya+81wqDFufMxt6+9wK2btywgdzKkNyYrK4rMdqM32rY2ua5rQ76FS+RHGCMwSiOMXvxbFNkubHturOZ5nl+Y5VOEUZjmnuFj3o1eFBW+Z/MmT44HDE6GONIljm3MVqMb+ssdBC5K2ZSMKOhgGoMwisgPF+ccNKBptVr2fEbRIurnB7NSLaBVi3M9D8GfAz8pJcPh0MouMqvzXlpaIs9Kbt+8hZQQeJK8GIOu8KTdEHquNV812pIT3U7vXv7j7GN+/UZRZI0tqR2X93odHN+hUhVVXVDVBUZpqlwjpcvq6irtdkiSxItrZ/7fuUQI7smE5gDWdX2UsukQjhPYP9cao+00QQhwnPkxsvKjVqtFFFvDUZZPqZtyoZmc6z4tUNczpjXHGHvcWi17D3W7XYIgIM2GdDpLFsR7CuEqosTHcexoXjiSSjU4voN07HpQK1s/W1YNUiiaSlEXgiRYoi4E6aTA8x38QKBcxXiSIkWXj37gEZ555jHywQ8fbP5DA8rrb1xldeUyXfUIo6s+nX4PYmyIp3Qohj5KSJSbI7F1icaFQpV27CkEYdDCoPCdBoxBa0nTGIrUkN495ujmHrtv7yIU5EWK4zfkaUhtcqIeNCrk7Tt3qIRES0FvuYMG1lYNTRGyf1fQjfoc73oIkdOOA/pLMVJEOMLF1CFJK2RwlJOPQ5b6Id1uw9ntmElTMtpTvP2Fmptf1FRaUkeGoOUTalhd03juTICLwQ8A6SIch7pKcCU4EdS+pBYFrqhAlbTXSmrt0vENN196g432JVZOPUjU3uL8xiZuucmd5yWRCXCVh6MUwjgoND4xiReQV1M2TtXsHxYI3SOQq0R9j7AjqauS9qrEi1KiGHCGuL5i+0KNHygeeHCZfk8zPNwlSiRvX9snzRom0xMc0cJx+3i+QXkuTtgwEIIiD9GNR1Y1PPyOR+mub+E6gnNPdcmdLuNpROWvUftweCcl3dsgm2g8R6BNhSCkzAWeZ/U2eSq5eyOlKgWvDFKKymVYbLJnIiY6YOdwgJIFJvJ57PI7YCo5uVugx5qTndtQaLQb48iQvf0JYadDEncZ7aUM7x4y2h2ijkLefmnAyqmYZ793g6XuJTpRyO27R1x9ecp05FqTiqwompTRuCaJl9i5VVKXHtlEs9bStJsD8uNDsnHF8Fizf2eE0hmez0wrGYCxzlulZ126ORwfV9S1RGno9dcYDAqyTDAa2XFLOsrQKqcua4osR4gSYRqKPGU0HOKImN6SpL/asLbR4fDwiGee/jB/4hf/FMPjKVU6JAl8Atf2+E7SKXgeucrZG+7ief6ihWQuGBfCZqYJnMUoCXFPR7lYiFUzc1T7uIG/GM/UWiFwwEjKHFTjEIURvutT5RVC2F5coyAKO4vgXq01Ah+tPJaWltnYWOUnfuHTHA6ndJIW/8+/80/5e7/0T9g+dZbrt48oPIVQmqsvTmnKhrqsmAwUCnt8pWPAOGRpSRx1Z7rMml6/TRy2+OxvfIH9nV2uv3UdlKYsS+LEJwi9BcBpqAnDmLqSTCcZYSyRjh0t+b5PGcK40Iwnx+gy4/lvvc6HPvJhdCun8Husra1RjhuqgVm8bt68SVmWZIMcXUESLyGFZca63R7g4LsBo9GIV199lTAMGRxN6PUC6tJFmJCl3mnevjLl/vuewHFj5MzJPddyNY1agIm5lMHzPKYDw2svHXD5wXdxsDflzs4ulx5q0et12NzcXIzb5uymdG3UTLud0O/3Z4unshmGefED39sGVbe7PXAkkyydGZ00VdksGNNer2WZH98nCltMJwVGW9ZwzojNF/6maRbu9SCUrK8tkQ0HHN25g1Mp2mGL44NDO4okm8WnTPGCZgYsYspCEgV9frDKbw5e5mNEy+gUM71dQxj6ZFn2A3l/DqPxMXl+guu6TEY1UtjUCykAU9E0BaoRCGFBETrgzddP2Njq4YWC6dCjyHs89MQl3vFel3FWEEQ+5cRl/+aAVrRGkeWYxmNwbPWPSdcCqH5/lbIsWV1dXeQeVkVJ4PnEUUQUhkRRwurqvXQCpQxxHC/enycdkjBagB3Ps1rKsiipZ5mTTaMxmoWJB6Q12BWVleWoEUKCNjVKW/NVEM7lAtYJfPL/pe3PY23L7vtO7LPW2vM+853f/F69KharijWRLIoiKUqiWpblKZZnuY2O7VbcaTsG0naCht0xEgQZELcTdAI7CdzdaMdxty1DsizLsiZSJCXOLLIGsqZXb3733fnMe157rfyxzj1FAwHCP5QLHKDq4VZdvHv22fu3vr/v9/Mdn6JXClbgK5SULBbzdaDvPHF9fh85V9Q7nYTBYLBSuKecnZ3h+z67u7tcunSJwWBAHMeMRpv0ej2n2mFodImnnP+1blzRwnnLz/k97Vx1/UEf8Pl7ez5o1nXtDkZ1Ra/Xce+thEB5GO3sRqZVlEWDlD5F3tBqqEqDNe5+GCfOE9m2Hyjf579nx4yM8FSENYIwSFfrf88hyKJoNUieD6YuILlcLjk9HfMn/sQfZ3Nzk7LK1y1S56v78+Hb97319d22Ld1u10HuheJs8tgdNIXBkNHaBVZkXLg0YPdCH6Ql7aeEcURvMMAACGi0Q4Etxjlt3a4ObpO1RcE13ymW45yNbp+akh/5kR9h6/JFTFv80APlD+2hTJ4a2CsXL5EvfKLmLovFgrhOaUzj4umNpD/0MbZEoKiqFtOK1Q1IokSAVIZlXtMKQ3Ea8qf+Qp+/8jeeZVpk+FrSG23xn//Nt3n1jWP2ntzECy2SLio4obFTpDTUZYvAUNegfEmxCBhuCmYHIV5VEduEZFQQX1yiioCNmz5pJ+PBPcFy4qb4IA3QVZef/tyP8Wv/9iv0L5/SNhGirYj8FrWl0EWLEjH7bwV89o9F3L99yqvfbLl6OSIIFY2fEUVQLeD+bcmly6vwg04QskYEFi1rerHg9rd85o9ahDW8+OOvsPvcZcYnM47236OTWIop5Mc5k8M5ofSwAdS2RRiFH3o0wuJLSb4osE3sTpbKnXJcqXxNGkqEqlEMqfWUIHQfxGtPJRzttzy6U+KHhroUxFGPMIamqdjavITyPB4f3OKpjzzBrTv7zGYLLt3YpjUKW3iEuuRssWC4e5GzkzF+Ijk7XRIqS+VFjLY8qoMzotSgU0tRKjwjSdIGEQDapx9usrf9El//7he5fO0pPN/w+GCffFGzt72F7Wuq4xnd/iWO5/tkRwtGvR1GOwP277+HCgMu7VxEez6zyWNMZaCKWZYT8DcRxRiTBHw0EAw//Ale+bEX+H/+43/Ayb5hc9jFekuK3HL15g6nD+ZY3YW4pDvwqTs9pMoYNJsUJaSXN2nKMx6/fxdhCvIsIvF6LMolRuE8S7rFQ4InaS14QoOVWKmxTYjnGbYHCR/dzPnm/YQsrfDxWM5yNrY61E2MbQP8eIFupyzGIb1ByMmhUw0vXd7lievP8nD/AQ8e3qM7rEnTLRYzSV6N6XQ6WJthjYfRFYHvuIzL5XwdxrFScPnyZU5PT8mzimgFn67ywvnqUBjcDVP63geq5Gp16fu+68P1A3Qj6PWct1DgE/oxW1tb3Hn/bYSoSMNt8jJDG00Y+igJbal54soNJqdnHEwXGOuhbUnaD2jKin6/C3gc7M+QSUW/k1DrnMEwYTmDprb4gQATg2ppVgl1N1y1XL96laqqODo6cuZ6ISibcjUYeVha2taFM7ChU3T8FkvLhb1L1HpGXixoKp9ed8BstiAvBEjLT/30JxmfnfLdV99ia7RJuJHSFBXSfnAGP3o8w/cF/UFC4EnSjmYyPqVYJFy+eoXtnR2++KXP09vwKQqFaWKirnBVh9Mpxko+/KHnqStNkS0oiwVSNSwWGZ5yypRTm+26VlKKgLATUs4WXLr0PJ/82ac4XdxC1SFp9wJNkeHrPl/98m+i2wwrBhRlg6BBaEUYJUhhENTkpVNDG+1WxNlyifKiNWLn3Ifbti2Hh6cuZR1FVJWzHgTh6kBiBFpDVdbO+L9adTsP6wddxsCqzcjHnqdKZUvVVPheROCnLLPFemhxCJd2vXJXStFqu9pyrbxsul37+MqywgvkapXo1LWi0s6b7yer+sQ9pospZ6dzut0uui7Ww7vRIWHYYk3AcjnlmWefZDYt6fS6jCcHTCZnvPyZTzIYRLz19uvMTnyaXNLoCYGf0Ot1qIsWzxOUzQlxsEfakat1v+Hho7soqfjMZz7L7dt3mU6nRMEqHa8ALIts6QDgBup2lQy254qYIvRC6rpcHxhdgrhep9u1NoShj7HOdnCOiKrr2nnklnOq2rGFhec60JVS0Fra1l2bTi2TdDod8nyOaWuk9Ggq5/1s2ppqNWwlQYivHKweyZqHu8yz1UrcqYFpmqKkz2xxSL4weCokjGp0Y6iLCumVq1khZJE5P6KxAi/wiaJk7Q8/PwSfq+bWWqJV28x5Wvzk9HjNQf2g3hGKusLBEz6oWtRas7W5jVKK8Xi8Pgidp7PzPMfz3O+42+1Tl8UaV5QkHUBSVhUIQ12XJN0E3zoEkAkk/qIg2h4ynRf8yZ/8E6Tblv/hv/9VLCHStyhZouiCqFb/3xjbuEBkqwRGGGgkaTRA+JY4DpkslsQDaLQCW5Okmsm4pdNLKeuWYW/IbDJnPnW4NGEdB7SpW6w8T6JrrIAw9JGeWA+vja1J4j51ZSjLnLTr4fmW8a3iDxZs/um/+rzd7fo8eE+Tl5rs0SPKfE5pIApjyrmh0w0JwpaqLmhq19IhVvdfz5M0hYemRQQNk8eSf/DfbPLH/1LC/mOFZzMuXnqOv/ULB/zKr0zZvipJkg5tE9CKBXCCLzXaaHSrCLyS2QSELxlt+hSZR3bsE1Y13Qsl6XZAoDW5Mmz2h9Rml488e4UvfOGL+Klre2nmfcbHCfH2CXuXa1ppaDKD8DWDfsA4E7z+mxXXnvLobd7gZL5POcvYu+AT9ASB0gRI3vy2z8VLLf2hxyzL8XsgmwiTC8K4Jbc+7/+upVMFyG7GS3/0M+RNSydUvPH2V9i4ajl7PaaaNFRZhfE84jRA4lM1OSry6Q26zKcLmiLHtuApl8RNIh9jNKNRhBFLlBJ0ki4QsVjMqJoKU8eEfogXaMqioc4tW9sD9vb2+OY3XmdnZ4taLpnlPn7skSSgrcIKQTcM6EeSvGy4e3/Ck5efRPuGd969xcWbe4goYTF9gFdLikVNPPAgzikz8CNBfwjFYcynPvUZ7t9dcuvuWy6AZkt0YwgDd7ILowgVNRipacousi3pxYrBaEgmFpwczlFGUI81bKdQVijjg65BpWT5mJ7t0A0ifu6v/DXuf/dr/N53vsJoo4fwDUKEyHaCtBfYXxzSmgKaEL+vKdqGTrpBvSiplcTvRkRaUp0cknoj5osJy8IS+z5V6xoiAs8pRrXWJH6ICobUzQzllSiTYrQl6Qh+4sNdXn93wdyDfFkiTEB3YFksC7Z2hyDmLM48FnOPILREccD29jZNbZkvSh49esgTN6/yiU9d51/88y+yuZ0QJi1Hj5wCI2lQouvSjnXNxsZwvfYOw5BKN2TLgk6ni+cpZrOZU9OFe1hbAY1pCfwIqUBhV/4lQZp01+vs1kqGwwFnkzFhGPEL//Ff41/+4i8yOTvEDwRt5aNNgx94lE0JrWHUHxCogEHaZVqU3Ll3n9H2wJnQm4KqtM7g72suX3mSIGo5OjpBN5aqdp3GnWSD6fxovYI89xq1bYtCEIRuvSqFT1ZmSCnWyB3n5VoNJa0CU6E8i5IuZZt2JVVVUOYuEZnlFSCp25pnnvkwRbngwYMH9DtDLl++yCJfsMwX63vidF6xu7uDbRsub1/i+OA2datZLNwQqwJoTE2jBUokaDRCGIS0LJdLwqhDWwuqvCKOQ6JQURZ6rQClnXjdgNI0DZ1Oh7pcYJVP7PtMphk/+x/+IS4+EXDnnUNOxhOGYUIz93n3jXfp9RMm0znSC5hPpzzzoecoqpzx2TFCOAuSNYJ2NbGZtgXxgcpYVZrhsEeapkyn49Wg5zA4ee4g4aPRiDwvaZrzVpYPuqOd9yv49/yUUlhaben3h6SdkPfv3OeFF55mMc85OZmAFSvguBsCzhWUc2VWa7MGp0spaZsPhqlzQDWiJopCdOsU6ij2yLIF1goir+vWz9R4yqebxhTVHKMVAo8g9KnKmjRN+dRnPsa3vvk6y7wgTfpMpsdIz4W/dJMTeH3H71NuVTkcJQhKTg4LNvcs01MXogpCwXLu4Nhx7LNYuMpTzwvQdQMYev3OKoT0gbJb1B/0Xfu+a1OhZR0aObe2nH+PS187oHWSxigl1ocDx4ecoaSkrjRp2qOs3PdZa1kuMkevCBVSGrJlTb+3Q9NUWOtaYaSICSOF5/tMF3MAYi+i1+0ymY1ptAvfIv/9gNRotLlW+OqyWWOPympGLx2598OO3YfKur9nUbpwStrtudVw6z4TUom1V/rcV2yMQckPPMdOZFHIlfXj/Po7vybPQ03gZpdzKkCapgjhUxTZignprf2Sxug1lklIqIrGNZ9p1snz2WyCRKACi2lrGg/6nS46M3zqlU+xP59y+413wMsRNiEKFU1ZoU1OGHfIiwJPxUhbukG1dfe7OIjBSqLUY1qXRL4h8Cq2r6U8OJrRGElISHYINooJvJYkgdl8RluD7zlMlVRg2pUVTYLBEscOIaZNS5IkzGeu01uuihSa1tkNmuP2DzaU0+pHRKnk7Oge3UHM8EIXL4SRjGm1k2rns4KmaXErBLXi4bkTk8UHZVBSE8mUQBg2vc8hZ8/T8QNGu5tIL+d4+piwZ4iCIYE0RFFNY0F5Gps1+EYSp5LIB6/yoUrRRURgh6RDn6aqEJGgrEsWdUtlYx6dnCEHM+ZlgIp9jNXQaqQ9RbSHjNIU5QkaWePHLYEOmd2WjLrwp3/+ebzpBuPDYwZpy3DkfIFpF/AFdT0i7hvG+xCmBj9UKB24k15SkJmara0Qj5zLT+7xf/w//Vfc/vabfO1ffYnsQPH0M5+gaT1kUDJbWFQQIyWUeYW1Dd2Bpa4sDx+fMZ/X7F68wPUnrmCMZmszRYgST7ZIWRKqLm0Vkvg3GB8Zup2IQb+D50uENAjboGRNt6s4PTnm+PE+w16I1RkqCrFINre6IARh0GNjtMXR8YTbd6ZU2qcFlm3OYDQgjEImj445uXdA4nVJOn3yTCJsh1BdI/RHgGFyLJkcwwsvfJLhcIM6b8mmC3w8fAI6SYi0EmNyJvslk0c1k9MxpbZsXtjh/Qf7eL5b+7S1wkt9wtzHthE7e1063R5NWZL4MYWuOVzM+cX//r9m//Xf5BMfusaNyx8mXpwyPsm5e2R55+Fdgo0B6c4uFy4N2Bnt0vVHNI9rbl64yoXNbfwaYuuhiCkySWQjehs9MC1pGK08Yh4i8AniCIUAe4LAYBtJlBiiCM6mGf/6WweceaAbgRDOW9pq2NnpMj0tKWYRSdynaTP+1n/2n/OjP/ITnBy5mrFaT9jZ63I6OebX/s03GQ57GFuBUfhhS5FrBAl5ka08TC2TyeTf8xnpqmbQ69FNXM1a0zS0WIIoXD0c3U2mqiqqskFrh7MQfID5CMMYsBweHhF4AXlW8o/+0T/i5OSIWhvms5xFmVM2NWVds7W5QxzHnJ5NaKzhnbu3ef75Z/mZP/wZEIY8c1Wd/X6Ksc6bt1wuuX/vkPksQ2tD4HVdHWQxp98brtem5yvscyyHkh6tNiwyt2aNk3DlO3KDSBrFHyTBpTPoCxyTtqlblIxXSoYiDhOiWBH6kOcVs2lGmsY0TYXwA5eUr+r1KxUCky8p5kvqxqKiDvNFThBKkIKo45Okvlv/C0m3F7G9s0lRuJu2ryQXLm4jRIvAMegMBYaMKLEgKjzfrc+SJEE30OltoITAUiI9w913Tnnz60fcevMY3/h0OyFFdUKYWIxpiGJnQ4mjDrPZDGNdmEMqRVU21KsgSJp2UdINI1VV0OkkbG0NyLIFZ2cnSN9D+t6aG+hCVzCfL6lrx5889yt+AFd33c8uzBCu/zuEYbmc8+DBAz73uU9z48YNxuPxOuRxHq75YFB06/mmqdbfAwYlWA+qvV7P/blwnuCirsjKU6zImYwXLGYNplUs85KPfeKjRGHMhQtdwkjRiy9Sly1KtSjpsbN9FYTlvTvf5/j0FESLFTkXLlzAszVe69GPN0hCha8spoaqtGSZC9PoVYVlUThSQN0UWJpV9aTrIe90OmjtsDDD4QZ15d6H81q+8yDM+VfTNLR14+r0+GDtez54n3sYq6rAYjCmpm0bkiRy6ly5IIp9oiig002IopCtzU2yZe4On5FDFDWVZDEv+fGf/CRPPr1BrScYExEFmwRBRJDELIvcDZ9qxUBcWRyEEBjaNeoojp039+TkiOl06qDtiwmCmqJ0KDfPT/C8IUoOsDbEWEFVu3CK8h1j8wc5uufrZyEEs9nsg1S/rrEYRqMRYRiu6AaOUiCQsKqhFUh009Jqg7dKpVsr2NjYcivhecbW5h4XL1zFGsX29u46LKPbhqqxBFFKEPq0tiVJfYpyjrEVceIR+RIhFCZI8JqA5dTZf77x+uvcv3sLT1XQKpcnIUCGProNKHNB2wjH4IxCllJgwggBKK3p9TvM24K4K+ltR+xc32Q4SlBVQHPWwQLXX7rG7k5KawrKsmRra4PNnT6NKWlbg7Xx6rDtQplBoJDKhRjjOETrmjBw6XhPOiXYk4ZB7wP27v+vL++H/cbdvT9EFRS89JNbHB5p2iqmEVOUKUEolPJo6tU6w5ynTd3No2laqpVXBOVRaUst4Bf/9XfhkkR2ZphbcPPpliLrEcQNi+WYxiqsX+MNTyiXKUMipqcT6Cekqktvs6Q0Gl3F6HYJQUDSj9B+iW1WUFKTEQtFsOzw1jv3wRrSTYPREpFKLj2rqeSMoOzShBpfBdRNg5E+9x9qDr37bDwnmWQZQj7L9Scsd+++Tlx0ef+1JfNHE3b3fLKFz/F+w/aVPmU15vihR39jk94wp9Zj9j6yyUef/RC/8ou/xvzU4Av45ue/zMc+9xJtNaCfhjys7lMLHyEUeBWWAERIkkAnTJhPMg4fzxj2Bgz6m9TVjDhOUK3P2VGDlJbeUHJw+ID5YuZSyTLBaEljG4y/8iT5HlEUk9cNUcex0IRSJB3DvXsPMCYh7ZScTcZYG6L8hrKq0MZyvJiSZF2E0ShhaPKG5ZlPUxeEkVt1tOWCyIQMNze5dHOT/gvXVycfj1aXjAZDpLJUZU7TevhhSL1s2Q0jnv/Ej1L2Xuerv/2QB3c9Ll+7QGJapkXJpesh79yeshOFZCWoZkiYTknMBE9CniaEquLhO0f8xV/4M/yhn/sp/vpf+l/y8oc1o/kJrzZ99vb22H9jn85wA7ETcvY4I4otk9Ty/cd3uHT1BvOjEu0XeIMN8tMJW+EQozI0rllFCIFpNNJXVFoT4oEReIBHQJEbEqvYiCzhYJdymYMC5bmOXSUjqrKkyqFYGp79yIhOP+A3f/Pz3L59m6OjMxdiiDXLRQnSp66EA6fHgkxDJ+2iRYLnQ+Apau0QHVVVudYUpZjP54BkPB7T6dREK65a2zZkWUYcp5S1e1+sNoiVAuXWlOBM4qB1hVpjWRTdTofWauTKm1nXDdHKr7gxHFKX7oTtBT7HpydUVctkNuVsMibPSkBhdcPW9oCr1y5ydtZwfHxMnPh0O0Om0wXKq2k19LojLlzYYjKZU5YlOzs7ZFnGaOQqAU+Pjt36X4DF/b1uPHGZ+WLBcllgQkNV1gSBj+e1zpaAw88sFhm9Xh8hNUKC8hRBFNHScvv99/nsT3wKa1u+/MWv8/D+Q5QwbG5vre+JD/ePGc8X6Krmte98l+tXtvB8d3BotSSfV4Spx7DfY5EtaStBTk6vN1ilfQumZ2M63YSmqlcrXR+sU/Ga2iktQaho2xpDS9W0pJ2YcpGRBIp3v/M9wkhw+eIm+X7JnD18aUiSGiGgaWs8YWlFy+nZAVZAEjuWapp2MFYwKWYURYVZ+e7O1a9+v7/uiV5X+XkeeVnjK4XvKbAK5QyIVFW1fvh3u13HrSwywjBcp3Q95daSk8mEJ596is3NLf7dv/t3YD20bmgtWMSKh6jWQ6haefUW84zwBzAx5x63sizJspLQ+ESRawsZDjeZTueMNrpsbg25det9yqXHm995E0nLg9sTuv0BV29skOk5QegxP3U+wVZoHj04o9sP8GTCclHQ6RjCSNJULtih2wKlLHleuWss7GJNhJVLHj5o8FWHpilAKKf64EQea+3aE3euRoehC9I0bb0exrtJSl6VSAtSCaq6RErWDM5zLFbTiPX621iXbPYDd1jMMleZmXZirDWrgdcipEdRlbzyyit87/tvslwuieKAqzc6vPnGklYPaVtJlAxpihbpa4wt0DpepeAtvvKwxjCZzugPu7BKm0dxSKNr8twdCJxFRfPw4X2UqvDDXUzrI5VBejmoJb7X0OQtxhqsdavowWCA7zerlpt2rXpOJhPH8VwFnJbL5foz6epKw9V1cd5Yo1YEjorzdhz3Z/5KqWxXlomSonBNfL3egMFgROCHDAYj8nzJYNBDeJL5YoqRDZujDSbjOUL4lLnFGEVT5wgZ0BkmoMd0t0Pm0wnzHEIihKepC0USNWRFgUAz2t0j08dcvdbj+I7ALEpGXoRuW1ePbFryyYRYgl5W5MRUYcujBwsMPp7JaRZwZ/k+F3Z6SNVSN5LZpER5gk4npfQqqjJHKQFyxcykXVlIWuq8BATOmeKA/9KHNA3dxumH/PrhV95/+edtNj+hKApO7i8oizMGwRxbLfDbiKKxVAVsbId4QUWRSUfVpwarQCqkNUgRoe2C6Sn8j/7sJn/7v5QUVcSyMAw2Ev6TPz3jdC7p930evT9jY9dgfEVlM4bzTV566SO8fXifPJ/g+VPnYVt08LubvHXriKuDGeGuz3Je4QWCKFHEBprpNsXSMsvH7N5I8eKMtmxBhYjYKacilYQKoiDg0a2GhXBK0/SspptCawe0VQbGdRRnxxp9qkCvek0Djxsv9rjxvOHtr8DtNye8+Kk9+k8uiTpbPKU/zb/9Z7/C1edu8ubdfRaLGbEXQ66Q/RlV1XBpb4fFImeeLbCiR2sK/ACsCdBVTS/dZDqdMhwlIFpkC1K4QacsBL7veH9YhR+4GimnqPkgCuIkoK5aZvOK8NxrJBWtCDAWrLDgC8pqBm2E0DG7V0OU0Dy4M0EKn066QdZM0DZHtbHrwQ1KJAEq8Il7lk7scXq84LM/9sfZ6j2FCY54eO+Er3z914ijAWVVEXdDyqohShPy8YILm9ssg4rnf3zA+986oT/ocDxv2dzYZfL4HTqDmPGDgAtbWzw6uQ+th1CSD31ccvxIMr47o5QQ+jVPXn+O+29lvBA/4pmXKuZZwL95NcL0BVvDhuGFj/DgziE7HcWj7Iwiz2kqp4KFIqJYLBFBB2OWPDG4zv7xfdCGxlhaKaEy5GWJHwYoC0YEyKZAKEXhKzzd8qmXL/Pm7TH5ZInwNE1tGW2mdKIdZot9olg535dMORnPCbweL3/swxjd8u1XXyPtKrJlRb8/oDtSHB6e0e/1kaRMJ4ek0S5ltcSaiiiJmc/nK76d48DN5/N1CrLf7bHMM5qV6iFxqqQfhlRNQxo4tpvy5BrUe66YuO5mB98Wnk8QeGvYdhTFZFlBv5uQZRl7O7tMpzPHwJvNCEKfvCzxpCSJu+RVjhIuIBRHPjeeeIpvfuttvKCgbcWKs+ghhFNyzjuNnfHernucz9d+5+svawXQkqQBjV4BvBsoioYnrj/B/sE+Srm6NOVZPN+yXGj2dnfxo4ps6RSxRnvUtUvCzudzpHDYkzDuYHQF9gPVCBPSmoqmKmkrwydeeZb9w4ecnk7xVJ9eP0Yoj26nx8nZQ8aTksas1tsrXxhW0jbnfjifZTbFWkEcpWh9XnO3ChmFIUXZ4omaKOxRFfmq2i3g2Rc3efv1A2Z5gBSGJLYIqen1+mityfMcrFsLDgYD8mVBazVpr8t0Mkfh1txb2yPyPF/71c4bZvwoXIceAj+iaTRt3axqBZ3Cdb7KNMbgex+Eu6rK1dZFUcSgn5LlJf1+nyRJePDgwVohNq1FW4O1BgkrpFGLkh8s0nzvvCWlXCeMz/1fnueBVHS7Kf1+n5OzCWenE37ypz7Nzu4mv/3bv82f/4/+DP/yf/gNjh6eMOj1WC6nblWuIjppQG1qljPJ3tWUyWTifG1hH8+bkWeu9zvwE6pS0+gaTwrquuXCxYtsb2/zjW99lygRYELAc13ejbNSeNKntSVte94F7RiaTeNab5bLOUnq6kXrukav/KftasB3lgG1HqSTJKGua5pGI6UbpAajaO2ZDIIQ3TgPph+4UMvG1haTyQQpPZaLktFoxM2bN7l9733Ozs5IIp9mZV9COX9wVRf4Xoi0imWxoJd2EVZi2gZrDV7gUDpNo7HCrpFV5wcQa+2a49jtDqmbDLXC1yVRTFacsrWTMj1pyaoFQRAwGAxYLl3Kv1lVHTpfrlqvttM0pSzLdYjNfZ7E+mcHQfQD8H5FmqZ4nlyHus5tEwB17axMaRo7RJCVxHGKUoosW6A8hxz60LPPkcQh3/ve67RNs24OOh9Ie1uKbKnppxFbuyGnQY0+MJixQnkJSz1zjWOqpi4zdNMnGSm6FyWVmrP/miCWNcNuSqUb18dNiC9ivFiBZ6k9SakmjLYuMT44pa0z2sq974MNt8koShc48zyPMBLEqUC3Ocf7zeowAt2OsztkebmyDzQEkUIqu2JiGszqEFQd/gFzKB+99SbbI81WpyUp5+zuRIx6XYyGSlfrdOc5KFRK96ZjXZdwGCh8KV0FlA1RScwXvnTK6azL6XQLZVtsMyLPGpQc8+jhIc997MNEPWhz49aJfsPH/oMKoRrOphV5WzObCW48cQ2/jOmZBCEMbVkRhopCW+pW0xKyfbGPCiuaOuHhbYnOfCIpYOET1tAqweS0xZcxZQXzo4Sjb7f06g2evLhBJ4hQcgpGYtH4kaU78lxdXqjoRB6qltz6+pj97yjqcUUvTHjjS0fc+t2agy/P+fo3vspUpbT+iM/9yEcYeR6LSUEVTElUhNARyzkspgX10qfOGwI6DP09OkGXSPl4siGOWnQzoylrqqZGeBVBZFHK0miwskH6DUknJEpCptOcvKhQXspi0VDWgl5vG6FCrFLghehmgVINyhPUrWZz6yqx36Hb9zh4fMThyYLOMGXniT7GK4n8AGkjfKFIOz6oDrY2ZGdzJgcFm9sbpJsev/WF3+DzX/jvuHnzKZq2wY8seZmxtbVFEPmowEeblmHX4/BgwtmjCV/6V/fZvbBHGjfMH5zy/uvv0HqbHN6VBMkZf+bFe+zVS0SwTdTPmS9CxvMZrdfSj3roLOTe4R2S66fImzFf+X7AP/9ijq8biqOcw2PFw4cPGWwosmoOJmUY9vFMtFLbFYP0Khh4/mOf4VFxiicVVgnCyEdZTUtJJw1QaMI4wIsrZOiUoJEvuRFKLi+WXBYtW7GPrwKUCmg1HJ+c0ek6PIcfBKjQZ2t3k43NPifHY1rjeGpl0eD7AWVZs1gs8VSAMil725fAeiyWE7rdFOU5OPJgMHLr4lWdm0slKudXWqVfXaWeQXoevW7f2VJwJnIrBUhFWTfU2vH4zge2qqqQymJ0ga5d6s/djF21WVFk+Mrj4cNHGGP4xCc+4dafQq3X1dPFhNbUeEFDEFacnc750u9+g8GGpNdPKXLNEzev8exzT6Ib6dKUvqXf28Dadg08dg/RZl3td44taZpmhU2JMa1ASodg2dreXtW1QbebApambknimNOzY/KswvedT6qua3rdAXnmYNN11fLss8+xsbPFyXhMJ+6sX8vxnOV4jtEtQsH+8SPHxvMCggCOj49ZzkoePnhAkeV004g4jGgbh09pSk2R5TRNzd7eLp1Oh+29gEtXezRmjtEFttWEYURZaAe1FiFFoVnkMxoL2rSESZ8f/5k/TH9nk7RbgHSVjL3uBnlWrqv8dCPce4yk0+2hG8ty4YbHunWIl3MmXrDi9IJLaVdlgx+4ZGueu0E27fXZ3t5GCMF8tlhjYs4bRqqqWv+7e2jXFKXrezYGHj7cd5V9Vq15em2r10zD80PDD/L5zodakAyHG7z44osEQbDyJHpr7qkQgiSWxKngy1/6Gr/yy7/F+Czj4EBy4dIe2JbZIuO5Fz7CMy9e4flXukSdOfMso7flg9Jcun6Vrb3u6jAliQIPIyxaZGiTIYRjoznQ+oLXvvsWvlJEwQbKEy5EWlmMLLCUtBTrPvlut0uSRCvEk2WxmOEHLkiyWCzcnn/1GZOet075t0a7oTOJkErQ7XVI02S9Ak/Tzirw5NFqt3EI4gQrBEEckpcVeVnTWkOnnzKdz3j/zm06aY+026VtEjzP49L1TQabDnvW62xhrcQqvb4fFWVGrWsaCoyoaNqSJIkJgmCVAG+RWDwpnDVBVwS+h24dgqzIG8pqiqXEaJ/FrCbuuO3HxsYG8/l8PThHUcD29iZ17RTEc9tDlmUUebl+easB/fz35tibPS5duMjezjbCslaCjQFjWN0jkrWFpigKl25fre6Xy7m7rryQxbzgje+8zztvPeTKhScJ/JQoCPE9y2AQsLfXZbTR5WM/usmlZzzsqEOBon8zZuOah6hqBsKDsqWqBF43QAdLgjSmmNcc36rZ3N7huR97Gf96ircbEyQRdaGR3RDvYszSg9Y2tBOP/ddOqc4UbRW5MM0wXA/TRjsvPKLG8wVNo8mzlqQT0+2mxHFCWWha7fzjZV6TJAlSuQYhY8Ag6PeHbtP8Q3790CvvVz51if7lXU7uzgg/HdK7kfLgWye0944QXUVb1viBWp1UWoePEe7kWTcVCA9lFcov0VoiTUXfS7jQEbz36B67r8D4rGZ2ZvjcH32aYFSTtSXBnVc4uvU6T76YosuKf/3rb2JEyuXrHtm8g/WWPD6a8Rf+7B/nrdcnyLTk/v2vcPfsEWkSIDXcebflcXkEXsZiUbF1vcvxsWBvu4sJJkgvJVgKBn7O+NGUVgZcfmKJ3w158GjK9edaOgOFKSM6HUmRQRT26SWW/FaGbxW51sjQEDSWe69PsaJL6y3xQ0N2J6IYLXl2NyApKn7uT/5Zvvu9r3Lj5oscfv1rCOtTLGsaXXF2qlxoQlpQBcq2dENN5YFtW4Qt6XZ9wihlNrbkywxPSBotQHgYu8T3BY22bG5ucnxywM6FLstljZAeCI+q1IQBYGpa0xCIgO2NXSq9wHgeW1sjlgun/pTlmCTaprFQ6jkXL44IA8nsYIqyKeiG2lbo2vCx557l6tWr3D74HjdvXuXjn3mF17/7LtVRj7fevoP0JaPNLaSJ2T84JO3ExL6irFpKq1DxgkEwYPPagPsP3kM0XdAVtIbldExCipZQ7jtkRtYcMEx2KYolo60e+opids95bqKkS+U3vOcVRHuSq1sXuffuMciAdlmACjiel9i2Ri/PWCYRwlc01jJeZnhlw/ZOn0fv3KdZlEShR29nk8O79+n3OuTdENsYxNKA0PTCBJnA6WxBsyyJNzos7QwxgsMHFYlIETJnMq3xZcThQUmRwYXrIQ0zoM88n7DMU+7de0SvH+LrAN3WSM8jEF3ysmKhM8r6+ytkSk1eTjDW3TAcwiXh+GjMYr4kTVPC0CkGRjeUi9KlLVetMmVZrhUo5Xtrhem8/cFasxrYaoIgQojWNdiYlrYxBH6IbR2U3Ui3So7imNk842tf/yZ/9+/+r/hv/8l/y/yt76ON+z1ZKxGELOY5Qnj0Bx28wGc5Lgk8y9Zwm9PJlNaWlGWG1gprl3S7LkyRJMkaHwKuDSKKQleiIA1YSbZ0ymroK4q85N1333Xr0EjStK7OL1/U+KGP8mB8WnHhYocoFJi2om0KROtBK9BNxXdffZXIj+lHMQ8fPlj/7Bde+jj9fpfP/9YX2Bz26UQxWZXTWvClRSrIyxPXnZs3RFHF9auXeeutd+h2uxih6fa7tFgePXrkkrhbA6wOuLh3g9Oj05Vh3qC1YDw5BWkJPJAqRSofiyAv5nz9q7c4Opkh64Be6rkmL5tgzNKldqWHF0q6vR7Hx6d0uz06/R7j6ZjesEvoB8zGM6SULOeL9fAmlCSKEuRKaXIwak3bGhf6y7M1uicMIqo6X9c6nq8ez9VDd91JNje3uX//PmmaslxkayXc2JY0cX6tczi6Ui4Ecr7i9lS07qjPsmxdiRcE3krQqPEDya1bt0mTAeiYOGzY2I1p24Bf/+f/jNATPHFlxIODCSJSNJ7luBb8ib/yFzl+cIAxAb1+yO999XepNARRQOB1KJeGbrdhmY+JopC61AgbEMcDsnyB57n2pbZt8L2QIPLRjaSsJ2ztbFFlNZ6fYIwhThwmZnGwj5WgAo+irvFCj5YW3dbUTU3suTYcY/Tq9+GvW2jOVWQ/8LA4T+DR4YQ4jun3+5xNJ1itAYNu3ZC3WFZ4qzBTmZeEoWvsOTw8RDsZEVpFWUIQuMBGVs5oGkMnSCnNEgVYoDtM8SPByckET0m0aYii2PlhsXS7XYS01JWrAA1CSRJ3sGZM2wRcvvwkg1EA/oSqXnL3vYI46KzSyWq9lu73Xc93t9vHWkG+SoF3Oz3SnZTZbPbvqY7nCe62bRmfnSBwtIIg8Dg9Ha+UznYNSj8PkwErWPkquKMrdOuG5yduPEnbtkync06PZ+hqusJyOaV1NikZDLucHhQcPdJsXh+w93Sfxk4pMaTDCOE9Jop7lKqgtIqgG7NzWTM7PiUNa65duMSsbrlz+x2MLhE4awqh20C2s4K2rBnPShAhnl/SypatkUd/wzW/TcYlvt+h8UD5nmv/OslJopimChhuxYxPx1RlCy3rilLPh2yZESQSa1Z1kZ6HEgZP/v8BbG4YcvudfcbzHLt3haPTmGUeI0VL2NaI1Wxa1672KIp9x7oyoLUL6ijPgHUnz7QTM59VLKYtP/aTuww6GxSZZDm3fPN3FS89//M8PpnwrTd/n6s3AopxBXHF0RiOj5d86MbL3Lj4YbqJYlE+4IvffJ3uxpCwf4P5SRdfeATUVA9g0NYIVaN1y5WnQp74cIdlJonjGCUtdS758Ed7aN+j8WPGi5pZadl5yuPmxyTaq2kXBeWpRRmJ5wVEaYQ2CRWaSpY0nkDbhiARkMQ0skZqQ+x7mLiBVjOdTpFyzD/8+/9nHtyacHv/HTZ2Yshb6lIgbAhewQsfvcTm8BK7W31uXDN8+kc36A0kRiuaKkBXhuODKXXZgG2x2tkWBB7GepSFjydTlsUUi6QqDXh23Zyh64ZssSQKfaRp0EXF6ekYJQKKzOCFEj+AydkUrCbpGIQq8VrB7Tfuc/jgDBl5FH6OTCzCE/hK88xzL9MZXeDFF/4wr35lzPjRgMNHPq++9S2+/o3vMJ5MmIwzKt24m1pl0WWOaFqGwy4KBaliXNZkxxHkgjS9yKi/S1xbSA3l2Of3yx6TiwOicIFBcbRfc/zuEQ/fn7Pz1IC9JwTGy5icTBDTiKi/xY2nniTseMhCM+gNmc/nNIcNtlDIToduIun4mq706Xc7RBsLjqaPyatDQiTTouDug/soCUpY+ttDVCdkOBq4h68pUAJGG31K0fLaeMbvH8J7jzWxGKy8PJYoClG+hxAemzsdnnjGcvn6gNnY8uLLH2EwChkMQ/xAuGSy9bBGM50uUcECK3K6PZ/usGK4EdI2TsXrJClnJ2fMZjM+9rGXuHHjBr1ej7IsWWQZRVUSxAFWAFKsvW6RH6yDED/YQXuOKzlXA7TWJGlEkoRrJcIaN9g3usBBkEFbw/WbN/jXv/ZvOBmf8d1XX3McPU8gRUCrBcs5VHlAFCukV3N6VFPkJVcuX+fNN9/mwYMHpGlIUWbk+ZKyLF2l3ErpOoeeyzUCwzgMxvqfLVL6YN0acDqb0GqBpaXf79DpdClyQ1U7tdPomJOjCdJGpHEMtsU0kjTqkcYBnsQ90JuajV6yfp3tP6JczLh6dZt5tiSrW/xAUleOKuAFiqTfgldgWp+trR0e7+8z6HedbeYcfmxagtAjigPms5yzszM2NoYI2eD5BuW1RLELxPQGI8IoQXmasjwjVB5tmfP7v/M1nrgUUZeONoFcMp0f8fQzV7ly5RLWKOKOx2w2Wbd9tNbw0Y+9xMsvP09/2KM7SFkulyRJsu6TjuN0jScrS4fQOUfRxD8AunZDYMtoNHKg+JWyeH5dnfsolVK8++7ba9SPWSnPnq/WqlLbftDLfX5NCiFo6nYVorDrQ8+Xv/xlF5ioWzCCMFAICzs7W64RzW8Bj5PHU4pFRWdXovol02VOEKW88+73mM8m3H7rgH/8D77Aa6894Ctf/Sa/+Eu/RitzhNfgdTIqnVE2R0xOCyK/i67NyoseApIgVFgK9va2+MQnnuTChStYK2h0ga59FrOGVgdsbGxw+fJlpPDcinIwACCMfLrdyLVwKUHTVERRsE4gB3G04jE6hfhctT1vxXGMWYtuLGkyoCwamrrF98KVtcVzzXTC9X675Lh771x3tmv4alvDxnaXKFFgQqoC6iYH2VCW1rUuSYEKPLLSba1ufOgSURqAkusDqasNNdjWEAUhvU7XJfZ1RCgThGiYzk747nfe5+CRz/VrrzDaGTGbLZwCmpWkaZemaTk7mzAYjFguc0xreeqpp+h2uys1Ua85mM4eUq8DQmEYumtXV5ycHnF0fEhRFK6tZ+VTPW930qvBu9VizYd0djGF7yse7T+gqguS2Mf3YD6fopTi6OB4jTN79OiQLJ8TypRHbxxz93vHyMrDLgrq+ZzRzQFnQrP3vGWwC8tsRlvG+GHA7o0nKX1FLWY0reYjL3yKy1c+RH84oNOPMLLFAFdvXCcMukShRCjnn83zlGwWspxaknhAVVUugd5UNLUhTVIAOj2fKPKQHuxd2GJre9PlAaxjdXZ6IdJGLjlKiW5y5tM5TflDbbvdDPLDeig//lf+mL16ccBb39zH+FCMZ+gyQ8yOkGJKVkI5C/GUZOeCjzWasvScfCwbApFStA1Sa4Rp0EGHyazgr/3NkP/537uACEu++/Uhf/0/OqJoAq7duM5n/9B13t//Jkps0RRj7r9xH5PsMj89Y3erS5xsI/VdxvoMm40gC7jz/QlxCNeue+SNZbYQeLrAWEPUCRH+kNFeSFYEBFFNb6h4dP8Bz/1oj2Cz4fb3FkzPEqoip9cBej0n+88aRNilDjMiYykrj6P9FnXmuHW0BisCPCWwwvmswtpD5zX+sMt/8FOf5eF3/y1eFPPNBwmmmuF3oc1WzQKmRHuK0UbKIPC59d6CZ17Z5cmntnj1i6+TVT6e575XNw1FUWIBJUN3sw4USE2tLVqzqnmCulwgiGiaan0TAtbDQhQ57hy+IEk6VFVDXtYo38OP3cpjZ2+P8XhMXZYIyxrdcj58XL10mUen+wQm4JnnX+LVr7zB1ZsDDmeniCzn0vaQgRfhbz/DW4evIRaaXkdxtmjQWUMcGhqryWpJb69PNplx/fqTRJHHwXv7PJycsre1RdnkdAcxeblgc1fw6H0PXTREoeTaJ2/y/ndu41tFMgzJ5y3z02OefH6LuWkY0Of0YEybG5rCYHBJ/KrOiHwfL2iYGkHfS4hSGM9yrPaJUwi0YZbXbKQxoq4QwlJZy6IoUVg6cQR+Q6sFadxlMpmgNWAhjhP3cG3ACxpUmJAvDaL2eeL5ba7uBlx/8gr/6B/+NkEosEbhy4CyytyaXUCcSDwVYymx1Dz9ouT970nmE0HaCQCP2mRgBVXe8MrHP4OxPg8f3ePh49vESUhbVsjzPmUkrAIYvlTkWYbnuy76phWMRhu0JufsbE4v7VKVCxoNg2G6HkR1w9oLBlCbHCk8ED6mVeztXuH+/Yd4SuOHGmtc5V/ke+TLhhdffI5Hjx5zduaYmiIy9Lu7zGYHCKNYZDVx4qEbg1AShKauDJ46R9ysUvOrjl+j3dAHTkE9VyqcwtoShIJer4dpHJtvMpkRxzGz2cTdCIWg0/VJOsoNLrWiLCxKrXxWtqaocpSM1/fEjc0OGzuW2Vnggk9dn8FgwN3bZ8SJhxWGxuYY7WN0xbC3w8lkwad+9CO89u3v07QLPDVg58KQ/Ydjl1hVtVM9Ku3W+pEDk+dZidYtvrBEsUJ5llYLdOXYjI2oePnjz3H/3ffwwx79zU0eHjwmDEMm41O2h0OaMqPSFtn6CONjlcUEhsl0SoBHrCLm2ZIo9un1OqsQjRuYmqaiNSVKOXSPFC70cX7YUKuDR7fnvJRZVqAbp6gh7dqv5nxvDgkThiHL5XKtJiVRiBENVdms2X9RlKBWPeICA0Kt31djXF3o+UCWlxm6McTRCCk8JtkRWEjDlDi1VGVJYxRl2dDrDmhbS1YsCBJBf5AilKTJMnZ3Orzz7gQrE/YuKnpJQ1UIjsYNEQFCwGKW8/yLTzCbTTg6WBCGActlxpM3P8xirvnoxz7Gl77yeU7PjklSSbe3RZX70EZcvznk/p3bjMfjVc1pjfDB6hjpNUjh07YhyAy5oi30OjvUeoFpPlCLzpE450BsbYxroJONs5Z4AcoP3EFM1bSmpMw0mBTllyhhyPPG1akaQBk84QZcoTS6bmgahZAtuq3odYeYRjBfjAl9ifQCLIruKELXGflMEXShWmaIFhTu8ySUwbQS6QlEXRMFl1mUx2jt0YiMON7g6Wd3eeLah7j//ju8/c77NHWCNgtMW7O1cZmqqdFWc+Pyk7zz3vfxfEvbWsrivPxAoHWNxFsPhP1BxHJREsYRi7lL///8X/pz3L3zFr/1775O2olRMiIIe1TtAQBttfKtrlTK8+v2fFCVQtM0DmSu4oyr17eZHhvq0qOqTmm0wBclQdShO9wlGnkklwKCRGCnj1lMWmwTkrVT4oGgmliUn+BFKQ9unTDcihhe8JFtTDWHxbxiVmaEiUXWEZ14yMOH98FofOXhSZ/lMl+JABYrWobDPlUzc4SKGpbLwrXx4LN1acTZ+IjrV55iPL5PVTYs5xltEwI1dSmQKkB6NWGQ0LROCJw/zP9gPZRRYnn1K+9xuH9CXYzxe6ErhE+6WCSRH+AHEmNXRP4gwfedP8RqgZUNaRwiVUgUDfCsjzLw+u9FCN2A6VHXHlVds7Gd8nD/gKN9Q19d5/Xfu0O5fIpmlrJV+dzsXWR2YPnWW9/l9XuwOIsp7Jygt+T6ywl26DGtDLQ5nq2oWoE2AbZNmRzOGT9cYPMFxw+WnB5O2b3c4+E7M04fLvCrlI1NSSuhOJJkd5fYqqHGEoYxnuqQewbRc+spTymQPkQhrTRoYbDKYqWiVpJKwbKYc2EnQCdbjM9avLpiZ2OTy8PLVG1NnEh29lL6fZ9iWTFflgw3Ex49GPPvfvUNhoObXL16HWMEbWtQno8fBASB6ynO85zFYkaZS8zqlObuCi1VKUE0a3bbOYpDCOFOgkXlVjUmYDKZUdcl/UFMp+swEkEQcHY8psgyhBU0ZY1C4gmFlAphLLfffx+5bJkXc77/5hu0I8uDyRFtO0etktJPe3dRtoJlS6Ak4zahClvSzQEi2sarfbbjHovHE2I/wKssr3/1PU7GY0bdgGIxppg1bG4IPvsTN8nGDkQfxjPiWHPw/dtEIqYta9pjeOWZG/y9/+1fpD+ISGzJgwcPqescXRcsZnNC35L2fLzQZ1FqqmWCykP8qCHdSJG6JKRG+j2MihG+x6LV1EkISUyRV4RC4QmPbrcPsk/ZKKYL1xSURilpGNCJAjwlqY1E6oggbzFNieoUHL475ne+8Da/8flvsDGICZRBCXfz9jzcetMPqUpDVS8BibBdLm78FMKM8ANJpyfJ8yWiGeBLh6n51re+xmRyn6qcIix4UjDob9JqCPwIJcHzBVGsMLbCCw2ID2rTFovFKszjVJQwjIjjiKpsKYuaqvxg3ZYkEaCRbbyqgtMYs+Tg+BZpr8KLarel8EpE6NOIGD/p8O7tOzRoLl67SNHUeCrh+OgAtE+WZSRhQBqkBFIgWxwySgYEMsRq583ypEQKi20NTVMipViHE9zAYdG6JU27PPXkMwR+wjIv0Qaa1rXEXLt2g6ZpCcOYTqdHGDjVzViNHwis1QipCSLXz9zofP2aTSsOHxvSZMjOzg6tVizzM6fa1i2LaU6TpWADlOdq/JQwfOebtzBWMJ/E/MWf/+s898xHKcsaqRoUPqbRRIEkDn18PExjicOIJHaHoLquV+lMl9AMI58qa9ndvcAzLz3L48dHjA9vkeiK5aMJfm2wNURxihd6IC2NKTFtgy4aOnFMkiQYWi5d3sP3IibjJb4fsrnVx9gCqVrC6Hxt/UElnfIkH3r6JtrUWNGu1e3z2kMh7NoLed7n3bbNOkhzvhaXElprVr4/u/JMhuthsyxLzCpJXhTVGkMURQlS+mSZa7Xp9QbEnYKdKxkX97qM+kM63ZCybNjYGrF5yWew6aNtTlnNEBgiMaTJYuqFIjCXKY3Hz/yRn+bS9gbtPOb9txWTsaKaGIq8Jc8qNjY3EEjOziZriH4Q+Nx6/12OT4/4wjf+Ff5ozPBKjA09vLgmrx/RtGccH51S6oxOv0eLwkgPawPSgSHueFSNpjUlnpI0NYRBStVMsbpBSjdEW9uurQTniKUoCHDKUk0n6bow1nKBrl2dbyceMBpt0rQzdF1hjGPNdpINOp0O/d42nUFDd+DsMKZ1KqOnEtd1LgS2XRCFPRoRUpqSqso4fXhGvqhpZEYxa9jbidnejfG7UAmNNi2e1pjjhrL0qM2SziDB+gWeapE259792/zTf/kvaY3iEz/yKay0rt/beEjVsrHts7XT5fXvfR3DkqpqnK1DGp58eptOz+IFMYPRAOUrOr2IxVS6Z2Yt8D346Iuf4M67hyiZECeKKGnZ2ArpdQOioI+0rr8cnHJ73qt+Hm50qniApSXPl7zw/MtsjLaRUpIkMWEYE/oRqA1Mq5idTKjrqUt0nzbk9QjT6VL4kp2rzzIafZQg2ma5bDk7rtjYihnfNbz5xTGzg5LLNzWL8gC/jikOGyb3pxw9OmB7NETaFmtcDadDcWlU2CKVaxm0bUxZuOIA1+ees7UHJ4/PoA54+3tvcXKUMzm1bO9cQXqCuglQYUvaFVy8tMkyW6x95D/s1w/toZy9NmXzYsRi/hi16DPLp3SDiOPZnAsqZNnUhKGkkeCpkKZeeZqEqzer65oQCcKS1VNnwmhh74qhlxgWVBSFoZPsOPCmV/D53/wye3sxiBnzye/T3evy2T/3cb7064e0J4+5FG4TpD6vXP8o9/O3OVw8ZFqNMMrhW6xn8GOJsop5tkQkoGSDbToc3JuwLAyh2mU0hCtPFeggYHq4oCkNW3sJRZ0jRcgsr9i7EPHjP3WJX/3l+yhP0et7zMZnVEVC0DNkVU0kFMpofOGYV4uqpvXh2sVtXn/z69y9dcJnf/aP8ieffJqT08f80q/+HlaktGaJMQnlIke3AlP7xHGECmowHkXjMzndR7eCQW/E6ekpFgjOT584L8S5of7KjT3quub+vVOUgDBKVv2jrqj+vNNZyvNatIi6yPF8hbdiEy6XS6TyETi0gkRCa4jDBGEdJsTqFiUVURAxbyuUnyDaJU215NqVJ9jdvciXb+3TImHrSebNlPlcs9WD5ckjiqKBZEkohkgbkrc5aZBiGsO9h+8wGvmczgqWmWW02UfWmpPTOctcEicdjsVjhA7Jy5JUeFTBGK8Ts/d0SO/6Dr/3nTMOJ8eUE8vwgoc0Fl932drtcvvWHeTSkESKi7sRVbtk8bhgKJ+m56fcau6SRhHTg4cM+s67s2wa/F7MyWRGEoZ4BvAUx/MZug0wCLzAI5ABgdFIWup8iodHFHj4SlFWOXGQ0LQtP/O5Z/nV3/kOD29NiXwPJX1aK1DSg5X6EEYBvnGrpMBPwYZ85UvvgnAIjMODOXECG5sLJicSX/lEQ8lkckBVCXodV6NZ2xppQUhLUzqVtW1cuOa8EUdIS+C5oE2WFUR+RJaXSOmvmzrcdlMCNXXt4NZJGtFUNUXusxjXDEc9UFCVmjTpsVhO6eiEqmlomjlJxyV1y0wzGm7R6fbJpjmWmlYExHFKEEiWZYHyFbaxYCxx4NMaTeCptToTrdLpfqDWCqUwYqXaqPXgMR5PODk5RQjB0dExxhjOJhO3WhWCvMqpjgt6nYThcIivlpRljfLAEwrd1ijPEv7AHVPrmsW8Jc/uU5Y5ly5tI5TB9yrisEtZLdHtAtsqhAmpG8HmBUGZLfH8lu1gg7jX8Gv/5DfwA01VagQRnidoViDlpja0rfOJRqFHrVyjicDx4mgbmqYmjARf/8q3IVXIdIPBhT6Tk1PaqaWtJSdHZ2xt9xACPN+uVnrgKZ8iq/ESRWtb0o5HkozY3z/CGMPjx49BGKxRhDJxDysD4FRCqSzL5Zyd3Q3G4zFNU69W4Ibz7df5alEpsQ6J1XWGlN4qoOOsOOdp/ShKVoqQ6zeu6xop7AecxiQmL+bOSpUbWut4g0r5lLUl8Xoc72cURYOUDWVbUlUwW1h2Ll9g2JdMTmccH4xJQp9OR+P5huWiZNqeIZoO7956gBaWsp2zeSmgtTnDMGJ+JkhixXA45ODgBCWdImuNxhjp+JympSoCWgQoSRRGjMc5woMosCyLKX7cAVkwGProxuF+hjsp+/dywiREVxphY3xviafc78CaACn1msUKlrp2zEdfeRjdoLyWwHfPW08qlADfV8Seo3Hs7e0xGR+ipE9Z1ChlUa1luOUOybaJCWPtGrN0iLFzN9x6gassjSN0pUl8QIbkFrzIQ6mGUNREG4rJvHHwbRmAlfihR2lbPvozl0kvDji5f8q7bzxww3DTUhc1UkYMBx2++Y1bhMnr+GFEt+/ICteuXeP23Tucjs/odiKscKvpuq6pK8H4tOTSxWvcvvuAs9MxSsY889GP861vf4UyL1BKM+iPuH/vFg/2H9Pte/heiq5bJmclnlJ0hjHz+RG+cmqva8jx1urvOa1ACoPnBXhpyhvf+T7IiqaShEGFNa5wobIt3cgjICAbWw7vnOGFGhNN8GPF4gDGZzM6SZ9ITrm8t8X9B4LLe0Oeupnz2qv3CMJNhumHyea/ROLNoS157vkXuffwAeOjKUZ76BYXoBHQak0SRdSmRFeWMBRUVU5RQCdNiCLLdFwRhpKq0kSRRbcCrwMn4wO6mzHNaUldhFSTmjCShIHbUoTxDz0m/vAK5eBKyI1r13nm8hV2n/AZ3WgpTI6gT9E2eEogResuRCmoGqeSeZ4Es+q1tAYvUAThAIVHGPh85ctTzg5iImFZLCUqdMPOoLPDpctDwk7G5WcvsHv5GoeTit/8zff4zuvvU6HojrY5eFTxp37+b/OxZ/9j9Ax6cYbyLI1p8aVAScN0UeNZZ+jUbcPG9g6f/tQfYXPT5+H7xzx+b47nQRjE+KnhymUfzxhEuIFIDds7IWVh2D9+zNM3d7kYKx5+q6JdpCQDg65LQuW5n+cJhCcRVhCHMRGCZFkwXXh85voG99//Dk2ww6UnPko+P2aQKIzuUOQVkg5KeHihptYLqrwgDAxvv/v62ng8mTiYa11pjNXkxRJLS1U1+AFUVc1sMiZfLugkDtTsKacW/GDXM7i1+Lkq1e9F6Koi8EJsowi9Lk1laOqSyJP4UmFbl9w/X3MKC7ppsFrTSzqE0lC0Nb1oRGNnnBU1290Rve1NLv3oX6XNY7LJIcLGjBJBKBSDsIcWBYXVnC0WRHEXKzWmCTh6WPD0C9fY2RoymRVcuNrj5rMdzsb7HJ0cEvhbSD8mSlJaz7C1lZIGkk7o8eo3f4tvfOl3UW1Ib9AlCWPSdJPN6wPkzpgrH9mlP9qgWFiWJxGYDh96bpfHZ4d885tvuaFChIy6IwoaZNWijOT06AzdCipApSkm9LBxiOoYkm0P0TXIjnLXgCewFuKkz86gh5EVYhhCECJNy71jn2vXRsRJgCDEWDcUNTrDyAY/cg/iutSr4oCMZTZhf//eqpc3ZG9ni4+8cJEgkPiBxvMUTS05PcsQNsKagCxfMF/MsWjn1/Mkoe+DlYRBisDHW7VKKCVQwtJLezRNy+Url4jj2K35q2LdMOGKC+wafm4M9PoxH3/lRXdN6QZswbDfQdmAeVERpjFbF4a0pnJVc8Zy573blIs5rQAv9JChQCuBthokGOk5jJDUaJFjqQgjhW5LDA5QfJ4kB1w14qqaz3mjaqqq4PDwcO2xFIIPKucQhKELHIVBirEh9+7uk5cFyjOrQ5ehrhv8IMBTyfqF0HiBI1xgQwd4bjawaIpyTn8YsnsxIU4CqkLS7QtCPyBbGBaLlhtPXeRrr/4qKizp9lxzirXud1lXmrrSKxXPoyxz9xlXyqnMyqcoKpRyOBPPCzg+OmP6cEmblVDFdHopXqL5uT/15/jxz32aLJ+S+j6mbYgTjzBxvkylPGxr0FazWCyo6iVBCKa1+F7CaLjpHrKZZrQ1Iu4ka1SLMYZHjx4xny3pdQeUZbXyaDrVzPMdhNz1GwuWK9TR+YH2/FCrtV6tFjVZVqwaT6p1yjdeqahBoICWwTCl14+IU0kYCTrdmO6gQ5bNqeuKVstVT7Wmm27TG0ZEHcV8UnLwYMbZcYUgIEk6FDnMJg5v57Vdzu403H7zHY4ePcYXAapqsJkl7URordnYHDJfTFcpZLHyORoCP6G1LVEgkFWAnXUI6gCahnyhqcrA1fMlCY1u8eIGEc8JezXalhwcTLGi5ebTfa48oSjKOQL3efb9yD1bhaHbS2l0Q5yEJKmPFBohWxCaNE1AWrem1q4y8caNp6i15eG9fb79jVddw5Z0eLCqdkPobH5GEJckw5zWNDQ1NLoiSTpIYppK0uqafLZ0gRWpHewfQei5RHdegWxKRN3BthH5smTUj+luxFz8+A2u/omrTKIZIh1ja0mxAClTkjTF1iV2WTLYLQlSS28QkHQU168/ybdffYNHDw+RXk1ZV5SZocw1EkunkzCfNXzrG2+zWGR00x7WFnz1a1+kqgv+8B/5GV782BP8wl/7D2lMSRB6NI0LPTVNRaMLltmE06PjtVfXGLOyjH2gtp8Hfnw/cH5tWoyR1KWbV+rGdaIrKen0QhbLgqopqeaaO9/NePyOYPreHovDHlvDARs7JaMtjzwPOJhMoXfCmb3Pw8k+157q0Jhjfulf/Abb2z2C0KLLHpk4pDvQaEqMsEhlkWFDlNakfUFZlPjK3UeyPEcpj8BPHG5K+VRljMSjLGo8L0RKD89vsMbH1CFbGxv4QYDwFCfHZ2sovG5+6Hnyhx8oDw8mvP69t6lNSeH75FNLs5hg2hlL2cHzxWqA9CiyfJ2e8jwPqUAYBa3AGo+mbqkaTZBo5rM+b7w1ZkBMOfV5ePcEEQVk7YK68PDlFabVkt/8pddgccStb71GuDwhm0x5eP97PLWzy69+5f/KL//2P2MwvEm3X3Pz6lX63XTFo4M0EoRRh7wq6PUS3nzjTe7fv8srr3wcZM3jhzmnd/vc+0bFxz6xxeiCj2otV64PiOMNBn4H5fn8xi8XvPnVU/a2Rox6EXdfdz5CDNC0SE9RS0shDbU06Lom1vDCpet88uUf5cHhjHJa84Xf+A2mi5KPvfJHufqhlygXGWpVCD/c6KOCkKIokTZiNi7Y3B6hBAhryPMlyyx3MOSmomlKlIK046OkZLjpsZjnTMZLvJWvz/cdV7CuP+iGPQ9b/GCNVxgG1EXNdDzFl4rt4SaB8JHGEoWOadW2DVVTugeEkpR1ycc+/jI/++OfRhlQYYxpSx4dzbj/cIxfTuklG8jYoUC2tnc5nWQ8nhVYX1BXE2wdUJZzdkcdRskFBA0/8zPPc/nKNhevDnn6IwFJaNCZT6Cv4ZkY2fpsdIf4FCQhqNDgCx+lFXe/uyBp9tjqDlieNJzsLzi6u+D0cMx8ajG6z7JZsnGtw5WP7LL94ZhZU3J4ryDsBcSDCJ21WBryUmNXBnwfhbI+RlsabTiejslK1zAT0EXpmHxWQmNci4bUDLoS2gUnizNsT5PIEqVmxDqlthWhSGEOlczQtmRnb5vRaMsNg1ag9Zy6dt2uLsSQ0x/56EZRlTmtqaiqmjvv5ZSlZj5fkOcQeBF+qAljw8svvczzzz23Ap+7BKpArXmTvu+73uRG4636lc9xPMtiziJf4PkQJz5CWIrCeXYEkmJl/q9aw9/4G/8ZH33p00xOChQxnlScnTwmiSwff/Y6P/L8Czxz/TnKueHq5WsMhglxqoiSCE3llFOpV6u1FtHWq0YlSxg7xIoKJMoXDDZiRhtdWlPS2ALTgic+sHT0+112tzcwtsEPJEEoSFKfIHR+uU43ccGspl6nZ+umZLGYu9CP8dnavIRpV60sXkRda2h765dpPVotVp7BnCI3aL0kClN0Y/FVh2yp2dgRXH4CUC2VXmJMShRs8/DxfQ6O7tEbGuJ0RBR2CGOPqqlJO0OaxiXjPd8ilcEascY/CaHWB7swcqnnIAjYG/a5sB1zcnTIxb1L/OX/5OdpwgV7T13gQ598gXy+II0jyirHCEtZF3ieC75EfsB8WnNyPGe5KKnqHN02TCZu9ZV0LLPZhGyx+AHvngvuNE1LlrkBcO03Ww0t1jpF9Dyd74ZR53l1Hkiz3rCcI4M8z6PX66y4gK4dpWkqsqzh4or56PtqhRcSNKZCtyW9/iaj0QbICt93wbamqRhswMaOBeYoVRJFGk9VFOWMtq0o6oLGNBhZ0B0mpEMflfiM8wVeEuPHQw5OMrY2e+i2oCiWa096kVdIqaiqBiEkZZnTH1k2dixVvUQpS39D4QUag6Axhq3dmHnWrBL4Ff3BNsKmRKHP91875d3vZbTaQ1uDsSXGltSVddzStiWOQ5xK3OCFAt06XnFdNwSBj5QW3bZcvnKd6SQjCCIuXN5la3tEtnCqt++HdPsxRVFRFRJtNZ7fMl80mBZQmmxZgo2QMnAHz8g9T5raUJU5nt8Q+G64GvR2KOkytxnHRYkN++wfZDx874w3f/d9/vH/4vOYZsDmdkoU+MSJR11pJtOMsqkxJmRr9yKXruwR+kP27xluv38X5dd0OilN0dI0rnlJCDAUWOMOdP1+H6zHZHbqwl8e/OSP/yx/5+/8F/zcz/2P8cOAR/tHJFG8Unc1QhqMbWhN4YJg9fm9wN0bPc9b22fOUVjNKtha1+XKUxkhpYcQrRt2y4ximhPGQ1SsUKpiayOmbaa0kzGP3y85uWdYPg55fHvK6eMcz4aMhhGRlxL5Pe68mVGXSy5dNuxsjBgO+iRRTS/1eOblDa48FTLYsqQjRRAKytLSVB5x2KdpXGlDEAYUuWawkZCVE4oiY2Mr5vBgTpXDdOboDEZrpFXoUiB1yKXLQ5CVyw5EDuR/XmX7w3z90FpmeuFNOv0R9sjDNNuoOqEQI7qdHl57gm2X7mTaWprGrbm9QNDqEiF8pKixyiUgrTUIfIz1qfSCX/xnMX/6pyymsXQ390g6Q4q64N7+I7byHp1Ll/nRn9hlWSxZGMlOso0I7+O3NcFpyze+fZvdpzLmZwJkzOTR+yz3PfauwXJe0Q2GLGhQ1sPUku0Nn/2H32W+vMClJzZ4dGfM/n2PLg3dQUY79Nm6UXF2t6GpC7KFR3e7y6Vih+/93ut8+qev82M/NyCbP2Z2siSMk3XoRUmJEgKjFISSZVHy7eN7PHWnT2kF1eKQ+XzK/+Ptr7G1eYl6/phOP0HXiqSv+NRnf5zf/Z2voERNWWjkCrId+M47JSTEgb+qt3Lr7taseIVVjjI+vh8QBI6DlnYcr6vX6xDH4Soxm61UGucVMcZiCZDSUtcOUr2YnFHHKW3TEgQRZV6s0rMShHInJOtwHe+99x4nRz16ezEnt3MGHYURErusOSqXlHWEVjWz+oiq0dx45ip33m/xhGDriYSH3zklVAoZDvFjizfxmasZe8+FHN3PuPHSEzyVG4oq52y24Mf/0Cu8+9YR80nJaLiD4QTp9ZnUZ3hhB98rOTjJibvbbF0MOHt8gt/3CYMOus7ZGqVsDa7wvdeO6Q4Sbj47oBY14wdTlwyWBoKQVtXEUiNkjDZgytr1ditFWRV4nkBYiWgM1s+xraDf6UIj0MJD2wQlu8xmczavX0Ucvk+3FxHFFTduBnwjO2Fzcoe/+Of/FHfmS37z136dcC+hyGbUlaXJSi7sJly62OX9W6d0OhF1bWh0Q5pqtBacHZXEScmNJwOKAhYzQ1NVzhweCBaLghdfeIWXX/oI/+v/zd/DWkOWLUg6sWtKaCtnaF+pz2HkUqAnJwvCUFE3GV4I2jrPYRSFTCeLVb3cuconwQT8d/+vX+Txo3ts78VEgcfhAdRG8tRTT/LM3i7TZcFGt8dnP/kj3H18n3le4HVizuYzrqR9RumI4/yEZe38SruXNrn7+IydQQLtlKSjyOYNddXSjVLK3GLbBGMrfGXWQ5XWNUIkq2HD4nlQVS3GaPqDDvPZwg06uGYOJXys8ZzR35RcuXqD8dkST/aIogVlNSPyFY0WNOUHoYgojNEaBqOG4VZJuVSosEBVIASU1RIZCA4eNgwGA1RwzPbOk+xspwSBx/3DbxPHPXqdPb73nX16aYgfe8R4bG1t0XviOu+89zpltcT3I1q94uvRonXpQmVFuU60LvOC02zO089d5nSWMat93nz/PseHjwi9Pm2j0dIgPcXHXv44b7z+PYJVurqq3DBXG1eK0O0mfOjpa3zn1dcxOiaMQoSs6SYpc70EBAK5GiAdnzDPCtJUURTF2mqwRv+svH4fHGA8mrp21XJSIkM3HBvdghCkveSDTvZwFe60lkG/z2Jecnp2jFM+BUhBGnewUpF2W+7cvs2f+tN/kpOTE770xa8SG5iceJycTAlE6JLSyyVJ3HM4oxC8WpEXGllYimZGS4BsIAzdNqeu56RCEYSW+XyKbV29ZFU2JEnXHbSaFiVDKlNTC4kKFTaCRal58uYNVJDxzhunVHoOGUgDXru5IqEodOOsUrYJCHzB7oUOo82Ut16/TzkTJEmItaU7wIYelXb0EAdQd0l8zwuxwlA1FaPNHg8f3uellz5Op0q4c/cWEo2SAQLI8xIrLdpYdAND2aPMIuoqw4gGBXQHHQSGPM/whU+pFYEo2RoqJtOaLIPKtHgmIA189h+7UGx/0KOqPbRqwVb4mcLXHW5/cYx3bZtFfsIyLxmN+rz0wouEnQD6c9A1xwczjg+OsK2isUsizyCMQrRdpK9d65AUVFXhgnk0ZFnB1etP8vTTHb77nfepG8Noa8S/+Jf/mn/2//7nKA+SJKK1S+IoJYm7nJ1NVvWXAcozlFWBIlmjdICVPzJZ96Yr5WGMJE0D8jzH9+KVP9gihIc1IE1Als0Qokfi9zk42ucnf/JFyrP73DoOqecVLRAn0FMhxb7i5GFBEJQ0bcPiuOTjz38G3884PH4HrRTWNDx4L+dsEpDNA+raUpcaXVlAYdBou8AKQ62h00m5eK3DfD5eecktBwcHxElMFAU0dk5rcqglSpVUZc7Z0YQnPnSNYTpwHfDSNS41xQdYpT+wgfLC9YvsjPY4sCkPHrcczx8SeT4qDFm2KRtVgFAtumxQIgSjiCNJXrgTtaXFBgYVNIiijx+31Kakmwi+8G8z3rrtUduSYlzRzlO6qaVNBjSZZXq8JN7dxmOHtLacTR7z8Z98gZc//iTF8UN+7be/xfTQEgpJDex+WHBvbqkmAbEXcppP8MMOWKhLTRS3pJ2AxWLGix/5MJ/+3Cd579F3eftLZ7z48ApnGWxtetx5/ZRBcoVkGHL0aJ8oOiNNI77861OWVctOkqC2NWfHDWEskdZHtAZ0Q0tNVbeMLm0z+sh1OnnI9rXLqE7N4tY+Wx3LiT1gw04JraIdXOZHfuJnqRtNlTV4okV5KV7goUTGYl6xublBknQ4PD5aG9o9z3Ecq6pBKcfIDCND4Acssxo/MCgVrLEr7uETrD8syhMoYzBofE+hZICSECgoFnPitEPke0RRj9l8iZKKoqldF7SE1rgVgSdbyqLFUuF1YrY6u5wd7GOtYegPCUWHetqQL095cE8SNJZLV3fQfsuLf/gSb71xl0UzYXej4LndKzzYnzNbPsSebcH7PR7eLkl7HpYznoqu07DP8fQOsd9DWsssa/B88H3np8IURN4GJ49LRtsdzjIDUpJ2Wu6+f8Dy1GMwjOj0ar71+/f4+Z/+y9z4hYj/y//tH2LOaoajLo+PMmRr8UYetjAsFhU+ijAI8KSCFeZEKZ9SCjw8kiRlNpkyMyXWtPhlyVK0fCztYT40YlDX5FPD7GRMv9OhHl3hndu3ePPWmG6SMjm9TafvU9YtWaGYTl3aMowsus3RGqRI0Doj9Ad0I4+2OaXXTemmgNEUXolfdxEmpJwH/Mqv/Nf8039q17gYpQTtCgaOcEgv58V1K2/QxFHAdL5gb6dP1TQUS81sNqXf75F2YhbzDD8IV6zSkiSxlNUDgjAjyxqqMkD6ligS5NWUb9+v+Z/8T/8ata64cu0qf+e/+HvkB4fEePhC8spHP8neYJOzNuflT73CP/jf/300Ei0Ljs8yPD8hjRN8NHVTkC8zqqqlNxiRpBGP908wxtXBtRbKwiGHlHIkgzCMWSzmLJdLlCdpdE0QuLaSqqoQNiQIfHQrODw8BBPyzrvvEkXQ6ycrBmJA6xXre2LTNAgSglCgW4vyNFWzoMhjkp4gCDvcu3vG9Q9tsb0T8Ma3fJ68cZPD07d5+P19ti4Inn/u03zi5T/K/33xX3Ln1h0CE2CNx/37d+l0U5IoZrGswGqE8FZBnAbP85HS2VaqqqJpGqLYJ59mHB9MGW5f4P7dR4j7ml7sCgoG/Q22f+LT1LOcxw8O2RluUlYZk8WcOA4d6Nk0lHVNm9W8/fY7xHFMqyVNU2NLQyXmhGGM1k5VxEA3TVxKO46AFmMcCeA8ja3+vyg9cuVNOz/UlmVJawyeUiRpimkbxKo6j1UzStu2lLaimdWrvuaaMI5WASWQK09h0lE82r/Lo4cHSK+lLHPiqEe3YzDWoVWyZUlrcuqiRZQeQZQgRYkNOlx/0aNiynjfYJseDx9npEnKzkbCcjFBCQ8VRGjdUOkSKUCbGrFC0URxwPi0Ilj6dIcDDk5OGJ/WXLjSwXBCkiomkzlxHDOf5izmFd1egECiDEi5auAKDVsXUnqPQBcpbZPjyZC6tmjDym+aUlUN2bIgDAN0m4FtCf3QAfcxvPbdb663SkEo8PzA1WRikUi8UOAriMIu09kMY0uUCvCCEN1Y8ix3A1nH4NeSIFHYwOBXIboAWVYEsc+9g0ckSQzacnIwJYr64EuEl+DhISk4vPUuPZ7j2vWb7Oxu4iuP73//bSrjsfNUy2x8xt3XJGnXUuRzANJkEy1PGWxGVJViPslpWyeYPPX0JXzf58GDY4aDTa4/9RS/8J/+Tf723/q7XLmxzW/9+ueZLx4SKA/fC9ja3CPLck7PjtCNO0BvbvWZTudYI2hp13WO5wQUpdQatWSMg4GXZema6HRDGMa0bYXWLUp2UF5AbA26HhNtdPjf/Z2/z8Gj2/zGLx1Q1YLWVmzsbjA9W2LrEqNmYALqpY8XNWxsbvLerTv0IkMYbrpaVFuSLUumi5w48cizmsBTWGvo9boI2VKXGTee3KMqPKaLE5Sn6Q1GGF0gMLS1QHkeRVHRGkuvH1HkDUpYwiDm5U99nC//3hdJexFSBavnjSt7+QMfKKvDFziatUynhnL/Ab6GWi3xfRj6XagddiQIFDQRunKgbs8H04S0tYfWlkgEIGoao7BGEwU+h48F/+gfH3Phw7vo7H0O3jmhDEK68Q5pqjB+l9msoS6O2El3Sfpw8OiE/8NvP+DGzRyRKx5+ecboiYrLr2iKySZXLjbEfsjjo2O6vRjTlDSNJu2k+HHM8VFFbUq+/Fuv8fHPnnL96RcYfO4CX/6Vu8h8g8GWYNCXBCqg193h9a89QISnBMD8/hzjSR6LjKDbEvkxranRtISej7EufRkrj+ub22ymfV578yGtjAiLPqoTk433OVm2+IHP1Rsj3p5MefW732a+dHaBNErpjrossjHtwvWiL2azVRJbI5SzF2TZcoU2gHwJUeRRlTV5VjLaGJDlOUIWGB26+jXkqlZJI2WLbi1pmtDf6lCXDctxRplXmAbiKEJJw3h6TBINkMZJ/mEUuZO0dSuWQXfIUx++ye0vf5Wwq1iWFlk9xvdCLm9vIGpDIDxM4+N5KctxSVPPqd6yPPtSSnC1w49e+gnO3r7PpFJMxzUdOecjTzzFoR9w9xvf5tlPXmeaPeD4eMm3fv8Ya3wubG7z6PaEYa9Ht1MTS0mtp8j+Fh4D8uWM5576EPdObuHXmfuQLAdo3fLSJ7b5kR99iV/+5d/hwt4V/vxf+Kt879sPePz+f8Urn9hg2NljUgp6UUS+3GeZQej5COvYb51OilBglUQbi1mWCM8yW57R0hDHEbWxtLohTUJef/AuS68klikfTS8Qefe5/3BCgGLi3yXuKrrdlssXO8zmBZOJJQk7runJSMrC0Ov1uHhFcLA/J01i9/6oCdOxgrpDVS9ZLBuGm4ppXrO1PeTpmy/w6ne+TpwqkjhkPC5pDQjhY62k1S11rQl9uWq2gvlsQRD0CUPIizlx0qESdrXWqxBG0B90qSu9GtoUorXQKEIvdc06deOq/jDMssc8Wrb8N//0n/DpT32Ob776DnHQ4X/2n/51fvcLv827777LV975Pj/zmc/xmZ/+Y8i45eOf+ARvfO81NvoRzWmFlYIsW5CGA3QlqOopnopZzmp0LXjuI88wnU55cO8xg0GXNE05PTtDa0NRVKv0pqKTdqmbirouCUJ/1bCy+rsjiaOea7XpBiANRVWgCkvod7BWorwPoOq6tQjbYlpB5F0ga47peX3mtsXoiGWtuXRtyI3rH8KyZPPCCQdnt4hCB20eHyl+7wvvcO89TRR2ETi2ZxIGGN/Q7cacnpywubFDVRYrQHqBJxRFlROGsRtClI+hpancIDIfZyS+5sJGTFZpFnmAGmaI1JCVDTf2rrD/7kNOHxzSGwX0kph5XgEBmJAgcAelqpAkqc+nf+wjvPnm95hNaoS0tFpjDMRhRFUV5IWDxTsAt12jhM4B5O1qPf6DlAmsRQko8+wHbAp92tolmeu6+WDVJlizFltborwQ3TbEcYLyPUBTFEtabel7u1y7CbfvvI1uEoabIeXSMUvb0ud4mhPHM8D53HzlYW1DUy/c6jIVjPo7zKuWI1HT6wd8+KktionieHIHrV3TUF0ZEIYwDMnzBiENQSAxQhPFEk8WSFvTTAWhEjR1yfvvLhBey3TilDWjG7CWre2QwI9ZLueEMuTSlW2yqibPNJ//t48YDbcwpqDVltZUCCXR2lUPtm1LVbb4KqLINcORwhiFQBGHEdZkCGHwfB8pFV4I2bIAYUhSD6kUrahIe5LD48cE3ojtizA5naN1n6ouEMrVxs4ncOmmZnYE7UIRehFtp8JYRdMqhp2UKhd4UYsRCYtiiWglViv8oAK/hTbk8Owunu0yPhkTJ7hQWWhpDgdMZgs+/HKPa1f7/M6vfZ9nP3KN779+QFNLrr14gdHWJoqYF196jiefusLXv/ZtvvfGHT760YtEXfi937vNd75zyNb2Db7wxS+TV8ekSYdyWeCjeHD/gH6/SydJUb7EGM3h4QGtVkh8/j+0/WeQZml6nold532PP5//vrSVWb6ruqt9T/dMj8fMACDcwIPkgmaXIBeAdimKIiJEUqTE1UohkSEtdylaQVy6JZcAQRAECcKO65me6Z72Xd1VXd6kN5//jj/nPa9+nJwmfylGEauKyKh/GZVVlSef8zz3fV1KF/XQqNSHZII0rdWEdbQlpBH0qYocqK82QdBkd3cLx7FR1QLDCRChS1WVxKmN5Wzw//5H/w3ZLMKyNO2gy/zwiCILcISHZ5XoqkJamjyx8FxBFg1ZHBrMSgs5MJFIxMkzuEwL+q0Ow8mURtPHNAWz2YIyqzCwkVaFKkz29yqazbrQWZ6ULqN5he2brC6dZrDqE0cp9+494OzmefYP70BVsbLU5eB4SLfXIElzquL/D6aczqBJJTXZUYg/WMJrxAxHE/rdASmKKmsQFHMibaPtFF0YpJmF55hUuqCoNCQlqajQ2kQKEIZJoSTSTvm3/2zCH/rTLoZvUnptmpZJZcTEqcRwwZddOut90sim13QYHu7Q1wbpgaKauPjtnOOtEtN02Lm94Bf+Vz9EuDjmwdYxQVeTTjUqb2E1TF787Dl+43/6gKZnUVkl5WRBmAwpex2WAouta9vcvxrRanro5hbu9oROwyedV2RqgeWamIaBaTkUWYk2Sgzt48qK3EigqrBySW6aDOcLot99jTjM0BfPcPz+Q37ur/wCt27c4eFv/huSfp/7hU3Z7XB96zUuXDjFaqtLOCkJK01pNbCWDZxpAobCRRATogQswhzbMpAahFt7TtdPrZHnKXt7OwjTwBQO0pAYjqZStXoqLwqQFdow0KqW0YyjAk+5FHlKqQuEU/NNq1LhOSaGaWBKjzwOIZ5gWQ2y3OTiUpef/+lnef/Q5dHeGSp7yvu3Dlhe6TBPY7a3My5uNBBI8F3SccTnv/gonbNPklGw9aBg/M2Eyf5rGNJEGB20kTK1QmJlsty/gOEuc/3tCac3zhOUI9KjO2ye7TCZBkh3Cs2MK08b3H3TQytBHM0x/Qqv5fBwd4LjW+SWR5FqFukIaZqUluSrL1+jv95jeHjAz/0f/zzD+9fp9QQHkzk3dwsM1cSWAww3pX+myd17D/Cbzsl5TpNME9rtLp4dUTZdhsMprmtjuQ6T45CltT6DXsCDBw9YiBbd3KKSgneqBc+0G7hTRdOAorLoeEu0WxMCO2DruEJYCksVOLaNykZYAgzTZ+P8Zfb2X2FJeuwlFlYZINWci4/FjA9T4t2KtcGTGPMdlgeP8Zf/D3+VX/j5n2Y0mjCdJLQaKycIovqlRBo2nudimxYYFUmiKZUgmiS0u60Tw06O0wjwhaTMK8rIpCwtbMsm6LZOQOkZ80XKZz75PIvZgveuvYf0JGXuUqoI1zHYvvsOL1GyvHGZJ58+h902GR6GeD3I0ilXHx5y9Jtf48Xv/QTzXoNLjzzG4Z1vMjwlMWIwpYnvWBTOHNNqkqSK7sBgOsw5Ot5iMlI4ToM0y5CW4MyFNnvbC+JI4Xp1635ldcBoskc+rl3fpQhxzSaTKEJKn8ViVkOXF3OW+6sIfFyrjSEXgMD6T2LnVeFSZJrFBJrtipX1NYwsBHVMmTmUuiBd+Lz7xgOsxhiETZUdE4UDHCmodMJstM1o/xCnkeD4DpIa6G4YIGUNCS9VxiIMaTWXUSo8USO6VGWFrkqkZVNmCikdsCSLOMKcbdESy0SZZO2Uz/FwD7fwMRuKq+/dQLQbrD92mixP6K52YTpn786IbsMlUjZKa4w8o0xsHFvw/IsXePXrbxFHDlVpIIVEG2W9LQkz0iSvLSNVxgnqlqIoahxRlpBmWb0BzhVa189+pVT9Z09ifN/HMk2yNEYoTjKTJaZps1gsaLd7pGkMWpDnCtd1aLZbzMOINC8xpIOQmqPFEVeef5Io/oDRXokrKkwkWhXMFjM6QRfbqZV76BxV5TXBAhPTNmlaBt/+6haVpXjqxQZbtxbcvDnHpEFROEijIMksTJ1hyoJFKXH9DoFtoYySeTokmdtYnmCw7JOkCqHgYH+PoOWxutZgNJxh2ZLllS47D2ckWUWu6mFTxybDwzlRWrG06dMdVAwPFgReim02KVQK2kKpkLLKSBJFtxegypjN3llyXXCwt09g2xRqgZCKojDRpmYRV6z3POxGwHgSIT0QsoDCodtaZdB0ee+De5RlQb81IEwXeCsmB9sm0srQMZRlE6uhKcM2cT7FaYk6K70fc/7sRXa3D5jFIUErYNDsMJ+NaglBIVA6oNkskRjoasrOboo2ZM1xriS2NeGJj2xw7lKbb718lY9/5ikuX1nmwqVHOHv+NKbrMh/XRImHu8d8+UtvcO3at1EKfM/CsVqE0YRcGQjpYZgRZ9ZPYyJYXV0lLUIcfPJsTpFrLO0SxxGGOJE5GCW2qKMtg5UOIJlP5piWQRjGWDa4VguqCFMUQG3JCVoVjaZLmuZUStL0W0Q6oSoc8mjIX/yzP4vjWPR7XVrNAYvFjGSRYZkZucpJhxWeG6CFpO0PUFbO2sBjOi6YHymsRQhCosombiPF1R2spsIPLfI0ZhHFtIIOpx4dkMuCo50pg16L4eER2UKjlUHQcChKkzMbawgx58HWAU7zHI888QwPbu+wO9zCdiv6yw2OJ3ukqY2XVfiBzcFu9L/8QFnR5e2vfZPd+xMuPvUID3dHdCyNjiWd0z2KYsF4sosbOERhTqUVZQlOKyCOau+pqjSm1EhTfpjFK1WFbUvCRcGtq5JOfxVJhnAEOrMxbAddmvieIFtAmU4xB6fwAxe7qbnktHnp/VfJTZvPfO4c3/qd23z+J8/x6Pet8T//d/dxlEZPDQzDw7YLxsOM7b0Rpy43efj+Ea6tGY0atFXFw6N36Xc0jUaLxz5/nquv7RA4HoEDw8MFVV7z0wwElT5RNgkDYRiUVYipJFLULTLH80jLBVU+ZqUl8Ltn6V05zyd+/Ee5+NizvPH77+F5YKkJwwlMdMRTzz7KxuYqr758h9kiZXWzxXJ/lflUUfkLRqMRZ9aXyR42KeNjvEaJLS1UZJIkKf3eGp7nkMQ5VSWYT6coBUamCZo23a5HHMf1FrkyMaWFoSuEKGmZXcbHY0xTIpQNQlGicE2bOMlQYkavFdDq96kSE8OB7eNdrrzwGZYuPcm6SHlHvg2mxGt69HobLHaO0HoPC0GWTVDHET/6U5dQrs/Occj9a0eMb09RaLq9gFKVJMkuZZbSXemiLJeFGPPpz5znaH7M++/e57mPPMVk8pBYxFRBxOqFugk93AsYDNrcun6XtUfaTGYK6YLlTbFtUImH3YzR0ibLEo73QuI4pdfpk8wE5ehLrK90WcQmUVxwbsNCRRXj8X1SGdHwfHqrDeI0YWO5y5kL57mzs4fKpoy2DYRMaDR8vIZHnuf0lnq1kxeFF/Qwo4hIW4gqQZhNrk0Nso7kVGedYjdhcPoJjg5uce29begpRJkRdGzspsGtW5LlnqCvY9770jtYi5TiUg9LhDSli7PcxI0meAFsnpJMRtfRleL2+2/wI1/8ITwp8Js+aZwwn08RVkVl1Gfu76j2tHZJkoRWo40qKno9h7IqyYsU33fJdIHrm6iyxG16pOGCKM5oBB0sS9JuBBwfTXl4cB+XJqSSUysdlKGZRJrv/8EX+dV/9TKPPrIOIqPZe4bbb98kzSdgtbFpceftt4gGLr//73+Nztoyn356meOqIDnyMITCkjZlbDBPEq48fpGD/RHtDsxHIeNhiq5MpFmhK5fjoxmzqXmCrjHIshzfaxNFEfNpgSUb2IHCtZpkaYWdm8xnBzz9zHPcu3ePtJxiSoVjW3zq0x9jvHeLWzfvIP4TEURllaSqItcVepbh6CWUWeI1mpSZABWTF/VmXGQlS0uP0Fvp8/7777N6psHWHYHvlVRVPWgFQYP5PEQphWVLjo6GCEMwPJ7y9FPPsbszxLQ8mg2X2WxaF2NOyi1SSoQhSYsSSwrCRcJgxebiZpOzl9Z5+etb9Nec+krQTRgN56SGQ3t5mbKqWB206bkd7j68gesEVEoS5Qam7RAtYoK2zcr6Jndu7IOoAflCgudalHmJe2JPWixyoMaKlUVFYRWAwDYtqkrjOC5Jkn1o0TEM40O3dW0SE0Bth5GyLgFqrWg0PCy7Yjwt0SJGuiWlETGaTvBct2bmOhb97oDbH+zzuc9+kW998xsc7ka4Toc4HdMZBLh2wHR2TBLntFod4qT6cMurqhylHJaXGxyNZ+zcTmh7XRbFnDItgYKsyBBYGFaOYVlUiSRWc7Rh43gmOm+gqhSdO1QssYgOqbSB7TjoImc2BWnWebu97ZgsrfB8QZkYqNLDsgsWUYTlNcnzFMcT2G5R8wXjiIqKZsuiTCRpXL9wZllBNNOIKkWbM3oDjyidoqoAvwFde5Pd7V08mUDR4uh4jNOqG/X9wYCjwyGHR0esLp3jv/6lP8y197/C7esHND0baRq0BwVF7LCy1mNra06n6yNFSl4JLMPG8TK0UjzYuY9pOAgMnEqiSlCGhetKCp1jGBFF1mR59TSWlfNf/vz3oki4//AByQKKPOPClXNsnD3Dpz7106SJ5p133uLe/R3efvsPmM+nDHfD2mYg6gJhr9euOxtWiGkmeIZNtyWIs5AigePREE1EVtj4/oCyiMlihWEUxHFUbx5PWLO61Jhmfcn7whe+n42NDX757/89skRiSl2bmLwabVWmEgwTQ2oODvbptjq1ktmtox6e530IRnddlzgOATgYbqGKFGlKtGGhNViORV4WWKZJITWrp9eQnSOC9SHnimXefWULszBxrJDpXBM0FtiRpN13iFKbMo4o1JT53CYtC5SouLe3jxQCNY7wbBe31cVvKEKZEo5i0lxxsH+fnbs3MaSHyECZFapIyFKBKRzCqaLddmi0v/ua93c9UH77d7/C0cExnWaD0f17rHRrL2S306W/0iTrPkYy3GU+mkDlIm2TqlRkcVLn9lyLPK+RM1LWEEoB6FJj2QbzsWLnjqK/2sKpCoxWwdFDgekGNP2Ao3iCE8/JqgYTf4QfaMrI5LU7HyDcmKarOfvCU/Qvr3Pof5Nf/rdfxje9+orjWRhxSNNyyMLasPBzP/dD/Ld/4Z8iTbBdzd3Xjrh4pcfVbzyg2Wly/uOPsHT6FEtrPr/y97+FUSmqokIa9TkI48Q2Q336NQwDYZgn9gZBlqfYgU9SCh6MY049ucTGqSu88fI1/sU//+c0i5jTTcFxEeB0FR0hOdxeMN4J8SiIywVG1ET6Eqmm5ExpdAomk2PyvOTpJ15ge/82YRYS6gSZ17aO0XgPA5vVlT6VTjg+miMNl2bgY1oGaRyxurbC1s4xZVXbRgLfRUVzHK8ip8RAY+oK3zEpqwLZ9GmgUGnK6DBDNnt0l+HS2WW28wO+cuMmiwjamwHFTGFkOVfffZu1zUcIFzZVETLOIp54aoWP/kCH3/w3b/LGlx8yWHbpLwVUrsR3TPZ2xgRtidINbM9EepppnPK9jz/J0mzEnbv3uXN/C2nZCKlYO2PgWT2iWYPD3QVluoMXNJlOEspKEi0MhO8wGY/rIn6qWV1dJa+OSbMEwwazkXGqNeDOBwsO9nP6gx6T+ZBGt8Wt/X3iPGbz7BLt5QWGY2KEp9kejkjvPsDIYpIStFtQlRWm4538sMyxbYMg8NAGxJlCmYKqzPHSgvG9Mb21DWy9zcpKl/vS5oWPfozf/u1t5mFG14fnnl7hnQ8OOZ7mrDYH9D0DES7wgpTv/+R5fv1qiKEC5m5Ij5wg9djXBeHcpOue44t/9HP88j/+ezgtDzKJ4wQEAYzGMxq+g2mZSENg2/UDVFop0lLE6QTXblOqBLSFY3VIM0XpODh2m1kxZT4+5tFHuwyHMYvxgjRR+IFNo+kRzxtceuIsWbLNC898jHOPtdnaWXD20U1++PsCXnz+E7z+5qvcv3+Nm99+G2018aTFfDpHWjnrGxeIsozZbMzDh4q49EE5vPixT/DWW+8SZxMCv8XW1gOkaXDv3hy/aRHHJkpnCKDSCsuuVaSOW7ewMSWj0YxLj64htOD+nRHPfnwDz7P45le3uXD2NKtrff78n/8lfuEX/hxlkjEPFwyPYg6PpkSRTVg6CPkfIb9hEuNZdVTEsDQH4yHtTkCaQhYnXLx8Bk3B3v5xrWwTGa9/7QYf+dh5Vk5Jtu5c5xOf/QjvvXWH4+MFq2t9wnBOpWu4d5JEuI6DEPXQ1e8rtGjh2ILj4QG+XzfPy6LEdRrkRYVWKbYjmE8T4kVIeyB45+03Tk7+ECtJkqYs5mO63Q6dvsFiEZNri8JTrJ9f5uheCFVWMwqtuuU5Pp7i+42aFJDXqB9BgZS1C1opRRjOEaJeFuRZrcRLkxzLrk/8plHbTWyrzqatr68TJ/UP2nC+OBk6K8qiRgilaUpRFKytrSJkxWC5x/7xPYJGgDAChscxnXafZrPJcHKE6/sYRkaRKx7czJiOKpptk0U4RciAOMxJxZy8yOkPOuRFSrNlE4U5ZVlhWR5HhxMabUnTa2CVDXReofOSPMkxbUlVgialveJjSIv5PMMwNZ1T9QDVaxfEYUEWlRxu3UMpRaPVxWp7NQ4t0ViOoKgiHCcgLyKq0sYwKqSMmMZmHR8wCpSCcFpQKYn0TCy3RAPzWYy0XCxTUpYZhqho9ySmcwSZRiUurY5DnMcsb5xjcWRhWpp80WBsjHCtJnk6p7+yzJnNxyjKt6iqkv3jHb72dcXqeguvkRDNFeGipN11Sb2YKEtxfQVGgWVXUFmgLOJwztIpl0ZTsvUgpN1sUpUFcZ5SaYPSyFBC03Ta5HnG1t5t8iSn0gPOXrjM5Sc+SZzOidMRX/naDdLwA7JsxPhoF4FFkpQ89elT9M9nlHONlILKqNmQdSZas77RYHcrQkrJfFYipM3aps/4OKTKLeI0YbYYg64QhouUAs+30JUkT3MqQBguRVkhLfj6Sy/jWAaLeUTgBigdY1neyVLMwg+82vhU5QRBwPHxCOvke+H4+BDbdvG8gMVigeu6NJtt5vMpqpQYokVVligKbNvFlBZa1PnMKD7k4Z0St1uS5Aa9Zo7tS4QOKKoFrnDItMaqUipDkKR1uSuNJGMi2l2DRqeDLWA+rbh8ZY3+csIrL+3R9BvMh4f86B/7QZa6Dk5T0pAev/Effpfb70V89Kk1xotj7lxVtPsVs+OEK1ee4+p7373L+7seKFVRsXHpIsPjB9i5x4ufeoai2aUoS3bvPKQ9aOG3lpiPJpjSQZVZjQqSgrKKcW0PaZmossI8AWvX9X9dw3ZNm3BRUFgFTtDisSdDtu4eYNvHGFYPMZFUhk/QhOc/BYZrsuxfYvuwzRu/AWXL5Ztfu89mIydYX2Jr9x7hsYPn2lTJnNRocqhi3MDi4HbG7WcKPvYjn8bI5zTPLXGw95Brh0PWnjtPt1fwcH+XP/6Hf47XX79Bu3OdeBRjnLRCK60/VI0ZojZ1CCnqPF1eO/eEAMO0Eb1l7gwfcvfGW9w42iNQNrEh6NoF/fYS17ZLBnJBlluUhUEeFagypt1usYhC5tmc849scME9z8PdW+wvJixf6JCgQFlko4xWUyKERxiGBA0HpWA6HWM7EkvaeG7AeLLAlALDcFDKQmlJUVYsry6jdI7sVwRhTjcTKMvjYDzGlBVpVuEaNu5SgzCNWW/Z2OtNjmb7lLsm21nE9Xfvog2fP/EnfwTbLpkpg+ztI6LJFmWRUjS7iMSjd2Wd2zsRH3/uE+zeKrFXNMcPJ/gLg91ZjBN0sKwWVCPyQpPkBafOSV597V1ef+l9nMBEdlK8oKLV7FLMmkznJY5VoKqUZuMU0liwKFKagUuRag4ODhDSpt8VHB2NGJoZlahIijFPPP0Mu3v3kOUqq+fPcHD3DtF8n41Tp/j2a/cxlM1Sc4nh7QKjWOWRyy0+eHAXe5oR7me0VyXR3gK/18BpWcxGGZUSOI5HlmVU1F3Ydq+HzmNMPE5dCji/+iQ3tu+ipkPe/fp9+pee5OWvfY0Ht97Ga3sE2uL+UUYqTGRscPkxxe4s5U5sc2nV473ExK98htE+fq6Zh12OTqXsz+tT/Kzc54P793jmo48zjh+SRx6zaURZQqNhU+kCrS0c3yaOagNPWRm4ThMpCsoyOcEG1XmtRrtFVFTkWYau6nb3zRujmlVZZqysLZOWEck84cHkBmV6zM//2R9j627EYm5iM+CDa3tcurDBe+98wPmzTzNf7PD1f/eA7unzhNNjojjio59eJ5cuZy49ys0b19mbpISLhMsXrjAcHSDcHRqex3iY4gWaNIOzZx5jNh2ymIdYdoWh22hZkKcZjpmTJi6OK2pvfcfg7dfvsLxmcf6SxeHukMHSKZSO+ez3/yQ/8RNf5C/+xf+KH/qxz/Jrv/qrZEmF55pkoeLhw4eE+RGd1sqHz8QzZ9c42Nqj6XVIqjFu2wERkuUZaxtnCRpt4jgliYasrp3CdZZom3vsXh9x460QXRaUiw6SAVF4yIMHDwh8h0ajz+rqKm+++Ra2LRHC5v6D2xwdjuuNh1e3wMNwfrKZFHXWzTAIPJ8sqz3Q+3t7rGz0aLlLVFlUbx0nI4Qq6XZbPPL0Ixwd71NpweHOBD9osLxygaJ3yGh4DynkCapJ4WgHszJP2IVZPaSLCn1SYlCqOOGRVvi+jxDfQQEVdHs+liUpigKtDaqTcs53Wt9lXnyYr5Qn9/Isy9BG/b00D0MwCipauL6F6zmkaUxRJLQ7fs38bLQpFYz2Clodl5df/U3aHZ/hsWRz4yJmcMRsWjIZLk7oBxFSmCRxim27KFWRJXPanR7Syhn0mqhSMJuOkZIayZMaaCza3YA0F8ySMa12gO23cLoJqhS4yieZGfQHXcJwgRQ+gdvmeLhPVRU1piWqGCxdYDi5R3/QJmhWxAuH2axC2hVaSJoNnzAeooBK5DS6y8RRRrZQmE6KadWKSq1c2n2F45Wkc8HGAB5sRSxSTWWabO/s4bklwnYQXowuDaRV0gp8xsMJ29tfp7Msak1frBjNRiBc7tyAL3zhY+zsbLGzf4uG32W2MHGcAFP4mKIiUfuUWUmeuXhNk91pjuNBnGiMIGG5ZzDe0yjpoTNNFBa4VoprNnEDya1b3+KDW9/id34HQCBdiV0VCAOcwGBlvYVAc3ScYBou4x2fJMrxWyZFZiBMg3bHY/PsgFzPSNIYz4FKgMpMjg4WSCyiNGV5YCNFl9lkgmFUlGWKLqEsjLpkdoLiolJYpsXRwR6mKQgCh7IoMMxatCAIWF5qUhYxrmcQp1CVJZ7noKvaG/8dt32W1QW+un1fKxwdS1EpSV7lWI6u2+VpjO1I+ksNtrYjGtYxbfs0+7cPSN2yvlzkIwolcK2cMoHBagPh2ihVkRxXrJze5PS5ip2jA46PdqHysG1Jd2nAcHzA48+eRZolt67H9C5e4cc+80n+9P/mTzPbn2PmLu5axdVXRvzS//2n+a3f+AbvvfmQ/maLr730DhjZdz1QftdpS9OxadgCT9q4jQArCLh59T6vvvQK779/j5e+8hKTWUzgBFAqMCRJFFPkGr8R1AwybdRtvBPyfKWNE4aTgSEK8mJIPJ1z8/4u0+M5v/BXTrE86DI6imi3PI5nU9zWkMA7x3P9F9gbDmmfGfA9P/Ui49kWa5sdvufHv8gTT38v/uwSxsxBaYhVCyUqfKPOtuztDflXf/9XSI0xnDZ4/e2v8c6373P0cI4nK6ZDh/1xzN/6R/+QL736ZZymRZHXCrkKDeLEoSpqWK8h66LLd6DPUkrKSmE7guFkD2/Zx2nWOaZ78h7TVsQHM7i5qHB8D3SPeFHSbffY3NzEdTvEecVCRTRX+9w/HLI9WVCkAb5TMp0tuLd1Hd8XSKPFIpP0lyWWbdBq9vHdLlXpEC1KirICmWE7AUVhkCaa6x88wJCC7lKbg/ERYZrhNDvkwiTKcsqGi7faxbVsWg0HoQom8ylJqqkMKMoQSwQsRikUGrOqHyi/+Qff4NqW4t4w5mM/ssazz1+AsKKRacZyzGwPrr22w7ba4eIzPvP7EZXhomyTjrdGEaeE0Ra6VCTzhMsXH+F4r+S9t24wWFUY1HgXVcJwdMjWgxmUEt91iNMJcXZISYjt1Jq7JA6RRh2uNn2XjfObmOYy8dzFNizuvb9PduiTjibsfnCTjm/T7vSZDeecai7z6WdO0XISNIowTAlLj8lBRjDoEnQbpPOK/vkmeVqcvLVDUdRcSt9rkWUlvtcAKiJMiqLgrau3eeXGAzJ7QdMxWG5COZ8zHEXkGZjdglwrFtIltHu4nsnBQjPNJZ5XsD9L0PYpfvgnvpdBq0nh5oRiwav3Q6KFDaXN6soKt2/fZGcnZP9WhN/pkMQpUghMYSC0Aq2oVIEwNaqqoJQUObiejeMptPJZWVvl3MVzLBYzbG3RsJbJoxzL1GycOk1elhhWyanTm3zmUy+SpAWdvo8bLPPNl7dJSshFxtH8mE6rSyEKWr2SW7df49d+9bfpn1ojHT5AuG2ef+Zpnnjsac6dvsy7b1+lTBO+8D1/iM3T54nCI/YO7hD4TVTlYLkWrU6NzkriFMuRKJVg4JIXM9ygpN3q1OBfM8Z3A4oiQmIReA06bQ8pNKooOT48wHct/sHf/pv89f/mr5PNDb76e9/Et9o8feUjvPDsU7zx7d/HzSdcaC+z6gcffpxdeQLTCdBOSbxwMOIKxzGwTYs0KdnZ3Wc4PuLcpTM8/sRH6C+d49mPrDI4NQZKfuAHv5+9g1tsbd2g3bJII004DxkOJ9y/v4XneSdGG8XR0SGaHMupsB3J6HgIlURo80OAuyFKVCnwvS6O41AUKR+8exupA5YHfcLZlDKvneBXHr9UWzDsHstLp3juo49w+mKDNDHonVqls9YCIIlykiKnEgpbGB+atKSUSGFR5KqOARkGjlcXnOI4RFUFSpfYLiRJfHIGdE+GT/Uh3DxNY1zfxbRqFWOpFeXJcOq6LqXShHFKqzMgySpajTZalRgoWu0mujIpc0mWKvI8R1ol0TylN/BOAPqK2WKHJ5/rU6oUSzYwpYttmyAKmi33RGloE/g9ml2XZrNBFEVMJ0OqqiTLy9oZ78C5c5t1QUi0WF9bxwk0eRHXXNgs5WA3ZTJUHO7PsUyXaDFnd+8hRlUhKoVnmhiVTTgf4Vhdur0+puUxXUxIY48Lj2xgVApdxXhOzRm89NgZ8irFaUsqCZYrcRvgBid+bGFTKkUpcoaLErfpIiRYBpTzgOMtD8e1kbYG7WGIvP67cArWNgKCIGAR5hhGQBTmjEYZg42Yr730Orqy0EWPaK5IkggvUHQHkv3jXTq9Lr1+gzxPCcOQShdUuYlhZfzh/+r7eOIjazRMF9MC3ykJfEhKyCoDK/Bodxos9RqcXuuxPuix2moQeAOaXgNDtYjTkvGwoNlf45lPPcMf+/n/guc/+QLzOEF6DkoWYOeMZsfs7gxptnww6wWVZecksUJpWN+0SbOIcJaAdvC8mnIihc2FR86T5TkbGxs8cmmDOKmtQI5bD4Cq0hiOpkRw5dkneeq5R4izBdoAadUluryslaNZUXyoDf2PjF8JVMxmk/p7xoRK13gtrSqonBNsVE6WG/Q2mvSW+8T5CGnCU09cplIRpJKW45CnBoaGjzz9/fyVv/Y3+Kk/+gmW1m2m8R6VaPLYlXXWNjcYnHHo9Ru8+vuvcevVfW6/e8hsXm8aJ0c7/Mq//LtsNtf5pV/6ImQRZy6usnbB4Dd/9QZCKnyvy2KaIWVGq/fdY4MM/Z9kgv6//br8yae1RcJslKNkQXdliTKXhIsFGkFRpegkoUtKlkJhQDoP8ZsOhl2SpwW25ZHn5QmioiLLqVtLZUmeJ4RRg/56D2ulDWHFpWdPURTbzCcg44xnv2+VxBiy2vwYneB5/tVv/C2CvMXP/td/hG98/au88coNjMBHeyEff/LjVEclv//vf4tW10OHc2wzoNIKoQ2ka+Cca7FyucHw7pS27nEQ5+ztjTEtQcu1efTiKR4e3mW85ZJPJyfMs9r6Y+gT9zGKSutaTYdBnqcI00A6LlFRkhcaQxnkVY4b9Fi73CItIo5vhlR5Sru9UhsXzJRec43JeMyzzz3K3sE+42mEdCw6A5eiWtCzl3n22TV+70tvIqqKZfcsxwe7aHuOacFiahD4bebz6ER3pihLhWkXmJaDymsHr3RNlFDYTRdVVeSqopjlVCY89+zTTK2K8e4WV9wO49mUMIrBzHCdNTJhMIqmZKqi0ZLkaYqh+5RxjiKh1bFodLpceLbD1q2E566c4fTKZbJZj1ujV8nykN3DY555Zpmb39rngxt7NDuK6czF9QxMC6RwSPOS7kCydtolDEOSuYvtaU6dXuHOrX2ieIxrdQg8n8lwRNBpsLlpcufGHOnC8WFMYDlobZCpCtfzOXuhw97WHMuxCcOYlcEKUrtQaj723BWyPOWtd2/QX20zG01Y7gZYZp9plLE/2kYmBZuPtni4t0N70GR/G2wVM09DLFPjOm2U0kRhiu00AZjNRrXj1TYoRYGODQxpUVkxvn2aS2sF5UwzWjTYPbpJ4Nk8vWKwF2puuT7BLMUIbIpMQJGSzhSf/siTfPLyFf7xb30V11ZE4RGpUiz7HcK0wMDGbfsIM8bJUkrbJSusE7RUhlYZRa7xmjYVijQ2KYsE02wyWDFxPJcLm5/jjXe/gevUG69G54j5KODwYEq716CqDOaLMYYBhvahSLFcD0wD1+4znxzwmU9/khc+tcn+cJuG06LKBGmesUhj/sOvfI1Hn3mecjLhB/7of86/+Ef/gDgOmc0iHnv0DKPDI06fP8c0nNLrNxkezsEwOTg+wLJhY6NFtAjx3A7j6S6u3WQ2qWgv5Zw9/yh3b+0zPBrS77uosm5xV6ViOimwTJvlNQvTNghnJnkR0Wi0KQuD6TjjwoVHePYjT3P69BlAcfZCj/Rwh1kkuXzlyQ+fif/d3/xlbt79FhsbAbvbFbZb4bkli4VEKQu/aVIZFR//xIt8/eWvMp+lrHZX+OxnP8f97Wt0GhfZPf4WZblgtGdz9swK71+7Tas5YGllmbX1Jd555y1WVtYYDoekSYjr1Vy8JC7qH0RFfSpDQKVrj7yuJJVRl2NGxxmXH3ucL/zgZ3nr6hvkOmU2jDh/+iyHx0O290Y88+wVTp9Z5rU33kAU4LZsLlxc5yu/+Q2K0OF7f+RRomiOrT3euXqH8SRGijqvaqCQhiTJ6pOwNGperjagyGu3sDQEruvgeR5FURGFCUIIgoZ3YsOpTniVJkrVA7SUEtt1abb7PNh6iGkKHMfClPU1azZPMKWDFFa97bEsHMcmSyJ0VZeEpAgImpr5tKI90OSZxnUCwmhWO9MrQeCZKFWBFnS6TZIiwqhKkvkJPN6qKFROlla4noHQmjjW+L1l1jYGHB4+QFcCO2gTpbvkqYUtTSglKq83UFLU5dOyzJEqodINKiNm+dQqiAVlWaFUwWJq4dma8ShibcOjAqrKZfPMBh/cucWpMwPKsGB0PMb1THzPYjSMyFOTRlNiuglF4VGmDp6fo5KASmUkixLLBscrySIbrBA3sJGuACnQhsNsPidLLISfUkQNuu0GSi3Q+UkWNk/RlU2r1cJrCIbDEboIMI2UMlVYTkBRpag0pzfYYPWjPZKjLfbemNM+7XG8H0EBTuCjSTGsChMHXdZ4QSGdGn8lJI4u0dpBy5IyFjz28XXccyaPnv0JnPk2f+/v/SMsUyJMhS1FnctdaBwPitQmL2eo0qQzsJguEpb6PnmUUaYWpmmTFxEGJpcuPYVpSd5+6yovvPACd+9d53B/hOc2USRIccJ5NnLiDH7u5/8Uh/vvcvXN+wwPIlynQZzMWT21yvb2NpYw0NrgzJkz7O3t1f+HbZv5fP4h4D9PFZZlIKRC46AKG7epwEwI5xWf/uwXODi6w63b9zh38QzHh3PSOMd2Z6wu93FaLvdvjVnuX+bs5Rf4wvdv8uYrX+L1d96nkh6f/Pwa86lDp9PgaGePaLpFPOywvNniK19+D6o2Fy5v8vSnJCLa5MLj57n27jt85ZW3cE2L+d4EIQSu7bN+2iHLQ3buOiTjmfHdzInf9cn78aef4vr1N5lmc5JoTpJJ+oM20aLkwqVNtvb3MV0wioqsnJFVNpYpydKKbsfBoEIYAlGCpg6TA+gT765tu5hhiGm28HxBnBt88M4Oaxsez764xDvv7TC8XbD6xHn+w298iYd3/yWONPkjP/cMX/39q7z28n1QEUYxwx07eE9ucZy3sUqNL10SW6MKRaxTguY6f+YX/zi/9ge/w2h8gCdXGGyc5b2vvY0vJfNZTBYZPPXsgKcHG/zB1ZeQUtabnBO+mjj5XRuCUitkJTGEQoga56MNE8e06wegZVGVKblZMrk9BwSWGeDaGiFjTEuTYXO8GGLZBlffv4YUFo7tkCUxB/eHbJ7pUCjF3rTAXaoHqfnBglaQcnHJ5n7SJ5yOSLMQKQ0MUWKZABJdVRgaMEpc7yS4q2tNXbPdZDaf43TaCC3Z3hsyz+ZUOiNeWaeKY0ynIFUJsRqxqATCsBm0BoTlBLuqMKoajHz6ylk0JRsrXdy8oJgsUFXBQXZMq7PCG//uHvPhXXorLV4Z1RuMYNXBbypkUzPcFywvC6L4mEZ7GW3HZMrGkB2EFTGdzbD2WkjdIE+P6XcMJuNjtNQ0gi4Nd4ki/YBK5HhWH1umCLPAxsE0NfG0wHU0jj9nNkuYTWzm45K/+Bf+T7iO4O/893+XvEoZLC3z+JWP8aXfeYmVgUmYzshnBY212rHrywGOZRC0DrF0Fw7h4qNXODg4YB6PWN/ss7c7RgqHwPdBCxZZjmF7tDyL3DLIsjmOM2J7OyNLJE880iefQYOMFi5GAA/TErlqIKoIEgPXXiM4OyG6+ya//v41EuliOusIsc96x0OUDZpOweH4kI3eJpnKKBxFOBQYnkDYNioPcUwDoxS4hkElIK8UtpSYpkuSROTKZmd/68RpWwO0x0d1eccJAJEQJyXrpzaI4hmzyYLAb6AMQc4cVSgsp+KrX/0K9+5f4rFnV8jaMVfOXiBTJjvjXX7xz/1pvv3N6xwZgt/8tV9hPJnQ6TaQVs50Pienot3x2D24z/7wCN9wOfdYi8kCtJZs35siBeSNfdZXLrGztYuQFZUyuX51n7WNJpaE7ftzgkaK5Zgo5fKf/exP8c677/Pg4V1MM8exfUzT4vBwwtJaj8qK2Bvd5sFvXyccZ/zoT/0x0krwG7/yrzmezTh7/tkPn4nPPHGec2cDrl99D2nuoIw+igWaEtc3ybOM0Shi5/4RP/L9P85v/Nt/RSHm/OZv/R6mXeB673H61Dl0vIpWN9l68JBux8eyBbs7W6TZjF6vy2w2I1rUbdMyywmLEtd1sT2X0WSBtCySJKkVcTqmVKIuC2LQafvcvvkBg+Uu49mElZUl2pvL4DiYgWDjok9jOeONd79G02+zvrrK8TTkxt1tOhsDZjsLXNMkM2AeRR/yJauqwqgqDCpMuy7p5HmO73q1UaSoTmxKdUEnSVIaQYAWGtexSNOU+TTHduuN5XcMXkmW10B6U5KrAmlB0ApwHKveQJqaRRiCNFAiwm069JpNwlCRpSVukCKFg+1oHNsmzRb0Bz3m8zGe7xKflHHyJD8pPxmUcYnArk1RmORFjZlSqiCJNI4r8VwPKoWUFa5no5I5V799xA//5A+wt7fH+1dv0+wGuIag0+2TK5Pj8YRGYEBZUs1TyDNyQ2BYIcI0CDpTbn1Q6wJts0Wre0w0lwjpkeWC2SLCdSqORzuUuSKe13lUy6/zwqUu6PRdJkOFZQ+wTIkyxmRRBUUCIqbIXCpbsP5YTKu5xM03ZsQpuA0Tw/CYTiK0kWLZFm7bochNtEwJFwVNPwArp8xEDRavTBazMUurpxnOjijyhDxSNP0OGBGuY+I0LIpkzv7NBZ/+Q0/xf/vL/yV/6Zf+EptnT/Pk44L3vnmHJGtSYZBkKVKUZJmuM90OGKpE5AaFVTHP5wjlkTGhSCK+9do/Ye/tYwzDQZoSQ2hms5z+ckWuMhpOQKkcTp1qc7gf0+x62L7HaDRm0DGZJQVVkXJq/Rw/8zN/gv39Q77y0peJkpj3r73LYh6e2KlCPN+FE82wYdgYOuH1V79NEGTs7h7imT1Ms47sPXz4EANJJUxcx2RnZ4eyLGm32yeFOQPD0DSbDcblFG1ItFGQ5HOkaLN0aonzl5a4fu02X/qt3+bxJ64gtaDIcsJoymOPnyHNNNPxlPPrm3gdzTR9h5e+fJvj6VM8fblDy9C4Z5q88a2H9LuPkZcLIi2oOm3u3t8lHuU89Zk1zCJjfLDAtT7ChY+s8sZr1zm/dpFq/5tMy5hWr4s2UlZW23T7NndvChrt72qWBP5/OHm/9sYbRLMSoSsct0ceZ+zcfkA0nrK3u0WlDbQUGNIEoamERGtFUSioNKZlnICUa9XWdzyulS7rt1INvrCIFnWORjsGVAl7u4oHt0sqOSEVDaL9UwhZ8OnLl3D8Ji+9c5s3fudlTOseoTHiYFrg2w0effIp8sFdTNclTo+hyMhMFw1oVVG0TSZjB/YFjabDpTPPEU/nBGZJ2+2SpRNe+oN3eHhrgigFhhBoceJtrcoPob1CCKRlYpmavMwQlk1RGtiGg5AVvTMej3x+A7fZQicJocqIszlK58TCRGsbbQqMDHxpYpuQ5yllWbKIQ0pdErRbRHHF3njM1at3yKclWeoQd0bsGTbXjyGOMgyhT9qTBrqStNoBlU4pK0VVGlBpDFF/k5jSxRQB4TQhWYRkakYcjzk82qYnTLqmzfUHD9mbTphXBV2/i2k5mNKlMgStgUuh47psonOUViySlMPhjOvX73PtjYwf/UOf4wsvfA/vv3yAFQle+Phlfuhnv4g2fSZFwlwk9Doett/AKATNRk4yr1hdGdBp51w+9yTZ3Gbr9g5ZCrLqcLS7j1Z7NO0GWRTT8C3ajTbCnnD9xm00iiQygBmG0CymGtMU+D2HXFe025vsbZus9tZZ7rS5fL6FyPb5n/7FLzMPD3GchHu3HvDqV1+lIRLa/i4rzZLSErR6BnuLBTMxZ3urYrV/BvICr7nE7sOQSjksFgvCeEy318CgwLUNdJkQVJIqmpDmE3xRcKq5RNvv0WXA8kabpuPwvRcHfPQx6F3xmQR1EUrnNuXYxRJNQkLyhQXuEtqwGXT6nOprZOUTl5qD+QGiUnR9i9nuAyZ7kq1Fm3ljwTxN0KaogfQaGq7PxtoqtjBQac6li5c5c+oc4SLGNpsskiMUCbP5EfPZIRUWTz37DKbtoKq6gDYZx5S5REoozQJERpUK+q0ObtBjeXOFyWyXaCiZ7Fnc21uQqhnz0ZBgcJGzG2tceNTn9q0P8GVMHB6zNFhBOHDm8iq+u4JTBqgoxQnmTEZDorBAlwVeUKFLTRY77O48RFoZeVIxnSTEyYSH9/f4oS9+im7HJ0vr7X0Yh6ysnqK/tFK/8DlNoihEGh3OX7zCeBTRbTVBVYhc0ml7fPMr/4Y3X36JMp1x/tQp2sHow4+jyTUagzYvfv5jtFsWpCOq0oPKJ4srpDRoNSxGxwfcun4XI/MQhcSxZgx6ikuX24wOD9jZvonrNAlaEkREVswJGmatBawSVtf6pFleDz9l3Ywuy4I4XpxkFWvFrUZi4IJhIo2AsjAwzZJmw+H1116j3QmwGy3iKmR79zZ5MqXVgCSPOP/oJdYvLJOUMYt5iuN0kUFF/5TN+koPqpJZHH6o6ayHxoI8r3OR39F5FoXCEObJyb1me0pZD5yceK+BE51iDQI3LVFD9yuF7wXYngsSlleX2N3bodn0MaRBXubMwzpbGQQB7U4faQYoHGzPYbDeptM8i8pa9HqbdDo9JkOF1urEzANBULfRl5eXcRyPNMlPzpQVSTojiTJULkFVlFWGY7codMHZc6d57PJHCRcKpQ1UUdFpekynOffv79NybaJpxovf8wKldplNE5pBvS02tIt0TSzfQnqKVi/A8Q0WM49G02Q6TFmMU/J5gO06GKam2bFwPdBGRRjOafg2cXLMLDrCcds02z0s1yBo2SgjQ1szMhVi5Ca+W1IpsAMLw5MI30Gxyc17Q0zHxnd7GEiGwxHCcNCqVhq2W10avo8tG7Q7Lo12hSo8siQkjxXNhsuFi6fxmw4f+cQGP/7HL/Di588SxTMMmZFEFZ7bptl0aBc9NOf5O//jPyFPRrTa65zf/DixLlFOhNksSMoUy2ngegFZloEhMIRLVWrMCnzVQEQaF4e13hXOrj/FmQs2jXaG4yfEcYIftGh2MgbrJvN5iOnP6a7CymmI0jGzKGGwPEAZAsevkVVnzz7BrRu7zBYJ0+mU5XUPp1HSaHqsri3juCaeHSAMTaUKTCw8S/LuG+/yra9+UNu5qpLV9W4dQ7Gsmrlq1OQXpRSu6xJFUZ0F1po4jmuKgV1QKkm7u8pnP/cp0izi9q193n//Ac1uhx//Iz/MNDzm0vnT6GRBPlQ0q9O0ZJe261El0O33aLgBp5Zy7t6/xs69O1zoNknChO5Bhjm+x3jnfebDITs3x+RTxc7dmMXU4cKls/yFv/RnaUmDq+/fY33NYBbdZfVci97pJnmlWWQJdx8ecft+yR/6sR9j9Vzjux4ov+sNJaViOltQVSa6yJFohGNTxhmHOyNanSZRnKHslHajxJjOqWwfnebo0sKUJRqJa5sUZUlZmcgTH6eg3v4VDUU1HzG65dJZ63Pq0bMkOufBrR26Sw7Xr73CxqX3ObPxCO0VH3ZnHL7yHnK5TTltI6oIJzfo9pZ47foO0zgn6JQsZhZFVdC0MoRy0TLhy7/3+8xmH/D8059jf7LDv/33v0Oz1+B4lmOUFRYelQx5481v0rR9KOq3cUuaGIZEGwbSlKRpViu4VIWyLAxlY8uCvA1ZbuMkcO2Nh1i+gYh8vFzQ7LVQumQ6npFIjTYrTEt+iNKwzBoSrAFDWFRKEkYpna6H43sc7Idk2Q693hoFEWtXHsMYH4GyOZ4MUVWCowSiEvSXLUYHBVBhez65KojTgkZTkCQjUu3gLQ8QpsR3WkiRovKCYuLCYobpW0gZMDfrYSmLYqS0eHDtLhrwlgLCdEZzqUMexhSlokimWI5Hu/8Uf+1v/M+c2VhCSMW9/T0eW72I0fXxqpByVjEPBe6gZG4oljsdZKuiSA0skfLwwQEmiuVen+F2wlMfucQ7197Cahr4g4AsiZHaI40zktgg6LQIxzOCZoo2V5jNNJaYobVmOgmJZjGxVzLo1NmrLK3oNdq8+q336DZMiuU1Pv6JZ/j2669xcDTi1NoKOBYPbk+QfcF4vmBpY4mbd+/x0Y+s4+dneDCJ8AYV8fgBWa5p+B5FlpGpgjQDyzJxGi5us83iKCOXLsXMJO0pjMkxm0GDRWHz1ls3aLQy0qqJe2Rx5yjBtw3m6ZSgaKIsqIqcQpU8yAs+/cTz7CcmanwT1624uL7Jq9PbCNuhazXoigV7QlIFLYyxonRcFsWcCyubNJIhySLh4f0jlF2AY5HENrkzRyBo9DwW00MqT7C62UC2+hzcOWLj4gX2trY4Coe1VjSd4eV1HlpWFYNGl4mIafhrBM0Ft+/epilavPrt1/niT/4IZzbO8v4771IFfb789S/z1CcusqJ92g2PG+/us300ZGdrl1ZgMjIj3n1ti0+/+BTn3dPcfO81pguDtabJRqA5yH1aPciSHGwTVI7X9FgkGYN2i+PJhPeuHvC//au/yMDb5Fd/9Ve4//A6f/v/+T/gex1cy61VipVFKTN8r83HXnyGd97+Fr59huWNAdtb95hMJpy7cJ7//X/712gHMFjZ+PCR+LVXXuK1D+6ymGyxsXkOYU8ZjY+wXBNKmyozMaVLWWU83L1BlMfkU41jL6OdiGK+wscur/CevMnD0TFNY42V04rtBxnzpGQwGACa3a1DTBsGS+e4fH6ZN99/j8OdOY9e2WB+OOQ4zLHdijwTmFKfqApTTEsDGs+3mQ9jytSk0Wlz7fpbnN9cocga6Kq2sNzf+4Be8ymS+YztgwM2LE2/b7M3lfidDqJcIp7dp8gFVWEiZYHWJpW2MAyDvEwwgLIAYeboysKULkrlNXIFqIwSQ1poI0XpWgFZaUFRSHKV43keQbdJFGY4rqSsJIZloY26PBF4AWVS1N5xrzadYXiUaQWiosgmuFaLQswpNBwdzvGbDlGYIbRFqRLiIiVGE7n155DCJA4jNAZFCcKAiqp2SyuLvBxiSJuth3t85qOPUVGS5wW+34BS8c3f/wp24LC0MqAoMq69usV4EnL6/BLbW3s0Wz5psY8lSxzfw/IcDg4smn5Fb3VOkgoCN8BtCUY7CRcfN+gt5exu1edSz7WxLIfJfEZ/tUl71eHgOuxPZzQbJpN7OWne5MXPXeHaa69iNZoshgLT8pmNc1zTpxIZ/SWf4czDcxySuOR4mOG6LfJyhsDBMR32tw6pTFhecTi1ucSt97YoE5dWY4XIGhP0LWxtcfX1B/yRX/o8Vqfk9s0dHFegMolQJbNFzPIpxcZmmw++/htE8Ryn63L48Dpv2DP8oMV0PsERHkHTB3lyobQMfCMgtgxKc4EuwHF8xHLOcGGxJtb54OZ9XK9LEmcsdxMmlaDshySlYPl8k82nLByl+dp/mLJ6yiNLwdJN9raOEMqmEaxRGTPeee9NxuMxWhU4do31GY/nZJXm7KpHnlIXcQyNZUmU1sRFzpMvPMZwa0gSxeR5wo337mOYBpbjoLIcA4lh1ESJosgw8RGioNFz6Q9Mdm/ExLlmqS/J5glFKun3XbLYwZMWd94fMbx3m+XLHZ64+Ajj8TmeeK6JU3i01i9gtAVHD++QbG2xUDHatTDSKe/tVjx6/jH86IjpBriehSi79Fcs2iuCpZ6HMiwqu8CwPF569dcZHe9h2yahOaDMujz9/HOk5S4fvHeEY67w2GOnufHeAf/kb/9TBmv+dz0mftcZyuD8hnYNE6jl9HkY4th2vdKtIClT0qKi15IMOjmLmUZVLlmc0GmbYGXoyqRSgihOAXmie6vIswrLMQljBQXgO2DbnDtznv7GgEonLGZzHr+YsX7lNJ7/PXzpt77BaO8OrozY2j1AuC2ErEjTFCcTnH7e49mPf4bX/vmbTIf7aEtSVAmGkkjT4TCP+ez3fRZbuBwc7nD1jWtYwsW1a+NPq9knHKdURYopS/ICvuOOlabxoW2mLEscx6FSKYWAQHikecLKlUvkpYFv54iWQZ4q8hAOHhzTcJ3aDmHYlGUFlSLNUxzH+fDzG4ZBRd127A36SKFIkpwozun1ax1ap7nMwcEdhCHpyAaZlIx3D7h4ap3UKJnEQzZW+owOJqR5hbAhy2xcz8YwFdL2aA7aDOdzXFlgiiZxWtIdNNg/fsDF8+e4d2OHaFKiRYVt1psEKeoTW6lL7MCi22+QFiUbq+sczUe4ac5zn15GmX12X91HOj4f/Z4/wr/89X/I8f49fOlTyZIiyxGGzfK5Fmc2lrj3QcbmxQFlljIaHpDkkqCTMzpOITZYWVkDMWcyS2n1TNxWxPGBQ9AUxKM6m6PLiCKTOA2PaG5RFRF+S9bWkzTCswbEyYzHn7pAI+hz54PbdFtt2t0mo+OIPJNoe4dm2+Da6xVLvSbLA5sHDyZYbpMihzNn1/mJn/4iv/nvfocHDw9Bhqyvw+FehBuYrKw12Hp4SJGZlLrAsiWlLAjcLjhD7LJBmJskyZSnV3qYgc3td/ZoBjbxUFEFJlZ/iWE0pFlJOr0uh+UQmRu07YDdLOd0Z5PPNUy+ceMah2nB5mof24JH+hbf2s/x7IjDRFCIAb6OyYAqXPDkuY/RKg/Y3bnN2oUXuXn9HSpR0BysoKoxplghM1o8/+yjvPTatzl9Zo3h4ZzReI+15RWGoyOKOMZ1TMIoxvZc4kjXmVxf0LVbfOqZ5/m9b7xBLMeIRGMKiygs+WM/+ydZPrvE+HDCLA4Z7t7mlVfv8PkfeY5rb+9ht21mWw+4dC5je7dguXmWsCjpfHwT6+ENRg9S4irmTG+ZrblB83wDNc/xTcHD6weceWyZNJpw50bB9/zwJ/jxn/4ZDHuA57iEO/v8P/7mf8N0OqXdaZJk4YlBB4rcIIlh40wPxymYzWb1M69hE80aKKW5c3+LuSrYPw4/fCZ+9ff+gHBxl+nOnNvvXeXq/ddrRVvhUpRT0JIiF2AuaLebzKY57Y7NZFQy6Lc5d6nH3TvHlEVEOIkQ2CytBCiRcvr8ed769m0soRAYmLZEG20+/UM/TjE54j/82r/ho5/9PsbTbeJyynB7iOu7CCHJ87I2oJT5h2VBLW2KUvP8Jz7GaLzPs89dZDG1ODq6x2wa4QSaNK2YRzGe18b3bHo9yb2bM372Zz7D7/7W19mdTFHzjCyrM5BK1dv/sqwQhlm74UsDaZZoLU5OmQam1Pi+T7Plk2clURShtcayJemJxcN0a+2n7TdBNQhnc1q9Jq1OwGQ6otNaYTjcQ6LB0MzCBf1+l7LQLOYJtmvR7TVQVYZW9XNK5QqV1V+/KiOEYWI7JpZlkue1eacoM8qioirr605pVMjCxW8IlOnw5/7iH+XmtW/z2//6XdIU/L5dm2Eyi3QWc+b0MvNkRhim9BoDJtEx0vJwg5SgVZ/pW60W+9tzVAaGU6KUZuNsj83zgm9/uaA7EAyWDe7enLOy4lEZNr3VkpvXZ9i6S6sjGE7G9NebRFmMZ/R4+DDB9WJaMqC1vERnrcXhvTv4QZvZYock9OptdaVJspgsdVk9I/H7KeW8x/adA9otn6r00DInylNMV9FprbG2omi2bEZHBVs3DpHaJWi7NNo59z+YUZbwiR/4KLv7tzi4Bs12ySIvqDLN859RKGVgWwE3r2ckccH6+iqWU2FaivkQGq0+YTgFM2d4PMcyWihV4AUa27BYzCKcpscizAksl6RMCIuSXr+NqWc01zw2LwRsf7DAmUtyy2SYzHD7XcKtMeHUwXZL/PaJda3RQMj8hP8psTBxpc1iFlKhaxtMqdGVAFmxsrLEdDqnUhmOHZCmgsFalyvPnGXn7ogPrm9jGjmOIVFmQaoKHKnJEhPfFWSlokJgiwDpGRQi5flPnudoa8bO7R18v8WgbxLOS2IlWDp1ivFwl/BghhIuq2eWafiKvb0Dzm28yGJywNoFG9G8wPH4G7hNSRxpZBlgyzlF1iI3WohOSH6UYLYkpwY+hlDM4wTPbOB5Hkm+j7AaWKZPr69Jk4rZNGNx7HPpyS6TxR0GzXNc/faU0fGI6WhBkuScuxxw7635/7IZSst0MVsNpNaIsmRSZRR5gS4MEkPgIyikgWX6RGFMrh10WWEaJnGc0Oy6KK1QutZsCamoqhzL8pBmjlIaS1iURkleptjS5PrbH9DYaXD5mU3WL/S48sQV3r3xgG++/rdYa5+i3bYIJxZZUeEUCZZXomOb7rLFmdMbfPW33kekKZUvyMYZsiUwDRNKwXKnw81bN5BYZEnO6soZTG1iWhqlKyZHIU5VYciKqCgwpYvWJ+xJIVBFSVnUoHOBgSl9ijJGCwPH9hgdTolTxdqKy8pKn8PJhCRVWL5LluSoMscOLHRRYn7oUoZS5RjixLls1W//89kEW9oYpibwJdFckaUZi8ktGn7AYhZz/uNPcu+dt+kJk8cffYIHRzvcf7CHQ4JRmTiOqstChkua5ZhVhTYKdnd3MRwL0zNpdxX5NMFwXM5c3uDunW1k6eJrhZI2qlRkRYHlGGhhsLy0Qnupyc7eQ0SuqIwZvq3wgxaNUyaec46f+bFf5Gtf+jbH4wMs6RL4TTqtLobjUJQRyWwBpWL3cITomVy9/hqedNk432WxO2E2A6EcTLvgaHSbT37is9y7GzKPtun2z+JYMdNxSLs1plj4FNisnW6QRjmJOcJvuBhVraIic0mKDEXB1u4OvX5GsCJRRkplNrHbBZODISudJeJkweoZm1PrNvG0xHV9pFUwPpqyvrRBv+NjBYr2asbxdkgpVkmNBVVecjRM0cJFYWB7DQyZUOWgkhlol8RQCCPDNnrsRhbVUUbLsDhtZ8QeFE6Xmaho9Zt4pcnufA7KxC0thlZGoOHe9l38joltNnnuiQHR0X26fYvrYZOj8ZzOUh+Vpgi5gzaX8QjQgWC6GOIwh7YL7QErK5e4uXOVZa0Qlk17+TGOpwteffVbqGhINCw51VwjHhfM9w7ILc1gcwBGSTNtsD+Z49sCESiuPP0ZZqMh7+1tE6dDHNXAdEuyk4bwO9df46Mr30u8SPBbkhc/c57VtsvXX76Gjo+wMhuntClmS7TklNNn1vnSS29hXWvyM//r7+ft198nniToHKZfuouf2dibAeP7R5x7bh1tVDjhgB//yXM89vTHeOdr1/j6N3+LtVMbrG9c5C//1b/KL/7in8X1fYpcIIVFu+syn6QMBl3Go/oE+Pkv/CBvvvUtIKMoY37hF36BdHLEa2/d4oP33/jwmXjh3AUOrA1+4Cc+xd/5H/4vrM6WuHlvm2bgU6kCxxFIS1Mqm/ksRVeS+SLHb5pMhxEfFHsM55rVnofvCI7nBqPbc5baPmudBKtyCHoOKxur3L22xVq34rWv/SZPPvlZfuZP/Rm2x0fo3OTs5kUcs8nB1h6mWdalGAS2bZ8AwiVFWeNYrr5zlaeef4zRQnL7zjs8cWVAFOYkYUSW2zQdj4OjEQhNmDdx2zYHw110mVCGYEkXZeb1C29V1Y1ay0IYDqpKMRCoqsQ0JVJqtFZUlabRCNC6wjBqBaaUoi7tCJNO3yfLI0zbxDA0WVLSbHZJkgW5Cun1elSqIklqDIvWinarQ7jI6q9VlwhDogqN63kki5LZPK6XA1phmQVr6z0m4xkGZv0yfHINqlR1Uoqp8/Ed22acZmiziWuVvPqljCc//qN84Scv8a//2a8QxA5llZDGMZaArYf7KENg2Dmj6QjbdqgKged2CBdDqoq6cxDX1qDGwMexCn7kpz/J1dfvIrzr5BI0AXlpYbkWh0cRkzDDc13ixYxKezRaFuOjmErW0apG2+Xi4z7xUUyiEiyrz2yscZwpVR4grQhTB5hmjuWYzBQkoeLcuXPMql02zrhsP1jguQalmuPLNqI0KBcJC+mw3Fqm39LsuSEiUxzvHFOWa/itGYG/wTuv3UBWEUHDYzgOwbA5tRpwvGMQhyY/8IOf5+pr/w6lTI4PE7xAYfshyA6LeJeKkrY/QC777O+NcF2P4fGC9aUGjabDPJsgrJqMEVgeosqRUYxtefSNFfLdmGdPP8p464DjhcYf5oSzEIFJ4KfYnkOr6aLyDL9hYzsWWTFCax+pNRQ5VAUIgZQGnuPjCotpkrBYRFRViZAWlu0gLIesypnEd9GVwYUnPOJhiV74hNmChuNRFjkNt808HdPqNTAlVFnNaRVVk9GDBT2/xaIRECYZaekS5yHRXDAdbSHMOQ27RV5mTHZCuo8M+Ohzj/DSV7+BSZtmr4fp3KDr1Ya6bi8km9h48hSLLAHTIJcVDbtke1ez1rXQZYnQHQYrDSbTIb7ZJswUybyk3fZwTMH60gYLd0HQDJmNl9i6fZ/7d0e0Ol0+9tnz5Crn7W/f/27HxO9+oNR5DImF0WmSlgbaEAhTYjUC8mmCsB1WV5ao4hBZueiiqv9RdI0ckpZJhUIogePYSLOiLGsFlxBlfUqvMqQUWMLDxKbTkChV8vabN1g72uD21RGeU/G5Tz3Fq9+8x/7WPWxhkKFod21s1UYOYpYftTk8TunZMEnnCBcspwFVUT/wjPqhNJtMabc7LJIZ/YaNZbbRlUESjzBESZ7lKF1SCXnyIKub3dIQFCrHNOWHeABDS4KgSTgJ6fQa5GVKusg4rlLiJGM6XRA0u7QbbUbhEb7vYwgojBJtaHyvUbPdMMnzvM45CkGUZXS8Do1AkpcGWV7jGVzHwtAu4+MQVWU8eHiPJR+SSrB/eMDh/j6nVi3ifEpVulha4dgBpqsxitpFq8qIyrI4/ciARXRAhQulQ1lImn7FlSdOk+65HNw5Zl6McR2HNMlrY4BpI4TJ3vYB0rTpnPGYpDmqLInLCa/8+4Ik/gq/8j9+g2cff4HLl8+dDOEpQcNmMg2pRIEwIZzFuEYXqSf8+Ec/wgfb9wnLAzrmgMm+otUSmIFHWmZ867XXkeYA6QgODhSmVDQdhygqaDsBtkUd0F9kdNubaBkyPYiwHIm0csoqx3Vc5tMMzT62o5iNJfvOPr7VRYsZdx/M8DyPU6cFR8MjxtttOt0eR+N9PvmDT2MGAbtHE7SRMJnv8syLZ7l754huy0Nqm8PdI/q9JphzUClaNWgEgmwiKMOSwhJ0fIdCZWgnoGW75MJAmCVnLwqKls9rNxbEWuMuf6eZHzMtXHztIJcsun2TB4dwYWmVgC5lfItb7wqS6gHLywHhLMHyJI48g4pjmnbGSFdE5FTOOndvHWG2D+mvOFgjmDYzguAMu8Mxo/23kbLAsSXzcIbR8+hdfpTjnVtQFhwdz/GCAGkkCDPDUBaDjXWORzvcfechhir56R/6Pq5ee5Pt0MTNSzIvYzqaUo0nrC1pLr7Qp+wpRg8Vz3xkCSt7yNbeTYquyamNs0TH3+Jw7006nRxzFhJNNbkM+IEffpbrN7bZeu+ANIqIxzaPX77Ay7/3JudOryDNius3dnnpjf8XtmPwiUef5e7+nOW1VeZRzCc/80muX3+fRqfN8fExfiWZzhU/+7M/we071/nmN7/Fu2/fIssK5scxf/dv/0N+7Ef+c1orFZ9e7WA3/6M14vqt21y4+Bj/+B/899y/uc/Zc4/xM//ZnyEINH/9//o3wNAYQkFpIYRGGIIoi9G6Ii8VT116HnvrPmWcsrTsczSe0G772IHJ/igj6Nu0O03MapUku8U4LTETuHfzDsvrpxhGBzz59LO8e/ttWistjg/3UUXtIrZsl6LI6k1dVWGZgrJSTIZDyqwgXByyu3+E7Rg4TkCVGzQaFmUJXsOi0fHJUgttWOShxYXTy0zG75PmXUwZ0Ok5HOyPEMKiLHOSBBotk6xUWJZZD4+AMASgwKjq8k2psaRJnGSUlcL1apuRMCVlqRAl9SBKiuuZKK1Is5D5dEK73aRUGYah2dxcZzZdsAgntDs14H14PESOan94Epdklkm3a7J5epXh8ZhFGGGKulHu2g6qLP+jClIImu0uVhOWuiaj0THnNs6wN/sdPvinBrsPUhpNGI+mfOx7uzT9DV75yk0woLvUxvZyDndmGFLhSI/ZeEKc1QO0YYBpWyAVWZFhmS6/8s9epVIpvt/j1JmKeOTy0U+c5+7t6wSDFJ03mBwqTOmR5BGqMFkedFhEBgeTY5Y2mkxGPllYEeczrKMUv5kjWcGxC5bbG2T5nDhJaDYkUTJH41PMPNb7TyKWM1znLuPRlNX+JU6vrjDobjKLhmxeXCGaxLz51k3CWOEqA4lJIXL+2P/u+9m5M6HfXOflr1xnMbJ48tIGbpBzsJ1z5+YU2/L5t7/+NSzLoL/c4/h4RDoRDByHPIuhqjAMwf72IVobeK4JKDrdJotS0gs0Hj5JUaItQS7AaUqMWLKIHXbinMunJNHOgqlOcRpNSgHCc7B0iSnbGIZBNC2xTJ/h4RDXbmNIBy0K2q59Ym1yMEwDx61Ik4zZIsRAkJUG2lAYpkOSJGQkNBoOg81V1pcuMUv32DW2uH8c0W60OPfYRQ6ne2xdv8/m6R5pokhCjTQEURJjmA4bm4/iuZLjccRisWB4lGFIAy8QtCqDUjeZJiGedCiLCe+/N+H4oEW33UdIhzTMWW5q9vemePYKruqiRQamg2UabCzlPLhZ4vcDnnhqlVylSDfHacZIu888hkazQxEfsRjDXmmxvNKlMDSuI9i9axInMQcHJSsrK8ymKd/48m36gwZ+47sv5Xz36sWy5m3po4TZ0ZQSQX+phxMIZMcmPlKYQhPGIYFOcU2LKMswTBctJFlaIE0Dy7KoquJk01fzy5RSFGWJFJDmJabZwKg0hqmolEKKgGwcUTqKcZaAdMFp8+zHn2B8uIveK5lHC5pOytPPX2J3cp8kjdlsr3CMQBolsyyqs5BaoIyU1VOnaSQ5+3vHiKZFv++QVgse3D9kyV8mKVNMR5Insjb6aPUhyPw/fTOvdF3QceySsjKotMZvB6gyI4oqvBPSapZkOCJiHsZUBqSlxsgyPNenEpKiSLAspz7PFCWO52JZFs0gIEsSLMeh3VxGVR5bOw9QqcGgv8Ljzyxx+4Nddre2GVoWK80On//cZ/kH//SXsTxwHYOQnCSpQKRUwsCQHrbpk5OjREWcTDFVG3KbyeF9EAWTeYbp2MTjmEbXoxHXbDalS8q0wHJsjnb3EVYNWZ6MS0yVc+HJLu+9dp8f/PGP8967Dzl6ewt5eUxVnqYsKzpLXQ7Gu1SpSRItOH32LO1Oj8OdB1iBoKpKet6A/eM5zYZEdjOKzKLRElRFC7sN+3u7tPw2rl9hmxLtVqRDjziZUVBQlCWuCLC8KX7DZecDxdplhVlaaOXiOIoiK5jsaXzPw7NLLKmospK8BEe0yIsxUVSRZwElMBxv88gTJheey4hnF7h/sEu7Lem22tx9MGO0WNBt9XHMjO/7oS/w8tffwrSbeA3BfFFgaoG/4tJteOyNpzR9Azmpoc5Bz+XhgYUROAzHQyLdRvsmrUbFbJLS6rqUukNgJZTUGbb06Bi0hTm7S6b6GOd7rM1d0nzCdqQpLBtZVeR6Bm6LuNCgXAq34P2jh2y0LOzxDe5WDnbVY5I2wdGweIhvSxxtUToxKo2xVMT2wYwor3Bw0KZFhYEdNCFa4AxM8jTl6GCEK+HcxSU2HtGY7gUe/vZdlBUizQZZWbD5eIOljZSwLEgezDl4/w8YzSSWdxHBMoc7Dzhz0cJ//Ccpjid88vmAKjjm5u37GEaX3/3SQzqrBs//2BO8c3uXi0ubjG4fsyQr7m5N+cKFFn9w5w6GqXmke5r9NOOdN9/m7bdfZ3mly5/443+G+3fuIg1YWu4xnR0StFy+8crvMRpOeOK5VXbuz1lZWcLzHF55+QP+/t/+Yb74fX+YzVMDnOZ/hPzu7x8yO05wY5MrV65gdHJ2drZ487X38AMHpTLSWGI7FXmR1CdXV2GqBtrK6PdOczicEBsVcTblymrAe9sLVJUQZfMTt3pEkR9y+mxAHJWUpsl89j5G/ICnPvl5tKo4tXKK+eGQU5dPce+dOwSBjzbANkySuB4qDaMWMHQaNu+++h6bZ9e4cmGDKMs4noe4lkFVlpSGhbBMTG2QFwXz6JjDI8mpZZ9Hn7rC9fcfMp8qzpxbYzwekiUFQlZYlkW4CGuVndIYhsZxbJIkww9MHKe2yni+yWIYoipdqxsbDbyGTZZpkjwjizQGEVEU4wc9Lj/+NPcffkCSLshzjaUdsiJlh0M67R4YFodHY3q9Po1Gj2Q+RUqDTjcgjUOU0hweDBkNQxy7SaVyDCBLU3QFaAOtK1ShmU6nGLGg0w3I85IH93ZYWVnhU596lpfUSyRJiwvfZ/HTf+pPIst1Xn3t/0w+SZiMYy5fWYHMYbo4QogYQ0Cv4VOoHCFrRm2ZO6ydanB4GJHMIkxPUClBPBqgTc2N61fJsoxnn3V5+5WQsgLLMSnS2pY0nQ1pDFqcbVzAbIRMRzNmQ01RGhhVio/FzevbrKxdxOhK8rIgijRlmRIEARcuPs4sHFPNCpYHTZ587BGiMIdijY0z6yilaDfWmUxDDsZvcepywP6hw2I05hOfP8/BeMhrXx6yOE5o9j0e+cwTFMaEViV570sHHG/Na3yTn5IXsLGyQZzFOK4gihXHe4J2H0xLUKQZgd9mOomwLUGpE9rtFnuTEboscWwfqSu0FuQlCNul0Wkgw5Q0LKjGywTOGdL5LZ67coXZ5DUOZns1LzXwmYVzLMtAVTm+D65dUZYCiYUqKhzfZXo8w5YCYZnkZUFjqcOg1WN35wjXt5HSZDabYlsmFjZH2xFbd97leG9Et2sS9G3SacSDWzvE7pzHLm9y9pmnyJOQl37vZSSCynDoNZcYDyOCdoFYjmhUE6IDC0vaFEWOtCzyLKMRuEjDoEwtArtJuIgwpIPrKUZHBt5Rn5WVAckiJ/BdHK9g//CIZGGgowoZ2qw/YjJJc8q4id/QhOMFfW+Z0Npid3vCpUuPs6V2aHUCpDdm++ER5849wniyjSoL/I7L5CBBihkrfYe9hxPc1nefofyuB0p3pYcxK1G5or86oLfkcuHpy8zinJtv38XuQkHJyuYpuvYS+3fv0mw4xKk+YU3mGIZVD5HUb4W6Ai3qBp40QVSSTClUFtMYdJgXOWWmcRxJUSmMIqYsC27f20VUCrNc4emP/xBPJCFmVXE4u814lCGTiwxaIbceHOP3+4SjXWxHn7S2SpAwHA4pcxOUjcRk0D7Lj/7hz/Hr/+rf8fLvvM1Sp01WLrBk3Z2sqDBlDRGu0B+6bPP8Ow3LAqVMlGFRVA5o8D1BmiaYpUm70SFwHWbzkLysyfmVKshyhdVsQqbRqqTdbDFVUOYKxxIfZjXn0xJVjFk/tcTq8jrDwzGWo5hOx2iR43d9pPAYZxl393dwLAuRVQhhog1IyRFGiSoctKhIygRpuvS7NrosmM0XHIYTev1l2kGT4XzObDFjfXWJhtvkzrsPUUrhuDYYBrZjEi5ql/jx/hihwe1LZncWnDt9GbU84fSTq8QkuN0WVW6wmISYjQQhJaZh0gq6LOY56WJCki2wO5t85e0tug2HtrdEnOX01wfs3j3ASHOMyiVLQ9ZWBqRpzGS0gxf4NFst/EZGNg/QeoHrQTyVfM9HL/PeO9u1e5g2Wk4xbQNKQbrQPPvEo1SqpFKSo+keUTLHtATRYo7jBxw+sGgEDo6d4qs1PAVXX39Ae0XSDlzm85jZsKI0DQLToYxzornBdDllpe8zGS9oyBVag5RZtE93zSIfJ3QaS0wm27hyhSSLCeOIi2sewwlQXSIu5gRnfMqFhW6NyfKS+Sihs1Ih4xS0hSHbVEWCXDqDyDLeeHPGU5dbHHaWCPMFhunQQ5NnEXNV0mh55PMcI9ckhoG/1MUlY7yIMLoO+WhBlWiWHc1+leNaHuPc5+LyGb742dN8/fVbfPN6hCFcDKcEUSKMAZ5jE+uITsPnv/ipF3g4vY9ybSw7Zjw8JDVm9EwfGUh0suCdtx7yfNAjjQzu/8E+B7shYbfFeHSXcDrh+z79aV5/+SqbT38Ev+tS2C1Gwx0WRU5uzTBFyP33AgxR1VliAfdnMwZnL9EID9nObB594gJyKnjhuReYVVt87MXLeE7CcHTIv/+Nf87myhp3H9wBYWBqG8+vLVPzcMz774R0ug6bm4/x+revcnD4AW+9+/uoYsyf+VM/zruvvPnhM9FvL7M/fJ+9wyHTPGew1Gb3xhY3rt3l4oV1RosJGB5KGVSqNnvoQoJpoIyCX//Xv44rBMI1SbKczz7zOJ+66NFrr3Ht5tc4OsyYpzn9fsClcy/w8le/hdWM6fRXaQcBwvTYurVPVKUstS7w/Mee4CXzd/j2K6+w1O/WWBbToCgzTNPE932iPEFicPf6Lk+8uMrySp9Wp8Hx0X0mYY6hYgqVM4tKrEDS7CmCgSQuFPuHGd2BgZACx1METUGR19rZUtXn6zobaVEUKRU5pilOAO311SXLsroBXqYsr65hOiYKhRe4NNo2sZUzPN7HMAwunn+Slf5Z9va3EYxZXRqQxhnFPGU2GbFYTDFMhedbNXO3SpHCJAozbHNOr98iXMSkyQIqDVW9wKiUwkBQlgVKV1TaYLCyzP7hIUtGn9nBAteyWWotce/aFlWxwZVPbLK3c4N777fYuS1449VvsJgoOj2F0gnb29t0Gj7NVkAUK1RukiQxzbbA9cAQ9ck5CjPWz/rs3Yy48uwlRscJO3cO8bomi1mO41W8+vsmtmvQ7TbJixDDUKi0RSOomB4Kgt6QqlC4tkNw1iQvEg7u2pSGxPYNWkuKaTym1eoymg6JRjZLyw53b99k45FVYMK9g4cUucGjl5+k0hO+fW0X3zUY78W0HEFnc4Drt/jUJzvsz/a4/NlniF95nXYQkx0vqOKYZNLDbDX5yu++x/RuhtmyKdOSbtDAcU0Mq8JQEStrLZApD+6GLCYCbZScP3+K2ShmPs//P6z9Z5Bt2XmeCT5rbb/38Sd93rzelr9lUUARAAGCIEBvRSdIItUt73tCmhBnWhpJ0RqpO6I1HEo0EkUZghQpEiBoABIgCqgqVBWqUPbWretNenf82d6sNT/ORbE75g8nhhmRkZGR+eOkiW9/5n2fl3ozwLA9ojxlft5C5HUMWyDTBCOfpbVliYE0pnSW6pRlSbFlsONu4UYVnpJ4nkMxUNT8GsPJEGFAXlUzlqS2MG1NnisMWSFNm/FkgONYqKJEYBKHGf/w//oP2N5a5xf/7b/HrnUJxwNs06HIS5zK5vbrkxkBQZkElUVVt3F9m97WAVkCadLgi7/1IqaYYmkD06sQrsk0GyKCLrlTUUw0q/fV8c/cR3qQ8cY3Xkf7HtrSxNMMpU06czVEZeA6XSoFWb6PbdToNlusnPC5/PohOzs7BC1Be07j+gX7tyw6S3VSR4AvKKqCSQq1rkbYI86depyNg6+yPzjg1IN1bt3cIMGjlClXb9xm7egK6xuX0CMfSsHq8RVcP6e7VmcwGP3ZN5R6WuEtOJx78ARBq06mJON+ytVXrmEoWD52hFF/F6/eQpAg78V2OTUXo0rBFBSFxjBmOa1KSTSzs7htuZRVTpZXSGOWYOH4Ho70qXoTTFHh1eYJ40MC16PbaFBmOYf7Id944w7dukK6CtdeptMy6A0L1u/s011uIkuoKodMKrIiwbEcTEwMJEmWQiUwSsnLr7zOrZvrCKWRcqbjRDkYosQyMtJCI+8lRZiWvAfiFbMNJCCVSZEWuH6TaZjMNqa1FoYjsQwIJwWlVHgNl6yfI6oSz3fQQpFGUzzTRcoZ688QEtOZaX0qNWsKTWlQZhWbd3dwbI92p0Y8TZmMKroLHkmZ4IkJRuDy7CsvYihBsxEQRjEyVzQ8Aw3k96IBqyoESmp+l+2DbRZX57k72WGcJOS6JEpibFMS9iI2+3tYwsG0LCzHRImS4aSPYRgkucD366ThlPbKEaZbh3z428/y8gsHXHv1q3zwOx7m/gdOcuPyCM+wiAcJQc3DLPXsZwwzsmRC7gUYt3vkSnGYmviOTSfw6V8fUGQ+YU9gWTFFZhNVKVor/JqPImcajrEtB7MmcKVJEtq0liRxWGO4W1Fr5exs9mi0HNodD8Ms8eeaNGtzHD+5jOO0EDLgv/zqLyIMRaulyDKTKm0zjHaoBx0aczbCkJw/9QTPv/Qya0cWmIxTitLAcSTTaYEfgNuo8eY7r9OdDyiVxdb2ISfOGZQjg6tvhLheRqnHGMqg4hBhGxhihRMnO6hrY3BATzvk2RjSQw63Mlp1g0ZgEW4PaDk18AS2B1SSUeJz+uSDnDn8PRCHkBgcf8RheD0nHDhEkeToEZfBNMRwfUyhEWXOjpL0lKT0KporNeYSl8n+DkND4ba6DCch49Lien+K0XmQxpLH0tBj9+AQFWuqKmQa3aRUGtOoMexn/OH1Oyy7Bt99/gQbPYcjTzzJfePnuPLWZWrNjCy0ubW+TfOIwalja5iNdTrH6hxsShgM+YlPfQ8f+cS3cfUfH8xSgJghU8LJBMf1SYYKKXzKcY7lSOKpzzfCOxjNDgfRBNmZo2xZNJoWf/nP/W3+y2/9Z67ffo3HHv0uqqJPGB1Q5QrPT2h2K9CSMIIkGxBGPrpqYNszbMzXnn8BQ5Z84fNf4G/9jR9l2D/gs5//DMFK872a2JpU1Oo+NeGzODfHzrsp8/Muax97DCqP0aiP7Uny3JnFwZVgGg5JGWO7Ng0Fy615jp4+wkc/+aOYXpN2EPDqq6/ykU98gs997nN0u3UOt/t85jNfYHV1jmXf43As8NpdjMCm1arzwOmH2Lx9lyRK+Z7v/0m2NvcZ9fewLAOtNaUS72HOHNsnrsZ4TZPXX7nL8VMe86sGo2hKq9XCigWN7hyTUYFdd9gfpKiqwXCwhzBiHrr/Itvbu+zvHyIwgJmjG1HO4mbvocuEBMFMrmE7JppZFKMhLcoiRwoXaTqUqpwxCZMcz7NI0oiq0jiOyduXXmN7f4NKh/euWQbSsDENF2xJvRlQFAWtRpPpdEpVKKSwsS1BrWGRZRlCmEgqtMwpqwiYReEJIVFCo6VJrV6j2W5wMDzkIB/jWRIpoR/vYwWCnTtfY+sOmHUbjz6/8m//HcNeTD3wCKwWwyjHC6DQY3JtzLBCyqDd6TAeD3DsJn7NYmvUw3BNlMrZ36lothRSeyRJguX4BL7DsJ9RlQWdToNKKaIpeIHGcCcoZRONJ6SFixfYOI4gaNoYVoblTzGlj6oiDg8sTp1dpVZ3uHnDot0tGY8GBIHPwcYuWW5SmTn3P36M+pEmSTbg5FxA3TdIKAlHHlXa4cbVfcwkoTZf5/O//hJHGqfI45wTR4/QXXQ5rBK6yzFnT63wxvodRKVozrkYfkmeVywtLXFwoLh+9RZrJ3wWl2tEYw3aYDQO8boV77vvOJNJyigcs7TaIIsFWzfHLHYWEWaFTAuyMsNEEo9C2pZH54gPg4rOJOChhfdz2BuSZSXt+RV0NqVW88lViBY2urKxbYPRNGR+oUOWh2RxgmVJVKmRlokwJL7vcPv6LV78+gtgGMRxQlVp0kqxsNhlbiVgeH2E7zfI9JTd3gEPP/BxHnjsCJeff5lLV67Ry8c888kH2Lh6je2bCUVSUmtYpOOI/Wt93GZAro9ze3zIE0+s0u0oNrduzwajymNlpcX8STjYjemvRxiGT3+Y0Q06hKFiZ+sGonWahbVFiqxJlq0zPEgBi1bHpiLhcKAJliVeIyFLfXI75/Lm6zxw7mN84AOnuHFb8NpLhwirTxw5RBODek1y+/ohQnh0aibxWDAZVZSyIskV586d+VM3lH9qDuWZhwI+8K2PMs4Vb759h5tfv86rn3+Ow50+USrYXr9LESbc3dgmVYJmY568rKioMFUxM7KUiiKfiaHRM/zOLIJrFoMkhEZRIEVFFKc05udx6wGiLPEtcBstlPBnCTVuDa9mEg9GjPopZdUlGu6zfXnA7t1btBsVkglhPKR0FbZysW2HUlVQSsgU7U5ArWHhSYk/Z7O7vcvO+ib1pk9UpRQ6nsWZ4czO9bZBo1mj3W4jJYDCcW3KqgBtURTVLPYvH+H5ksl0yGQyxLYlSyt1ussNpKsIfBvXtGZFTUsMNTt1f5NZBZDG6Sz7855ZR2iwrBJTGqiqokwNUAGddhOVNXnmkWdYOf0glruMKzwGacLxh76VnEWkB66RQ6mxLYNwmt4DCSds3N6liGC4PsHBmTly0wRkhWEbJGXKyQunZulAUhNnIYqSIyeXaM7X8ZsemcpRbsXuzSHTouTXfuWLXHv+6wQU7N9KuHG1j2MxSySZa5N5NkXLgbZDremyULdpmwpTFGiZkQ/6DDdHXNvpE2lwUZh5QbtRIo2EvCrJKsFwFFEkJmWqKIqMQkSUStHpuJhuzle+8scUeQa6hm9bVEXMdDTGsTzSLOLKlXU+99kX+Mxn/4i33n5ttl0xuhhqkSLVIMbMzdWoO8vsRod0jte5eX1INkoY75dcv75Hd96hnIb4tofOXCg0jVqT0b5CFxLbVOShTVCv41klTs3F8QXNlk+908SuK2zP5rVLfU6snkGrgmFvm2oHhhOLxoJLqW2kIzACl6hlcRhmmLoCy2U83eb52y8TBSXDkYG12sVKuzT9Fq2WxAkcUm1iaotkktCfjmm3DZSlmdYszj5wnPRaSrqbIe0Wu6lgd6QZGDbStSmtiv/tl7/EV7/2FipJKGWM13YxmgGW71JIYFJQVRXDYcaNwx7X9w4pdZMvPvsK7//WeZaXA/JcIm2YbuzScGz2t7dwz1+g9fiDPPmJ+3nkzz3Eu+Mp7YXTfOTbvpXSA0FJx6lTmAtMpEc6yCmExsZlNBoRpUOWXBcRjqiSjGJaEmYholigLvZo2QmNIubGC7/B1179OnGZ0p9OWN85YDiOOBxMMZ0WmA5llVExmT18BmPWjszhmC4P33+M+84+zAcffIJnLqxycmXhvXflRmxsb6HHAe2oy9p8TqNu0W0d4c6dmywtzlPkoElwXIk0FJBgCYksDCpdMWDCjfVdfvU//XcWV06wGQ44+/B5/ut/+Q0eevgDPPWRb2fhwgV+4b/8Avffd5wbdwe4ZswPfvz7aLaW6YVjwOfhxx5n9ewDLC6e4Ed++FNEYYLnzZiLpmWQ5jlZkSPKHNe0KBOXlu/R377KOy+9yYnF47iui9GxqByYTGKMShOYNbKw4GB/hO1a/MHvvjBj5I4keTZzagtsDIOZXpSZHlHKWRNrWda9M3jxHhQ9TVNMx6Qsc/IiYTg6QMuK0WRIvz+g1qhTqZwz5xfBGDCe9hBK0z8YE6UxtVaT+cU55hfa1OsBeV6wOLdA4Ji4rkm9BUWZkKUz3WZZCLSyKKuCvCwQhkTaFidPn2VpZRnLtbi7vYlpWwSeg9+xKBHY0qEZOAQdiekJilSQZD6PPf0QzWVQIicrc0xL0dtLmPTsGXDeEbj1gkz1aLZNRqMx48mAo8frGJakCG0uPLrK3ZvXuH71Bn7TpoxzxuMp9abB8vGS4XBMVUbUav7M6OmaGAbUfE3gWUTTlN6mYn+7IEsDvLpDoSwM6RJOCq5e3uH1VzbpdAxWVju4jklRxMTRlHicMtou6d0NuXH5DaKxQ386xe6YvPH8Lr1syqXLLzPcv0ls7VC5IYtLDUT7kFhMuHOwxfPPXeadr26x/ZrDeD/BMMFwAoKgyzMfPs/cQsD+/j53bu9gmS6HO5I8tZiOK+KkpNdPSIsakzwnWBQsnlzh7nZF/YhL+wiMkz5RFqNsieF7SEPjqRabez2uXh4RehJvqc1OY8qd9IB608PUIZYpadU7uM5se27aBpWcYrkelQxpdLs02y0c28Z2TCpdUlQFge/x1S//Mb2dPRa7nXsLHgOETa1b4+j5BsEiqDzCDtoEbou80Kydex8L86fQRYnwYOXEB/nuH/6LZEXK8vElpGmCUPQOd9ncfBfpm5A2IZmjMGw6J45iWA5VkqHKjKce/xg//MN/AZTg8HDA3/xbf4Wf/qnvJclGZBObumuzMu+w3JonHbn0dmYsb2HktJouZDW2bk7JRjGBVWA7LexOhy9//Vle+NKQ0UHI4oKJLVwCx8ZnlccefJpOvaKKSwZJCU6CyGz8rMO5o4v0D3f+1A3ln3pDGUqLl752iWlvShgNaHo+VBGLnRZZFbM0v0JZlrhaMBxPsSKFVZgUSU5k2qjUxvamaC0oCoFlJwhtYEiXoowRuaBAYZsOFCWTg11EzcLxLKKxYO+gB6JksdUhn06w/ACUIDAqDJETbvUI/DZukHD+vlPsDXawnYDVo2PW39lDBEChsUwDpTVVJcjGOWGU0ai76KzC8220Y4LSeGKW4FCiMIwKIRxsy2M8HOF5HlKaZHlOXmSUJbhGguHYVGaFY7noStOaM8hTzd5OzNqFkl5vSitYRjkFURRhSUGRJ5hOE5XGSDkLlM/yCN9xKKucvKiQhkWpMwwZUKlqFvsoBK1WQJnnpGnIzc1thBDEowmZkJiWw7Nf/gKqKKnX6sSiRkpOWSkc16CoJFLamJYkq3JMWxNHY5baS9x//wO8+fI7FFmJSjT76wcoWd5DHUmQFsPDEK0ERVbgWg65cjB8g0ZjHrW9hbPWwc4E116+zNnF8wTHjpNZOTot8TyPKpyiS4Pjq0dYmDTQnZKsmSGreS7fLXEZs314SJYnWGUdLTWTnovIJcJIESY0OgZxXLG40GHQn6DKiiIviPY1hoS641EgkYVAEYFpkVglh+kO7eYyk36M780cpjdvXMIxbVqtFkk6InC6JNkEoX2kG2LuaV787FVqi00eeeQRXvqDd1g93qaYFHiBjyoklpfiyhZSuEzyA4SviGJQvYzuoiTomhi2QZa67A9GmJbg+LE1wihD5fDipa+gtUA6BkJkLDQ9MpEwLGKquMQyPWRu4hkVlrZIkympbdE/SPG9JqYl6fQkdw/6JIOClt2g7tlE8YCaVcOqwzQOMaih8gnddpvJQcG5x84x3RyxPdjHDWokyRBXzWFZORQJH3rkQTJ7RHkIS/EuUTLh1k6fzFR0azUmWYJrwvG65N3elM++cpvTR3KeemSJ+9+/SDsw+Ox/v8Kdm3eYRCWN2iJnLq5x5c13WXMe51d/5Zd58Hs+wLFaRX80pd1p4m3vMQwldFIcURFPQgrXgNxh6YhJtV/ju7/nNBvv7LD7XMHJhxe5cWWHne2M++ZAmqs8ef4pjq80+d0vvYovM9LEYsFTqKpgVNrUWxZBUCOpMo4bTeaWfQ73r/HEhx7iw08+yvBgzM9/+ne49OrbnD+3xpGlY7Tt5L2a6Fp1HH2LoHLpBgknj36QbHiDZ1+7yQeefpD/9ulnyV3B8nyJZzTZHINhu9h2iFEVeIZFFuYIEpxig0tfe4m5+86ws36bV5//CqPhkO/8oT9H0/Y42It57o/fot4uaPpdHv+W7+QX/8P/yrzrsjLXwVv0kXbApRs3yUr4wDMf5cVXvkqn00DFyWzgUYJplN5remZZ2qYx2xKuX1/nyLFj7KRjPLtHvb4ww3qpkHcu38KUoPohcVJxeJC+h0ybubtzTNNCvYegUyBMkApT6lmdyCVKFaCqGRxdG4hSEU7HnDh5jCQu2en1sC3I04S5uWO47hINUVAUmmgcEyUpbc+lqqZIOePx1hse0pAcDKYYpo9pQDTRqGq2HVVKobRCYgImWilMwyWepmzeXUeZOUoofL+GaXhExcyZ22yUUFQkaYoyJHbdpNmuMV4PuXHnkKc++DEuvfkOQghatsSUfdy2j0gyTASG6TIZK4TrIUVOrd4gik2iSY/5kycZ9kOEtHFsRTJUFGWJ4wmElRCNPVrNGqUakMYlQvqklYPSBWGicZTCVDaVqUinOekUhDAJAotKF1iuJAoTFhaWiJOCcOJSaRPHbqG9FOmH2LFN22/T6Co29zeo+Q5f+cwlDCzWX95lcbXO0kOggPGBwDBtXMdFFALcDKfh0NufcPfaJoatOf1Qm0zlFFHG5tYOewcp8XCXRsNlPB2hC5NofwoqJ4tsmkfbaODyV3dZOO0SFgPa1Ni7NqR1Xxvnes74aoy9VkdUUwzXJi8KmsJhMp1w51pFtFzhlwccW13A7vnEOzlW2wY5QToBnsppNuqEsUQ4KbYvyL0Qt+kR3c6QySxnW1QSi5j+tEAqzSTt0W22yIWDIyTCcXj+yzd59PH34T7SZHP9FUK3y9zCPEUUcfmVN5FCoiYTXvjtL3Dxo/cjqFhePsed5CqFTHGdCiO3qIZjKBVmYDGMbOqd4zz1wRNoMs4+9BCmE9BumRw5u8DhnYrveOYH+L/8o7+CaWq+7UPfilzr0d/WHJt7mFZboMXnsO0llBWTZJLmXA1LNUjzmKSsUeyWmFbGXKODt2Chq4xJYrAyv8idO0MWFzIuvfYGVgtqDZc0SXjwgbNE2Yi4NJk/s8jd3b0/+4ZyuJtjiQGdtoVrLDMZhSjh0Y8yTt93lmOnj3Pnzh3S0ZRkOkSHYzyhyBXIUmEoKHKJ683gn3legq6wbYmUJqUsQM/E3CXVLI0hSSgNNdMeVVDlms3JIaunFlGloB40sUsL1zKhSNlb32Su3ULbkrnOGgUl6STBEPIej202PVumRZjmTOKUhcVV+r3hLCJPaVRZopSGe1Fg7/mbREVZzqZsKQxqtRYHBweUChwnIMkK2ksBkbLI0hLTyJmGJa3aAlr1CfcUVa4wgoI8nZ1mPM9F0ub0maOYzhRNwcJcl6XlBdbvbnPt6m0sq87tW1tUpSSNUhwvQCKosoJRNKVRryO14mB3f/bapKTQJcIwsAwD03MJ4xjHMEEYSHO2lLZthzRNsQOXwG/UtvcAAQAASURBVA3IshRDesTTjBvv3kLn4Pg+QmmKXGMKSb3VJclSOq0WSRrROxzgWC5xHFPqnOZCA8M1MTwH6VhMxhOCxTYycClGA1bnF0nKKWkRE7gB3738MJ3elB1jAEEbe2tAurxKMH+Ch86OaE6/zpWXd/FVj+HUJRsb2JaBoWpUeU4mFF5QcbieUOYFRVFRpTVMaSDNgjgCLSOkUWJZJtJihmKiIEoj/Hqb8WHOiVMd8sRmPD0gnCZMoxHtToCIfIb9lFpQ0V2YR5U5SWXxxMMfx9UtDqNd3n1jj8WjCwz2D7HMOSaRJmiHnLx/ifU7WzRqHoEHGzvb+P4ixAmGUbGw0iScJkgrpz03z/DOXfJUML/UZnFlnnP3H+PV179GPNYEliQvNYFvoZSAok4SSgxcpNSzSDsVc+zYacbpPqceXEDqkp2bQ/a3LGotqHwXg4q6r6iqjKrK6e0OCaOShfef4eJDKzR3HV75xi1WV5dJpjmry6f5u3/3f+TyG5f46NM/wauX32By8Ba+X/Fvf+2PSBJJmGbUGk0Gw4iL5xf58xeX2L2mmG8YoAte+WpEd7VB974Gj37b9/L2pTf5ylf+kOvXzjPMClZUxNHdTeSV5xke/zCmUWO/P2JsVsh6Sa8vyJWkO7+ELfbp3wzZSHOW1pZ47e09Go0Vzn2oJC9jGn6X9pxHLC0GyTb1pSY9c4X7Lr6ft7euMri1zoceusDt/XUqw2HSL5gLWhT7W/z0P/xp9raG/JtfucIP/NT38J9+9Tc50m3w0Jku25s3uXDsPJvjEbvl8L2aOFWamncStxXy1pXbpNdyOvOQuxkDpXj0A0fJqJCViZMdUpvrMN5MOYgttGkjLQNLCMaHI77zB/8Hzn/4EaRqcOPqSxxruhjTEVaziddsc3vzOv/j//Dn+exv/Rr7Owd8/sufw1mdw+t2GIQKc9yns+yT9nbw5wK+/Qe/m5e/8SwiS2fJY6Ymz9Q9uVGJEOK9dykl0+mUG1ev0lhu43lNCiFJ0piaX2dv94DAsiizfJZLnVdk+Sy+lap6b/P4TZSQUgJ1L02s3WyS5zloCVqSFzlSCpTISAuDWn2OP/ejP8XP/dufpVZ3KQpBXpVoo2R7Z53eYBPbctGUNNsmli2x3YCiyEnSKUVRoKqZnlCVBnE0uPd6wLbtWQILYDsz/aSwKvI4x7BMgkadTEcUVUaWlyhTUyQVWAbDwRhdFSzOz5GpklIVkEss12b35ibd5lHyEqosJMIhM0xco4E9J1FpShj1MesS1ytxqoB+f0Q0Fcx1V3Aci17vAJSHkBpEge/XybIKz8+YRAlCCPy6wKsJoiQlT0uyvMS2JFmWYFkWWguUAsdxybKUKIowLVCVRkqTwaBHs9lmMEwIGgFp0cMs2hj3NLyvfP0Sx86u8a3ff5wrl/scO90gHoWsHV3ECVJKbUAVcOJUwHQYs7+9QzqwWWp2id0xBBmTITz0vuOIesK731gHYq69AatzZ+mrqzi+TRRZFElGbhhUJrjdOpUaUpN1Tp3qsrU/YuFkl/ufmmNjcxud2EQDxcLqAkk8plQVIp5iuy5ZluEFJpqE7Y1rmKbN9vpNHM9B2j7T6ZB2s4nWglrQpVZvIsyKUkmyGLpOk858h0EKW1f6mL5PEZfklcZwCkpVgrLJQ1AlnLl4kSc+8Umm4Rh0isoU9cUaptPCLLv09w+4+Ilz3Lou6W8UXPygzVc+/TsQOLxz5UXkNKfpNpCpxdn3PUL7RItHLnyQKi8pqhDQVKXECepE5QRTWBjCoN1ssF/d4a/+tZ9gr7/LJ3/oR9D1FreuTXj4wgcw3T3ygeLCQ89w8/otFptnGU/3mJ9v8cZrE+ZW6pjSZ3n1CEJDFG9ysDuksZhj2AV3t12efvqjHD1p8MUvPUueB0yLnBOnzqBFSeAGqHyPt74x4uIjJ/7sG8qjJ1r0NizGhxFx1CeLQwLLYzyesnXlBo4hufPOTYSULHRNpoZCKY2qDFA5QiiKzMI0FY4HWaYxpIGmRKsZViiwTMIwRkmFRJCNQ7orTYZKQVbQaNSxfZvtrZjlVUmr7UNZYpQeWk1YXVnCMCwSlVEpk4ocoRqE0xTbmWVsV1VFluc4Xp3jC0tMw5Qjayv09g5QZYmuFFIDYgYvR6h7uCCTOE2w72kI+4MdhFRYFlQ6BWnh1hRFXJFV4PoWeRGTxFPOnjzF4WCHWsdlEkYY0sexDA52ezS8Oodbd1CWTzQN2a+VvDi6zmQyIU1T4BDbtml1awgh8PwaWZKzt7OL1JLBYAbodSybNM+wHIeyqhCWiUJToFg+eoQ0zOj3++hcY5kmpmnOkCJFNcsnVya2IUijgrxKsPApk5IyyrFdj7KS5EkOaPb2ZhOL4zgopfFqPqUy0JVib2+PSpW4pkklocoK5uY6hGFJbzplbtnn8eXT1Esf15rn7mRC+8RxrCc1tz43xY/HeLXPc21zlzSCxSWDolzAKnMMEeIaLbIoR5cZkwMfhIFplVBJOt0mzaMVO7tDTGOeigS/4aN1RZanpLHANXIMGyzZJBpn1Joew8khgdXFdxqs76zTrC8zGvYIah6WIUjjhJXFo5hGQe/2Buk0430f/DC/9ulfh0rhWQHNuiZLCyQxybji3Kmj2EHJ3EKX4TilbSaMDyxGA02WjjGYR1YO67eHVNWIhx9f4/a1nEk5ZNHz+OrzrxGOLB566FHubr7MIE7IpvOk2RTHNrGclFLlCBGwuOqzu19yZ3sPL7BJ9scIL8RerHG2fYysv02Uh0jDZTzNOXbfGlbUZPv2Lh9634e5dec2J849RKd5mvn5XZJJzHd87BO8+saLmCzxv/zLT5H2C7zFBukw4dWXb1CmFq26oDfIMM2ETs3n9ddDLqw+xhOPTdjbHTKabJHsbTDcEOz1DpGxyeLCMv6ihSpSTi61Obx6i9WnO7y7E2HX75InY7peh8P6lP39dU4vn+HN59/COjfPyWMr9K5cxy5dwsOSgQiZiyb47SblNMIwEvb3dlm1bVq2zTgacN/KGl5kMt3a58f++sf41d/4A+JhjBCaPJWEvasUWcZrb17DczKWpcOVP/o1llTC/vWrfNuP/RhzS/NMqj32Zc6dePO9mlgYJQuepr8dkwdz9PJ9hrcG5K0EY+zSaK+wvHaBrzz7R4SxyUc/dpGN/AZ7tyKEyMC00FVIuxHwn/7LL9M6dQTlWhh1m7//z/41htDcHu4wGSXcvHuH1fMXWVg7zmTzCr/z73+O7/rUT/LHz36Bb3vme1g7fZTteMjZ84+zsXmXI6fO8w/+wT/lX/6Ln2FxoUOpcqqyQilmGkchsEyTSikMNIY9M9QMNg+p2Wt4dR+jskErijxlFEU4poUSkjBJqYpyRsxQYBjGTMaE/pNGVRu0Ow1M2walieMZzLwsNbZrIK0Zdk5Il3/yP/8vNFsutqsQtoupFK1OA4mc4U4CKFIL2/JJsxKwqHSO61igTPIiR6uQPBe4rkuSzBqyOJpJe2rNGoY5M1C6jsFgmBL4NsKWmPjYVg3b8masSCUo84I8T2eGSKFxXRfPaxKGId35FmWa8M6lr7B24gSFYzNOprTqXYo8p+61sVyXxeNjbr0bM54ITEpMy8BzXYoiYTwoaTRqhOMUwypRBYRhSKvbwLQK5hdmJjG/FpDEio5fJ4oLzHwGjnctmzhKMQwbQ0jyPJslulQVVWZjWhWFTnHtFnu7Q06cXKEUfVqNgP31MUI08eoJXVtRVkOe+0M4fnYOdA7a4Mqbuzz0+EmEgPW7d+k8eZzOvI9havZzxe7gkMPtITUvwJAJ73zjJuN+iGn74Eg6rk3dq5hfXeOtS9ewhItds2jMuexvCBbP5jS8ZbTUtB7wmLwy4eB6xamzLu974AEuvXyVg90eztEjPPK+Y+SWpn9pwPr24F7GekVZVfi2jdYVaZyQFiGVAMsKmAwLTFOCkzKdjFDAyTOnuLN5nTiD8533c7IzoX/1v1MVGqSgKktELvCFRygqlAmBbTEe3OWF5z+D31rAderMdVdRxiLalKhsRNPpUu8+yMr9LR582qdlNjh6oc8kUvS2t5nEGSfPHOcjH/9W/sMv/kfOjr6dx+9vEJbboGtokbN+cImFuROsLZ9mmpuYpmShc4zBwoBJ2MNq2Nz3wKMsH13CuFVnv3eDK3e+zKmjH+Txix+k3nyZa+9cZbhvcuHceZrNfYQOaNTquK5kOh3RbnfZ3xkwHluMezC3dox3N17n2ZdKHrjwOOM7G5ThHuvX7uC6CtdpcObC49Trl9ndH/3ZN5R3r98mG0Dcn6BVSTtoE0djfN8mjELefeV1XDNAVyVuXbGwtsr1G3ewDAdVFmBqlBKkaUmtaWNZs3QFhEYLheu65HmObZhU0qTKcso4JRxPqCyDtChZXjRYOR4QqohoMiR02wRNC60UgddAGgZ5oUAUaFFgmAoyF8dysBxFHCVIQyCkJIqmRGmFEBZlXqDyAlVWCDVLYvhm5qyYiSXRegYf17oiz0pM4aOM2WufJhFLK00M12V12WLjzgFx5HLu/ENokdDwAo7f9wBvvPUOUSaZ67qEkwGGLRB2Sa83IFU9JJL+/h4oxfLiPI1mB69uU294XLm1S7czR5yltP0GtbrLsD/7Q0+nU7I8Iwj895BGVDM9U64q9vb28Nwac4sLRJOILI2hUghTgNQUqqLIc5QAUwoqrUnTnIbr49U9JtNZOkiW5RiGREtothsIITg87CNtA8/1MbVBPI6oNwLmgia5GRKrlHg0IdeSSuUMp5phy6CpPa7t3+ZKfIfvmf8Qi40Odx+J0d0J97UCeoNT3L7ao566bN2ZImy4cP8ixaiGyjxu370CRkyz5ZJkCWbZ5MQFl+U1kwu6zZe+cIt6sEB3wWI6zpEWNOsNqnxKPeiytzOliBLiScbP/buf4Z23rvIr//G3cByLvCxYWp0HpVlodtnf3SSoS8gFc602vUGf1ExZ7KxwsLVNq77C7Rsv4LlqZrpIPF544QVOnFvhrct3WTmb0PYDsomgih06rS7jYUqzLRkNJA89+AA1LFYX9rl9U3Hn7T0KMpRyuXLlCo1WG8fwydIRrWYLpcYUpQVFh8pOSRMDnWXkdobr1qHMSA4N6g2J3dkhmk5p0KGsBLbpsL8zJE4jLOlw5dI7PP30U/zg9/0kP/vzv8yDD97Pk4+fp4xdfuy+n2Qw3uedyzuYNY+r727x+MP3017LGR6OefTIg6TxFuF4SNWK6R6Z54++8g3On15iaTUgC0uaHUH97ALL6X3c3d5nr3eIs29SZCaXr29S93zmnnmS8IU7yL19fLeNaY9oGRP2Yg9TKeYXWmSlwduv3sWxDALbJQwNgrkOg+09qskCQWOe9tmE8N2UY0fmyCqfSgW4VY3OkuT7fmiJdDyA6QHf/thjLLTrzK10WOvO8ZkvP88rb17lX/0//gELbp1TT36Q3b11br35VZSriUgpkpjt4S5V+SfYoMceepI7N99mb3+fzuIaK906J06d4stf+yqNo11WljyuX95lsLeOmRtM1z22Dw+wyHAyA79WI6wstM6wpOaPfv03eOqZD+HUbOIjbVp+h5OtNb7wh7/LdHuT9fEIs7VEfv0KrWWP8VSytHSE0w8fZeMwR9sGUTEmjmNu3bqBlg6PPP4E1y69RuA3sG0LLQvyPEcpdY97O6txQs8+d6XB3es7LJ2aBy053DnEFgY5epa2ZRrEUYY0xD3kjkYKAXxz8DYoq4ogCLAtlzgO8RwXIWauasO4p5uXGpVlnDx1mm63y41bb5NXFYG/RFVUDHopZ06d5vTJnN2dPlWlKPCQoiRNCio1axqETvEsidIFliFnYRuVQsjZBq/V6hA0AqbRBGGYaGHS7FQIAXGe4AYtbDfAMME3EoQ74wCv+AvkZYGiJMsz4smAVq3OVCcYHqz4NQ62NzH9BtJUpIMhjVYDx+gzmfZp1hapwox2B8ZDzXis8AJBEFj3Mp5zDLOiVguoSo0TKCwnBdGlNzzA8wIODnJUGQCCIGhg1xLKUpNlGX7NwzIcwmmKgUAoQZVXWF5FURRQmqRJgWlVtLoue/uaw4OMo0dqDKIRK6tnmQ4mM7NWPuP4en6DJIow7YC9zRHTcMDSkXnWr03Z3Ogx127hGAWV0LhOxdycwfWbI6pcUndMpJ2jDI+DXp+HH3mCn/kn/5Cf+OG/xvreVdoLJY7rU6vV2L4xImlNsBcEt66E3P/Ice68tM7zv3WZ6WPHSfoSxwOcArddAzujs7rAsFcxCSOUAMusMGyLMI5mFAGtqPIErQR5pVCGoogSRlVGqQymoy38msfaw+cQfhvb9GnNddnZXcd0bEojx5IehZ0hp+XMGb5Q58AeUW0c8LHOx/jwhaf42u4Nxge3OX3qaYzlNsPxNvHWXTKG2CsfpWau8hN/7/2sv/kSv/Mbn+Ev/U9/gWd//8ucefL9/DgZX/6NV7l8+TL3feAc4ajEsWucrl3AEi2uXn6LoN3CXJjnyU9+gie/5Vv4+X/9r1mrrbC7MaR1dJkkus3e/jaGtljfeo1Cb5AnBaUZMn9sgY3Nm0TFXXzzYZI85Mbb36DVapDkbZbWAuKojXC2yQ+uU+kJNfsMjXYDp1hD3amoL+eUKTTry9y8FtKsr2GUxp99Q+kUFqsnXG4yBXOJaZxSYmBlkq4fYDQ94jhDFBXNZm0WpdSokacS1DdPIjOnY1VIHNcgSysMMevI0zRFVgKpZogeIQzkPceg02xgBQZhXHLtSsjqWgedOJiypCwDkAkSjzhJMWwLAwkKVJYgygrfkeS6IgjqFEVGpStMw6DR6uK6ATdvXsMVFlSKmQpIz16v1EgxO5WYpvmeYcaybMpcA4IojVg9ukxzziDKHOLJkEa9oEo0jhtRqxeEvTE7m4rd9QlLR5aoSHHrdeyupMxMbFvgWCW6Kui2j+O7NuPxELvVBMvk1u4B0qxzMJgS1OtMkwyhodHuIKVk7eQp+nu7bG5szwC9fv3e2V5jiFn0WDieEE2mNBsNRDVrjGebTIOg5pPbYqYXKgoMW+JLn0JVoO89KKTEtR0MyyQtshnWyLVo1OpIaSKEJElybNPDsDwOtvZACXzHxbctUuXQ8edxdMnG9RuoTpegU6OYSg4HCS/96qvITp9G5HH1skaUUx4+VefNjQHOXIBTQZqb5HlBkSk++QNP8vxXb8yc4rmH2ajz6uu7ZF+26Cz5NDsNOs2AwWGfLCmYOzqPUTns3j7EtiSeGVD3Ciy3w8btff7oi18myQZY5jLNruAT33mR5/5wA1O4eJ6D55hUKiOOtgnLkpV6m9HhkDwqGfT2kcIkyTRIjbQjLlw4wnjSIxmNufV1G23N6AW26dM/TFE6ZX8P6o0OQhbozjyPPHiCOzu/y8LCEhubtyirDCMXDHqgSp88jdgZ7dLpNJCmwKkl4KQIaYNwcAUMboZ0Wg1MQyDMgEEvJRoFTIsQZIVlC9xCYikD2XApcsU3XnybSy/+DO3FOpOpwZW3b3Owk5AUJStHW/zuZ57l3GMP07u1gV9f4z/80n/jiccf4ZmP3If9fMnJ+z7Op3/x03QaJhcfnOOph57hzp1LLDeO8v3f+3383O/+AltbAw4OdzCUiVNv4Psav2mw0J2n7dax7TFZr0lSGRTmkLpTYCGJyxK/WWe7d4BW0LADYldj1SSjUQ/T84miiDCLsG2IE5PBeIBwI0oiCm1g2wWGJekudvm//fW/TXexhsakP67wl3xWT+3wu1//Xd69eYfTjz5ANR3Sblk8852fIpnmZNk+NBdY0gWvv7H9Xk28/s42eWnz6FMfoKgUDOsYJpw5doxReMDd/TrTsqDbkgSiQb075Mix47x/oUka3eaLzx7gz1to18EVBcl4zHf8wA8wmGjSEMbTgnpdML/s8dCj3086SbFEk+LCcfZyh0ky4MLD91PZAVG2TTnI2ertcd9jH6TV9fnS534fS0scbxYH57gGYZLeq61yhs8RAkMIuHf6NlA4lsfe3Q2kkNj4s3hdDHIUQkNeVjhCUpQVQgmQs42nVrPh+5un7yRJGA4niNbs91WWJZZho2H2vULjuJJrN64S1GzSOKIsMmpBjWmYkCQGrr1GHIXYniYvIyxhzeDhljWTJakCIRx0ZpOXKcKQszS2UlELmigMkrjEtmp4gYGiwvMbMxOfFmhhzqIqqwohS9AawzXI8xzDNjAQVPksdjaPI/ylNoPDCV1vgblWQj+eRQcWVcZkVHLfmVO4Zp1r715nacXFNKDZdih0QhJBck9H61glSnp4XhfDSun1D7CdBZrt4wTNDvE0Js9jkkKT5zlVFeF4BbmaPYOkLkFJJJAXM3mBaZpUhQJhUakEoSqKQrO13sd0HFptj7yK6R+Y+J6i2RTcvTFi+ZRBkuUsLC9wZRBx9OgKe5u7RBPNsNcjyyJa8x5ROMKsHSMZjFDS5WCaceT4IotHa1QyZ/PGHlY6oj23xl/5Oz/DKB5z38NdugsnePvSDdKsT6JMrNKlyiGw2gz6B+RjG6fm40cFwyTG8VzOHl9hoWPzzpu36U2meLbm/LGHiOOY9Z11Jmk8A/g75r1EPoOm26QiZXZGBiltdJUjzZgiydg+LKnbKa3FTYIjyyzf/wR3Nu/gOAJtGCRVjgtkpomwYS4PqZUu549/EL3RxJvz+cDcg9yea3BYaBplTMMPqJ19mrW6jZ3DQbRHttHEWVjjJ//636A+t8wP/+gx9rfHrD30Yf76mfvoTV3CUYnKFEJbmMyjtMXikQ66tDCET5xrAs/lp/7G36Xh++xPd9ntfYlL177GqeMXOXP2GNt7V3j3nVeYa13gyNppDvtbIE2E6qKLOYJ6mwtnVyizGkKGJFFJyw3wF+pk0wOWjq5Rqjl2b9/Fr2ma3TnCMMcwHEbDCEjp75s4Tvxn31DOXTzGdCei3jxCOBohixhNSW15CbPuI2yBU69oSJu8GDIZTTGFJM1ztFJgSBAFSsF0UlJvGlRVghQWs3o2O1fkYYTSsxisSlVQFNQth7C0KeVMy7ezeYBvd1nsLGAok6ISaFFSWRWVLhClRAoP26kTpYeUpkIoQZrmVEWBYQsQsLq0yu1b6+iqohISkO+5t/8k43z2scwrTFuitUaLWbRYmiXU2w6Pve8+9vcL9gfXafiCRx56mldevMPuwR5LtqTWWKOzdISbdw4YTQbYlkezvkKapBT5hOkkwvYM6vUmkzhhOJoSJinTXJIVOWmR07BcBqMx0rQoy4Isiaj5wWzin4xpd+aYm19mc32TIs0o83J21pYGCoWUFbZtE08nM3yI42BIg7wsiaIplilxPQtlg9t0SQ6LmXalmrHj6rUahmkShRF+zSeJIga9hGajTZJHzK2uMDzsYVk2AoNxr4dXbxBOpyitMfIx/cMBjmdzdn6emmFyc32dWGV84fbXcOsZD4j7iAcF5mDC2kMLHHh7OEGNmm+wdVNz2KtwpMPRMx5+x2YU5xQjgTQgmk5pdusYc5KqmKVsZGVMVSks20aUil5viFFZSCSBVacbmOxOxvz6f/ssZ+4/wSSJ2Fqf8PGnvh2hZy7FJIo5sjoPUuI6db7vhz/K0sWnGN4ekKcZx4+fpNZ02Ny20GqMQUCj7rC7uUWeNfD9BtPpmKq0qTUEi/Mt7tzYInA9oixCGyk3Nl5nJd3i3Zckc75P27OJ68dIqoL51TYH/S2Ghzt813f+EMePL/ML/+4/0V6oIZwJeW4QpwfEhqTp2Mx36yTTlDAbUmUpnqhx8cIRtrfHuE2b27dv0wqaSC3RsgJDsNJZYL5zhN50n6qC61cGGDgIR/PupW1ss8+tm++gTcVrbxicXl4mmF/mWz/yU0z2f57t/Zjv/d7H6BW3OXLsB3jkA9/PYQy7u2N+9uefIwkanDjbgP42qYaq5tGZd3BME9f3SYVNp+kREVAVKaQGeQFYgtIQ9Lb3aRoFEsHkcIjXamO6BlZp4VsBZ4+f54V3nqNjNVleajAa5DgqQhaHYCTUVEFZdBiEGrsBO3tDatKkJCaL1zi22OVTH/8wRwMo90MmVURfTyjKKfNLi9zeuUW70cYxA9y6815N3Ni7y+LyArs9xY2br9AWR7HziGMP3YcM1xkMS6bZENx5dMOiVjeYW+uwttrh/R98kGH6HO9cfRPH76AKzfrmOv/xX/0C3/JD3w+DIdTbjCPB0+eeYnsUkpkZYRXz8Lf9ONO3n+fK88/Tar6f8TTFkx4vX3+ROCq5sr7PxYvncQOH7tFlTj3wnfz+576IUimWZYGWlGWJRqI1M+OfnAU1lFWG0AWeacxMkwiUYZGks8hclEbpCmHP0n/4ZpmUAhTvaSmTZJb37XkzHePM+S3J8xTLcfG8AKTg7voNDFMhtaQW1KnIKCuDLI8J6i6jyZC8inAND1ODKR2qIgSlKNIS29IUWYIUNqDJsgJVzeQ4fr1GmuaAnKXwSANpS9AmtVpzRotQOUkyptFokMUaJRSOYWGZ7uxsHsfkacFco8X4sM9k/QBf2PSzAbbr4JmzAd90HeI443AKwnJoLMPSss87r04pYhe/UWNhpU1vY8DKkRVikZEbKVVVEMUhhmUAJoPxAaYJnbk5ppMDJuMdbM+gyCRJXFBIjWPN4iuFEOR5TlkogtpMEoW2yYsppgm27YIymE4y5hcdpsmUw11Js91FM6G3I/ECn34vJsocKEe4gc10OiWouZiyYG8n4ujRNZKsR5V5RKJPlE6pihy79BjsC0pt4QQVcWzzzP3fzpgBf+1v/xU+9q3v4+svvYMpA86ev4Dj3+HaDUk6Ckl1jbJImRc1Jpt7ZPkEr20j84LSLLh1LaJcbbHXy1DKwK6VNNoLDIfXUGhs36QQmjLLZkYraUAlUXqGyprFjua4jgBsFJpW3WT3xhXuHrzB+QcfQ0YlogDf8plk0xlpBoVpliw3PB4UAStqEeuOR/P0ed78xhXm6k2so5LgTIs0rsCEyhLU8gZRFjKcbnOw/ibt1TOcXDrF4eYE0zPQUhH1NJlZx7I8qjym6bdI8gTsmTSsWVuY9T2pwjclGrAXFsktiWvlfOP1L7Gw1uXuztscDJrUPEXdaSJ1hK4cdCkJOi6WazGOdplmt5BWSZpUnD23jKkE4/4Bd3tX6TYbiDjAtaeobMK1O5vUjIxKFWSyxG8ZxBOPoFZS8+f+7BvK3es9RrsTLARhFIKwWDu/RnOpzcbBgDoWlm8yPhzjCKCQBJ5DHhdobVJUCq0r0AZpUsz0abaYNXOVQBgwDscz5qKWYM6atzgMEUmK5fikWci5M0dwvTUUU4gy1DTFps00G5JXBYYhEJWg1BllahEnMcO+h18bz1yyGBRVhIHNsD9m2B9hCwu0mLF5mBl3NNV7RRAUpjWbqEsqKAUSSRQWPHL/Q+zvj7mxPqDWnmPtyFFc9xiF3sU2NfXmKm9/4xpL44pGq8Z0lBHYPirPkIVAlxGBK4jiiMNJiBQOgVPHNG1UWpJlEQJNqQ0WunP3XMkpJoIqL/GaAVEUUUb63ibRY+nIEQ729jElFEV2b7ta3eOyGTjODOPheT5VlmAZFlmcUp+rkyYhnueifI1FBami3exQqhkaxjAMVDlz49fnGiilKLOCpmehfY94GpOGEaudRSJdkVkOZQm5MHCEgW/BIB+QiNmDZqndpnAVE6EYjA9xwg7dBZtEl0xvO6zOuSTliJMXakwGGePDHjs9uHJ1iON7mJaiVqszGg0xnYw0AseUFGnAIApZOa4Jpyb93TELS/MkyqNUmjLs01lbZj+uuHuwTiwm5JXB8lqTa1dv88d/eIjWI+qNJllRUuqC8+ceJkx7LD/4AAdXvsgjj57nnXfe4fatbUzTxiRAlQn5xCdJFbW2ot6ymEQBskqpEoebl4YIUcdwDebaJsPhhMDysVsu+jCi3xeMwiGp6mMGGbuHhwgUnQWPhcUjeMYJNBamWzKNc5KswsTFVEOyzKGx0mLpZMBHH/kIn//t/0ZjzmNnNOZHf/wHuXTlKnfWt5lkGa5rQwW6yrlw3yn+xt//W0RlTJpX6CKjLARxMWvIkzAhG6WYdYPf+83f5dobb7P/5g3OPfAUf+fv/SOe+dZv40d/6GkeXfHYuzvhYLfH3c2Q5195iahwOLLkkB5s0PYb3DicEK0fMtn2cb2CvckWx061mEwUc3WJocFUEtO2UIaJa1qYIqexsIRTVSh/ShqWxAdTukdWKfIJX3nja3SaTc4fPcOd8S263WMkaQ1traBMyAuYqIqVZkCUJsggoNBgmB5Xb+/h1M9z+v4mt0Wbo8daGOEm1996kyOrD3J99DauX9FL9rGdFVbmV9+riZPDMV4ypcxyFn0D25LUdMD61TcZJy4rboMlkbFPyt/76T/Pzq1D3r35Ao+cMLl7Z5EPfMsZXn7xTUxbYePT6Wqeffa/M65N+b4PfS/1oA6Ox/bGTW5sbnJidQ27jLly7Rtsb2zw+CMXCMND/uAzl1isGaSjIbXVLse6c7z18rOYNQdtCw7GGSfPneDuzVszdmycURT33NamPdPeVTNTm7RsqrLCUBJpSMI4wrZnemuVZSjxTfMNWNKgujd4zwbtWX9ZVRWGKciLlE6nw+H+wb20HoHl2GRFykJrCQzBJBzjegFZkmIbNhJJpSyqysJ0bPIiolH3CZyAMIFcpZS5QsgUqhJ174Re6oKyLMlLMaNwGCZxHOP7PlJKoihDug5FWeE6FmWuyMoMwwJNSZElCC0RUpBEKTXPRwuBaTlUCkZxjFWv4eQKQyq0lZFVEk/4FEWfUpQEdZf1zbtIs+DRp5dotSeEwzpvPZdTVinxNCLPc4aDEU5gYVswnIxJ4ox2p854ElLqHMtUTIY5tqkJ6jUQOULYKGWgyHH9AN+yiKYxtm3iezOzim3bIFMcwwLtYwgHy46ZX3QRQhMNLEyjji41k72UqoAo1dz3xAkcP0JUAsep6O1kzLXrVNUIw6zo9/tEYYptaoo8okLiiBrJMGH+RIDjp/S3D/AbNe6MLjMYbULuItSQfm+A5SYc9DZYWKhzZCWnWtX0BhlZnLDaPYNhR2wdVuiyIhmERHqKXzvC7kFKd9Gk1hA0O3O8c/kbTAcVqVJoKXBdB60FWlaosgJLobMc23EoyykKjSrBtupYEnI1RfopdlTn3a88R+AazM/VyIqEsspnfYPMcS3JsEx4K81xVo/TnsS8ffUFJk0Le9DF7LRwjA6GqtC5gsBCF0MMI+bk2jIHtk3TcQnjEsexSI0KQzaoVRNiHAwSHFsRVz2Ea6MrMA1JnoJpGbMtuzljdIvCgFygdcD7P/gJ9g82GPfeYnXhUZS8w2CUEIYhUegy6Sv2dq7Tanuk2QTLy0gTDSJj1A9pNpbYGN7g7PH7GB4M8FnAwOH4mXmiyTo10WAa2mhrQqEiuvMeyXhImRb/3w3h/78NZTEsaPguValZ7p4kSUOEMMiGJcu1DkWcEZYhzU6b8uCQaZTSrQdkurjXq83A4NIw0FrOnE2OQxIXSFwwSkzXxMSkKCqyskCYEk+bHG5sotvzzK/M0+9PMe2csw+c5L4jbU4tWmwf3qBlX6DEIJyMUXlGpafEhSCo38d/+ndfJi4KkDZJHOP5JkopNu5soEuN41qUlUKj0Bqqqpy57xD3NpYSrRTCmKXtSKlJwylrR+dpz3mM0j4nlkom04TNt27z1Vuf5+jxY8isxeWvTrFlg0svxLiuh2XVSZFomYOEeu0CeZXhmWqmKypKVGGQpILpWJEX1uykUaTMLzbYCqcIDYZwsO0m455GiPYsRkxpVFWhc5uyWKREkyQRYTRFKI3vt5GWde+EUiGj2URWRBWG0SeeZkRxBI6mSC2KqsTFpMoqcnJcx5s121pjCgOhocgKbMvFKDPajk3btJlUBfN+i3w8IAtTAq9GFE5I3ZSaaTOtLCYamu02aRoySlPcXJCv1ii6iqo2JRvaMJKMbZvr1yxaiyWWcJiGBaqcJ0kmSKOayQ+qIbZlkIwrTOEgLU2WR6R5Qa/nMJkkHG20yZJqJq2wCqQ0yDOB7ZoEdp0kMxHkqMrg1u2bzLWXeOD+B3j79evs7u4zv3iGZlfyja8ecGw0pdffRWcGtXqdYVwxGu6wOH+UJJ0wGpQcOTpPkaf0biYsLygGw4yGEzC3JujtZ9T9gH6v5OTxZeI4JB3EnDuxzJXJNnl5yMJanb3DBJVEuHYNIXy+/OwfQmWCnDIeSzJV4XkOZZbDtIFTm9C/PSbvLLD8Iw9x6pE+h4Mb/MB3/QD1BZ9L//130WWF0glpEkFlMze/zFdfe40HX7nKfQ8+wzDqEbgNKunjNBWGYdBZgDwpOX/+GH/wuS8wf36NFeHwK7/wS/zUp/48f+/v/mX+zS/9Bz7+yFM88oFFvvrS13nu5ZcBkyzOONVs0lOwN4EoA8evkDpF5TkrwSJOUmPci2E/pbAVqaVQkU1VTtGJidNssLG1iShc8lxRbxv4/phinNI8ukpFH8eyubG/wcbekJY8QDs+kgoVTwjqDZyyIMk0RaiIjQzDmDkp72y9jD15iWikeeT7nsGsLdP1z/Hw4xmvX38B3zrFYdinsdihLFPu7hy8VxMvrhzhSGuet66+zanlx9lI92kpj3ZTcfHD55nqMRcf+Pu0GgmLRy/wz1/9Ff7WX/wxtl5/lmg/4n1PHOfP/egj/Opnb9HtxOSlj+EIBtdus/ip09y9ucH1K2/RbtWxswnj6Yibty8TZlN6o5C6cZH77z/GsRPH2bxzyNrqMm99/RuIY4rT9z3O1s4N9g73mU4zal5KUGsRjabvSXhM8x7jVoh7TOASLQw8x6WqCrI8x/JnJ+oiTrGESaZnuKEyyxGWNasj93Tms0G8QEiNEJIsy4njePaAvHdeRxiYts1kOmRuaZF8VOF7dfJMU5UGpm3heDZOkaN0gTRd0qTA0iFCTyiKAimtGY6IiiKbubnDeIqubPx68H+SKFVVSZplNFuNmWZUzS5LYTghqPtM0xnyKE+Lmdkljal7PlWZIw2DKI1QaLzAwzItrCSjkBYGAabIkHZBSzVRVYEUEl2VLC13sJjy4h+U7Nye4tV9pG1QFgLXbjGZjiGqKMsZysiv1ekdRAS1JranyPIpFCXS80jzCsPSOIGJX3fZ3x9jWbOTfJbNpE+2ZSCwSNOEUhXU603yPEXpGSR+d7MgywoMw8RrVIyHE9pBCyljTpx1OXGmxubdMdkwJMxKikwiBEhpUm95WKZHEhp4NYEa5XT8GtIVHAxK5ufXOHtxnhsbV9jfqOiVBzzywDk+/PD3cuv2JkEQUJQ5nuMTRRVlL2fllEOrLBFJQqOes30w5fiR46gy5YXDK3S7ixgqZu3EPIGRMk0TNt6cMIkzwmnKwkqXJJ0ShjG29LBdf+bmN6BWM5hMJjSbi2RJSqVSSjXFtVw8s05WpKAU/pxNoW3GZYYlJJbhg1RoaaOxKYoSq5vzhcGznFv7FublSVaGBi8evMmj8x+hEJpK5niigZE7TLnFs899ngdPPMx9j34LyTCntDK0lrjCwShSUtvGUgAVRSXRsoFRmBi6nNlnPZtcl5jMnrGGcGbUhKrEtFOuX3+Z3e0rLC8cYXd3l3oX4tin1QnY2j/E9zvcf/8HuHL1JpbMMaWF0BnRyGTEiCzOOfXgWeZsk9WFFbZ3B2zu9/CkyynvBGtOxdezKzieT6a7hOOUJK6Qf+ou8f+XpJwypTQlwnepTIXt+kzihHQwxLFdkv0eWVURSZeWHdH0KpxC4xNQkFCSI4SBYczOEuFU0+56mFZGmVWYpcKyHdI0p1AVpmlR5NVMByihLBIm+1t0u2sURKThiP3eUT71I3+bM8csvnZryiQv6PeH+G5BlQhkdgNzOuSzc68x2ZqizWSWISs90DlaRwhMylKhhURVGsOUIEo0CiFchDDQKkVIl1JVGAJQNlp7/MN/+l0szXWJQsUo6UMi2dwd0qw/wJtvX+UPv/B+RocBgW9QNyUmJrY5OzOpUjOd5Bz2ZnnOGAaoilrdxvRtzEojhcSz5CxdgRKdVTRMjRQmSlVUxUzTJOQsQ1wiMQxBPNWgBQqNY9k47VlKxmgUkecCU4IWgjSukNKabY6tHu3lP8JoNEinCbq0sISFbZv0BhMcJ8ByBY7vkOYZYKALE0sbWKagaXWYFgWGPWIhm8f1YF41iWsRUlXkRYVX2hTKxKg5BEbF+s1tGkc6FJOMZGjRmbcR0Qbj2GUYZ3gaeoNDllfb9A5i8qmFLVwG++sEdg2VlGhXU0oLW0lC26GYJOjCxmsn+J5NMi5YVEucWV7gpWvbCFPhjEzqwRymgirJ8OeW6G0cokyJb2ncdo35pUXyqWRusUk7qDM9XOcbr1fUGnWcKKbebjHMJ9RqNZaES6UVWg5JcxPT1Rhehdu0qLUCBoclXr2G5ToEtRq2P0tfkO5N8jKi2azTsm3ubK2TWAMCY4V6/Rjbd3dgBPlKji4qqrzJUw88iCOm3B3toHWFSiPAZ3lesrz4JE98/Al+9n//j/yv/+ofs2TUUWaAITRvvbbFT37/j/PS85/mtbeus3zyccp0yPU3X+N/+mf/glPnjjIaDBFSEsYlUhWIIkXpHIMuk6SHv28RjoZEu5Kf+puf4t8P/zX/7F/9K86fO0a3doJeXvDmpR2+8twV3v+R96Oqksn2Nnmc0osNeuMMg3moQpLCJksy6i3F7uAu0teoQlGFCkOlWGaONHxGOicpUh58+ARrvsl4us35J87Sqnd49evvcmuzT5q6TPOUlQWwDidY0qDuHzJgRDuYY5jfwqfOMIqwumt0V+ukl7YQDPCkJGi30LbHJMuRhwes71zj2LFjrLhHeOudq7S7dXZv7qM9wYphv1cTE6m5CbTPPUo1OWTZXeKTz5xlMO7x3T/5j0FtMtwb4QWS//f/9otEwuL8w99G3TU4Oy/42vUhd4cTFusKlI+hK9y64Na7d/jNf/fLrKx02LvzDgPHoxIQD++gCsHo4DoN2eHuO6/ghCNiYTLub+G1bAIrpj8uaS2dZDSNmQx3OBwksDBHtzvH/tYeNd/H1SahTsmrHE/7KCSGkAgUcRrNgOimSZmXSGkipUlRVZhS3jP+Gah7Dm99D/cG97SResaelMKgKHJ81yWKIpTSWJZE2pqiNNnr74PrMc6jmTHTttFWSVIoFBrXk2TTmDKZkikbISW2dCjyFCwHKSy0UTGJI8rKwK97ODUHQyryZKbXNC2FHwQUVGijRDDbptZqPoZp4uqKMAkxDAPLkrS8OklWgBSkYcR0OqHdbmMJiTYFVZFRlArpOGRZiSo1QldUElQW4pk+h+tT3n01pN1eZOGIJBxPsZ0aWVyRpsOZU1kZBHVvVjdEhe16IDRJktCpz1HlFWEYkSQJc4tthITJZIrnV2gxodatkamKeGAhYtAqAiWxDJcky3A9m2xs0GzYnLxvhfqcwWg04ZUv3ObhbznDsfsN8tRmablLWg5YO7ZA0oxAlFy/ckCU9EiLgjzxISg4ehrCocRraZAGlQjpthzWb27jeYLjRxdJ9raJhgpZzvHOzX26R00unD1HfzwgzyKm4xBHLGKUh9i6C2YXo2nip22ieMz+7gEXjq+hKolhBEx2E1pHj5IO1hkPJphWxVzgUrMKhB1gDHOKPCfzFFARJxrL9LDqPoN4SLNewzOalJlDnsRICkyjhrRdyjQj8FPiIqLCQ1QGihJTOpRVwtJ8jehIRk3a4Oxj9I+yI8douclo611WltYYCg8lUlwBtarOB578AN3GSeLRzABmKROlQAuFMiyMUoE0Zr4M4172NCUVetYbVSVCaIQooLRmG3Ij42ByFZkJwlRz/PhxdLnAVnaLalRy/tgx5rxjvDX6BnF1yNXb+0grIK9KpoOSZtvn+GlBNnUYj9eJwjHB6fPMHw8ZX97imfs/iXATrj73Ot05n/pek6s6olv38Homou2wuFD/s28oi6IAx0cVOUvNDkmWzjYnlkmUxBS+Bb0xhmfinjlFEk4Y3LqNSMHs+qAdinI6Ox2jQNtUpUIrB9POUEqCyjEMgbwHpZVSooWiLBUd30H7JgejIc1Gl7QoePvm1/l//ftf5t/8i39E1Rvxq5/+GlW9YHFtAaNwaS88zNkz8zjdL2HubxEEPoWtKcqESuUEQYMorGbAbjkTlUstMM1Zk6W1nmGEDBOjMsBQVLlGZQkXn1hjo+fyW188wHEcHnnoYUQlyPJDtGcyv/oY0/6Aet2j3fTpj0KklLjuzAjh1SzQKYbpEPguSRxhmRZBwwNd3GOMVeRZSlEqDMNGGhopLMpKg5hpoYQEXRUoPdsKaDFreoUUs39OVQEGhoRW3WfQCxlMY2ZybglotARVdImnPn4zwXN8UgVVMZMjGIYBpoUsLaoqQ5oNlNIYuqDMM44+cJwHH68zGc3jihQhLcaTkvOu5oEn2shOTLgXM4fmoBihNk3OnV/lY3/1o/zhb93g6PEGFz96lrfefoOnHjvPkZUutrXIH735PM2zixzuHJAmDkfOe2yuH+A2W5iyTW9ngOc18Osmvd4uj33wfXzkfU/ysz/7c1SGjeGUdPwmy65mwVM8eaxJfzogyT2KaopQKX5Z8e47V+naK0RmnzjMcN05Lm+/yWMPnMWUkmkV0u8f0mgt4rpw443XaQi4dOs2N24d4PuLrB1fIM43abct8jzAdd2ZZtT2sIyCsrCZDCCPCgxbcXvYI/DOYFkmP/A9345fl4R7hxTViH6U859//re4//T9rHxoias3rjK/tEjAEi+9/gYZAWeOniHu3eXY0Yvc2HuX7/quH2J7Y8of/+Yf8zd+8Ic59/4nqAqbty+/wue+8EU6gcVP/K1/wYmnzjP/a5/mU3/zr/LiCy9x7fWzfOg7Ps5B32SUbtBqtakKSVaNkZXAtU2yrI82bPK44szJM9y98Rxvvf02P/xDP85v/tpniYdD3EZKXpkcjkIuPvMAsm2TxxZPf+fH+PqXfx8zq7PUqjGZHpInBjljanMNyiLG8zzqDRMz9JG2Ta4FpVESJyEPzT3MrUu32d8ZcvG7HybfNknVAm/f3mPjsGSSFKycDkjuFmytR6RFC99pIUONUSSkeg+bAFmDZFDgNOd548Ym5wTYskYpakwyA7NKMGnw8usv4jRsluQcCRbHzh3jyju3WFhd5PAg5OM//H3v1cSb195k591bdE8/ghUske/t8MINhbAlV/+fP0dnYY6tnUvcuLROyKxR/if/93+N127y5P2nOZyMaLXPEcl9At/BjIakpSTo+Lzw6h/zLd/9CZzleeqeTT+PGa3v0Wy2OXX/owyubPPIh59hmo5wBZx44EnWb9xh5egxoumAV778x5R2hnBbrC7UMaRkHIUcPXOS9Zsb5N6MSODhk1chwiyhdJBypq+EWbMo36Nc/AkSCGZnbXWP1Ssl70mDqqp6b/OpdYW4xwCecRPBcixKMooqp1uvM4pzELMYVqU1Ki8RwmCu0+Xq5cscbO3g2RZKl1BBUeR4XoOiqMizeLZ8sDyWluYpdYwUM1al65hMR1OkgiSOMaSD5QpMa5Y7niTZTJplSFzXQ2tBlpVkVYqWAstysB2TeXeeJInwhItlmDQbAeNJiCEUtWZAFidkaYZpm6DhsD/EdT3cmoEyS9x6neZcjd7hhG6ngSqaHB70EWr2O3NtlyiaohSYjkdVGPT2BxjCxDAMbGN2Ep1Op8zPz+H4HmkaMx2miMpEyIKyShHCIS8qdCFptTXJGGodl/d9/BymO+b2tQPm55f45I+cpbPaIBMxptEkn2bcfDtkcaVJVlr47swo5dgNiHMcV9Od9zGUYndrj7q/hBCaIk8xDEFQk8zNLbB994C9LcHciS7PXfojHj3jcO6RZxgNvsCZs6fY2r1LmpvUm02ScZtaQ7G3f8jWtkXNN0iSkmNr57l9ZwfPk9TnLEwn5s6tXaJsj1rLYzpWLDdXkFbFMB5TGgqvMcfS8TX2N2+g1YQwDjFNg0YzIK8yHM+cDfLzs6Yo2jmkECWVUeL7NcQ0nznDRTELMNFTHMtl0hdYNRevlVPlfcJ0nWlq0TteUpW3cPMBrruC0ClRlmO5Ldr1BvqbuQfaQBsSIRWVVpSUCENgaA0CtObeEKZmDaa8N5Tpewl6UuF6LtJI2bn5HAe9Pstzj9Cca/LiCzc5dnKeRrOiGJ2kc3wNsfMa4X6M4yiKIqe7YBHnAxQLCAO8lkFr/jSeeZpjx4+y09ui2azY2LzNwxefYHHxBtF0QubWcfZTKnJiI8YuHQb9wz/7hrK0Hdqmx8lzZ+hNRkx7A8aDIfNLi3QXlum0GhwebJHnJsNBRKMzh/eAT3Ewon93naDpAT4Cge0UVGWCUrMYxTSd6T9mTaSBaYh7GicTXc0yYuNRzvFTx/BVQTQs2LyzS4nml17+Jcpwyi/+7/+SDz29yGtvj/ni26/QP7zOwfWrhHtdxtt3sZ0Z2qJSJYYhMCwQVFRViWFYIDVa/gn+4puF85vmHG0VuKVDalRUtkUpV/jcly7z1qs3+Onv/hEuXHiaX/+DDY6dP0PaXeHGtWs4DYntmESZwMDE8w2ypMSwLLLMoqRkNJpSqzVZWe0SJTkZEqUEmtl53fAUlBFpNITSIMtTSmUAFXU/wDUdPK/BaDShVAqBoNQKrRWGyEBKDFFQYJBEBUHQoJZrJnGEYUnUbK/JTFBn4pgeioqKEmXImZBdzmLjEiJ8z8VMPVQ+xXEFhWryfZ/4MZaP5ty+eoVw3yRJFB//jo/Q9s9y4b4WX3nu6wx2N/jC1bf55I/+GJ2lozx4eo10csCDJ+6y2cs4fd8yFx8OWFtp49pdYulg3Am4cXWCTH2kmVP3Frj/9AJf+/oVSjWLPouSHuOJINIV4x4898oOjaMPEu3d5X3PXKRz5Ax/6fu+k+He5/CmBScf/BS/9Xsb9NZ3+fEfP8e7uwPsVLC5YXLyIRstx7z0wjbrNzf4Sz/53Tz/0iX8eolhnqPIFVVU49bNPcZOjQfPHePh++eoz5kMJg5X3ikZDFPKKieZjqAyyePZScr3fdAFqlCI0iGrUoQYgrb57d/8PS489AwPPfY03brP6WaX3//CZT71t/8yztxRqk//ZxpWg8lwwv3nTtHvHfDEI0+we3iWH/rhH+Nwb4Nf+KX/wM/883/BjSce4vTRR7k13kFNB4x3ExrtgFJpfue3P8/K/GlOffw7+PTvP8uZ+ZN88qf/Ki+/tkt/b8TZcw36WzGG4yEcBTIgmlZYliRJKrIcbly/xTTt8Tt/8Ht85MMfodmd4623LtHoLJKGCeiMb//kJ/j6m5ewVYM7NzZYv7NP7rgQDih0DV3F1JpdKisjHsdo0WBuMSALI4p8iECjM4GlNbrSnD55inp3k69+6Sq9A8G5cyaHow0Cp0Hb1exej0nCirrjk0SHiLTA8o7jBC2EEWK5ddCCRlCn3W2R372LU59jUCaUbo4R5VRmxCjZ4+jJLpduf4PoUo/Bzi5PXPwu5p6e551bX2HOa3H7lZfeq4lriws8v/Mi+ypjSfucO3GcL/ze7/CXf/p7efHWi3zptQOc2CEwYqZxyLnjT7F4us3hTshvf+Fr5LWUVquFTAp8OyDSHpZQmIaiP+nz3Bd/n7/8N/86X3vxeU6uHieqRxw9sUw5TmgcP4lsH8HYhCgfMVQ5TE0ev/jtTPvvsLf+ObJyjbUzLeZrFnevrqPMkuFGxcOPP8St9dvoUQEStHSRuQIPuOe9+eaJ+pt18Jtv30QN6f+DdlIp/d7XleLeBlOgtaCqKizfJ45jQFAUBZYtKZXgYPsQ061hSIOinJmBLGkQ1FzG/R7T8YSa65HFKVpX2K6NaUE4TSh0jmkLavUlmi0fJWLiaY4hBdKQVKLAa3kUZYxrzqQSYRghLBPPC3D82RleAnmeIYSk0WyTxVO0EMRxONPzmybNZnNGwBCalufhOA6TcYhtmNS7XQaDwSxO0rLodnwMERCFYyb9jCIvsWqAdNncu83i/Bxu3aOqNEl4D2Du+EynIePBEMOQVGVFvdkgSVOqSjHuT6mqioO8RyUkpnDRukJi4TsuRVVRqZKPfNcjTONtXvnSFhceWuXM42uM9Q4dt0Gj67J01Gd3+4CXXtrlwv3nyMpN7tyNMOI6Ny5tof0ONUfhOnXSJKfRqBHlA0ajEfsbE0QVEIlDDBXgeCaCijhO6e3vkSQheZ6zf2uM69bZ393jzTev0h/sUN1QoEw8oekPdygLBzeLMLyUncMYgaJmzzEuDjncD3Fdm2bbJk0zpskhhnTJ8ohmt0tRaXRZYFoC37MJRykSQV5UWLZH3TDJi5S8zPB9h7zM8es1pDTpdObQRcHenQNqtblZ86ZshNSzq2FhklgaV+eIpCIZGBDYTCufhu1Qb0eMyxHn1p7ENX2qPEGJHCfwKYsSHSts6WCI2f8WeobJUve8a4ZlovLq3vLnHpZQCBAKIWbLHWFwj5F7QH+0x82blzCDIWG4z0ZymV4f8nJKnBjsrA9oejXevPkHJElI3V8iLwYMBwlBEDA/d3TGs04Sms0uaSoJ0y269WMsdB9gsXEBVZlUWUYyzJmbtzjc6NEULoZlMkwjhDKQ+k9/8/5Tf6c1SSk9n6uX3mW0d4jh2ayeOj4rFK5BPxyRIvGCGk4RkoyndOfnmVYRT73/YV57/RKWZaK1nJlxdAUopFSoykJgolWJEsywFswi0rTQSFOQFBOGoz5m0KDeDPDNBmsri3zs/RY2KZ/52hdpxEf5yueepfaIi1M1cPI95molwirgnuBVJ7M/MtKgyPJ7BVDP9JFSorW6d5qxME1z5tMRiqQoUHUPObVouD6RtDhx+hjlWLJ0fo1U2DQsQSAt8tEWrqmwHZ+y0kgLcvXvMTKQQhOOU6pSI1CIqqJ/WMO2FphOI8ZxiiHF7DQDWJaJbdoYsmA4GmJ7NqqoaLUaBLUaSguSQuPWQaOQ6FkEodKoSoEGYUjyNAazYBhWGK7EqSKyVCFEA+n8+dnDQ2l0oWZ6UqWwDBuqCqVLDF1i4mELi9LOidGYXhdTOvzFT/0lWp0md9b36HgNgo5LL0z4/JdewVMnUY0hL776q3xp/e/wHd/7F/g3//Kf8k//5+eo+222em+QVynFQYlwHKI85No7rxEVIxr2HNc3rtM9usK8f4qvP7eHA9TtOlgGYSbIihyMAr/mMonW8QaK+OAmk7DkwUc+xtYg5DdeuMVj5z/C5cu/zzPuGmki2E1uc30oCMsu6TDktWuHrD68SjZxufjYIzz94Se5trXN8QeOc/3NQ+579DTLjXlu79ylJ2Ju3prw0WcuUPoT3vziXbZ7u+z3JveyjV3KzKCspnheRaUzJtOQuudimgZCKFy7RhgpdFXgepqv/uF/5YWv/B5m1UQ6OZ7X5Vd//RLS/SqLfkUVZly82OQDHz7DZJoi8ahyl3j6dWpzAT/zzz/GH/7Rr1BrdNnd2OH1S1c4Xne4e2eLoVsx78KLX7pKkrU4jEIWGg2SU1vs71vcvnSHh59+hHDPIBkbmK5Amy6VbiHlCFRGnGSk+x3W5lrEqzWoVdx+5wvkaUaVKqJpitIJcdiiTofscMJc22T75mXSwTYyWOG+tZMMGTDeE9i6Ti/cwSodlAEb7wypeQrL8bGCE0gzpeRtiiynv3WX+tISZ87Du+9+FcQq7bkmr7x8mWbQoj8q8FsmpdYYtoHpwjQ6QCVThJEyCguWWgZuvosenqVrNHGtHg1H0chAC5NxYnP26EWwcyxTcjDew1xqcDg6pCxznrzvB/Esgyr9E7D5+u3r+DWb7eFNFtbOUm9qVhbq6LBkY2/A8rnzOJOMaHeDujpCbtg884EfYhQesr9/wG989r9yfK3GybNnWb96i3qnTZQMqMqKwJBUgx6f/2+/wQMXnyLpJ9S6DcLU4uSpE9y89jZ6PKDZqpFOQwLL4sj7HmBlcZlBK+Ti8CN4NRPbnWNp9STtxmtcev41zp7NOHH6PEfXTvIHv/3bOIbEsS1yI0FKG6Xke1vJqtL3tHT6/5SG882vm6Y5c4vrP2kiDcOgLBWWNXOBV6XGsmzgXiqPFugCqDSW5WLjkKYZrUbAZDJBIBkP+gilqXs2upgZAizLoigK/j+0/XeUJOlZ54t/3vAR6TPLV3W192N6/Ghm5C0ySEICBFqEYDHLLrAL+4OFu8uyu8DC4hEgPGIlhDBCEkgjaUbSSKOxGtMzbaZ9d3lf6TPDx/veP6K6B+7uvZd7Dr84p09XZfbJUx0V8cT3eZ6viRMdzYyoVcooVSBVIcIy0JSJbuo5JzyLSZWk4BTxCg7tZheZ6dTrVYZhRrc3oFAoIDSFWzCxvdxnbzjo4doOcZoyMTFBHKf4/QFCCgxNZ2t9g8BxKZYrmKaD45WxdIskbtPrRViWol536Hc6qFRRMDx6nT5FbBzXpOq5ZFlGt9vFtVw0BMFgiGnm9mvdbp8sSbFt58b5NE0zB+qpRJM6SqQgYhxLEEUJWSYRwgRdo1AzSTTB/W/ew/6DB+ixQd2psrGSMDJl8+JLp5CyQnXU5ZEHzzIxXkNIF5GlxNkArW/SDwyKZYPQ76AbDsXiKJ1mh1q1ARnEYYrQYtLQwjAc6jVBs7WGMGH3AcHmkkHB1RnEa/jZQdyRMu6owfbaNqZnISILq9DF8SrUJip0+k1kKBhs9fBMF02D0dEJZArIAtXREL/n4hbLRGFK3+mQhgaeV8YxEzJHMjM5xsbmNTIVoqcSw9IxDIv0+obN9UgTwVKzx22veQfj45e4dPIZNK9GKOw8ClHECBFgCguRSSzDRSiNRHWIdI/IKtMZXCaxLarVE5jmKGnUQSiDZJBSsDQ0adAnwjQMMEBIdQM8SgzSRGDq5nXtb36ofFqZ92QClWZYjiKKA1584Tk8z6KzWWW0uI8gXqM130faJmdfWGOsMoJXa6OvDwk7EGktNC2k4DlsrPZxOxqWEyOURjsc4nmKcjnnB8eRj5Y62JZNEvbobyY4kyPMjq+z0YV+v8eu8QKWY7K17v7zA8ph0UD2BwjLoLJnEn+rTdL3GZ0epVKv0Wx3UX4ISLyizXCQ0VnZIhwmvO3b38f6Rsj65hWKBUEc2VhWShzHGEpgGBpZqu9wAyUC0HVBll1XF0kQio2ra9QmFaXxImGYkgY2b3jtO7kyt8wTTz+O7PSYnnTRkjoFw2fyWBG9LNg7fYhvnLyMJuL8d6gUSAFo6PqOnUUa5qsbnRtTOZAocmBmCA1Cn1YS4bgF3nzTfVw8PU93VWNxocl2dIpXj4wTLW/y9ZcuoxdL+J0QU5dkMiPNBsRKY3Z2EikFtmPkFxACXYBUCs+rMK7KO7GUAqWDkpJMKgzdo1YvsLHZROmSd7/rHZw4cQuGbvLQl7/KF7/0CPfcew8feN+7ieOYi5ev8ZH/+XEyKYmjmMZIhbe8/rXcfuIEAsHMzDS/9iu/ycNfPolSO2stmXOPMiXRERhCI4oTdGGQKR1Xt0hiCI0Ap2gQRluMzx7joQtt7juoOHl2m89euMaBvTOU45h7XncLU7bHY3MLHJndy+Kl5xmduY1f/Y0P8+AjJynUjrJ+LaCdJJglSXM45NDuUW47/DZ0s8NDzz/Mzfcc4uzZDvFiD2UOwHDwnGmKjo0lt9nsdDBsB1dTrG8tMTm5n0Ztiq2la3z1k5/kV3/llzi7vExDL3PvrW/HdF5icelJhK+h+y22Tm8wtutmKsV1kq1J5i+fR9GnULFIZUqY9HGFZONynwvbMYsrpzAtD7cwwtylLf7qrz9Oc6PPzNgkQkuJtT5CK+IPBJVqge1rK7iejmbbdPo9dAS9no/AwrAsRkYb6IlBuVHE0yxC5YM2IOyv0vRXqY1NcvyN30pnbZO5+TleI27h0uXPoxkGtYk6G+EmZx9fot0MWV65RqleobnSZBjrjN11jF3HalgLIZ7p4cpt/KhDxRYEg0Wubad0oipdNri4cJqiq3Fw917GR23iUNJtbXHilgrdZhOlFQjFCvf84D1k4maC1EfEBSwP4synH/solZAEBmtrD3L/61JGahklbx8j1dv447+/wr9873dwam6DuD1HIUtpdzWSjktq+ugFRSq7aGSkSYHBMCFVIUmm82//7U/xmYcepRRH/IsH3ovPkLX1Zd584g40w6DV6mGIPJ5yIdxkslrHKRuYKqJREmh9QVN06Yc+M7Uy5nqPgahQFgEijglUicjrs9brsLG+xYnb78ErrPH3n/ozbrnZoLm5TOTqbCY6wsxu1ERv134OmkXsuUtsNzsMjheYObaLi0tbbG1sU67vZf/0OC9trDLXXub43fdyeX6Tha2zTI1OMWwvkAb7eOu3fwe/8H/8LF5vFadQQ+hgkJEYGnMXLqBFGvtfcTeba5u8/qZbsEdG8RaLXF6Yo1irMVIpoZkGz1w+S/XaGuuDK2Rxht6PGIYDNqOUB+66iZeeeJErKyHo29z72geo//A4n/jwn0JPoVUquRweuWM+rm6Amn+4rbnufPEPtzj5v8t9EIEbbhCaloPQOI5vGJqjWXk6mkqxLYtMDZjdvYt6o8DF85dwTQ9/KEnCCA0BQsMwDPwgwnB1amOjWO6QOMkYDro4rrnjZOHgOnmwg9I1HNPGsC0yFWNaDv4gptVso9kGmi530rMCJAFSyh0T9ghM8KOQNAPbsqiNNOh3umi6gVCC3mBApkAqQX/g06iN4LguXhwzDAM6fQ1UShymVCfLVMcmaLa6uGWLYS/DLZjsm97N1mYz9/c0DNIkoe2HOF4BoUyCIMC2TGzHymkAMh8SoHKOq6GbpJEkSyw0HeIsYM+hWdxixvy1EtUJOL94jakJm+WrPr6MsV2dxug07WbI9GyDm2/ay8mvzeEPBkQKJqf2EAz6JHHKsB9Tr1boDHq47jiubdH3W1i6jVnwsEydfi8kChIMswixQ6oyynWD8mhI4HfRsj2IVCfo99AdjULDodPuUW94xKlLa1NRGTVBGai0SOiHuJU8Z35tfZEw9shESLFUIU6HlMw6SoE9MSD1YbgdMhw6eGYdmUgGgx5pKkliSa3WAKmRJiHVao00BaE5HDy0D9sd47Y37EOoiMVzF0FP0RwDPUzzrWAaoZllYj1luB7zihO7SPqKkXoDq7qH3sYQzU9RZp+CLrl8ZZ5KYwRneoxhApYwEfJleoiUGbrQMbQ8TUqSezoL9Bv31D+MQUVZkPk0Krt502u/m2rNo91uI2MHTV/i/KmrrK5fJpQXidI+Z86uoLKQolkhEhq1ym5KZQsZw/LSHAWjQNGeJo4lSbdJN/DRwqtoWx6WbrD/8BE802R8pMLVS038OCapl6m5DpqKabVaaKrxzw8ovTDFbVTxe33iro8fBTQNyEzo9HsMWz6j0xXiRDHoxSRhim3q+MmQkyfPMbVHculybtmg6wpd94iiIVmq55YVmdpJonm5aOm6jhIZaaLQ0hRbZfibm0g0HM9laXGFB7/4VQ7cVmFh+Rq7Rxwae++hGw0RnU0We1dorUvmF07juV7ecbMTEyY0ZKYTBD6OY2IYGpoOuq7lHAYpUWQ73YSGhUHc9bj7tts4cNMJou4S4dYL+K0LnH0RWmvP8GwnYqJkoR9oMGgdRSiTaqNBFEZ023mX2e0OaIw2iJMMXdsxUDdyY/AwSdlc3STxfQxDw2nU85WR0InjDsNuG892GRsb4UtfeZivPPx5tra2+J0P/z5//Tcf49ypZ/nRJx5jfGKED3zggxw8uJur166i6zDoDfnbzz7MQ1/5Mmnf51d//de4trTy8mRSCHRNzwnjSf5g0YRA2/E7U1JHGiFCeTiUMJI2RgppN2T1wlWeN47xhvtmeeKFP+Pkl57CrRQQ42/iXfd4TM1YXJud4pX33MqZlTamdS/j9S5lTzJbHGO1t0K3F2B2+wzX5/jrM88TS0XfFAy2OqhmxFb3GoaZUdzlcHH5JU4cuZ2R8QlWNtawHZtUN6kWRrl06UW2Ftf5hZ//FS4uXuTPP/dlxncdZ2rc4fADR3C916E+18ffPM9t9/0gxfEO0pihfmQJQxSpunXMbAF/awMtMHBLFe565Vt58eI1RLbN/bfcx9pSwoXFBRoTU/zAj/0inc1LfPxP/phdM3uwrGkkGsJqsbS6wZve+A5On3k+z2HXKvT7Hb7pLa8nyUJOvXiOkutQKRXZ7G6w3tumUhXIUEN5JlPTBWRT8Rcf/W1kUsSrOuyfG+OuV/0ML3z+o/zWn3wGfcwg6Qzph232TY7R7XYRQnHLbTdzbX6NM6cXKXsFtJpGbddx5q5dyzOTBxnN5auYU6OEYZuzZzrcdvt+krRNZ9BgdX6JMGyjO4cQskY0WCZ1BcvXlshkiG5LVNDBrTkEaR89MHGKKZadoZIUyyyxtd7H3l3gU393Fj3UWNt8mMcfH3DkSI1ydYDjmgy0jIsXBrznPaP0+30Mo8nywjWGvYB6rcJmt41Vley6pcI9u3WCXpd2f4rXN+5l0GuxvjVgZnoPkdykvTYkK2ZkbYdHTrUxCgWs9pCsEzM2NcXorYd5en6BfnuIKSZYCTW0yf2MpBm2D6ZusbSyzB0P3M904wj7jt+JUVaURMz4rfdhzl9ma+3cjZp4+toar3rjN1GuaiysrRJ0tnn27EUO7prArZjsdj3qsUWv06Zgl7ELZdY6V7Eyn8HmPGW3wujofkYqB/iPP/VT/NVnPsHi1SUKJQNcHUs6pCrh4uXT+HrMm9/17TiWRXe7RbFQQ9MzPKfCxvoSIohojI3w5a/8BZo5ileQiESnFw3oJglZP2Xv8SPUVMyr3/h6/ui3/5zjJ+7ke3/yP/D3H/kDVlbWqI9UCPzcfuf6yvs6eLzOl7yu6r7+Wg4q/yHQzNB1cWMVruuCbr+HbVq5u4TIkDJGioxCwWRysoFh6iyvLlKtNWitNREq306BTpZmBFFMoepRqpeQJKytDTl0ZAan0CYY6uhazkM09AJCt2l3ujRGi/jDBD8YYGBimQXicEixoGMaJXq9nGtnmSa+7xNlElMvolsajVKZ4XDI5vYGTs+GLE9Jq9dHiOIeaArPckgSyWDYwjRtRibqOMMh3X6AEBnCtAiSlJCQVGqsraziGi5xXyMRGdnO0OD65NYwDDQEpmWRJgl+f4Dp2FRqVYbDIUIIkjRGNwyyTJGmCZYjyFRMqVRh7tIahm7i1br0fUkUJpx6BtqbPWYP11m9Jkh8g0YVVuZ7zIdtwjiA1CbxdZYXegg9JA77eG6FWs2jUTHptTu4nsBxTTobEWmiMDQbTYNiwSEK09wNpATDgU+xHpKkGdlQR2Gg6xANQtrLHnGko2VgujGpGHDxXJ+JmQJBtI0QZQymSNM1arUK9YbDVjPJhaVOnWJhF4m7gdDKlGo6RqwjdZc4kHR7IaZw0GRuLyhjRX/oUygV0S2XYrGCMD3sWpmi0tn0M0688p2sn/sdDLVFLEN0oaOUS2gFOJaOoIfecbGjW9m/bwa7cABXjjB24DiGWcLXI0zDZebgPjRSojAhA5SySNIY9FwgqzJJKjKMnY1sloIUOZVMKQlcd1nIQaZUASJJMHUHlGB5eRtd1zC1hFZ7neK0yYiapVqeJhGKltYj057BNYu02wWa7QV0ZhkbGWfP9E0UnUlsw0E3Jf3hOs1mn3JphF4/AT2fgIvAp1JIcXSbtbkqWSnAjwckokDB20UYrv/zA8riSIlhu0MSRZiGjR2n+EubaMOY6miDbODTXvJJpUGv16ZRHSEM+ozUa3QHPfYfD+j7DtfO6XglnyzWEULHMBxMU+H7Pkmad7YKbWcE/DJPR9MdEIok7JP1dBJZwMfiC196nFdk+3EqPda3IU10yo0CMouoNSxeeHKL7Y0WVkFHkYCQRFG000Hv5FnLBNOwyPmTOZFcCIFlGRi6hswEW1t9bjpyE//1d/6MK02dNNzmtre0CZIVrp5bYWvxGqfOnGF1sE58NSaLA1J5grXVDTR0pJJEScy/++CPcOjgQbI043d//3e46aYT3HfvK6hUynzhiw/zVPY073zHB6jXalQqNf7yr/6aN7zxdfzhH/8x48VpsiRiMBhSqpbpBynj03vp9Hoc3HcAEHmHGYSgMiI/IPAjdKFjOTAY9tGVw+Ejh1hcXGBhfgmFlScTod8wmNeEQZL6O91TDsBNqdAyDdPQkEojwCVOTF57/C726Ve4embIyqUa/+Z9b+WzXzvJerfDl3/zE7RedxAjHtLbjgmiKt948jN86uNfY3pyBq1QZ9/eg9RrGUYpplyvMhguc+HyHMuXO8zOmpQmC+h7Rgjn1/Fsj85mC10plldXUYFPoSQIAh/NLOMPfUbHHLa0lD/83F9RGB2h1lmi8I3nOPyffpqHTi2xtdAl8BXtVHChPcm//+lfxarE2JisrG9w6717ydpt9jYcjt5UYHxshl2H3kh93zYnz7aYLFuE0deYNCSf/ezjjNYPUmm4JKmF7RYoF6e4dHGeB175Bl79mlfyxje+lne9+234aUaqp8Shhlu9mX1Te+iFX8V2Igq2yXf/yE/xGx/5K7bmF/C3X6RQLFPQqzRsgVIeUvN54N5Xc+30l/nYn5ylaCUkdoTf7EJsUC4brG76WKUisely5sIl3n5nhffeez/IImk25KMvXOFffdNxxkoWKx2JjAJUvUjQKiNciR9uoeKMuQsLjDUmmZqssLSywdrKkPtvN1hqrWJpXUzbIgtNjMyk12xj2BJDFBi0A1A2/UFKwfVRRky7HbK6HHD7iaNszK+zf8pk6eI2Y7dIupt5QpaVBWwuQiYSRhpF7ILHIIgRsUkcRCwvPMn5i5cYdW8jC1t4BcFmLyHLmjg1RTM6i600SmWboNjBFaNMWoKSLehg8IXnnuNdjV2cP6dTuPnV3FTYxaVvPErkQrOzjR1VoWwTZl38dJWP//kfUiztYnK8Rhx2yPwRTj13kkFrHSrFGzXRcid49qvPMranSPXQJDVzArd6lqW1eYRt0pTzVNzXYo3XOFwd4/zcaWbHd2Ogc+9rX81XP/84i1eXuPnwK3nzG19Ps7XFn1z+E9I4QRg6iZ+QCEV9rEL76hyf/cTf8BO/9rtsn32Wb3ztK3zLm9/AXH+Vjr/BHaNjZAWN977xzXz8r/+at73nBxiZ8Hjxpcs88cJJpsdLCARLCy0ee/pFhJuysX6Ng0eP8S9/9Gf5k9/4eVY313Fd+0bN1XUdXTPIZE6s1G4ovMUNcJk/DHMBjxAKTRfouomUAAJdf1mso5QCleSm01nMrl37EUbCoJ9x07Gbef7pU0R+giLB0AxkKgGB7VhYBYtES5ACGpMVJIJKeZReZ5XmMF+xx3SRwqFYLKIpSJKMfqeH5xTIogDLNgiGEe1gSJoqqvXKywkzKcRxSrFQJEkSipUipbKDzDLa202C0KdgGniOhUQjSVOSJEVpud+lrusUi0UMU6PXTFE6ZKnEskyiqIeWKRxHo+yWuHj5MrZbQAlwPAu/M8Q0LIJgiMCl6HogFSMjI/SHgxshFZZlEYYhwhDomk0mE2y7lP/cRY35K3NMzRxi99E2ie7QXIywCyU2ljJe+8abuPTSFa69tIHjeQwGgqJjEYcphXLI6kKGW05wHYsojum0+xTLFWzD2NE7SHQbdANsSyOLdYIoRZFiaCaOXiJlwNaKyOuBJ8gcg+LYBPWphN5aDeI6AW0cewTH1QkCjUFvhVLJwRY6yPU8+SzKaPfaFIoOlZrOxkbMxYsLNCaHZL2IYGBiZxPI4ibTu+ocP3aEU6efZ2REJ1U6UZigiYQ0iQj9AE13sfCIw4xBp02ilWjJkM6wm08n07xxEZrE0kD6A0wTpicPMHv0dUjbpCd0LOMIZQTDJEHqCZpeQJHg2IAykSLByAyEUUAzDYSh57GmmkIoCTLDlBq6MIC8+cr5kyKf1CoBVowuQaYpGQmeXiWKQ8qOxtL2Mo89/Rw37Xsls4c9FlbOoA0KlKp7CfopIzUPKWax9Dqb23OUC2WCRKc1aDI6NkuhtBevHGLZFSZrNqamk6QGWhzjGQZaYYhyAxplh8FWC8M1idMu/P+DQxmFPnbJRWqKgmahHBspoNtu0+13scnoGRa2UcFMIgb+OoGQNBoulkiJ2eQ170i4dFZhGjZCCbJMEIWSQlGjUHQIggDd0NGEyL0gd2IPNU0jViEZJoISIlUYmoZuWbgWnDu5xu23NFjsbrAQv8TusVG8UZdLFzKee/JZig2b0I/Rjbwg5lYXgijK8mxqkRdLoeXv5YVQx3EMdE0j8GOMzKavUr7zfd9P4EvKFRtPFImDmPJUFafhsvvIbfSbK8heh7WFAYMk96FKBBgCXvXqV6Ck4pd/9ZeJw7ywPvHEE3z+wc9RLFf5xZ//b3z9icdQUtDqdPmDP/oIYRhz6epVHMtAAakwCaMOspXwPR/8Hh544AH+/nMPsrU1wPEMMiXYvWuGkZEGaRywe3yEDEEsFfWyjgReef8rOXvuHHv3THH1ShslBcKANMtQMlc4ZjLcOVcWpmGTECL0OsNsHcOsY0qHJNnmbd/2zRy9/wHK19Y4dWmR3bP7+f995zRDHLyf0BEIfuXX/4i9sw327qvys//u3/FLP/J9PPbYV/m5X/49Hv6zT/I/fuMneeyZRYxZhR2XuWn6BBXvPI4bcWFxmeZmyog7jZal0JYc2L2fcqPK/LV54khj355dLCwvoIU2K8sxr7n71dx24BB6fYa7X3GUYDui0+zQXllkqjbGxWiTwO+ye1eZQ5Mxr76jSrWyB9e7hW4GUaeLJgWmcFlfHfKD3/eT7Nt7kLVwkdmRXdhOyMLcVcpextbGQzSXqxSKJr3egOOHD/HRj36Uqekyn/zrL3P2zFWieEhBl8SpjgwjLl56hLl5i6WlNcquy4G9u4iSRX7xP7yNj/7mZ3ho8zTTYw08YsyGwxte9W7ufvPrWE1sHvwvv0pr4SQLsoVrONQ9j1DLGCxblCZ0gsgnGfQw6qM8eX6L5+IubrGB3aggfI0XFrqkicQplzD1iEo7o1JWpEFEwa6CSjENRTwccPvNB/jaM4+x2bKoTb6S5V6HolvGD4coMwM9wrFNROqgLEG9OkkQaOi2D4SEQwfXGcVwNghFm9HCOFUVspr18JxxDAZUy4rxA6PobgddFDGUiYwG6NmQeNClWp3ALRaQrZCN7SVmxm3Wr61TH6+RphElexSpNdFbe2n5mxiuQAsTap7CshRzKwNu2ruXXdP7aG6cZ+nMAoeO7+eO4/vIZJlnzp4jIWQQZ6ytXOJHvvdf8fVHv8HCxiLnz1yhONqg4Lr4S1c5snc/X3r80Rs18f4Tx2kDSVajdWmR3mxu7VWvWPibOhdbDZ5pnWfU0rl67gnuOnI769uKfXuPsrgVMrJ7jMl9JTb66yyuFYhkm8zKsMwSURCQiIyiZ5OEAcq12VqY48M/8wscffURigcmOBM3aS712TexD1GWfPGLf89P/+vv4ehNP04QjXPmxTNkbNGPAropXJh7CTOJWHxhi2EWoZTiwU9+gqM338lvf+Qv+L5vfxNBqm5sh65PHaXUECKfpjiOg2maRFG04wmZnwtdv74Wlzesea4DU9d1GPYHgIaWJRjKxB8q9LjEnXee4IVnz/L0o98g7Of2bCqVZFmKbdgkMiOWMYYtSLUEhUG1UaLd7DA5MoWQNrrYUZEjccsFwnBIEA6wTYtdM5MMBwFSV2QyZtCDOJZ4bpEklGCBbRlESGxHo9XqUK4W6PY6uJ5JtVBgOLQYdAekaUIwDDBtFwnomoVje3g2BEHA9laHSqGIChX790/R7Q3J9IzJiSqrVzaZ2TvNansbzbByuyDA9lwOjY+xtbGFkpJjh4+QxgmDTpd+d0jgD9A1QSrz9DjLNoGEOIswNUGamshM5L7GmWBh7ipH9t/BC4+/wPTRKl1fkQawdq3L3LktDh3dw9ln56lXa5hmRKyg2405etMIrU4LmRXQbQnGEM0w8cMBSqS4noM/7FMoeqSBhWlCpnyUtLAtj05ngGWXqLowDLcw0BHBkOVLQ7pdie83KRfHmd47wfnTfQqlDNtNMW2NLPOwbYfET/MJnXTwPAOl9Wg1TXRnSHeli+45HDgCnc0hay8tc9P+W0mibV48+RRxKmi2DLIsQEkNIU1UnNLeXKXbWkMzbFrFKlcMh3pjF0m3izIkQpeIOEMXglTEJD54BYthmmEKn6WNOS6cXaFUE8hQYAoPVy8w7C2wubhIxW3kDZVjISxJkuiYrkeqaUQZGI6N67pYxo5Xq2ViaCa6buxwjq/fazooA2kYmFqKpkw0QyfTJGkWoJKQyalxjhy0eOncabyRUbabCwTtSZCCyZkScd9EalVsq0ize4lCYZRKucGgu0qst/HK45hU8aMIx7ZxbEGlXkTEdfw0Tw9UIp+kOmYZgyJZKWCjrf3zA0pNmYg4RiejWCvkCjXNwHBsLEDpBYSRkpIglYGFwFUpWRDlvmMDi+JozMHjJnPnoFSP0cIKKuuSSgdkkDNTlYlSOrZl593YDk/H1hziOEZoAUmg4wQmzc0VDu49iqWKbC0bHDpwDyIz2draxu8lLC4ug9KJVJ8s03LrHQG6KfLP1hIMU5EkkIiEgukgkjzfGlfS7qW5/9Owzy133Y0+sR+5LJidjBnZXWPQiVg4v8Jb3nGc587MoRKf+tRu4pn9BFafdsskd4MUpKlkz+xunj/5PNvbbQxNI04i7rj9Dt7+9reTpCmjo2NEYUKSJFw6d5ler5ebCGeSyDTQyM3hB/0hlmnwB3/wh3zkz/6UX/nlX+VLD3+BVrtDvd7gez743fzmh36XNNNAy01awzhlq98FpTh65DAf/sM/JYkAIVDIvNhrBnEaYVsKscNnNXRJKjMkEmSMkjVcCRExtlvikZPP8vX5GFSJyrjJ5b7kYMkDIfibr10mDC1WNwcs+QN+7g++xPnn5qlrAd/xwbfxb3/6vzFa0Jg+PEmneJR+d5tBd4uy/jZGzD2srD+NpTymd62ycnmJhqgw5eyiE2wydXWIk5lolkMwiDCGVUKng6YXqFZGGA0FlS0b/wWTxvFJdk02uKVW4O777+DLn/s0XrlEpOl8+4/+F9Zf/BytQYoeQCYBUQIBfidBiBjpn+SF0y9SnRjj7PYV4jjGtnRKs1MUtF0k6z2urq+iHMmFufP8p//yK/zMf/y3PPzoGV7z1vswlElsWAT+Godv3sfoyAzdzUWkCKkV64zXoDP/JBcfn+OHf/TdfPWpL1MrlXjPD72TP/ydz3No90HmVyWf+bMPce4bn2Z6vIqTNBj2e7QGCYaT4BUtwiBmEITY2giYRVbWh6SdiCi9ijfiUK+N8cL5NSJDR6Sb7JneTZb4bIcDSj6klQ6+MqgECtO+wP6ju2jUZrm6voIWWYxVSrQjgWe5aCJgmHgYSR1d89FERmEI/bCNI8r0ggDLLmD6iiCUjFZniZOAza0QM02peXvp1s4hkhLNwSZeqrPV2WJiJAcySrPJlEs0WGO6eiflwrPsrhyjubJEfXyU2AnQ+hbZik3Pk/SNq+iaxrg2QqlRZHFOMuuZrK62sGsjLK15HJ64mc9+9XOo+YvsP74H5aR0+21cu0Amlrn55m+B4gH62uM8e+oiCMGRukdt4hDnNq/w7PkLvP2Vr7tRE1c21ri4cIWRepmV7Tm2O4tMFEboBQOcakZjdA9mu8naSo+J6UOs9jfQ9JB3HfkuOpubGGmRolsH3eHKS9cIh4Lbbn0Fc2fO0g/AsxQqyGhaGuOxhjPqMnfxy6zPneLI6+/FEwU66RoXrz7P7OweDs1onF/bpL9tcu+9Eyxc/F2cEZcktVm6tsjYeAPT71E1JcurLcIg4QPf/l4cc4ozZ87ztg98F3/6m39EzSuSqgCpZyhytbYSBoapMI1cUJlbBeWCRsPQb5wTXc9Nt68LdqQ0MXa8OzVNoWkOmm4RxSmzuw/h9zKuXL5INMjXvkkSgGkiI2h2ImSW0tjj4iZ1enqXerFIlLikokWru41VcjAkpKGFNxbhaQLPKrG0soGUA2qVIkLGREGGbTcoFhOipIffa6FrZSBCF2U0WcLUY+rTVZIo97BcW1yAsdzwfIBGHGSAhow0omiI5bj0sw6lahEVGySJZBglhEoQxYrD0wf5xnPPYLmCe287TuD3WNtcxyuP5M4bQ4mjKhw/fBMPLz2EWSly+oWXaDQsCHUyS0OaJlnfZ3rfKJvbPVIEIk0pa2WiuEcqMuySxFAjDIItJvbXscaH7L7FRaaCLO6jC5NBOE+S2Ww1+4iCRi/zGTHKlG1FJ05otxRZWqafbDEzvYfRGZ3mZkJpskC/2yOJQUkDXdr4vk+1ahL4HlLLSFLw3BKaplGbUqRbCstLkH5M2O8h0nJuEegGiLjK9J4hvY5Fpegi9Zh+EGKXIvS0TMKQJIvIVAMpQuqTRTJpomcuvXaPtSsphl6iMuEy7EWkieSBt9zNN557npI3Qqbr6FISZgkylhimQxxk6BUdP2sipMB2qohBjAa5oElkJGGMsPJnYJoNMVIoVaY5MDbBgy9+lJGJMWSasdXucsft93BwyoVuSrHUQYUQyhAV51ioYtc4+eJpgiACLc+1l1KSa0EydM0gzlJUVge9h0wVyARL1HM7J+mQiTg3XBAJSA+sPu/5njtpRX3qtR4bixFuwWF6r0FGREFPmJ/3CYwVinaZKHIZ9AK6my26gUMit9ncnsP0G8zun+XFZ54jTm3i+DxHan32VXyaTRetHxOvWOhTEIoe3kaBuln75weUIvcY59AtN7O8vYU7PUq63SVsNSlZDmahxCDpYZbKCKFhCMGgvUnRskiSBMsdkMUOB28VXDsXYTkaSRCQRTZxJCmXHLI0QcoUdlSDSqkbpFWZpBiaBULgByFj2jhHbr0F3x/wivvuYHv1GqeffhE/7SEsg1p1D8urTUxissBAqpgkETd4mtdXNtc/v6A7yDRD6SamSFFNgeVmBLpJ0Wtw4I5bGKQeL50/R2gWWL28TdQfstVucXV9wPmVFBm6TM0oahUbwxAUSyUqpQJpIlleyZi/NscrX30fL774PEmsiCLJu9/9bv7zz/4s66sb/M0n/4qVxWtEkU8cDsniIbbj4g/79Fohum5Qq1coFjxcz6E/6CNSQRRHyExScD1+5j/+R37jN3+Ts2cvUGvUEJqNVBJT10Gl7N29i8XFBcJhF9/PQBbQRZ6ZZhhmnuebpjdUlcIQeI5LkIa5hlxBlAgwDITIDbB7rSXKXojWGed1d/war3vHXbz9X3wf5y5vUA4HoLfZXrrC4vxZ3nTL3bz1ne9jUFTsmjzAuXPrvO+Nv0a0fo4f+/fv4trzF3jsyRcwyhmj09NkQqASgR2n3PGKeylt9Lj4Qguz1MCQbYRXRNeH7D2maA5n2dja5My5JzirCiyf/xi2OYoiRK9OsWt0mr/6299hpqrx3LlL/Jtv+V4cz+FdrzmM7dq5YlVIlNpRpGo2Sgg0w6FU9qhPTOI4Htvb23iODSYYjsXmehfRb/ELv/x7nLjvHs7On2V0wsA1+jz42U/TDof0ej32zt5MFmdc7kbc9Mb30v70x9mzz+JS3+PZMybd5gFuv3COsYbG+bOXeeDed+PYNzPAQKfPtWvnsaRFEuq0l5poZQ8jM0j6CYHeJyKhXHLJ/BWy4RH6cZ8xz2KfIVlJFUU0dCcj1DRIDDwrwS9Z6AsRWT0EvcbU7jFYa1PzRhn4iuGgzpjXz3PIY5epEpi1Cn7HJ4xKVIqKdhzh2hWkkJhLJSrjJfY2bsGKSiRJl/tOuNx67BX0F3ucjB6lsudOXjq1yZLcwKl4vP2u76QT+dTqm+iGxHY8pOhQLNkkw4h22GdyT5lHr3yDiBaNIfjKQCs1kMEFisEYtek6/c01IqNLt1zm1mN3UCnVmIl6TM3sRtdCMktyy7FR3vKK2/mvf/Tr3HXo1YztGuOlzSX2Ht7H6rUmjz33aywvLfIT3/0jvHD5s6xHbR596JPctn8/1xbmQey/URMLxSaDtMf0mM0uirR16GUK268y+8ABzn3jK7zixKtx7/l2/vjPP8Mt+6p4yqHfH7A6uEovXKafzHD2S5/kO977Ji4uJMwcO8Z73vIufvPDv5ivVHULb6uJMyLopIJCvQZhwJnPP0R3/xHKByawSZDL24zZBygmkqeuPI3rVNFLo9h6hhZts7p8gcFmjdmyxf33TNEOFWfml/nIhz+LNd7g5j2j3P+m+zhy8Bjn5i5SdHVEECMcnUzoGCg0le1QYF6um7oudtbd+YMzjhM0LRfS5A9RUOQxj1mWghaSZDHlmsGDn/9zgiDCdW1MQycOIRUORpqSiSL3vuoQTrbBxKFxLrx0jeblPr2iRVoYsmt0P73mBoZlECYGvphHDCcpjFZpbW2ghCCTgrX1DoYJtuGSpAGxH3LPPa/g0qWLdLrbJAloIiZKfPr9Ib12Ec8pYBgaoyPTVMounX4XoVtohkUUh0Rpj9179lIqjXLqzHk8y6G7vkrZMVCJhERipIrp2gg3HzhMoWwx3phkfXWR2YlRYt0jjNpERkIkB3zpa3+PkpD2JSWvRquzSa3QwC06BGEPxyswOXWETvcMdhTSD00SLyMVAh0FkUGqt/DKGVnfYuFsRL1W5fJCl835Ibcc24VbMSGao7ulmJkYodXepttsYegOkZ+SRm103ST2dWw1za5JjfNnHmdiYgQhA4b9AGKXxIpR2GxvRRiah+lKdEsRxz6apnHxbEqxMIGmF0nSEN1UKEOSqQhhF+jJVTJdx7A8oqzP9voARxtDN+I8lzt2ECJj2BuiNI/meoxmRCSBwhAOiW8xjGMcx2RrrU21WuPc2Zeoj5i4nsagGyBJqNbK9Ps+iZ5QLBpkKibuRVhunfbGkMFGE10zc9W8nmE6GqlI0YXKBytS4roFen4Ty5HURmpYlkWh6lAZbRBnEaQOIisQJinoFobIreyCIMC2bRyvQBzH+TR7R5RmGh5S9tAokIkAJQqoTAdspDnA08tkWR4kEPsJhjZCSkR9bIZOv4Xsuuw7PsWlU4sMmoJo2KJY9HjhpQ1o61j1AWa9RNHQCPpt5q+eo2A0mHu+Ra28iXJKFKyQ5dNnaTWHJJlg4sg0pdsOQLxCKmyIdLx+Ec8z0D2DZi/85weU290eugZLSytolk04GDIcDtEsk1hIkjTGLhRzMJhldPp90jBCq5ukIsNxbYJIEoQpuikI4hTLMciimCSz0TQTXZdEcZZz+a7bTAgQpiBRikwBmsB2HRYWlnj/B76f+17zGu687y4aukmQbBMm63zjuWf5qZ/8BbLERxcJriMhNm/YWlw3Tdc0dUORGKcxVrFEoWjR3Wyjo6H5EplkvPf7fpLIrmAQ823fvJfMTCACQ0X0hwPCQcSde2q0222mRicpeGUqRxRXn43p9poUCw5jY6PMr8zzBuf1/NIv/QoyTfnt3/kQJ08+zy/8/M8zPz9HGAbs3j2DY++MyS0HAfwfP/VT/PaHfgtN19F0jVKpyAe/+7sZHx/HskweeeQR2t0u7/+O72BsbJQf+lc/hK5r/NmffYyzF87xoz/8b/jQhz6Mbho88MD9fPnLXyEJh9imSyQhQ6DtqM/Y4RQplQuYroPLfhCTWRqGpSO0mDiNmRiZ5v67b+fKUsjU3nEe/ezD3DMZILdP8qGf/6/cd+8baakVrNBg3Jvh/e9/KxO3HMJPPQRFnjnV5bnPPsEhc5XRb9rDV756moWFBd7wjmOMjo4zv7iNURxjdX2IPrWPS2nGQcukpEAGPWQ5wy5W6QxTur5C13wOGBXGinUMu0qzkdAY9+hlgnR9nandFR598TH23/RmBqLA2L4JzKIAzUEGPkpIBPk1Yds2QugEUYZleUSpIg0SBpFP4MekQYJu6dhFRTwIcTyX3//rP6X5ic8wsucAf/k3zxFuDDh4WwFbd/nOV3jM7vdYXFvn7MICx43bicdmeOSR01iiydJWiJI9lu+8B7dS4urSMr/w33+W8v5XYkj46hceJNxYYrRYpt+XHLz9GNNTJ9go7+XIoUkYvZ24uUYh7EJP8dTCnzPZGOMDD7yBm2ZrfNtP/ScO3iI4OnILlszAFHzjwgbbiwF3HptiOOwTJQaDYoE0aWNmOs0rXTZSxczkbsrlOivbbWaqUzxxts3xKYdaOSCL+4zWxlm85DI5PaQ8C0a8n2e+MsfI7oPc/6rXc235szgzNf788w8TG1ssb13mcqRz9fwSSW+e+F0WM5UR/E7KzbdUGYQ9hnGIpjRKDYcvPfE4G8kam62QzU6LyZEqqa+R9pcx4gicHvFJweyoxa37TnBtrsXmxho/9P3vYCYuUqqUuGnPPiytzFMX1nnx0ja7pg4jzYznW22G0RrBixqBsYJmJNTKVYzRUZzFSbSVLkeP7SeVITfdOcN6/+yNmlguFNFSi7QP+yYO8tTZDYLMp1qtEMUFDMfj+efOceSOo4yrPnXLQLc9FpaeYOnsVZqbW4wwpK9fIwwvEA03oeOx65V1xqoNooJNaxBwc6PIuYUurpsSD3poZgmj6LG8eJnw8nnue+AEYtLgxK2v4sDMTTzylS/yhS/+Bo1dMzTjMpWCzVtfdQfv/84f5cyLX6BsDPjMuc/w7d/6/bxq9zTrF0/xyWeewdBfjdUQcCHDdkfw6aGnEkfPiJIYTc9jW6+vs6+Lc/I/GkLooLJcwaq0naYsj93VdY0oyrB0lyw20KwERIjnuejKIo1zMZA0JIYFRubwirtewytet8anPv0873zzCTbv2OLrT86x0tJIy5J+TwM7oTpapliZIuhZdLabyB0PQs+uYgmL5aVF4jjGtAQGFs8+9SylYgPiIq6r0dvwicMM23bRDMkgaJNmMWNTDYZ+huMauJFFOBywb3wPw16fu4/cxp49s6huh9ldezkndZbmNyg0BJZu4Jk2frfP9Og4ftSl22zhWDad9gCrrmMXiuhaimbrFK0qveUmWt/kyH0TrGxaOMpiZqxKE9jeHKKnKdEwI+zYuHWfgR8iRBHLVURJAEluRTbcXEXs30+toRgNBQcPHWPu9DqZW+W2V8+wvZzhODZeUCDwUzKZ4JUM/CjGclwca4RuPOTzn1+jteXilTzcQgW0Dr3tmMFwR0ykSyQpum6hawZJHGMaGYZRotXqMFH0UFKQJSaGJ1AamNYIBafMhTPnGR3JBSrlqsDGx2+bCCwQAzQ9j/VN4wglFaChGymlqk7YU/lEMQ7o9jbYt++VkJhE3TqmMrFKMGjF9JYTHENDK2YEfg/wkEmRULToXNmgaBYpeBbDVKJrDnEc5rzlLMYwiqRJQLHoYZkOhUIdhEGaGhiWTmWkSm/+EhqSLI6QIiJMYsqWgRImg8AnTVN0oZFKSZqmN2gkfhhhahLdSFFKoqSOoQr0oi6p0tCljtRTbDtP85PJAEyT9dU2g15CNFC8+MIcVU2nEmsMtyqM330P9zxQpJjFdIKr+FFMuXgYIQocvLmF4wlsbQbTdLE1A2lKDt55HMP2qNZdTn3uyyxsBQhDEGchnlPHjA2icIjV0BmGzX9+QFkaG8FMJb2tHgYCvz3A1ASGZ4PKQZkpNGzDYTDoU7QchGlSLVQwkGQiIUokU7sKHDwx5OJpqJc1olhHaYo0UwhdQ9My4LpZrobKcl9IPU0RWobScqsB29b59d/+Hxy8627+4OOfZ62nU1Ap8UafF772FTaubOMUM1LlMJQRjmXtEMgVQuVrdF3XkDJD0wSWkHiGQdjrIXQd4Wq0tgLe+O5vxdqzi/lrGwzaWxTCEigTKzNwbRtMB0FMisnMoRFkYjMIDXq9AMM0cBwbUzeIUg0yjd//k49hGzqGppBK8tnPfZ7Pfe6L5HJyheMW+MIXv4gQGo1GAwT8xV9+gpGxCRR5alCSpHzkzz7K0tIiu2YmydKUgufw6c98mk9+8m9QSqNcKbK93SVTkt/6rd9FIcmCiD/8gz/AdlwaYxO0mr3cEknPAbymaTdUnNenC9eTL2pVG3/YRyiBZlpYmkFzc5vf/h//A8sZR1QLLJ9Z4nt+6I3M7BqnNT/k5rvHKE7fxKc+/hdM7xnj0OtfzbmrPZ4/02Rx8UW++ud/yUi9jHlkL70sopf20UfGeO5Cnz3xLq4saqByBV9qxWR9h0tmheI3VRhtDWksXmGQSepYBP6Qsi9xbZPt8xtk5jWmlIOuF7jjrntJow733/9mTp0f8qlP/xljsy5LDz/Pgb3j/Mh3vJmu0FEqn6To+vXED50kSYjCGFW0yDQIoxjDsVFpBoaFVALTMUEWWO2u0RwsEXYXOPRAFavcpGzsp2F4/NhvfAvW4DTEM6xsrOJlF7j4wia79zgcani88hXQbTUYm+yxNehyy7FZ3nb3Ba5dvchkvc5t7xqy3R/jmZcqLIQGe/ftZqZaIJicZWn7IF//08eIg0voXMN1YS1e4VCxwUarw2rzCtOHjvHMyTaPzp/lh995iL/9ykmueQ3efv8UY+UyVetOCmbM504+z55DExw5UKBoONxk7uP0ma/xzNMljs/OsksL+cL8KZxddzPpztCYOcDicJM//qsP8dyXH+SRR5/m9z/9GV511034WyvcdWIPzz/tcvLpa6yfPcWqmmdt2KNilem1IwyzxYsvfpJL3ghj9iEOH95LpVxjY7uHbdoITbLWbrK6tYhjeuzGYf3sPEPDyVeurYwT91SoHR0FLeBLXzpF1YlY3HqSXfUK+3btpaIHYKa0B+sUrArf9a3fyu5nJ/ilX/0FRm++lYgynfYawu6xuS4Zm6nyp3/735GDhMZokdBsMnexiZ7qzFacGzVRGQOSoc9NM6/CLseMLF1iqaszzPpkSw63TN3FIytXeeTURbLqgGw4QzLY5tn5ee68+Q4eO7/AeqtFyduA7YSlVYHInmHuua8idPAmLEwFR6cnKKtNTm3YmI6JFsckVget1KBiZzz3/IucPFfkoc/+GE0/o+m3EYU6e8wxzjTPg/TI9ON8+lNP0xl3cQY9zKRGVdmc2lIcfN338ZOveTu/+uFfprnZZ7RcIo66YCoQametbSFTSaZeFuj8Qzuh6+4cOQf95aQzIciNo8nrbJpFudAxVuh6AZUJoixEZiGWnW9IUAYTh3bR8tucfT6gqAxedc9hNrMqmWnzuc/16aeXSR2DirOXqBcgMh0hBUUno247ZLFiemqGhZV1yuUykR+AhNTICId9omaEaRTwhwG2ZqMbEk2mJKHENCzqtRplr8TiygK2a3HTsXy78IM//p/JEJx5aZ77v+l1vOP7fphDu2f54He8m2vzV1GiSpylaIZJqiShH2K6Jpq06Pa6SN2l5w+p2Q3e8tY3s7C+zOjoCA/cegevuu2NnPzGQ/zwf/ppioUanWGbPVOTWKHErXnEqs9d79xNvTJBwdN58mtXWDyv2HdsF1Iq5i/Pcdv99/LeH7ydB//uU2AU6AdDXM9j31SJJJMMq9vMXVmC1EYocAs2GRkjoxUwJHJ9QHP5KmE0wJY2STvCEzGb1waYuoPuDDCMjFqlTDAUCAySWCGUS5bGCDNC06DgFNAR6AZo0sA2wDIjRNYm6q6xsD7AqzfAtKgUMjzPJFUhWZZg6rkfqgAKroYfhXiFEjJTKGLQhjj2KEKMEQ1jbDOPcx7zKmgU0Es9AqeHngrCzMazCuieIKpCOrC4++itXDl3gSAKQYcsixFaAkphGTmw1cUQ24Xh0CcMY9Az4nRIr5NhYJMGEQBRmiH0FCEFSZZbIKo0txbLkhihBLqYQGKAMNANiS4EYaJAT0FahJGGwkGlXh4JLSVBbJFGJRwnJUssoiCgUTyIZIDWdZG2gW84+AFsfG0RIfN0Ma+moWET9HroBFgWBH6KYayimxpCmEiR5hjI0NDNFIY6dauOMAsMgzpRUxBXUky3BO0VXnnfG//5ASVplvsRSgjjiFKlhIZg4PcRUmGYCl1q+L5Pu9uh2qgjMo3m5ga3HJvF8ST+AJprETP7YPFKgSyz0DRFmMQEgcytE250vXLHViEvWKFQuS9XlpEFMZ5uEncH/IcPfg+90CTsboGI80SVskahUMinoVpKwXGJ0x11YZzdWM8ALxv2Ghqd9iamXUbXAgathN3HDkFplL/7608wObEPt6bR7bUwpU5XDjEMhyRJ0PUhiDqdbkSmTGyzQm/LAjVGlqW54lGlhAG54MgyKHguQhN5B6bSXOEFiOuO5qT5X3lGU64+R+S53Zmk2dzE82xMywGRYmgCXTdQStLt9lDo6AYMOj00TaAJhaFrCAVJJBmqCF3kPpyGqZNmEk0TIPKppG25JOR8ziRJcG0LyzRRmcRxy2yuLfLBD36AH/l372Fxbp54s83ZW2eIohLe1FHe/vbb8DL4pT/5GqubIc3BMt/xPb9KbxuSdJMg3cCr+IRim9ZFAyUkmuYgUxdNi3n+mafxvCIJPnFaoFS2kSpiu73N4qrLhFVhv1vk6c0lvv2d38Jma5Ev/tnfc3zfftbKEftmb+cDP/BOmhstPvX5pwjMGi/+zy/Q7LUYr5fpbWZUzFFEOKDqlNhqdbFt+8b0RdcFhm0QNQMUKXG8o1RFkWYJpsinMEqJ3AzfD9g7exv7rIjzp1+iE52gZI0wNt1grb/CR3+/zUzNwNK/Tmt1QLlcoNsJ2DNbQqYp+w5NMn92kUF7nGHHYrxk0ktMhObzefEmpsbvwp/0qR82qLS2WEwbRHg8+7Ur+Nc+zKHSAguBz9TeXehWH9UucOWls7zppttYunyO8lSVrYtLvOOtb+BN9/kMOxa/d1byxa83Ee1LRN5zGFGFI285zrnz65x6bpv3v3WCJ09+jkTzuP3uSaaPvg3bhc3BWab3v4lGUScxBPunppkoF+kPihw6cguvvecMX33sGm//pleT9jvMb61x8cwjNGmSppJj47PIyCfzUnqRYn1dEYZLyJmEWum7KLsVXNMCMgy9yMrGHKZdwvIsClGZu/fewYZ3hnY74NLSKu1ezNXFy5jVIvhdilMlpsIZllc20Owio2NHKJg1JMvUxkpcas7x15//HCOTJbbmlinW6ii2sKISbklhNKokHZ8gDnDiAv7cOpsLA0qehvYPcm0n9k4we2gPK90l4n7G6N5prp5+iYZT4EBtlF6vxeDReThQYlWZ3GHH3LnXxJ7ex2PnzlE1Il46dZLvvt9juPhF5lc32L13D1YywC0IKiXJ2qZAScl3vr2C/sSQZ05nlCfKiK4gbA/RPB3bKDHspvyLd/8YUxMu+49O8+jTFwhHapQdk/7SOp/81G+RbXeZveMeJg81KE7vZ9Pa4vzJb/CJhx/kzqPH+Zkf+nF+79d/iy9dXaRWKuIkMZElCFWKlSk0KRDX/fTkdU7ky8Ayr6k74gPNAKWQKrnxfq7uTtGMgCzWSGIbpRIkIaZVIIlNwm5AacTCO7KPVpDx+gnFZGmGlfWAytQIpUpIN7pCw6phhT32jrisbcZYBYHfibjn7hOoKCXyJeXRCda7PRw9Y3R8hM2lLRAJlfI4wXCAbuT8NElKrA1ykY3l0u102D07gwA0peN3QwbbPcKez8/96x/FsEzi1OMvfvN3c9sVAd3BHE7BIU3kzvspmqFDDL3ugKJbRgkd2xFkSiPodjn55BlSYbC1GrL0UsDXv7pIlF7i+Kv2ErVTWvND0mSIgcDRRnnLG99E4cAWiYqpNgoc8Uu0NraoWbtYXL8Ghs3C3BX+8wdPsueO3YxOCjK/S2l6hKXthDSLGTQLREGbSlGgaXGe357atFZzpbpTMTDTMnvHbiaMeqCK7DlsEIsrbK01OXSoxubqkO3tJqZWx3JA6SmalQ9MvIJBFGxj2YKh3yfLAoJAYegGIg0h9ch8cNwUxw6JY4twqJElkum9exk2r5ImAY5TptsVpJnEdcsYukmvFWE7ErIKaZIPQNxiCdvOqNRyzq6mUkKp4zaqWCogHmYMNgeYA7ArDrNH7ubWY0c4f+oMuqWjslyEYvwD2oYkJo4UUlpohobtOkShxLQMDFMgA5+gG5DJCNsoE4VgW0XSNETPJGmc4Bg6SQrN/luRauQfQSmh5ak4pp2LkoUSSCVRIcTqemKOQNNM/GGKyjI0NNZWFSqr5rDNVzkeuA4Xdo5Ok//lNfjfv/YP31vO88ny7zuwuCGxrQFFu01ivPh/hwr/l+OfrvKOQqI4xSl4FEpFkmGAEoJE0zCR+LpCaYosTbFtk+Gwj2nalEpFMi0DpWOagqVrGXfco3PzPRHPfjnGtEEKiziK0YSeT+rUTiQRL/ugFS2XNFaYwsRyBEkicUoVtrvrCEejMAYFy0OlebechALTsNHMlDiNdgrgy2rFG/xMLRe6xEbuw5ilMQIoVws4uDz4kb9gbHqUZ598npsfuIXx2SmGQYBu+wS+Q9kqkwURytwmDlOsgk0UbgMuyAqZ0tCMFL/bQ2ZBfhMUiviBT8lz85zanRX/9UOhbgBLccNEGECiJDi2wdTkaD76T3ziOCFNczsky7KoVSpoQjFaKzNad9BE3u2lWYYiP8UaEj+ArcxE0/Sco6pbpJlBImMyJTFtizhNQGakWYxlFxCZIgiG1CpVHv3ak3z7e+4l7m5jj9U5PH6Mh59v8vRHH2Rtq8+gp3joUw9y9GYDzRWsffZ/EpDR2DvDRHWagXQxjIxKmBFrOloUYBAjU0Vs+mRpDz2xqeh94nnYDLfxTJNCocKVbYk9MUZZhJhkvOWtd/HqV43y2KMxb7/rXvzU5Nve+TaeP7nJ3zw6T3+zjZlpmFHM3LmLZJZOFhrs3XcUXwvJsowgeNmgWUqJSR6DlakUS3Ox0Xc88jKEVICFZoCtGwxFyMSY4rb993L/wVvopDHPP75GxahjNXaRWjanenu5+6a3UTAvUNoVkz7+OMoUDFqbLF8WrGcGk+MaDc+lk4LqtpCDvTx0+Ra0YspeU0Mvmrxr4VnczOR/dkps6RWmpu7m2mCGqRMuxYrG3CWfsPg8sT7G0dtexbGbj/Hnp08zdkDRj1x+8RMnqY3fzv7xHp2NDfRazHorJS0LRJRSMTSWW2sce8/7WZRlbj9yhHd977s5++xZDuy/G8d0cMujmKYGMuJ3f+tjxOMl5q6dh4JNd6DzpUf+hmunnuGpjzzM6uQ8c+ESKo5oTFd58fRVPK2G6+iMOjW2t7sk9BiZOML45DRx3EMjTzvJshhLs7m7uptPX3iGaqPJlfl5fFzqUxbjs0UqmcmMu4ezG/O89s4jXNpaYWJmP5nZ4tkrTRIhOHb4Nr7y5CM89NxDfOTv/wADj/pIgc3L86S0qU3s4fKleUZKNtunL9MdJDQaJklvi27YxlYxjUqRrX7rxn3aOmfhjRpMT0ySdtYIbcGe0RrdzoDF/ha9uMj3f5vDly4usHZRUt8bMzaWcWX+PCvLNcb2mOypGmyLUSb2mVQmfLSoRy/qoUmXCbvEqtpG6w9ouwXe+gBsnbO4sBBTrGrU6jbtXh+rrGObPlv9y6w2farrY2hSsGvfKMFFn1RrI7oBCIPm1XmaC2tUju3iu977rQTtDBWd5tlnPsPoeIEHv/o1vud7v5uPf/RjVMoVlIwwdLHj+GCiiZdB5PVmPPdSvK4Elf9ocpkrWK+rvjWyVNvJMAZNT8gyBapCt92nVI954C0HaAY+S+shR/aX+aZ3vJMXr36DNPDYO+nx4ONL1EeP4EifmdIUd03PcDrtIQqT7Dnm0WnGKFNnte9TO1Dj/vtexR/+0e/y2je+if1HbmL76jJPPPYk5bKHJQxSJYhSsMwCSSaIsgDdlBgmZCqfqgoBtmFSrFZQZohKYlwRoMUxaeJjuCUuS5PWMEPLFCrOt0iDvk8UhyAkKlP0+31kFlI2S6TDTUSviSEkUvaZ24hx7t2FcdBh3Cyz+mKL0UoNz9AYP1RlYq+JVR7nK8+dZXJqhEuXLtDaDHnX976F0fI489e2uPDikPPnO3iihi1rROsDgtCGQo/1Cwleo0R/s0XFq5AFCYZRJo0zMhljJKCUznAokGmb4XJKZmyjaUUmJ/fgN1uUbI/QN5CZRbFcQBMmg4FPrV6kue0DCQrJ1K4idjGFSMOrVzFMj6mRWaLERxh1JvcdwDFTWp2AQsFE10Jk4jFSHGfdnCcMFUpAnPXxPJcgSBAqRco+vm+iUcCrWaQyYHzao709RKNE5Ke4XgmhSQxNEA9T+kPBq1/9Oo4eOszrvumdzDU7nHv8YbIsw3McBoMIXelkmSBNFK4jkCpEszy80giDZB2rPMQq1rHMiPWNDpoucYwCbbmJjHx0YZImQzQzRWU6WZKCVChGUGrkH1FEBIJMSkxH7Phc7zRkSuVuKyJ3nhGajuWY+H6ygwbVDlzTd7DAzj22c9ddxwv/d6DxH74u/tFX6saXOfTK72cpNZTSCOMa3V7h/wka/qPj/wOgDHCUgYoSEhUz7PZwKiV2HdzHsD/AHSlixtBb3iZMYqqVGsMoxil4hDLEGoKpKY7driH0jP3HLC6/kNHfckjTGMcxEEJHyZTrZq9ZJlEqL0axijFNFynTPLPaNghkhO0YSC1GC1wCP8Y0847TsBMypSDJZfpoMTLLf6lJknfNigwhFKapIxOFaXpoRkSUOuw+cpQrFzZ4z7d+M3ajwsqV85w58xK7DuxD2FUyNaRYLjEMhhSKE1hIhkmE0mN06RJlpZyMa2iEQUTBfgUn7nVpbvlcvLrKvfc/AFHAsYOjFKo1XNPBthxcu4ime1hemSRL8pvdkAjlAD66KCNEikoEQsswDYVhOGQqQDcK1Mc8trYGhNEAgJJXIU1jzMwhVQH9YQfLAttMWFnS+NjH26ysrt8AtJnKxVCDwYBKpYJlWbnRsdTzGycdIHQLxzTYbDW5cvkiBR02rl2B6TewtTbPE198jMtnr7HZ3uTOfWOMVWssPxNy19tPoKc+X330eSrVIrY08BMdWyXYmkWm24RiHUOrIYIaptGnVCqxsd7DjDZ55d5Rnj8/R3GmwnitQSBTqlWTr3z1EZ46W2W0MsndJx5geXOT7fYUv/ynX+dv/+iv8NNN3GJMc7NHwU655VUHuHrlLJurGbafkvg90CW6MEjTLM8bNm3iKCUMc7CpWRaRJpBmbkqbhBJdyzkyIRmWZbG51GTZm6PZCSnXXIp2RKlQRfYGaCKmVjQ5deEySaZz9/5XUmxcJU49rGqNrFJmOPcFNNNgEPcYt6okQZfnGv+a2kKE9FvsOuxzdP9eLk8c4/wjj2P1NkiDMu1ijVDXWHxkg7IZEyQxyUgXbTDC5bNDXvH6Q5z6w88z8LZZ3tJInNczVjoO9kl2HRVUpvdx+9ZZnj3zJPPLfX7otttYLI7wA9/20zhTBtcWzmHJiJmDN/Oxz3yCSytXeP6FRxgZmWXXyF5Gxhp0X+gzNj1Ko1bjIxsajcJBJqZmUfsrrJxOGGZFkCWGmwnj5b2srvisd8DRDSq7Rrlpn82p530eevgFmtsGw26drt+jUrbpbrr85dwqUo0x6nqEwwB/YBJub1Er7qfcOIov1gikzdb2MSarNxG427Q3ezz91AWunmrSbV1jO+tSbUyhfIlpV8nSMUbqRUIVM2xaiLhAc8vFc2qIZECnJVFRCSNrYNrbpMkYjmbfqIn1+igb/W+wudxFDJpM7Jsg6mYUzXHOb69Qrk1xyRlhmIS47pBe3KSU9nG7bRpGHVHax7PnTrIpBY2llGFXcOToBK2NHhVnAhnHxAPYc0KRxT6+b/BDP2LwxGOrXLzmsCSLFPZ4rF9bp+BZWMLAKdcZBCm9OOILj6zjeDbH33o377j1fsykw3a/yWB7ns8//AI/+9sf5r5X3M7Y/tuwx9Z5+Ktn+Js3XOY97/se/vzjH0NaAVZqEIWCFBCmhpISTRf8Y/7kP+RUZvnTaSff23EspExzo/MsQ4kEMCgWC3S7PaJAI00Vt959iPEZRaAbVGsunWGLoDXGz/3i4/TRuHjyQQ7MHOKRxYvsGr+T3SNlXn/HNF/7u69y0+HXMJQreFLDGy3z0POPcbW1wbX2FpPVcWbHdzM1OoVZKRP1Iv7lD38/X/vyw8xdnqNSKUEao+GgkhBluBi6jmWVGERDNMuj6JSIE0XJK+DaYxhlhySJUSTIuEupUSXa2qKfBNRMAylj0AS1kQbtdhvLFgipMTk5TScJKHs1xvfP4NlVHNMg6LQwjhTYfV+FLz72PNtbIccPzOIes7HdlDjt8vUrX0RJj9HxBs31TcZKDquXMl64dBLbgeEyLFzuc++9hxkMVzj71AU02UWrTKIXJHLQpeP7GFlMlqa4Zoko0dD03KDetlxkZqLUAN02IAuQFBGGyaDl45nThNGAxasJtWqFVmuDchUqtQbdfotMGlSqFQbdNnFkIVzYNVnn7te8FtOqYCJZ71wGLeTo3dP4TYc0XSGM26SZSZaFeHaBJFZY+ghBqFEZLVAfcTj93DrOeA3L1hj2wPZC3HKJXs+jXPe4cnmBod9kYnwMQ4nczH5QRE928VP/+T/wqtfczhNPPcknHnqRsXGbixeuYOgmKA3LsAmDAYZpYBkaSSrQbZ1yo8y1laswWGfPsRIJHQxN586RBxiZrnD+8RaeaSF1E5mFuCWBij10zdiJdhbI9H+121FKoRsahglplHu3SqV2sMLOBDuTxJGCMAUJGjpK7CQkqfxJveMilWPBfPaGuo4NNW5wT18GnzvJViLfeLAjxP3Hx84HCchphxKlCdp9+U+Fif8fbIPiGCUzhsOAOImplCscPHyYbb+P6TokQYjf8fGHQ3Rdp9qoYwwDkiCkaJYIgoyFJcFNt5lcPZOhGTHlqs2wleUXdQZpmpKkuUpQE+aN7hcEQkl0kZImMQgN29YJ4hAtNbHSGrHRxDGtHe5bDgx0XcMwAJEgyO0sAKIoevkU7igVDeGQyIhhElOrVjj9jWtkVoZZdSnYDtWpcfpffpIvf/rvyIwiVhoRK5PEATIL23QRtoGp6YyUSpiGi8wEMo1IghCndB/nXnIRCFS4xXNfNTEtgwunNTQtRWMIDIBtEDqG6WKaNtfPQA6yQbG183PnhsE5+f36/0Yis3yqJjQttwNSGyDUDneUfLWt62hCoQsdFGSZBJHzA00798eK0uSGOCUIgp11VZav1TFIU8XAD1nbWOPE4T3Yld0sXXiOwcLXuX2fwfL2BmNjGUEnJJEVjOmIo4dvp2R6PP/kCq2VFqllIaOEhJ3NPopMKmyvi+WmyNBgrblIv93hg+94DXtecwfh//w7Lp1+AW16kpnREqpmEagGTpTx4jOnOXzr7Tz56CO89OQio40GxXGTulNkOBBgWNzxiqMcPno7jmgwf/UhUlKkO4phhAglMYxcXKDrOkmmEHquuNMRqDjD0HWSNINUIjWJEBAMfDRdp91tsdZuM+xt4Wl1BkHI3NZVNBHQHbSQfRthKAzH4bd/9ffZ2F7glulRpFZCm/pmzn/xUebW2thmmV5X8PT5Mi+mY9z3ylEK2Dw73E3QUnzt78/gXxtw770z3Lp7L2k/II4TwlGDgqGTZpCN7yYY6zKIzuMZu5ixDZYGm3jOJM1uh+2zZ5HeNdZDmOo8wMHDk7zpviKfPvsFFge72c58DK4wMbmbc6srLEav4ZkHn+LyyudZCXp8/FMfJ9C2ublxgOrUNN32EsP+kP/0Mxd55tkqm5uvod+3Wd3exeJqEaVyBX2aGbTIbWdIJUGiyHyD1lVFmsLHPtIj9GcYDutIFJZp4PszJGmMoetcXdfJlEJleSSpHNo8uekySHaRidv5ylVBpgRpUkCqDEMcp5NU+NgfRgwGLqZropv7sTQdvw9STZMmIV3LJE7iPJFEy0GOVJIhoClQQjFcNzCtl0vmVS1FNwvYlb+gPjWJjBIUDh3ZYv3SgIMHE9yR/dRqNeRKn4JTJC4mqNE9dLeG9ObPMTfoo18uMTFrE2tbdKJZTGeCJG3haBPM2g4NxyAxd7O1dA3N0rj77jpve43Og48nfPXFTaaKBdqhACsijUFTCaOewcVLzyIEeKVJOpOHuevOe7l7j43qr/D+d76WZ08+xUbWZWLvLTTbo3xj8CLf829+mPvv2k9pZAzTD8g0MCwDUyjS5B+n5Nx4NuxMLHOjc3UjIMEwTBzHyQ27RW54bpLbn0VRhCYsbM/kJ37sX3H06HE+8YkHKZpFbp6aZvHiSWQlYXU15OLcHGu9OVb9FH1ijEHzAq14midMWIwiaLfQZYFINQmLQ0w3YXaqDIbg8PFj/MsP/itOn1xk/doWpd2voTY1yw/+xNv53d/4FRavLFN0HdIsQBO5wXmWJgRxhSAysC2bIAzo+QVUYiHjANNSKC1DSZ0wLLDVzlDRDLZoInHQjJRWt8TSpkEcl1GDGJWCMExK7jHSLGZ9yyPTJUJTJEGDqu1w4UyHIJygUa/Q8zPaiWDPnr2ce/45gjCnMNUKRdKeh1Wssmf3JZLYo9tbZ++xMjEDZu9zWT9fZHSzz8Gjs8xdVGh2QmoVkcMY6RQR5M/JYtkl9CW220BHoZspiaxjCCfXFugOputwaWWDxlRG0RilOjCwLI9KeYLeoEkUKjIUxXKVQb9LltkM+hnFUQfDGaPbaZKFbWrljE7UJeq22W7WiHohve46nttAWDmoX9lcIk5CNF1HtzQMW2K6Ibv21jCkQ7enUyhKFBqVEQhli1g2yTLFzMwMItPpx30yLJSQjM6O8Phzizz6ZJP775zi4K6YLzz593RWT+PYKl+RCxvTkMRZHr2sm4okNcl0n9GpMZq+4PKVBbyKjiWqzE6mXFx/DKPcIekOSBKPqb01/KBDsGViFGPiNB9YSSWRO9xioWlk2e+iiQzdEESRRMZgGmLHVN3EsBUSRSozNEMjTXa8XIWOYQjSDJTKgxWEJkBtcn2mqCQ3MsKVEghNQxM5RS5HmyBlRpa+HB37vx4FoIBCAyWIU0WcKvzN/j8VJv7TAaVlWWjoZH5CsVLFqpXYHrQRCsqOzebGAOIMlcVUxkbA0BnKlMmRSQRt6qMgcFFpRGNCEPoabiVB6Qa65hGGPSy7gCbsfJuvFLqRp7b0+0NAYBgCzXQQIh9PZ4mB0CSp6qJLG0EuqND0HDhdX3Pn6YkKXXlI6edASosoFSqkyYA4kijdJ4thvDZOsTzDe775DVy59hKLVxc5cOIB+qvb/PJvfoi9u+tEw0GeRBMMCKKYONIJ4oQka+O4Gl966HG+8vA3SNNpXMemXHJJ0hRXQZYlFAoeuq5h2zbVaoU0SREivzCkkqAgTXPT4DwGUu3wLXMOZh7ZlKdI5J3J9RW5QNPJNds7Ku38KhN5NjgKTRjsTNhJFfhhjMzyq9HQJUJpGIZAyGzHb87G0HRECoaRoBkWKItIJhQth7m5LUxN4Pd7JGGJb37gBNeuhZx66u/oX9QRu1xc6zjtjT5fufwojqqyd0JntCGI0xTHECh9SJpU6fcHFCsOTz19ktW2iURw7MRuwu46D526yJ3ZCN2NAa9/zQnKBQOpLC5dW2TYz9gOAqQW8eu/8EugGTg1jXV/m+xKAaTCsAqkWcDV+RF0a4WlzU1sx6BYdTGyCGKbTOvnXFZpIkmxbAOZ5ikHSZJSN3U0w8URIUmWENsBjlNHVMZYtGw8Q6dqg1ctEhmQ0kPrTaCkzqDlo8x1avVxvviph9AxGDGKnDx7mhN3nODaixcJ4zt55tyTOG6JYST5xNe63HnvQ7zvju/n0oUhH/6dD6EXDarFASMzJd75bXeTBV0MCihRhPoIVlbGUE1Cu4Tfybhvr81n/uBPWB+2EFYJMwLPSTC9IqGQaMMFms/8JNtzr+LSLGjFCb7x3JMU61Ve9c138cLyJmPlGi+e+jvOPdri4OvvQ7Sfxi7pVMU0//pfv5enzn2Wfqpo9cu88OImUupomkkQKvqDEJnopFLtFDqIUw0NkQcXkJPZRSYwhCANIQpTsiwHKWmmk0mBrnlI8rWQknm3niQxUSAJGLBjAUG6MxzTNROkhuG4BJFJHGWUvDGCMGDYj+hmAYZuYRgmMtMZ+hkaOpqRT6AhRaCjsvyBqSGIE/mPCrLQDBQT7Dv+FvrDBS7Ohwh7DoMijYLHhTNrbFxN6Q9ajE0UaRsV/vyMTbuZUh4psHX1GodGxhmbqvDSIKE8OkGo1plbBeVaLF3bZNQu8+hSneW1AXfdMcHukTJrnRWuLJjce3+RTz3Xwu/4eEUX03CQBKhUoEKLilMg1RPipMvv/97v8cfZb2FYIyTC5wd/4H3cf/ddvHpiL1HmMjm5lx+w7mL1WpPucIn/8HjKhm9hmOkOmNTQjet+EPKGDQrk4DFLr6+6bTI1xHFqCFKmdpU5dzrAMCzCcJjXdc9DGIL+dp93fsvbmdr3AF9/7hLtJKLSK7JZSkijAY6+j2wkQrZbvO7O17DVk3RFxuyecdZPLTF4aZ29x4/T9SVZe5uaCikMBFPlOgOngmaMMtzs8Ys/e5lOT0fJGpns4HgRStMwtHeiyY18ApumSJU37RqKuSsOicx9OGWasdgCyzJQO79+TdcQ10dCmiAIw3yytHNttGKTQcciy5IbQiVghy+n8iGuzG4sHcWKQOiCOE4QAtIsRdd1zhk6WfaGHWGUQeuahmUbXDslEdxE6EcUiz2K6d+zZ6xAa7VHz4+4600F0IrElyLssEhncwndcEmymJItsAsaiZ0xscvBXzdor6WEymfP4V2MNBRJ4rC4tU7P7zM2UqFc6HH+pS32ztzGzESdJA3oBC3afpuRag2/2yFNQhzDROoeVauMH6xhuwnbvWU2Nzu4BYVdyKhULeKRgMYej/56QGs5xHMqrK0v50OYJKK8V0PUTbSaSUVCdymjNlqmO8yQKqKb9khdh812k821bcYmSoRxQhhHFM0SI+UGzdYmXDxFRYv5wpeuUpqdwDsyZG5zE0foeUqcChBoWIZJmkqEYaBFAdXSbvYcupVSc55w8Dy95T5rWcyw+XUMR8OzHab3T7DVCejt6rCxOGBml4NejOjGIWGQEkbk03wdDEshwxjdUDnNOFHo2s51RIJAoumSJFHomkDTNRIgS/INqqZlGEKQ3Cg/ig9+8APcfc/d2LbNF77wBf72b/8W07T4yZ/4CXbt2sWlS5f50O/8DlJKsizJJ5PA+973Pu5/4AF0TePnfv7nWVtdRWiCH/rB7+LmW+7Ati2+8IVH+ZtPPsjs7BS/8PM//k+FiYjr64r/t2Pk1kPK7wzAz7tUt1zEdG00TWN1ZQVTVxgobGzwPAZhQHm0wmxtmpnZbSb3naY6AkK6SD1gaxlmZgVf+Eudqy/JPCPUFBSLTp6FCaSJQGY6UZihGXnWd5JkN6x/oijKfROVxLOvqy9zQKlUXvSuh7ILXUPJNJ9ExgqM9s5UTCdNM2SUG6ofvnUM15lgrHqU/nCTVr9NakiCSHLrzfeRBJJMxJi2hWEWEIYJhqRg5lO7olskCge88MJzXDp1CFOfRNd1Ou0urucRJwmu4+J6BTKZoaRC1w3QdqalmgEiv9jyqDLYMVFC0zWUkmgCFNdH1rk6XO0sra8LfYR2vVKpnVl4PsnME4h2SMGmhZSKXr9Nc/sCI2NfIZUJpmnSbjfxPI9SqYI/CIjTiCxTGHpGsVBHCJ2lhYv8+I//JP/i/e9lYXEFRUoaNil4dZIso5N2GPM8PvaXj1Oolrjzpl0MBx0OTE2DCvDDGFNAlIEyfGSSR2F2Ox3Onb6EaercduImhoOUp55/nnJpmpHxBhMzVQbdAMe00fWYwaDD3BWfKJL4fkqaKsI4pt8bkCmFlBmJikhSyJS2YzQfE8UxB/Yd4O677qE/9JEqAmWQJgLdVBiWRbPV5RvPPYVXaFCcHEHFGeH6OppbxykX6SwuIyyfhavrHDq4n34Y0B10aRQqdId96mN1VufmOXTLMTQBza0t5q6s8/73fwdPPPEYvc6QQtHk8PFpzp9Z4urVy4yNTZLGMevrGzRGjiPtKiU7pFwWdP1ZMsuls3mGyYOH0RyJGUukjBiKGDNLMWMHaVpoCG49WObpR16iP1aGrIkMZ9DSGENJttRVXnP7QcYyg4cfu0T52M2sDK9wwtCJMp9e/xKBO0sab3DLLbfwLR/47/zc7z+PmX6WY/Z55KDEXW94M73tVT75+a/SkD/Mtbn8LszS3KvQtHL/wRvZz9fXL5q4cX0rlV+3mtB2soqzl9eoIuccvdw0Xa9c/1cuEFzvlHI6y/UPFjc4fZrQdxq2fILPP1jXqh0CktppZv+3ZCTxv/lWCCxnSKYCsjTDcTtMH/giYdRjpDLGZm+A6ShG6wW2B32GfYORkSmUEtTrHqsrC2jCYuhHBEmMhk+lVKZc14miiOmpUZpbCZurMbN7dd7/yr2gdzhzrsUtxyb57T++yutva/C5hy6y3FdYJYtCCZRfQEZ9DMsmEQGm1sAWQ+IsJE01uu3cTaPg1Ul1wbFb7+KeV9zJ1N7j3H1PhV/5b/+Rx774AvVKnSDqE0tQuoEhcwsUeDkNJz8NecOrlMB2JWEgqdcKHD4+xsln5vK0syRAoeNISJwifnvIrbe/gt3HjxJ0t7i42OZdb30VL77wEltun73GCNuDJt2NFo3RIrUxna0oxeiFBGubVOsjbKZFnOokaesah4oVYtHlykabZjrE1cFJxnjpzGsRWl5blQCvWCSMIizLRqYp/nCAIn+4s0OxMoz8WtF1jThO0NEwNIM0ky9H1GovA0WlFGgi32boYierWZHEMYZhkaX5ZOq6ZycoskxiGgbsiEEUMo/9VfnFnaQhktxOqVwqMOwP0GwDKTPShFxsqQlkBiNHP8GBm2x6c0N83WD24CiaH8IwYWGpw+zucQp6id7KNk+dWmfswDjClYTJENPIaK2E3Hx0F+O7q3keua6x1VrBNqu0e20cW+EPM8KhwXh1kqAX0Wp2UTLBNS1iH6IkJNM0hLA5dOw4TrXDdmuZ4SBBSYOpmQr9oEMoIFYRphJoXYEINJLUIpNFultrpMrh+H01emKbTEk84dJeHNCaN2jsLjC1KyUgYX4+5tj0MZrzbTKZoKROmhWoFCdRVko0yDCsIm5BY3OphV4T1PY6LD53HkfpYLmkWUQaZqBlCM1ATxShgtHpKrq1n/LkGu1hh6DrEYd9ECaKjD3HZuj3h0TbQwq1jE6gUXIBw8DwIVhyqNYOc/HqLEqkKClIk9/AtpKdxhhIRf7cN/Tcqg8JEpI4Ai0Hk7qhY5j53C9OIA3KNzQWhtEm29nofuxjH+O7P/i9PHD/fRw+fJg//tM/5Sf+/b/ns5/9LGfOnNkRAsGhgwd5y1vewm996EM7Fy43rudKcYxuHzRd52Mf+w3e/y/+HaZhYFoh/d76/6X6/e+Pf/KEUg4CjAQGQcDo2BiNiTGWN1dRAupTozTKDbY3l+hvNrFsgWVJ0qjPxdPnqdol5iKLOIlBRf8nbf8dZllR7n3jn6paaafenaa7J88wA0hOkhQVAUHFhHjUE8SsRzAiwXiOAZWMigoqHPWYFTARlaAkyZkBJsfOcaeVq35/1OoeUJ/n9f1d17u8ZDrs3nGtu+66v4la3aU9o5ge1khdphS0iYAoTvF9m6WdZRmuU6YdR5QqFaSEKIoWEhjy3BRmuTmlIEDrDCGEnSppy+MRRXGwELpEC5cwaiFVgjQKJRVGK6RKmAtdjnndUl72msMZGZ7EYY4hr4JfKqFza58zPfE0blXTadukG5Mq8rbd3TZigcZjewi5yVg8UGHRCY8xvONR2i3Nqr2rdEJrEJonOdPTDcpBQBrbnWiUGVynghYSk0vm5kIC30G5DgZB4GjSTCKVAwp0qhlYtBjXL+EHHsbzabXngJxS0IPrGMgcPD9ACgsxBn4N4eRIaTPKpRK0Zw37H7qcP/7hD2zcbJXw81ZBeW4XV40hyTPKgULoKlGnQbXSi1PymI1SntnaYPu0bfjrpbWEufUYk9UhZLmbjnqU1SuXcOQr30icxci0wlxrDE+WMYmm6rlUgxIzM3M4rmbQlRx4XBUpJTMzMyx2Kxzx+tMIQwijFjgpUrg4bgW05MuffYQdW9q2ySj4H7bgiwXlnizIA4Z5rokm8GHz1pTHnmgUi4JnC3pxzlvvvAqt9lEI6SCfsYTrPOlBeT5plqHMfmih0UnCgw9DpVxGCUgbPlEcMT0paLXWMv4niee7lMv70GrMct21ik77cDrtDl3VCp1GnTTfn6WLT2Z0dAzH9dlzbT9JHNpJe6bY+MwkSWp5xUMDxzL8REZiXFzPQvSdVhOHCKEBZUgTw44nHHR2COmEBrkKoyWB49LJMzBLeaRZo+R4hO2lyGcrqKibDcpFOR6Lh15Fd//t9PWs5A1Hv4rjjlrBD790KYt6Kxy017F4nseubVP8/prbGdjzRUxusEVdSlVYUF1CmswXLBDsJv4s0FnEvODM8oKkdik6E/szO8xBObIQl72gk/x7NaOwG6pc5wuNoRWP2CHm/J+Wax8HjPW5FSCNhaWiOLKiEW1V/VmekWUpKPseC2wTkcalhcfM8gyDbdBMNsDcrgqmOyRsa8bHpqgM9HDY4AHMpk/TFYeMbd9Cq6Hx9l1NMpcQhRHKzXARZJlgRsTkWYCOXYadFlnHwRMRW7bPct39FUpemzzRDN82RTSTMOjGnP6vy3lgS5PND83x7NYU0xNSqyic0EMagyanIRI85RH4Cn9AkOQhgauJwjmefvBmnnjoVtzyKl760v0YH0vxSj6pSZCOi28M7TAqIDQW/HvnIXAhBBpbu40xKGFRrdmZJsoBXfDhdQEVl33NXJ5x7PEv57CXrmX9k8/Ss2KahHH6BlzGG3bzumgxBEISVDOQU6hWDn4/cV8Pk06DTnOc3r6EnclGdugVTKdt5jpN3FKNjoJqd41SxSNJcnJsZG+jOYcSkmYc4bnWTo4i+ccKEiDXGikEWWrr8wL3qEB8sjzHKWzWhCmQMG0wOiVNdgs/ZfF+ea6P57kI1ykEGwJPSnJtbfEQBlNMLHWBUrpeDdDUqtBqNjBCEoUpUtjnaEWkGVIq+rr24Il7n2Tlvr3UuwQ9lR7cfJrexUsZWDtKvauf1pYqq1/aQ2VZiac2zhETcvBh+yJ0SLY/9CyuEszmPHD/Y7ilPtIkQ8g20vUY39Fkz5V1kq4M7e3CqST0lcqYqEJ37yBbdgyzdEmVRdU+ppuT+JWAMMqZnpmj5PfiduU04pDmbE6WCoRwkCVIVIKqK6peianRhDi1riNjm0KU59C1KGVschrH9FAbajKwukK7KZhrlxgcqqKznEU9Q3TiFlu2HEEU15mZcrHkwxJBvQunnTM5PUJX3sv0aJOotS8tA0KqwgvSPI+DaK0Fd2yUlKt1hrfuTRonGF1wHTW4JYd1ozlJZIWrUmrSWDPt2FVGIhE57NglyPOS5TQawNQI04xzPvVe9li9mjzTnPe18zjy8KM46cSTKJVKXHvdddx4y4286z9OY8nipfT01Ln6f6/m9a9+A+dfcDXGzG/mIE1sctOyZcuYnp5maHCAfffdj0cffYzFg4Ns3LiRww8/giefXEdfby+u6/C6170Oz/P43ne/y3PPPcdll32zCJGBuSZoI/Ecj127xtA5xHlGHP9/EL0YNlo4foBT9qkM9DAXt+gf6COLYnSSMdeaJYpTBgaX0kxTSkFAksX0dSsOPGAJee8mcg1J6CGEZmsYUanD2gMzRnb6BEJiyOm0EyqVKlL41ptSSlzXRhdKaRtNpWye6Txf0hgDQhc7PWsD4LpOwS213J4sy/E8F8f1yHPLaXGqCUZXGd8pWb0vHHJMP9t3brZGp6LNTCvFTSCQVVKjkHVNuRxQyodwhY8UArfI4nQKKD6OQnzfRQJJFHL0qyo4jiIMIxynz04oy4FtADqd3Vm4uU+aS5rtOVw3YNXSAyyUlNQIWx5SzBClihyHXKcM9S3iicef5Z5772dk2xR7L3Y49KC9KHdFNNs7MaSI3AM9H5Vmm+1mq1OomO0iWOnVTEz00ImbpGm8kJRTKpWYm5ujUqkUJvA2Lsov1OZpGlMu1bn/oT/z7NYniOcaQBXHCBAZkgpTYUhd+ex9+N785S/3c/2v/0S1q5v2zBRxNEOauThJBk5EJ1E0Z0OCIMCQMDBUJ4o6GO1Qq/bj1EMqXg/NmVkCr7A/0opWVGPL+kMLVdo8/dhYiH+BgwuYgnPKfNEoTNxdQbnsk+dm4WemaEizPC7OOUu3SKPc5gY7JXSmkVoilcJkGTqXSNfB4JIbQZpbI9wg8JGk4Ehcx0OJMr6b0WpFaCNQbgAyIMkUcazJc4c89xBS0okERnvF81f0LeoBkWG0oV4r045n0HGOYyRhM8GG0gtK5RJaGnJjIZaKp8ilZK4d0dddoux7gGRsegaT5qS6w+BAF16pwtzOFrXuHgLfI0sEe644lJNO2JOAGndcfwvJzOOsw+Mjx36amQnNiw8QrFzkc83tOwj8EkrlhZhOF5xfjZDKLoILVI0XHkJgm1BhmzrlqIXpobaqPGwEqV08f/yTy/jNb27h2utu4egjD+G0d74ZjGHnrlG+ct63LJQo59XE9nHPOfuD1OtdfO5zFwMGRVLw/awNl5YSgaJc9hBSLPiyGgFJGtOcmyHXGcK4xedRev6wFCk8MpFiTM6yoR7WTe1EacWinh5GZlIee+JZQjo4pkUUd9BOwPD4GAJNJ+6QRx2UEnT5XbTDmFxLSF06MzFlXxFGGfWBQdphRJRkNCZz3vm247nr8e8yVz+Mev80U080edublzCRdPjrUwEP3r+Tnj6NKnWhk1lE7pNkGp0atEjBhXYa47tVai7EIsfvbOHu329DlXyEjxVACoFOE+qlMmEaLwhxrPJbPU/tDY5SKAGZSGm3W1RqtikyRuB5imaYU+ryaDdy1uy/knJfizvvvBnhuyxZUkXHMbWeCG9O2AYqmaJnoEIrjJiZMExNj9PVJ6j318iziD322JfGTMjg4gG6aiUmNs+y955L2TW+HaFLJLENv5WORGcpcfxdpNIgJFJYiFHMDx9ygxELnQVZrlHSQWA3SY7joE2MwPoNWp6abTznT2Rjcuu7qW3D4jg2jAIknSgjb+9OgFsQShRz+732ew2RchCkONJDOLaBLPsVDlni86cbf4fbBd1dJUZ3ulTcU0mzkDRN2frMLvp6NHoyYfuWBjP9HYxu428bpad7CUJOQjLLxmEY3LvOIpXQGS+RzaWYIGR4ok07VNTNLGv3HCIMBeVSH9MTbXI6HHXC4ezaspVWu0NJDZBEu4ijjMaUS1YZo29NhshLGC2YnJmmVJlDG0ljNqW21GN4xyROLaVvUQk97ZB2NOVyhQRB1AlJGlMEpoyvDMJTdA04pGmHNKnatB0TEuhF7NoxRWdaoPwKRs6wZtkQU3Mt4iQgy/qRIifPU3tV5hHNyRBjBI5wiDpNjLS2gfPDpvkJnSnqkBEgtEA6EMZNhHLwfEUS5+hcI5AIY/2xncKAPU0MRkt0AmmWI5W22d3CxXFc8kKUBnDMMUeRZ/CRj38ME0uEI/nzn+/lpptvQwrFFd/5Otff+EeMUYyNTnDeV8/HcR2+/tz3MLn7d7Xz7LM/yStf+Qp+9atr2L59F9VqlcmJSbZv38XQ0HaOO+4VC2tgmuY4jkuaZnzgA6fz4Q9/iOOOeyW33nq7rcFK8dEPv4vjj38J1157M7tV4P8cig3/b0Q5soeS30dvf50slXZXnjl0Gi2aMw10J6R/cA3KBCgTohMXnWX4tYCdk23q/hJKlRzluGhSlqyo4AVtunsd3HJGa8TH831inSBFDddTzMxO47iSdidH4Ba7XMsDtIKUHFBkmQFhBRNZaixMhkOeZSjH8uBAMjvXxHVKuO4i8jwljRVzjYw9XjTI696+D7OTOXk235JYs1KTBqRC01droDxDnqZkJkM5EoyDzjLSKEbrjDwHx/VptjKUa3crs+MhQrmYeHchlo4iSezO13Wtma+OQ+I8I6g4zDZAZwNs2rqNVmsWIyRCZyjXwXFgoK+L9c+u55qf/hpUhluC+3c12Lhthvd//FiqS8uAxHFd8izDUSWkA3HSoJ71UQoc8jxFGgedd+jvW85D900UUI9DGMakaUwQBAuLhiOAvIIpxeSxwpOKKMw46cSXsGJvSbMxi8HFZBLfs4W3qSN6vW4eeqrBHksrrD6mzNT0LBWvik7r1OvdVCWkeYbrGPxSiUazTZylLFm5hEanQZhmKCWsQEnmuO4SHK9oALVkw/qE760LSNO/2UWZf/Ct2b0TtSlMgjSDKBILF53lp1I0loUvmQkWoNoFcdQ83UAIMC4QkMeQLKRUpQihaLVyhKxBBs3E0G7nQAWT7o4VnUs0s7OdYvqTIUQXOhaMjcaYhReS4SiHet3HCMPYeIqjqsiStsb2vrsAFYehtYciE6BccFyUUvjKwXNq1pxdSnxVRkhJuRSQZ5rpqRZ5ZEiTlCSOcZTDlkem+f34M1QXDTHbNLzoxf/CPY/dzR2334MWEM9Bhs/wthQZxaSp9TM15jJgAinsdEovbNN53mua/3AkGmEXcz0/QzQLqkXmPx1hqFSqnHP2mbTbbXTe4N57R7jnnhsAGBoawPcbRFHE8y00HMfhJz+5qmhQxwhKdVau2YnrlKjXSyT5OLEQuI7H4kX9hJ1Z2zzkDmEnpdVssWx1P7f/8Rek+TIA2tMvsQpNbZD+w6ThgRZKdyWDK2tsbbtopUnjiP7ufiant6FEFz193ZT8MiZqE3dCsjTFVQ6+6CXXEc0oo6uvThQ1ra1OLIjTWaq9y+hf1M+OnWOUu9okusTYtMQPlrH3wScxvOMXOO2MuCw56LBluIt66asrHnhgF+O7QsqBxJBSqSiUFgjlol1FlqWYJMUYgXQlWcml7CuMCC0ykxRpNwoSBK7rLqA+z4e883w+clGRplbgkCQJU5MNgqDE7EyVLIPAdYlTnzROOPLFr2XLpgZJ7JMYyKJpapVFTDRHkK0uZrMSW7dHeOUA31dkuhsjuoimXLJJQaw1jYmUMFKUgsXs2tYhcPrY9oyi2r8arwLNiW507qFNjhQKIxdwCoyJi2GEeh43fV7wAL7vkmV2Sm157cLGRxYTbzD4vs9HP/wRli9fRqfT4ZxzPk1vby//9V+fo1wqkeU5533lfBqtBtrYSVdPby/nnPVJyuUSz63fwPev/gHGGErLDdUkwVWSJBvH4NM32EMWN+nuqXDKW17G9dfdjaoKDjxwP9Y9FiKUwBiHFXutYvHipWwfmWZkJCXLe6h1DRF4JbY91sZTPSxZXWLbli1sHRZ0LxpkbOswE5umKdVKrNhjL5yW4snhFssH96a/x9CYbdJV7cJ3PbZtarJjpEzZrRPGHiZ30J0U6cQkUcDMqEM8LZio52QsoTVVZ2BpSNVLiKdiVEvgCx9TrlAKfNqtSSamZ6hX64TNNr70yZIqJo3AS9nw7AhpG6r1ELfmk+uQemkCHXosW1Jjy84J1g4uI2rHtDoxSWbha7spVYAgiUMwhjyXuGWHjAxHW6QDKbAbHVuW5ocIQtthhM4tlcB37Q0cxzoc5DJHOR6mUF7nOidLMouc5tYGCGMHMAibkmOMLIqRZNXKFTz00HMkYTcYUDgcesiRnHrqawFYunQpeVoF7fP0M5vJ0zpox0YQP39AUlTFq6/+H66++n+45JKL+MMfbqTVatHb18uKFUtZuXIZYRgu3DqKIlqtFuvWPQPAgw8+zEEHHTC/3JFnOV//+g+44oqfcuWV5/GHP9zO1NTM3zzm//345yHvzttoh4rJnTFSFCaglvwEgNKGuTHJZBKjXIWjXJI0I3YkjUlDGMfkGbg+OI5dkS3R24pnGo3dHMB28/nwpF7go1ihyfMMOBe4UxS8qEIPXXBULK3F/syqEgUhBmllm+Q5IA2TYz7/8w1lITchCojuhbvH7oGII9/0GH5J4bpllMgReUjgllGiF9fD2hEZ8JwcR+VkWYrjOygcOso2ZtJY098gs9CzznKkkKTONIFXo1pxmBye4Bc/+S7d3WB93RSq4hTNLszUKqx/+jl8lVLyfZQj0eVempMtWjMz5F5Ilru2UVMOadwEJ7IT1BAcJ8NxJP1de9JuCuJoB7PNBspzF4RMQVBmbm6OMLRTw6pfodGJSPMMhEOcRphcMT21je5mzmyzjHEyyqqf1twUjpMSCxB+Dybuw+/uUF+cIXv78KQCWSfKQubyBM9bjMhChKcp1zzcyNCOFULXUXkL10lI2iVSEZKamEw0wcnw3DLDI10vaCa1/jp/202K4j8LqNXzIFZLjrc/y7MiVk783V284NsX0I7/L5u3+duZ5wnr9N8P6ICT/+52/+jIMxgft/zYhfPaPP9yn7eGKFi3QhBGhk5UTEwEjIz9fXloNll4zZKAiYmgeK9Sxidynn56J1pvZ95XEAK+uXEzz8OJyPIqmPB5rzFZeHMGBwfwfR8w7No1SqVSpru7CykV09MzNBpN+np6ipQixcTEFD09dUZHxxee47vf/U6OOOIIarUqt9zyJ66//nqWLV3KV77yZVauXMGrX23fwyRJi5QtWDQwwH99/jOUSiXuvPNObrvt9uJF5ojqMMIztKVLd283o5s2sd+eBzEzuZN23CYlQboOff3dLFvhM/rcJt7w1lP4yZXX2teYH2Z5n1KgaSOxXKg80zz41AgIg+95jDUbDHZp1qw9jE1bt7CoZxEib5OnMXkq6K31EbY7JLHB8SSZVLh4ZE4JryQJlM9c2GByboRlq+v4qUN7BLxSzs13XEsSNjjvM5dz9BF1an29xKLGpq0VPvix7/D7a67iNW8MaMd1fnTFpUzPjROHhlbUIc0h1z6+Lwm8Eo4nIA9JJIhcozC4jkQblzRPUI4iSSN8ZZeM+VQtC5fZfz3PQyo7ubeKb0WzkRNHbyZLFwGCJAKNYdGiHv58i0OuNczbnBS1Pkrr6NxFSEWaDJDloJRBKkmWaZSQGOwin2d2rGTIbGSfMKRRjtghLTUoc0nD8sLmzapki7VBlhDOTJGWZieJuqhvQijSJCfXOYji9WYZGGNpR/Zy4y1vPoV777uXuy++227IA4+XHXsMO3bu5Jvf/g6vffVJvP71r+Wa667DcQKacy1efdJJ3Hf/g9x488189IwPsc9ee/Hs+vXUHJ80d8hVSFBdSacTgXGo1XzG5yZY//CeCLGSxoyh0yiRxhaKl0ry3AOHs05bhbRIM6ZnJE2pmHA9NJZXPrrLITMHkucwvcUjKCmiNGF8R8rwJse6feh9GEVYW5vc4LoSkxvSxHI8y2VbG9LU+iPmWtPcZdEAJQQTMyl5BuFoneFNkyxaux7Pixmfa6PSGuF0g1JQp79aYro1huMFlKseSSuk5Ho4riEOEz74n++mt9pPq9Xh6h/9CCcImIsSunskaQfKTjeuA40xQ6XUS9Lyn1fjJTq7nCzvLNRFtINE4LoecRqhc1MMBooQFcTuej3PO5AOeWowSHQuMLm1/MnSYrqZW3eanloVk2dk0loHIiCOrI+jMZXiDgUwztatT3P44cdwx+1/tY9l4D/+48186EOfxRjDtddeuXB7rY0VwRVw+wsr9xiuC0kSUqlUiKKQJGnzxBOPc8wxR3Pxxffyb//2bzz22KNAim2ec5544jH22Wcf7rjjVvbaaw927dpZ/B4cd5o01SQJRFGTJNmFdZ75/2JCqRy0NpbInOsF0r012zYIZUmk1ogbK5cveGyYHNebnwpZwYkueGx5JvB82z7aneH87teeCPMw5ALZ/vmN5XzTZ8zfrJKFREXMt55mYapk/7Unj+d5LBrsR8hCJIou4p3mG1ID2NfZmPTI55bRaQ+TxCG5jlAqxHEaOKKCMuAFLsqxqqzcURhjOVexjvEJILXTOK0MynXItUE6DnGeUXKXE0UJUdqiUiuxY9d6phsW8qhWu2nP7UDnrj2JSVi5Zhkzcw10lqOjHJlLvLKiVPdppxHKsWN+rTVlX4OpIaRNqfA9a1jczkfIXEHQZeGcNM3RAYUqLF9YNLIsw/EUrqdRbhdh2kBrSZrGjE1OsY//IhI9RVe5RhxGKB8UFaRxCVt99Ay4TEzNMTyeo3xJJ0kQYhLcHMfkxJG1WUjnOijl46sSWd5Ea/C9Mq3I4HuzNqkncgkcHy2bZHmHXM8nlxSbCcz8HucFx/xnP3/MwxsLX2uNIfuHzeT8fZ9wwit417vehtaaMIz46le/zrZtOxdutXbtan74w8s599wvcc89D7zgHkqlgAsv/G/2229vfvObG7n88que91w6/LOH49jzOs+tG4Ajd3MKTYHb2P2VKbiEGkcqpLIbNHvh7b4eoJgYLjS6emHyVNwS37eigrTIcRYFRmS/3g11ZtnfP99q1Zribt+++31qNls0Gk2EEKxcuZxGw9pSpGnG6Og4xpgXNJMAP/7xT7n22t8gpeC7372SG2+8kfHxcT70oTO49NKLWbVqOc1ma0F5DHDaO/6Dn/3852zZvJmzzz6bRx99lJGREZSU9NW7cNyckq9QoozIYyYmd1Kr1ck6TUolD6NhfNskPXvvyR4HLWfrM+O85tQ3A3DPn/qYmpgCkVNWklAqjDZIJeg0HYKujGo1YGIWXC9n0+bNpKbFjglD2MnQrkuShrTjiGarhfIkcRbhOgHt5hTCkUgvoDk3h19eRKXsk7YUbTOFERlxR2GqOYtXLSFvtrjzkZ1UeiuEJU2Pu4Y/3f4Ml33nbj7woQ/z3NhzDKzaD3+8mzPf+3ZyJ+TZjRuYHG2zbv3DPP7kOO1OgPIyqoEg0z7aBMRZjCG0tKIsw1fzlCP1gs9GF7C4lJBlSRHdmhfWQKtJ034E1l8PoKu7hkHQiWJALdTZXBsEmiwXKCkh1yRxitYCJ/DIkwwhTVEv1ELZV9IOA7JU4zoC5UOeaZRRzF/MQoiFYYQxhrPO+iBr16xC65gvnPcFDj/scE4++WSqlQrX/fZ3/OH663nPu97F0OIheru7ufK73+OUN72JCy68CJPv5kgedcQR9Pb08K9vfRt3/PnP3PqXO9m5c4Q91+yJ67j09PbQCa2tmMFygZcsHuKWW29HCMGGjZvYb//9efrZ9YzPtJE6J+8YPLeJcHJaSpL4FdJ2D1PDOcL1yXSbMGwiRE8RTGTdDzAStKZUDuz71E6IWylW+aFJ4xShLN0nyRJMagVBnueDFkhtKSlS2imb6yiE0AiFzUF3LLUsz1IcxyE3mf19kRiWmRzlKIQ0KOnQblSo5j343SF7HdHN6KYc4RmGh4dxHJfBJcsZH5nCUx49QRU3dWnkJWqVbo5++Zu5+69PEVSHSFGkcYtqeREyLWGEYWAJTE2N4+s1LOoJCFOJxEOQY3SK1jH1Lhff80mylHYUEjge1YqLzjqkJrd8WA2IebGg3exIrMOE8gOMcYhDh3e+820cccRB+L7PzTf9mWuuuXHh/H/jm07l2Fccxfs/8CmE8K0ziBS89jWv5F3v+hcmJqaZmJjmv//7M9x11z0cddSxfPe7XyPLcj772Uv585/v58orv8pzz20uHG0c+5lhewhj4NxzP8AFF3zvBdfdJz5xJqtWraRcLnP33ffQ09PDfffdx4knvoof/OB/2LhxI+vXrwfgjDPO4OKLL+Gee+7lmGOO4aqrrmJiYoL/+q//Xri/T37yE+y1116A4IYbbiQMI170orWceeY/r/L+pxvKNM0XVG1CSFzPNnpiHluah0CMwfG9hQXHkYI0S3F8cF3IUjvUlBr8EsWOT+B6Vm0tFprJ4jAL/wHMC3KmLRF6nug/Xzh4wRTT/p4XNAnGCDCWDD0zYw3ABfYicjxBmmtAoU3OPKlfScnT93XYa/8uhDvGssU9VCoryLIMZIhR0hKllS2QWZZR8qxRq0OJTIW2EGcgpI3v04V6Nc9zMs8gRIBJfVynApQIAoPvB2jtUqrXaM02cF2FclyarRCpSuhc4zqgc0UYJcQRZLqE55fIsowkTHC8HFclJJFGGt+686uyTYPxXJK0xMTkNI5jPShd1yVJoiLhwi4WUdomR+IrhSMkvgpw/Sl27GjSCCuQpnSaLlke4ihJknskbsYzj05iugTTWwVdvqRncRuVV8hNAlphjCA1giLJkCxPSI1GmBxHlYlzTZpndCKJmxl04iIjRaXaj45DTBw8rzMEbcaRu6mSz/vMX3CavPC0mr+BseeUNv9wjMi9997Grbf+AYBjjnkpZ5xxGuec86mF359++jt48MEHgA7QeMHfZlnI1Vf/hD32WMWyZYtf0Ny67nwHPD8d3w1vaGPI0uc9HyFIkxQwBEGwEOP1wh1VMeHXOXkUIZWL46j5bRbm+V8Vb8T8QpvndnGYf3N0nhFHifWGc5wFZ4L5t0w50p4nRaP6t/PPj3/8o6xcuYokSfjsZz/PUUcdySmnvJFarYsbbriBBx98gPe97z2sWrWKWq3Gt771Hd785jdxwQUX/c37l9HVVWVycpqRkZEFl4c4jsnzjK1bd9DVVaVardBs2mt62bKlTE1OMjM7y6ZNm9h7771Z98wz+L7D0NJ+oo4ky+aYbc7SW+tndGyamU5ErkMykduy7le444FH2He/NVDtojFt3STyTNLb38305DR2h6wRUliUIp0h0YodI+OUVAWkT/9SMGkPzYYVYU1OzlD2JWncxMHBdxQGn1KXR3OuBYlDHEVonVH3+smcGUYnh9F47LF6FRvWb2FqRLB4UZuB5YsY2gOaSU4niTn8CI9f/fa3eCriuUf/zO33PMraRUOEdLjtka00OhF9PcvY96Ahjj5+Xx5/cpRnn93Jpk0Psn1rh6BiBTcOJaI4RCibXkUe2Lg+ZeNZ57O7LUdbLoif0jTF81yMkcQhBf3AsGjRIhzPI9eaJEoRToA2OTrLMNItpupWvayNoeyX6KrVCh64RY2M1mRBRpppUm2DH3KN5dgriS52iko5CKGRRixc6/NrwsuOOQKjDR86/TNIZxYpNLf9+c/c8qc/0dvXx0VfO5/f/+EPCCmYmJjgK1/5KhjD+Rde+IIiYgQMDA7y+z/8gauuvpqvX3YZDzz8GNu3b2fVyhX88KrvghB89MyziGM70ZNKsmnzZg46YD+eWbeOQw85mOGREZSEFf1lpCMIfInrulRLFQKvRLVapd0qMbIxJY4z4tS3XFZMkW4iilhYqyKPoxglFdVqF67j47o+ruda2zlpE+GEUOSkZLmFaAQZSkgyYydjVohqnT2EUkWt0BitUU5GHIV2/dKCLCmaMyHxShKyHG1SlIKx4YjhkTkO3r+fPVdqnrwnolytEYk2UzPTdNdLBF5Ku9FkYtRD6RiVlPjmxd9irpMiVYc8atNVr7Jk2VIcYfD8WZyemGeeidl/2RAD1T62bd8I5LY/FBIlFUHgECcZaEFvrUqepERhB53nu1FNIRdgb0FR/3NDuVQmy2tkiUHngh//7+/5wf9ch3IkP/nxJVx77R/JczuxXb1qhT3vE0ufAKyKG8kvf3kD11xz8/MrGRdd9JUX1LYf/nADP/zhFS/42VVXXf6C7y+44Mu88Mi48MIL+EfH5z73ub/72Re/+MWFr7/2ta/9w7+74srv0VWr0mp1mJmZY3Cwn+3bd/GBD5zO+9//vn/4N397/NMN5R6rViGkotG4kKDkQBEZpDFFtqtVZmaZJk0yssyqI7VJMXmbtJWT53aUrrUhS62XpFIWBs+yeRsRicXs7MkKFJwsO5GxcJaAQulWVArbRM5z5OZZtkUJ6aotQypI0hQlNOASx4Kjjl1Dz2JDrafMH3+7CULB2z6wJw/fP8Vfb9zFqrV7Mjx2EOSGZpiTtjaydp8epOhilpAwTXEyReppvMxgUsila3fDwsZV5toghEJmik6Y43klkiRHCUGWWREMaGummkVIBzqtNsKdpbd3DVmWMDE5ArmmXKmwbOlSJicnyLIUT2nSNEVLB6NyZOrTaeXIMjRm2+A4GCmRiUPHpKii6HcaEjeg8AacwKQr6bQTyuUySRKRFQbPrusTRQkgMXmAIxOSaA7f60aLJuV6jQ1PTRFNNpBBSBKnSCckCV2MaVoukAQTVvBLGZnQpGlOOx3F8+qYLKeVJEjhIknR5BiTolxrQq9UB6EdlHIxaNphVkxNU2YTgecFdIy/m5M3P6gWhpNfezL/8ta3gjE8+OCDfPs73+aNb3wTrzv5dbiuw5YtWzjvK+eR55rLv3k5GzZs4IADDuSaa67lhhtuBIrEj+KOPc8jjjuAXWyr1TJCgFJW8fqa17yGhx56kDVr1qCUFSCAbQ6TJCFNIx5//B6WLq0DvcDs866u3y98pXXBs10wpLUowPyRZ5ayIYQk1xKT/98ZLo5r0AaSdPet+hatYH5RtBYwdpEUwm4cPfco5tXURxzuMzn2JEmoGVi9BtPu8PiTD9I1OMTY6Cx9i0oc/tI1bNva5LF7PNpN2xxr43HMS19KlmV8/vOfZ3x8AgT85c6/8Oyz6xifmOTKK77DAw/cj1KKqalJLrjwIjqdDhddcinyBVMwy+19z3vew1FHHcmf/vQnenrqhKEVJIAoasRuGyCALVu2ctDBBxPHMUcccQR33303tVqNMEpZt2kHMvHReRMtNJ5XwnVCdCiYCTPGAUcoBro1q/trjG3ZwuD+R/LIY88C0ApfRL1aprvezWxzDom2CnehKFVLJOEk9Xod4cC2DQ266hnVoJexsTFc10FJcP0KQbWL0InoxB08X9IJHbT2cIUhatr42qg5TTgZoYKIwd4h1j++kVJPBTJozc4gtEuzk9KebrBUGpL9XI455gBunNpF/5JlLF+9nYrbxm24zI7vYnxqB7f9YReu6/HiQ1/KxMw4e71oiL1efTL33vEUc805dg1PkuQNSlXI0iIdyEkQxqB1hnQlJssL1MoDkxeUJQdETKzt1MxxDHEMnuuRaE1j+gIrbJECIg1aFH6eRSyslgTlAMjIcocsVuS5JkszpFDIYkIvlcCVILVV16Mjex+ZKBosXWzaFOgyxcWIMRM8ve4+HnvsLoxukadW/e17Ppd/69tgDMuWLsVxHDDw9NPrENpgpCgmd8V6U8ClrVaLx556knKti2fXr2fNmlX09/Sw7pln+cJ553Hkiw/nX9/2L1z1gx+BEQgUv7/hD5x95llcfOH5jI9P0GjM4Xou5Z5eK9ZsJ0DA2FxGnjfIsimmxyVxstIOObSDMN/AaK9A0wDdQAiNzjTK8ajUeglKVeZJW5mx6EaOodWOidLcNurS0gxqlRISaLdn6LRDjJCW/6wLwZyaD+UTkIPvK/K0QyfuIB1R2HtBFimUOgMhJVlkGOqrYUoJ4YRgzkwjyy5eSeKYbgwp1VKJ6emEwK1y2rtO4Zaf/46sy+NlL38Jd951D9rvY0V/P5HQzAw/xNDQvmROwKYNuzBRiVq5QioExgQFbSknCAJSo4g6KR/80OmsXbsGgK9+9csceeThHPvKV1EpV/jVr37NLbfcxHve/V4WL15Cd3c33/v+d3njG07h69+8ijxXGGGnsllu0SvXda0Cuhhqve1tJ3PNNTdx5pnvtacYxXyjKEOnnnoSJ5zwUq655mZuvfW6/0ul/v/vcBxnwcYrz/MF94V5FEEIYW0RtUVu5hGm+eGR3QBam8ZqtUKj0d7tTOC5RFH8Dx71//J8/ulbyjKZNkhpuVCOA2mmyXIIo5g4TkjjEKOzAta2Clglc4RKUV6+8CZbPy6D69npjEFTrsGL9jyUr3zlAkZHRgC46eab+NWvfsnatWs566xzKJVKpGnGt771TR555FE7PSveTAEcf8IJnHbaOxEItmzZzHnnfYkkTfCDcaIoJgggjDKWLtubVLc5/pQeYh0yPeUQh9OUgxhRWsHRJxzAXX+8i8Nf+VLuurWORGAmmyxetIJ6t2Dj5jFUrUqa5Lgk+GmNRMXkQuAIa0ruFCMotzBvzjNDIB3SOMGVCqM1vnHI4hzXdelEEb7vU3IEiQsmabFrePMC6U5J68+3c9cuRLHAWpip4DKZCEmApxzSPCcLNY6v8VyJTDNy1/KV0izEVVZJTZ4QOF1MDU8Blp+EhiAo0263UdIliTuUS1YhncQQlFy0zmxElQzoNGfYumma/Y5YwUwWYVBID0yukZTwVUCp5rJrchrPreJ5AWmmbZSjshNs35MkqYOnfHThyyk9ayZtpMD3ffKkjREuSglMccFkWYbzD4aJe+yxB29969s448On0261qdVqCAS3334bv/vd7wDDhz/8EY477nj+9Kc/AlZB/O73vAeA93/g/Tz7zDruuutu+vv7mJmdXWhUXvOa1/Ce97wH3/c5/fTTUUpRKpV4wxvewIc//GE+85nPkOf5QiqT53kvgJDn/12+fDkzMzO0WpajMu+L6DouA4OLForE5OQUjUaHvr5u+vv7+NznPovr2k3L5NQU519wAaMjo/Z+iwd43/vewztPO41PffrTbN++nU67w+TU5MKD77vvGt592n8QlHyMgQcfepgf/eQnGCPIc4GQSZEnK9jvZXvx0mNeT0XWydwp+vwKSZ7z9JZ13HfTX7j/rns4ZJ+jyBOfpx2JEPbvTA6rVq/iwYceQkrJ8uVLMRjWrlnLv/7rv+I4DsuWLSv892D9ho12I1bwYQYG+hmbmEBg/SmN0Zx/wQX4vsd3vv1tfve73xMnEX19fQSBz7LlS2k1m7TbbcAKdH70ox9z7rlnccwxx7Bz504mJiaYnZ1DuBWeenAUoSMc3yNKNDXlMdseo941iNQ5lYqk2tWmt+ywZsUqlO6nVoo5/JQTAPjpz9uMTkxQ7+5m8cBejG4ySGm5WDoTlH0XaUBpRd0ziJZmdnIX5UoF5SrCJKQZRvT1dhHPtBGZtM4TpRTXsXC4cAXSgygO8VwXR2rGZyYQOqfLE+Q6wjEeeZhy0L4DzE4Z3nziuwnzlFtuvYPWnKbZbiNw6XQgTyscfMghLFv+Olphg9GZeygFHovqb2R2CoTT5vSP7k3Y8ZkLJxgZX88PvnsbjmsjFTMNStgNiDSSVBdZ1wbAQgxpEuE4LkYIUp2hjZ3cSiVpt9vkWRvl2qbQDxRJGJPmkXUCMDaVx/etQjaMIjAKxxVIcrIowg8qCKlxHUUaZ5z18Y+xcsUK4jjmwYce4ue//CmHHnoQ7/i30wDBzp3DfPnLVy5M0AWGk048keOOeyVpmrJq1Sq+dv7X2LFjB5/9/BeYa85w3S9/RVLQO4yxzaTd5Kli0GFtqaRUPPHkE+y3z348/eyz7LF6NTfedBN93b202m3SLGeu1aRaqWAFYRD4HnEccvEll+F7Ph/9yOnce+/9xGHM5qe3I2RGqeQzNjYGhf2MFIIkrtNu9qI8F89zCVJFJysuaAGSnFzHlLp6qHcvJc+tfR3Cpkrp3JBpsLSu3HokOZmtsXnG+9/z76xesZwo7PDQw49w7XW/47AXH8y/ve2taG0YGR3lkm9+y06QdU6rE/LWfzmVl7/kSLIsY9Wq1Vxw0fncc+/9dpObCxyhSLOUcKSFWCRxSin9a2vs2NxCyA4SRbwjQ8cwsGwPlvatIk1zfFXm1nvvIo5aOJmD21Viv71X87IT30m0pc19Tz6AO1BnpOHSnp6lVE7o7etibFTiOJIszch1ypFHH4njSs791CfIc5vv/Zc7b+O3v7sez/X59re/zS23WOh6bGyMr3zlPEBwycUXYWQFrQM7aSxmWJ/4+Lv4wx9+wdfO/xLGzCCl5JZbrmVkZJwvfnELMMbzqY433fRrbrjhl0X9WwJkxZ39A27Q/+vDFvvseTyjecu/+f/Po41a5y/YaM8jYGmaIqVaCCloNtrFJsy+gDhOqFRKBQz/zx3/dEPZDtuWHJrHOI6HkYrG9CzN2QbC5Hi+Sy1wEa4HQlrBC4YsTVFS43iQxZDEEpHu9p5LU43nQaVmjbcffughPvf5zxSqO3sxRVHIl770RYZHhlm5chUXXnAhb/mXtywswvMwxEc+/BHe+c53MNdo8IUvfIlXvOKV3HrbH8m1hdzjJGW/Aw9k771ejCiNkod96ETy25/cxoc+fgJPPbqVLY8aHDHLO971Fu67dys6PRAcTU9fF889t5NXxMez/779NOeg4ncx3h4hjjzIGygBSrggcySaBUfDDITrWH6hUtZQPIcoj3HdgDjT+MpBZII0Akc6eIGPFImNXZKutbBQijRN8QMPoaTdWRScUyHKGO2gNThC0t/bQ0pGmscEXoDKNY25iMCvE3iK3AT4bp2Vy+tced0t9Pb2kucprWYTpXy01iRpVOxwMoKSJM+tDUKWJSjpobUmKEvuvPUx9j18JWmaYqQD0sF1EjpRTJ46dsBmfKs+nm2SZAkITaVUIk8NOs4QMkE6HiYX1ppHWL/RTtgii0IkdleVZjYOMokSy/XMXkh4NMChhxzKfffdR73eTb2rzujYKNVajaOPOpq3vvWt+IFPuVRGSsnjjz+O5/ncfdfdDA0OMjY2zve/f9U8ZkF/fz/NZnPhwr3pppu46aabOPbYY3n3u9/DBRecz3vf+15+/OMfv4C/B7yAnvGPjt0Npi36QhiGFg8yPj5OFNmmzn62MDU1y/T0HB/60BlEccSqlSs4+uij+fAZZ3DuOecu3Ne+++zDAfvvz9jYGLt27mTrli2sWrWKoOUXzatgZOcOPv3pzzA8MozveXzjG1/niEMO4cYbb8IYFxjGcR36BwbY+ORGfvj9C+ktV2g7PitWvIgly1fwon1W8/r/+CAnnXIKP/vupdx31xaUejtCqgUj8q1btnDEEUdw+223z4MKfPlLX+ZjH/84Rht+9vOfsW3HTowxtFstwrCzwE6ZmJgsJur2B67nkaUZcZzQarWZnJoqfEpnieOEnbtGCTvthffBcjAFn/ns51BSct55X+LWW29DAGVP8fKXLKPc7dJTG6LqVlC9gnrVpxRU6EQxvV2LKYkuYtMmESkYF6OqPLduZP6Sxvc9oijksCMOZGp7iyxLLYdMe6RRhklCvMDlsJe8iD/f8Si1njqddpOKG+DmOXk7ppNO4BiQQWCFbnEO5LbBQhBFCX7QhVGQ5AZpQhzpMTU8jV/yUbJMqOGEN72Eqdkt4E+w7akxOu0QP9BMz0wyNTXDin4P6NCu72Sn3EJWEqigQiYrDOtNuIslhoj1WYLxSuAn7Lf/Ghbd/DjjmyepVkskaQehgoJcK9C53RDa0ASLG3qeh84S4iS2SUR+QBQJ4iTGK1es5ZtOyLOcyGgqtYB2sw1G4ZcCHMfGW7bbbWv6bUmCeK4gTzLyLMb1PPJM47gOUgouvOwSNm3YiuN6BOUaGzZs4eMf+xi5Tjn33M9zwP578/gTzyxE0/3857/k9ttvx/c9Lr74Ih599FFKpRKXXHIJm7dsptPpUA6CF9gBuVLxyU+eyUWXXYaU4LsBQkiuv+kWzvzYRzmtVOLRxx9n6+atTE9O8/nPf5aXHHUUQgjOO+98pIZPfuIjXH75t1izejUfPv0MhBD88dbbGCmUck5Z43guyoXeJTWqpTqCgJIX0J7zmZkoo5S1oPMdl0jOhwVYmkrv4DL8oE4YhXaKK+RC8zsvoHUE6ML7VWdWEFvyXXxXctk3LmP7jmEwlgLzyGOPcf+DD2EywzlnfYJ99tqTp9Y9jVIOpZLkV7/8NdffcDN53ubKy7/DX++/D20E2qS4ypCnmixROJFPey6iFHYjWjm1DNzAJ0kVRkDqZvQv6+axjY8xF81SmxWYAZ9Br4v1Tz+B2yNpzxruut9laX+ZSvdqauyB702jVIBOPCYnRtBCYLCv1fc9XrTP3mzY8CxdXTWLbOaaI1/yUk44/jUALF26zPYQQvDMM8/sJtkXLYUQNuUmN2DMKJd9/QIwmfUTFdbCbHR0GDAMD+9iXtxCkbg0X/q1hl27dlgqR7GxmZ8snnbaaRx22GEEQcAtt9zCddddhzGGc845hxUrVvDcc89x+eWXL3CXsyzjoIMO4txzz6Ver3PyyScXp6jgq1/9Kueccy6OY6eTfX19fPGLX+L00z9kaQzFE9pjjz14wxvewKWXXsZb3nIqcZxw4403kqRdHH/cyyiXA6655ma0rvHpT7+XSqXyf1zD/vb4pxvKvnoVR0C7U6IVpRx24P689VNvQEjB408+ydU/+F9edcIrOfFVr8JVDlu3b+OrF1yMQnP++d9m2/b1vOhFB/GH66/l5luut/F+wpAXSts81ziu5YOJYvWxwgLYsWMXlj+p2LplC6WiGdBFF44pxszCTrOkkJSCgKmpieI+BFkmacxC0klZ/9RWtAm5/7YpKqWAXqfEE49OMrw9JQ8bzE3uRMsGwyM9SDFDUC5Z7olQ3PLbdaxetZwWITqfRhGRd4ZZu08vwlG4UqGltFCcLNTlyp6l3T01kijGmAwhJUHgkWMIXAelPZIkw5gcV7ro3KBzaylk9O7mxNqZWKrAvKLXxlQ6NNtt0iSi5AmSTpskSxCuIZeGPM/wlSDPEmZbMyBcyqUe1m8aZfPWKRb11ui023ie9ah03SJhSBpynZJmIKVHpx1RrkriPMN1PerdZUZ3jPHkA1s48qUv4pkN6+jpHiKPU3zpUPID0ihGYnAQ1Bwf4yg67Zi8k+NKhZIOqbZE+jw3mDxc4JdKYSj5JfIoQxnQWiAyEBl4ykX9DeArwJLMgeFdu5gHalqtJu9733s559xzyDPN4UcczupVqy3PNIlpNpuMjY0v3IcBhoYGAViyZAkAu3btYmBgAIDNmzfzmc98hvPPNxx66KEce+yxANTrdY455hi++MUv8uCDD6KUYvHixQsT5XK5/ILnCnDcK1/JaaedxsUXX8To6JidjmBIM2sVNL+51EYTRiGVcok0TYtzQzNvd+L7Pp/85Jl84xvf4MtftpybeW7bfEMsBGzYuB6kRQqSLGbjpg0sXb4EN5DkGQUHOCaOOiweHKRUTlBViT/X5s5rvkPXQIlrmhrXXcrRb3otb3nPxzjkpGku/eyWwtzZXsP33vtXXvKSl/Dd715JlmV84Qtf4M477+Sb3/gG69evp9VsWnW0sNeHklYwcOaZn+Ciiy8pJkT2PP/oRz7CiuXLcRyHm2++mXarRaVS4StfOY81a/bgwvO/wk9/+jPuvfc+zj33bC644CKOOupITjvt3zHG8POf/9w2OY5Drd7FngcexmynRTPKGZmaQO9QdKIYZTQrl/Ry86Z72LRrlHq1h4ryGG1OEWiHOLXFdWriEAvLmxy5eYYu1cNkbCk+cRjheBLla6QrUG5M1omIpCIJJUkYIR2BXyrTMSmB7yNETtX3iBNtP/+iHlZLJZJQI/LU+sR2BSQJ1LtXMDc7zFw4xeLFq5kciQgzh2Z7K8vXruTRh59l7dqlNJtj9HT5DC2tsmldShi30Z2QNAmodfUzO9tGqBnLHxMuWdbCL3fotA3jzhx7rulnenMbqbJiszdPTZo3hZ/3XrRCTGFs1rrr+4hckeUWkhZC2ZouFLgOnzzj4+yxejUI+OqF53H4IUdxwvHHU61W+f0NN/DHP97MO/7tPxgYHKCrq4sf//RnnPSqE/j6N79FJSgzO9ewxt5a88mPfpwkibnyqu/x7LrNlCsl6n39pJElcI7MC7yep8QbG5tgn31exPr1G+h0OrRbbd73/v/ECwJKnk/JL/O/P/oJSIGjHHSec9nXv4GSCtdxUI4iimJmZub47H9/eTfFyRiazRZnnf0p0iTCdaw1nOf5XHzxZQgB27bv4Myzz0I5btG0KAyGam8XMle2cceiVu1WkyxJac4oGk0PkWtSAxWVgHE5+5wPsWbNSoQ0XPKd77PfPi/itSe8glJQ4jc33MStd/yFd7z9XxgcWES93sX//OyXvO6E4/j6Fd/D9zzL13TsWvqxMz5ClMT86Cc/ZcPGjaRZiusptNQICVMz03aCrDykCgi8hNmZJgcdfCA7d+2iHFTIEk27o5BIstjQHtdUFgmkqCCDlCg2dC2tMDPWxnPK9HSnxLFheP1m+vsWUfYrSK3RUcIMGzjlX/dkWjfpUjE/+sUveeXJx9HdLZkZaVOvwa4tM3R1dREEZRQOuTBQmJVv3bqFgw46lL/eezfSEWRG8+ZT3soZHz6dPM/5+c9+WWyOwFjjMoywgjJpxMJQy3rpihdM+ezpZIccz/86yyw0buZPt3lU9nlRpVJKenp6KJVK3H777Vx77bU0Gg1++tOfcv/993PkkUcyOTnFBRdcwDnnnMPee+/NU089BVhkK4oizjzzzAUu5MDAAPV6nU2bNuE4inK5TL1e56STTuL+++8rnu08xx127tzJ6tWr+fWvf4Uxhne9690IARdd/Bl6u7uRUrDumY1s3fEcn/3cfyNEFx/96Af5Z45/uqFsdGyzMzEyxeJFdd7+tjfx+f/+IpMzlv/R6rS47Y47+N2NfwAj+M/3vp9XvuLl3PmXWxHYNIkzz3k3WsN73vUBNmx6hgcfvguRzxNiBaA5+OBD+cEPfsTo6CiXX/5NhoeH52kr5HnOK15xLM8+++yCDZCNWbMf6Ncvu4z//d+fkiQJDz38EI8+9gjzEVlZ1uH9Z7yE31/7HHvv43Hwi5cyOjFOT71OOwz58433s2xFL888PY6SDmv3WUSp7LNtS0bUbgKG/Q+KWffArcixblTNpdmZ4nWnHA/lfuLMZrNqk1ulmLJZ2sqxOwNpHPIwxiQxSrk2LN5olDHoLCX3XVAZRiYIx8dIRW4kCoFSVsE4r7yG3RZFQlqI0GiJEh6uDAg8hywx+F7Z2hYlhtlokv7uCo4LUvZiVERvvY+ffv9B9EyTuOQjhEKhyItc5HnLIGMMcShx3BylfAQCrTukxuAoqPf08rsf30lv2efFh+3Hrh27yHROUkD6quxgspCq089AuYojJImnyXRCLmKy3OCJbrTOQBoLseUZWhcNeQLaKRVCJ488zen26hhjKKt5j8jdl83TTz/NWWedxY9//L+02m26ajWrdi9X8Fwf4xhOOOEENm3cRBzHxXk0zyXcTd4fHR2ju7ubbdu2IYRg+fLlxLH15xwcHGR4eJg0Tfna177G+vXr0Vrz+c9/nmeffZYHH3wQKSVLlixhcnKSRqMBQrD//vvvnnwAvb29nHrqqXzyk5+0XDTgRz/6ERdeeCEjI6PMzs4+75XZZu+iiy5i+fLlTE9P8/GPf3zhvj7ykY/wm9/8hmeefbaAWZYjpWR6ZoY4SazgbZ7vXOye+/p6OfYVx3LOZ87BoNFaLjxe2Gmj05ijDj+CP939IK88+eX0r+pn7dpBUA22PLyV2370LW79yZUc/cp3ouR+CBHhOAqBCzh8/RvfpNjrAfDTn/2Mn/z0ZyzEISrF//74x8VUxIpavnb+BQsc6HlKywUXXLgwOZnnTDYac3zsYx9DShtuMF/u5wU99913P/c/cD9FhShuY5gcn+GXP76FshNQcqC7KwDjsu+eKxga7KXet5hSXuOQJSHbNm9k6YolDPYdhc4yxubs877hDm8Bxqx1KVzPY5HnI4GXH3wipWCOTjJH3+Ai2u0p3vqy1Qxv20kUa+aaTdpJQnsuJxPQSIu0FKnAs8IN1ykRxS3StsL3qqRpkyDwaA6nKE8w2/Z50b6HoEWDen0ROzdGuKVeOmkb023wTMBzj0fMNic54oi9SUNJFHfRHOslaeQYU6IxHBEEvWCywmtR48g6kVEIBbWBjAP3c/nz9c/S5UlI7IZ/3mZFCIMVY2cIowCNxnqKRlFCxS0V3HcDwlhfYJlz9BFHoXPNOZ/+FOWuGr4b8MBDD3HTH29nqK+H877yZW677Q6MgbHxCb52yaUoIXn6medI45TZ2TkrIDOGK793Fc1mi1UrlnP2Jz/ORz7+SeIw5ITjXsO/vOVURkZGmJttLiBZ8+cPwPHHv5Lbbr/dijOkoFTyiRLbxLmOU9jQGRwlccslsjQnS1N0nqPzHKOhOTuH61kamNE5iJyo08H1faq1mt0QChsf6nhWGGeMsXZuaVY0PzmIjJ0bniaajPFLAe2oQxi2KVdL+J6i0ygh2A8ciWMMqdYcc8wRaG04/UOfx68a0rjN6I6d3Hj9DShHcuU3LuG3v/ktSRiya/tOvvSjnyAdyebnNlPxHVxPkWaaqNXi29/+NjOzTZYtXcpnPnUO7//g6ZTKZU486STe/KY3Mjw8yszkNDo1dHQCKsdTHvWeOi9/2THc+9f78StddHeXKbX6CsqUwNclZiYE9XKTVjsjCLpoz2YMLqmThLDXQXvx1NMbOOmkf6erJ+He++9hxcoX4Zo6h75ykDX7ebjjAUNdZQ7YN2bztnFe94q9WVoJmYtmiaKQaGoMUktHkIU/pFSKhx65n4MPOYwvffkCsizjgksv5O677+Gb3/iW3dC2Wgu1c0GsaDRnn3suF17wTYzJLEfVCAYH+/E8l9HREbuJ+eQnWbt2LXme89nPfpZjjjmG1772tVQqFX79619z/fXX8/4PfIDFixfT09PDlVdeyRvf+CYuuOB8hBBMT08vNJirV68mjhOGh4cZHh5mjz324Kabbi5q2H0ceOCBPPXUU/T19TE1NcWOHTtYtmzZQg2dmppibm6Oo49+Cddeex2tVpNdw8McfPDBXH311SjHwRRccyHhda97HU8/vY4LL7yQww47jHe/+118+9vf4dzPf5njXnospVLAs89t5OXHHsRbTnkjZmFN+H8+/umGstnuMD0+jUOb4044mbvv+SvbdoyASVGeC1KwevUevOedp1GtVKhUKsRxwp1YlfVfH7iDJIY8hx/95HvoXJImVunteJDnhufWP8fb//WttNstXnbMy/jCF77EBz5oeW3CCJYsW8qHPnQ6nzzrTOZtgeYnlI5SvP4Nb+Cd73wHk5MTfO5z/8WJJ76aW265hZnpNi95xX7ss9++XPvzEUZGI07Zd0/WuKtpthO80mKOfdNRbHlqjrG5mylXfE55+4m00w4//U4PxmRoDC978xHoP7dYsmiQF79uX6ZmZglb0O5Yv0eEwHWtOhMt0GlqVdnCIVcZOs9wfIjSCIlPJ4kIXIdSyaERtzA2NwBHS/ySixJ2VyOVbaxc10VKSblcJooi4sxCwTYpro1SkmazifYCq95zDa6jCTxFtWcZrXaETBVTYyFT06PUqw12DM/Q1VcjakW20FXdQmBl/em0Jd7gOgFJ1LQ54rlNjsjSDEyO4wRIX/KTn97M3OzJLF3SzbI1vYxNNwlFguc4pCLH+IpWFqLjFMfxSUlJ8tgq/5XlaSoHXAypnvfG8uxUNjEYZRMMMp0T69hO3orOaLdyGTZu3MCNN97Ad75zBXme8+CDD/CHP1zP1VdfxXnnnUen01nY8WnrF8W8dynAB97/Pp55Zh133323fe2FmfNJJ53E8ccfT5ZlzM7O8sUvfWl3QRK7M6PB7kIHBgY455xz+M///E8AfvmLX9Dd04OjFCeeeGIhVhnnrLPOIksz+hf1EwQB73znu9BaMzAwQHd3N6OjkwvNkqMUF198MZs3b+Yd73gH73rXu7jooos44ogjGBoa4pLLLmVR/yK01uzYsYOtW7eyfPlyOu0OnbDDvLcqGMqVChdddBG/+NUvWf/ceqwVoLUAE8WUfWx6ioGVq1i2ajtzc032OPilzLUi9h6qs/Y/Xs4Rb0256/d/5N4bfoOUvQh6rHEwgiXL1lAqOWghyXJNlmvmw3LiNLECpDxHCI3JcqTIcZXBcT0QduLVaMbkOkVgihxlgclzHGWvsyQpY4xbvD8aeD6JvGQFcUVjIApC+pLFn0Q5Lp5ykQJaTYl0FQ89LdFP5cSdBopuMF102jXWbRaFPYpHmNpHmmoWcgcBf15XYa69+7O//6k1lH3rEmHWYTnBRb4zriHohQDo0VY1m2YpaZaTpAlJmhIlMWE7oVTxkVLRmu7Q21sjDVMqjn2cqJ0yvr0X5cLMLsGmJywXdX4s4jp7MjExR6l8GE88GDA7E9JoDHL7b/pwPYp0S+uQIZ63kbKwsK2tTy0RnHbaOLW6Rxwb3JJBR3biqPVuGzdrG2S/j/MYTzl4jgNaFYpageu49PYvIkqqrF2zB5u2bKbe10eWajrtmIMP3J9T3/QmlFIMLBoohGwpGzZuoOQqtBb4viBXAsd1ChW3QOuErqrP9OQkAkFXtUKa5fzxttu55Y93cMZ/foBjjz2a226/1xpT6+L1AUcddRRXXXWVnSIZQztMQOeFaEqQ5jkmh1QbstwUTbch19JC845TxPFl5HmGo6zFi4WEKyAVaaptdrMxNs5T5wuogkVbC94xgizMaHXGibOAwcXL6Fu0lh27tqO1pq9/iMakIssSfOUSm5yVq5byyMNPWoumdgPlZhxx8It5y1v+BSEkS5cspuY7+J5g06b11KsCKRxS3SRsS+JIITC4nrWXi9ttxsaGEVLQ01Mj14Ibb7yFG2+8mQ+f/p+85Ogj+cuddyGktcNLsoQs0hx68MH88Ic/I4oimjqmUhKUfI8ojkhc6K246DSl2RSkaZNarUb/ol6Gd4zx8P3PEgTdPPrAQ2zdthm/u8pcY5y50THe8O+vo9kaR2mFqghWLE+59eFR1m1dwWB/HR07bNl8h3V7CbrJspR5hxetFbkO+M53ryaJItI0o6vezS233MzvfncD09MTGCzC+cMf/k8x1LJONRddeAlCegV/cn64MAHA4GAvL3nJS6hUKpxxxhlUKhV6enpYv349d955J3Ecc/XVV/Pwww9TrVYZH5/gu9/9LqOjYzz7rFVlz/v5zttt/fu//ztHHXUU11xzDUmSUKlUFrjg7Xaber0OWI7kvEgmiqKF4dJ8gmAcR4RhSKcT0tvbg+M4TExMWE5pgeba01CTpknhAdxgr732LtY5yU033cH8hv2RRx7lz7f+BUMXZ5zxfv6Z459uKGdHR6mWXPxKhVYnIshByhwjBCa1JN9zzvwEn/rc59iydTtvesPJLBlaiig4MGEYUmzkENLybzy/uLaNRAhNo9lBZzYf9q677+Sss84toG1NravG+V87nwsvuoDh4V0L5NE8t3YRa/fcE53njI2PYbTmzjv/wqGHHsrNN93M3i/al2NOPJFvfvOPlAYmSZTDty7/EXvsvRyDQ99gmUWDQ1z3/Xs56qXL8dwyV13+W07+t9fQbMc40kbvJW2HN7z5aL7xydvY5+AhSiWPlkrwuls4xiXLIxIBBBolfUrKQxfRWKpURWQ5uTE4gSLKEuo9XWRRizBpUKrUUKqE41QQWReVcoBCkcSp5RKZeAH2llIS+D46TSG3k0vPqRBnDXI5Q9/SxWSpw1wnYWKyw8xog/b0NqZGp4nbDs3GLMKUETLBZDmVuosXmUKR3C7G+Bohdvut5VmEHyggI/A9wjDDiAzhgOtJuiqSNK5x3e/vxHFTTnjNkfQt7aVU6SLKMjp5RCNukDsdPEeSZu1CoQsCD0kHtEbHunjs+QYtwkiBMNbfrtG2usYkSXAch7lIY1hsn2fxv3anza233sadd94FGEZHR+mq13nooYe5996/EgQBu3YN091dx3EdPvGJT6C1YXBwgLGxCb7//asWYGQopsFCcPXVV/P73/+eNE1pNpuYgjM2D4Uopfjyl7/MXnvthZSSiYkJLrpot/3N29729gWce8Xy5QXpuczg4BA7duwgzTLiOF5wTeh02tTr3exmW0JXV41Wq0WW5fzud7/j17/+NRdddBGHHXYYe+21F9ddcy2O49DT08Oll17KhRdeyIYNGyiXS3TCjm0gBJSCEl+/9BLuvPNOfvXLX2HNgFNcxyMz9hzL04xdw9tZtc8BLO1eTuL4RM0Z+hcN8GwzJRhPSDpTHPSG19DffzB/+GFGrpMiy1vTiXLCJC+6F0lu7AQfdnOUlCsQWpGrxKrMRUCrE5OlKXmWFv6gBm00DlbBq4UkTTIQLojnlbAXIlKATelBW09cpC2oWQJZnJCqjFIpsLyrOMVxXBAa5Xn29gbqlRqO54DJ0JlBte05OxOGuz/3wIFOujBV1QiSrIBeEOTGkMaxbT4Kfo4RuykP1i3AwfE9ShVBjwCjBcoRhFGEYxy6u+rEaUKWajKdo5yAJNGUZAmdJ3iFE4KQkiwDz3GQQuIqr5ge2sZRKVt7jSl8GeenvgVGZ21XrD3Prl0OU3N1ql1lZmdClNNB41ohR9Gcz1vHGGU3EY6w14sKArIst5ZKWPSm1WqRJiHbtm3j0EMP4+Zb/oznuuQm4T/+7e2ceeaZtJotfvOb6/AUeK6iHJQs71LYqah0FJk2pIXCvFQKaHU6dNW6EFIyNTNnOdiug3Kg3WkTRjE61yglKZUW0W4L9n7RGrZsGSZOhOVuG41b8hGyi8CtkiQZUmVo3aK7WqKTpvQMLGHtPvvTVa0zOjJBo9UmiiKMyXEEhK02frnCoFthvDnLyOSojRQWGXkYopOEgUV9hDHUu3rBCZmYbDG4eAladBAC1u5zEFlukK7DyGSDMPbxPJ88L+N6Aa6r6EQRUii27xjl8BcfwN33PET/0BDlwOW973s/X/vaRWg0F194Pv2DAwSlKkGlh3KtnzzNicMmCInvuTZNzRiUSan399C/aMDGBRorqtXkaJ3SaDYJw6TgR2eY1JDlCXvtvRdbtmymNT2Gclwcxydst3C8UiFG7eAlHkke0R0sIsqnCZsp6x6ZQKqcapfDYG+VZ5++lSwcoOKByCKcrhK12nIy0U3JTXBx2HeNzyNPP8XmZ+/kmSjHo8bSwX7Gdw6j4zYCB6nsJDqKO0QTGV3dPSwaGERJh8BzcR3F7EwLR1m6ROERsHtuLebtD/OC82hJeAAve9lLWb/+GarVKnfeeRda5zSbTRqNBsceeyyf+tSniKKIJUuWkKYpjUaD5557jtHRsRfWpCJ9bmhoiEqlwhVXXMEll1zKFVd8h5tuupl2u73g31utVpmbm1tYhyqVMs1mc4E65bouixcvRgjBX/7yF5pN6+l77LHHcu+995Km6UIzec6553LBBRdw11138YUvfIHDDz8c13X5whe+CAI810Wp3VZyzPNz/76s/h+Pf7qh7OoKCColZqZDHnvsUT73mU/zmz/8llazTbVWpdFqEAQ+k5MzONLjNSedxJNPPkVeLMw6hyS2hUhJENIWsTQDP7BxbYv6BuxOwAgOPOAgZmdn0Dkox+UrXzmfX/ziFzz6yCPMs2bNfHOqNRPjE+yxxxpqtS6ajTkOO+zFbNu2DYAsg2a4jHd/8BzSZISoM4I2s0xPT9LptElmp7nvmREG+2F4Ypha/zIOPGaABx5aR5L0kUvoRII/3Xwvi1Z4LD/AcNONDyG7y5hyGz/rpb/XpVwtYaRBSIdGq0Gl1lU0HT6ikeC7iuZck9lWB+mV0HoGmUXsu+cqJqancVRApeJg4pSpyRlLDDceeaTBtYourTWzs7P4nkeWZbhCErgeeIJKrYfOnM/WJ1NGtk0zvH2CmekmZeOj/Jyg5FKSkp6eXhJt0I7A6DI5MwSe3fHnucD3S89TgsUFV8/FkCKkJs+gv38pnWSWLG/jOj55pNB+xmC/D1LywD0PU6mXePkrTkWHEYuqgwxvHmdidguVShVjbJNe9hQyV+ROG6VcdC4gsxMALTTaJAglFvw955s4RymESGm35hVs8xNG+7zHx8cWVPBSKpqNFs1ma4HTAoaZ2dni7+zqPjY2Ufxu97Rp/uKfbyrBUi8sVM7ChR4EAe22VZTP7xjTNMUYQ1dXl4W8Kbg0mW2q2p02zWaDH//4x3z+v/6b7du20dffB2JmgW9p+ZT2pfX19TI0NMCmTZuAArbftAmAK664giuusF5mq1ev5rLLLuPss89m8+bNrFixgqmpqeJVG+q1GhdeeCF/ve8+fvTDH2F0jnQUIK35tNidGjE1Pkwer6GrXOah7VsZWrYfzz07RVNP0V8u8aK1a9GxIRhcyqqVHtu2N8jzHMd5L694Uy9HHLqSbGaaJzeNsXOqRaANUdJmtDGH1wqJZyeZMOD5Gpl5pO0pnHqHilen6jogKqiSoadUI+hzePzxKaKog3RSTGOQqHEwORqFROgmWbaFgrQM7It0NBUHWnFYwPAwMjk6n6xgY9oKWHY+XQhtpxzKETiuj+e5+F4Z37VxcwA6sxM+A8TNGJ2KhcLbmQ2JREqsc7TQ5MaGMIAskldEUa8teqMLL13X8xZI+0IoHCWtojcoE8cZRsPk5DRagKMckjimVK6wqK8b5SjLzTYSZLZAq5BSWM6fkuQ6KyxF5jVnBf98gS4gEGY+y95GyXl0s3TpCsYmHqa7VKMTJ0hHIQouojECIVQxaRNIo5GOhXMd6WCMLG5nCt9Kl/seeJjDDz+cb152ATrP+dqFF/LAAw9x2aWX8exzz9JqW89bjSBKc1rtEIHhox/+MBdd/A1c31I3BPBfnzuXWq2GkpIrv3c1Ughed/KrOf74V2IM7No5wr33PowQgrPO/iDf/973abUTjj/updx22z22wS58NPsHltMKW4gkIuj2UTJg+bIDmZ1tMBSU6VkySBqmrNuykanx6SL9SZOlESXPZ3HfEMozTOzcQVD16XZyent7yHNN23FJXGGnmSZlYnIWN2jj+3Vmp2bRQqNEwPSObaRZG+nYsBaFw87pOUy2DVhMUKqghEAKxV13PcgRRxzEty7/Eoacr1zwdf58572cffYn2bxlK81mi+npBp1OSNQJSdOENEk4++Mf47tXfofR6QbGuFRKii987hyq1S6UUnzve1cRtlu8/vWv57hXHovA8u7uvPPPaK0568xPcMlllyEFHHfssfzlrntwSz4I6+EopYOWGpNrXBkwmY0ROAqVj9tIQjSpblKtVmiFCVu2T1HuWkJbzhFlGlIPN3D50VW/xC1DX3eVydDl1Ncfz2sPEdz64H2sWFVDtzOyTod6XzdTM3XmrQYFEs+113OtVieoVGjMtYg6CeWSx+TUOFESLnAkd68Iln9p64DCJmTs9rFetWoljz32MFu3buXwww/njjtup1qr0Nfbxwc+8AG++tWvsmHDBn7729/a+FfmbQ7//kjTlJGRETzPY8mSJQupdGHY4bnnnuPFL34xjzzyCIcffjjXX389ABMTEwwNDdHd3V1EOlq/6+3bt6OU4oYbbli4/xtuuJHBwQFGx8bmyw0XXGAnpJ7n8ZGPfJQg8BemqEJAueQxR7pwrbquawcA4h+9gn98iL8lmv6fjoEllxmbfXwpyJwTjzuON77h9eR5xsMPP8J3f/AD3vi6k3nrqacyOzfH1u3babc6fP/7l3PxhRdy5VUXs3nzNrTOeed/vJ8NG5/lnr/eRZZCqQyOIzn+2Lfy+pPfQJblhFGHr192GRs2rOfEE0/i3HM/XTSItmn4+Mc/yuzcHBdfdDHnn/81pqYmOeXNp3Lqm99Cnuds3rKJr33tq4RhxMq1e1FbuhbfH2RwaBlqD+gmAAEAAElEQVQoTXfvIpYtHaB/oETUmSOeG2Fc72Jiw5Ps3Pokvb1L2Dnqs/OJV9vJitCs3OdOssoMg70VRDxJ1AJV9jGBQGY5rhugw5BWs0MkM2jB7JykXtVkBnr7KuRujhdB2MnwPI82Ka4jUXmJkoyIjQt5jZHxnfieItMKt5XhVVzmEkFPycHkNbJsFqfkIExKhMSLElRQohE20dMxXn8X1aBGhqIsMmSeEJZKJHmLaiVi+ZK1jO6cJOoIlJOhAp/AkSgnpzWXEqY5iS7RV+3Qbrk41EE3mM4ySEOUK0kchcoSPAK0K5EmQqsA0ATKkDk1DjroNeStGWaTjMmpTUyMb6HsKZQMkG6LKBZI6ZCmMeVKiTgOcV2fUlAmTXOqZesvhttGmm6b5qMzslShEMxOrWLr0y9ldzP5aYSwKjql1IK46wUn/fwV8rwLRWtNkhQWDAZMMUnr7++jq6trAUIeGBggjmNmZmYW7rdarTIwMIDWmna7TXd3N1u3brWKfN9nYGBggbw9NzdHs9lk8eLFTM/M0Gq22G///fnSF7/IeV85j507dnL55d/kwgsvZGJigvHxSeLYPq9DDz2Ec845c0Fgs2vXLr7+9a8zMTHBsmXL2Lp1KwB+EPDrX/2KSy+9lO3bt9PudHjve97D3ffcw91338U73/Vu3vvud7N58+YFGOT2O27jZ7/4OUY7wBk2Ek9DpeuPvPf0UxiZHmdw9Z7sfdiBbB0eodevsbM9xbI1+5GFKY/f9Ax3X5cxMRKhdY4QU3jVm3nz+45nr8X78sT9TxA5EZOjYwx4JVYcOsj7/u09/Oryr3Dln55G54pyyaNW60Ynlk6IFjgqIIpbLNljKWG7w8ZnN+Dk4AqIozKt1iuKD0yi82+hjWB3Q+lRDTT9i+rMNpt0Oh2b+kKRpKJsglCWphidYYwAaTlvWmdIYzCy6Dczu7CookHK6VpwcSh7OZ10t2+mozogM2vNIhwrWhESoy1UjxJFI2vwA5881/QvP5QsERjlEM5upRyUaMeGOGkSTjaoOL3stc8B7LdyiGY2wS0P/Jma28MRhx7Kq155KFPjOxD4aO0ThVPUF5f50Y/u5MWH7k93v8P0TMjsxAgrV6/E98s4jkfgOkCOF1jhH8ph14jkj9cLUA5a+Lz2jYY/3vYNhjdvp9xVQmR6IXJxwYBfigUID1cgtIPILU0ozZYSt9+AkAqv3EWefgPXKTaGgUeWp+SZRqAKaoP1xatVq8RhSJZlLKSlFYOE+Q2gMQZHKdJs3o7FXtuuF6Bc1+IVuaLTtjZtUgiUiklSd6ESKKeJUhnGKJauWMNMs03gO7SNZo/91tAcnebIA17KHmsX09Pl0mnnjD31ILt2buVPj44RlQxLFw3iK8l0NEWFbtbuuYpNG58hbDeLxdkh0xptMoTWOI5LFEXkqbYNljT4voMfuPT096ETzcT4OBhBo9W0GzSxgjh8Da7n2uSq/LvEkVoQfzh+SBpHqEIAKoVCKEsd6equ43keAgg7ISZPSeKYJMlxXOukYCklmVUyFAgFFCiCckgLb+KFwilMEZdp1eNJkuJIh8AvI/gIyhXMzs3S3X8zrhqxU29haSZzUYSWHXzpkOaCcq2OElU6czOYTBBnTXp6FnHUy4doN1v4gUtzLuGU132YKJnikUceolzuYm4m45lnnqZarTI7XWPb1oNRyiXXGcedvI04nGS2tZj60D689U3H8sYTVrJ9+yTnfflzrH/ucdCGJEmIk7CI1rT57rkIwF1DZ+b1ZKkiTVyMvoyXvezFrF//AGNj45x99lmsXbsWx3E4++xzeOMb38BJJ53EE088wSGHHMJ73vMe3va2t/PMM+u55557ATj33LO44IKvFe+rtUKbF91orbn55pu59tprWbVqFe94xztYunQp69ev59JLLy3+3k4YV61axWc+8xnWrFnDunXruOKKK9i4cSPLli1jZGSCLMtYsmSQkZHRIqRDsJD/Zwwrli8vggkUjqOYnJoiikLed/b/8pv/HSPXhjzL6V/yAGPbt9NuvoI4/Mo/1Vb+0w1lV99XjVQOcXgxpoCusizF93xEoWCaN4O1focJUkLgJgSlGNe3556SNpNYG5AOFv7RBq0hT+o0Z9tIhZ2EpcXWQMzvqIsnXcDdC3DNwnSq+KoY6dh/BIcc8RJ6Vq4hia3pq5IBUxMRSgZ4vqDeVabc3UVPMIheZOh5+M8cNfEMOzs+lz5zPCgNWvHKFXdw/NoYnbUZT2vErmKqP+SoyhBbp2cYlTGNnhIrnUVUNg4TH9THVI+LG0aIso8yHt2+pLtcZkpBe6YBsaS712V8UhK7M9RwGJQlKqpNJ/CJpnPiHh8RVBgeHaX53DCyJInqNaQU7FNy6K70Muq3mWi2GPJ6qdJhS5ygYkl7eozxdooyNWbShNXaYcwTTLcjsk5EHtfQJqbmBaRpSOwJTJSTTmbM1GNWqCpZ6jMnIlyZ0usrwsQwjqY7FVQENAQIT9spZtYilz10uR3cvMIeh76euekp8kQwPbOFJN+CI1xM6pDmLVBQLnXTVVFUK92MT8xgpLHJPHkO0iGKc1SeUg4CW5iMQOcSqTImdq1ix3MnLSwmleqlrNljMUrJBYWenF+EFnCNAvITBoMGJGEnYds2C01obYjjuYUCaky6W1XyvAnn33/1f/qBPZTj/s1Ndp+z9ry2cHSeW589pRRJYnN4dx+7F89/5rATWmkVuIUrgON5xJ3QNgRaL6RTKVeBcfG8M9HGWC6w+zve+YHX4fqKRqNJVOlh5fJV7NG9hF1JyMzUGKFymNhueOR3gpkxy+0JKiGucw2tiad4++kfZu8jDiVTLpuf2MxRLzuSjbuaHNyd8ctvXshDoy7kbRApYTsiiyOEiNFphuv65DoijhO6u+ukWUiSJJQqNdJ0iE7zpIWJn9GXo7WdtAIoVWLlii7caplmKyFNO7iiMIXHTu+ktNuLOErxHYnnCDphRBxFZFqTxBm5yXj7297Kq1514sJkzvNL9PZ285oT/52Km9JOPfr6evjZL7/FVVf9gD/ceD3SUZbf1umw115789lPf4p5AeHSJUu48OKLue++BzBa8+rjj2X9kzuZVVDtKTPTnkJ5HsuXrSRuT/PUQw/TmYzp7xniX055G7M7n+WWp+5m5bK9eNupr2afPbuZmpzAUYYsTFm0ZIgvn/dD3vkf/8LgYkmWxzi6i1bUItcROovI86xodG0zqIVmw3bF736zEi1cwijmi18+jK1bb+Dbl11ItdpHq9O0/EHXJSwaPsdzd/viuhITW7Nw4SjSTi+d9qkox2FoYJCp2a+BsrY1wthrLygH5GlCc2YGU1hF1btqtNudwsljdxrUvOofLM/XpqVZSoRN9ZKFeNHBCzyMdokjzyamAUq1SdJgAe4vVTKqVcXE5BT7H/Ziwk5GGMYsHehmerrDkScex+cuuogbrn8MX8zRXxmnuW47d9zwE66/9ylqqw+g0ZylPbINlVXRrsJRsNdeezAyMkIapRaN8T2EMARdXbiuolorU62WWLbcOkgoGRCGmixrs+G59UVEqEuz2cR3XXQ+SBq9tqCTSLLsCqDEfACBEA2ESTCFGNRCujmVWjdBUCFOE4zRxKG9vsA2sUkak6YZSnpFupF1EpmfnOe5LuKTeR4SZD8NKQSeJ+mEYZHeBaWgTrV2LmmWMjszR73nVpJ0C4YUV2nyNEMGJaRK6evqZfv2ERYNLcERZaJ2A+U4zM3NsXzlWl5x4hKicAYlHPr7FtNd3ZOR0e00W9NEUYvGrEeWVDDEbN4cMTJ8tPXPFZKXvnQ90+OPMTcLO0ebHP3yU1i712Hsv4/HjTf/lGefeZhK0EWj0SLLO0gpbJ3NUhy/Rq4HGB97NUIE6LwGfB1jNINDgsD3McZGSHZ3d1OtVonjmFKpxJYtW+jr6yeKI9qtNsYohoYWLXAw562FhoaGbJMvBHNzc8zOzqKUYsmSJQRBQBzHTE9P02q1GBoaYnR0lEqlQl9fH8YYpqenF3iW9rxWdHd3MzXVYPHiAcrlgnIiYGJ8klzndHfXmZmZsb1WZuki1WqZmZkZEPDpi3/JVRdvY3R0Eq0TFi15jK6aw87NBxF1/rmG8p+GvF3p0mq1yNMQTEJBASfLDI50UVKBTlFS4biSSqlSQByaLG0Qh2EBpVi4SAiBKuTbSkGWKYyRRYHLrMKu2AEX/WOxQd39uuanK0V5YWGZNkXBAmv74LhMjLWsl6ujESoiqGZI1USjmGxO4Axn7KqU6Bmvo3uWk2+cpt1q4yMhd+nkOQPtCr1TPoOJZMf2WVYOVTlitofNIyMsVilHreom1C47NmwgGYtZ2zVA0ozpSj02TGaorhb7m36GR7axfe863TMx+482mA0FotsnGEgo7dSYUQe31GGfxT1syAy9SyuowV2c1OhmV6fC3D415HCNQX+Gzv2K8Z2TvOhlPvv3LyW76TmiQYHsL9PSHV6xdYjZkmA7Y6xpJawJ+rg0mmCRjnlF0EOSRTw9aDiw10U/mbGzlHGfzjhyVY1gDn4jXLpcw6ndZZ4cy7iXUZYqxTvkHjzpJmx3NH7WYJlTY7oV0y6vpC9pEpeqdGQXKtZUlMesCZGOIgxDqn6VJE/xvG5KFZd2K6OV57Sb0yRJkdIjy7hAEiV0eyVSmYB2cH1F4Lv4Xolch9RLyxjZKItEIsGFF1/Hae86gAyPkutZq6EMJiLwdIvuuiKlhI8u4vIMRrs89MQmPvjeSwBBs5myY/tNFroxBmPGkcJGZBoDbpHDbqlnu1vDhc2Z2d0cavPCeMO/O3bT11hodIsCoc3zkl92n94Lwot/5ph3QXjeTxBG2rxaqTBFdrcQBd8Qa1vjup5tRLOM0e3b2OuAA9mxaQtdvs+mdQ/w+Og0Q3vtzdj2LRz4spdw25N3EydHUTjKkKSGN3zyy9xwxX+x4dENxK06Ndfltce9mj/eeR8TzRoPbX6QJ59qkfvdGL9MpzVHV2UAz1dIoTFGkCY55ZJHlxY0W02yLEabjDj0kGoRGHc32UE4xftk+YFSKhqRIO20bC0SPomUuFIiTG7NnzNBnKV4IqBcDoizjEwoZKlCxZEs8h2SKOT2W+/kd7+9ibKy0YsvOfZ4Dth/76Ixsc3Kf57+Dh544HFc38UNXMIoQQmFwGF4ZBfnfOpTNFst+vv7ueJb3+KpdU+hXEWegzPURU86xPTGdXYzJQzKUbTjBgND/Rx53DGse+ZJGjsm+c7Pr+DNr3sdbzvuNXzv2p/h9ij23LacA/bYA01KpzNJaWgRsdFMRDM4LZ/mXBsjExwVF9ZjEqWqYCRxHqKsfSxVtwwolOPR0x0wvmuY7soSktRKnqxR8vw5Y23aKGhHruuSxZlN0BGQ6QSnsK0S2MjQONWIXKPzBLIUKRw7PZTguj5pliKfJ3CzKtvnXQOiuJIMmDy3PHLm+W8CwXyiWkqrGYJ2rVhMzs+TZYFQ2M1EuVojTq0FEbnDnMkYWtzNxFY7wdtrzRF89sOX4blzSBcqQZ0N6x7n8edGCQZWkjQa5FOT9FZ7SFKIREauNWPTE6TSsGb//RjoH0RK6wEJVlDRbkdMjTUZ2bKBucYsWsZkeZuok+GXS9Tq3czNzNLb24/QhiwrMxc/v5mzDgPzCXLIHKGKDGpluZGVwAfp0IktZUDnlv8olbEG9caQpTlSQJ7FpBgczyFJE9tIFouoLugR82vq/EbdIAg7lrMblMrEYUSapERRMUzyFSuWGHoWDaGkT1dJ47kdAn8Qz8sYHCwx11xGI+rmnr/uIspL5HFkxacqZWhxH4IyOnfIU43jzrF69SCl0iqGFvfx7ct/yY4tEyxfMVhwsq3GwHVc3v5vp9Kz+CTOOfu7DAQBjz7+K27/44844JAD2XPtKuJQolQbx0tJwwwpA5QypEaQpBpEah0Msozn9dGMj40jVeGWUTR209PTz6+uTE9PLXwHPK+ZfP7PxhaGG/MFOs9zduzY8Xc1fHTUBle02x3a7c7f3FfBD89zBCVcv8rktMBMptT7S3g1n56ldSuOy6FvaS95YpidCMlymJkz9AytxnElqncNcbge8gzPkxjTYmwkJX9+9O//w/FPN5TTk+NUKhWq3V0oF2ShTrNrpcH6mNsdjQZSrTFpbkeumQvYfNZS4GF0RJqDjiSGzCqHjSLL4gVupdF2UAtiAV6xJ83uxXu3PtEez1845+EYY2DTlm1kagJETJ56+MEi/JKH6yuk45FmIUFJ4TZSWrtG2drvcdiAYmQ2Ys4kmEzSzBIeFJv44QM7ODBPiVZUOXltDzesG2d2ERxWrvLoE7uYCqcYeEWJ0qo+Nl73BI20yVx/ytGL+igndf4yNkczmMWblIQTIfc5ig15wKGjTcyWPXhs8xidAw1DS33ufnCUZ4zHHk+v45TeRfx83RbUSzSd2YwVT7TYtrlBS6TcFs3xxk3dLPvpRppLBuk+tI7ZspX9W2Wuv28bK/q6OGrPbry0zbrxiGPqdU5ZtpKbHhjHUWX+vdTF3HDMb58Z5Q1HvZwPVHayvuURhoMsK+esWlRm0WSD5TNw6EGrWDs3xR5ZH/s2GtwWjHDckiX0T+c8cdAKJjrDnBoO0Spp/jDcZEZEOI6DLzNWLq6xZGA5UZjTaUe0wyk6DYnnlFAlnzBsUe+t0ul0yGKrdrPNRUZPuZtmIyLPMhJhaDVDXKfE2PCs5f1hC0pjDh58bMf/j7b/jtYsu+s74c8OJz3p5lu5uqo6q6VuSS1aOSGJJATCZIMwwcxg8+IxM/Zgj2c8g+1lzLJZ9hiMDcYBY2GSEEIEAQKlltQtqbulzl3V1ZXDzfdJJ+7w/rHP81R1g430rvUeLXVV3fuEE/f+7e/vGyiBbqK55+QRBj3NuTNbnLr7AJMKxlc26S91iOKYzc0d6sayNxqztVvT2AZv28HaBlNgsAHJdBYdJwGJEa3Aoo2SnyW+zHl47QDu/A3rG8GsuPN/7oad3c/OO/RcyPAigs/NL/5yH9v55lokUrS8WKkkxpmW+xaeN6mCetNYB4SkH6zl2qWLHH/F7fSWDrCS9bj1nrsY5Tv4yJJ0b2d9aY1XnjzFw6clsjW4Vlryx//+PKb6Vp58sub8uQ7bu9v8yccfopyO2do8h5ALeL4RimD3oZRkNLW41uR9xh3db21dtA72QNY6jBBImWBdrz234H0GVHzjN34t3/Vd78V7wee/8AQ//2//K+/9pnfy7nd/NVGkOH/+Cj/9z34eYy3/8l/+Q86cOce9997FBz/4x/zBH35sPsbP+LNSimCl5VPqttB5+1e/gV//1Q8FUaLz3PfKlzGZ5ly9tknTWOqyIZYh49sKiUdifJicX/7yl/PFLz3OaFSEfVeaQhxi7bZj7G1P2dkccuzkKntuj7TXZ3lxhavFhPUTd0JyiezSPh/4rd/ibW9+Gz/+A3+LX/m9/8qZJx5j8va3cebscxy98wg71ZThFHKpOD+9jmk0xFNUY7CFQAndIlEGHUnGk5w07XAh72OFwhuLUtD4kvH+EK0zDI4kScjzvE3ZkGRZRtXUNI2d241Z79HKIbyktu2z4Bx705KlY1+H4izD7edRUmAbx3Qy4cixUywfXmPrYoiV/eZv+To+89nPMRmXgY5iPJ4aJTy1MYAEGaIuAaQSNFWN1B7pLMO9Cd2uIp+s4e276WY9EB6VPsxo9z5mba/1ozmnn/0tuv0DXD+/iV31bFxPufu+W7j9jjv45Gf+kDT2bF5u2JtKhlc3mVaXiPSAMvcU5RZpv0PRRMQDz0JjmRQ5tjZoFBuXr7K7sUVT15TTnPE0XPNIKZS2oDxax8Qqw4oVsoUK4x3FtAznsg6RuaYOdIAQMWpZWn0jzr2l7TDArS97kjKeUg4FW9vnA/fe9oMNW99irUE50P46sTH040U2dna44577uO++k/z+h/8AZz1HT91BURQoqefiwJmhtlIKZzWCGhFJemmXa5cuc/D4Ub7u3e/kiw8/yOcf3sJYR6+Tsb6+wN/80R/k+O1d4qhHJjRIS10psiSmLmpO3n6CT37m07z/V/4uhw/eQe5KxqMRr3rVN/NjP/JPePKJ5+glaywuRJh6StmUJPEgJOzZD9HtGcb5Ht6n6EhRVYYsilhbPUGmrrK+lLA5bLjn5a/BnyzZn3g+89DHWF6JmE5LqtwRJ2nwWm48IpJgPNZ54iihcsGZ5saiRr5oke7n/7l5fP6LBu2/4AVf+TD+F30JCBn8QfsDGAcrJCklWgu0DRHCWkuk8jgUdcuTnH+SDWh/uTvG1DVCeJwBY6akSY9SfPk7+mUXlFI5alNx8q6vw6UVxSinMSWmmSJMQ+0t0loiIzESUmkRXuK7Cbqu8GWCVI7v/Z7v4swLn+DM2ctcubaI1x7bTLFGo6KSpegcsaiwTkElqEyJlAERstYSxxHW2jm3JmRXRjgfDLi1DHwOpSQGkNGAxgny7Q0av08UJZh8l5F3qLhLmhwgSgbIrGBJdLArCXHa4QudTfy9K4hHOkSR4Gi8yPf/33+blf4uUi9z8PAqz26dxj59GhWfJIlSihfOc3e3S+fkApMtR35qzEq8w11HFsin+2xuxaTScHKhT3J0jWeeOE29a3j5kiFLFI+evs7Sm2/jxDuOcvH6lCt+m+WdXV711uP83pcmPO4v8tWvW0fZAU+sXOD57Smb1ywnv+YQ5t4lfu/JR8jWVlk/ZTi7k3HtkV3qpYLkjT0ef+gLbItVrmwPefORI/wfz5zjU/sVrzu5wEce3aSqarwf8ctf+gSvWV/gn37pWV6nB7xitcdjz3W42uRExYS3RSfYH1f8eH6eWwvJX737COXZnMe2BHfde4o3imXGnzvPU9sXeeCvvA3/njfy0U89wsXTY5ZWDMPxJnm+x1LvICtrS0Go5UJkWy/p0lQSfIRTNaUviaIMVIS1DVHsoAmRj1lmsS4njaP5KkNKyWQ6ZTjqMPJTpgiyxrFyoIMfeD772LOoXHCgV1Fe1PRXInb2C0rb8NnHc6wPhHnnJb2FHyGJ+ySJ5sd//DAve9VxlhdXuf9lB1vzXAFOcXl/Gz3dZzIes5enbG7t8PnHPsvzL1xhvTMgU5p9NJ1DB/nMpx+iLiRJtEjtLU7kdGJFgiRqCl77+pdz9PAS3/2t30qZpvzSf/oVPv8nz/GJTy9BywE7dfvnKcbPMBkHPmBtqmAZUdRIqanrGmfCBF41Nd/w7nfTX1zgV9//AZJ+htMSJjmmLKGlBYg4ItVRGy13BDeSc4pJnGXsbe6iG8Hy4gIXLl7kSxsbqE6P47ccJhYZH/mDP6a7fBjZiqAQMOjFSOlJBn2K8YSqqen0+uztbdNNexw7epiNrR2qcua7qDC2wbtZjjlzZMu1KIw1BqV1y4Vr/VLnA30oXG677STf9V3v5Ud+5O8ymTgWlxcAz59+7LN86MMfRQnBj/7o+3jb21/Pn/7pp4FQOP7PP/IPcM7xgz/4HZw+fY7PfPoLN4pt73HOoqWk36ovb731Fj7/uS8BHq0lP/TD383f//s/zXd8xzeio4gkTZA+cDHLskYriTU1XsCb3/QmPvaxjwf0Q3ikc9TFiKYyLHR6bNa7TIqS1RPH8UQ8f+EK3jekaYdbbruTenHI8nOn+fgn/oTrl0f8yLf/CL/1e/+ORx/6BErFvPAMbJy/QJPXXDr3LKu3GapGI5s+hdlHq4SmscEL0dYISRAyuJrpcI2iqPB4lMx4zWteSz6UZNKgU7CFnfMlZ5ZBQVMkQuSos0gXvHG1VtRNaCFZHzoCyhdMxlfAOpxJiRUkvYjhsMB39pCJxDqHU5bhZMh0WqF1Oyb4BqFjhIgQSmJNaNMb2xAREacRSseMd3Y4euQWltczHn3kOsJ5yrIMvErjkIJwvznH1Wc/x+pijBYK4TW6UkzqitNn9jh/8SF049jPd5nWQ7RZJmKPuixxqiFSCU73aGzBwQMHyXe3aaxFikBVSdOUyXQUnDkQxEqzuNLHmKq9f0P+e+PAUGDsPsomqNZlQ1rLidtu48DqCp3erXzodwCvAidXdhCqi3fB7sUQ8/RnPkoSxTgbitZ+vw846rGjLCyemsGBdcAzmm7T6S5y250vI8sGnDx1K889cxrra8pyElTuLbrsWgRZSokXEtMUOKkYjfdx3nLs5CnOXtxisHwC73cDp1NFxFGXVB+nmyxgaairGhEJ0CmVjphGBU9dynn23D4Hjx6n34+Jk1u563VvpHvwdn7s7/8HTj93jovnL/AD3/duXnHPcWS9z/7eiL29Ids7e/QHXcb5GEQP1+a6Ozyf/fxD3HqLYa23wrNPXmC4nWNwLPcUadLBNB6l6oBQNoF3rZXGlkNQDZFeYdQ0eK9fVLzFUZe1tQHj8ZDhcNTSMW5a44tZkemQMsX7RW5wugE22uHKIsSNNwVQ4kZHKhjeh/NvjW0//8+Xa0p5jp84BTrF1qYFO0LBm0hBpGX72RYvQPkbwNyss6OVQUsYJOX8GJASYQVlNf2LO2v/ne3LLii9VzSN5fLzp0nTIU7dOPBpY0gaCwoaa6m8pWjKYC+xr8lkilFj8kkPleygdUw+kYzzS6RZH2cbss4CUkfQCIytaYwglSlp2gkGr90BZRmiAHUcBpQZcimlQgK2rue2OtZadJKxfOAwSbZOesuUvd0RO7s5Ds+hwwsYX1OUOxw9uoizPRrliFxMM/UMT9xClUfgE7zV5AU8+NiQ4yfX2LiwwZkzD3Pp9OMUwzHpwcfxUYR0QYThS0ssEnrrA2gK/BciGhwH1g5SignufEH87IQFlSCiBQqpsHaR+PYTmMzy3Kf30WKVl7/+FJnS7LqM9Td7/koHhkXJepyyejzlFe8oqSiwlSfza9z1Y29m9+o2yWbEXa/oc/t9Y5T1rC+t0n/FiCPDPV6ZOdRUQz3mr6SWk8tHuXhhTF5dZSFOOHTwIOc3x3zPG+8k8oL+wkHyjQ3YnHL/nWtsTSY89Ox5lg6+hcXFlBfKfZ7cK4mO1Hz+4mNsXtwPN2qyyJGDGcOPfZA333E3r+mcZG+0wbA8wdLqGzh/wXFl9wz9foZxEl1neO9D26tpSJIIYwKfSXmBShOyLCBoSnbI85KFhTWm+4FfFQjGmjSSjMZ7ZEf6aOMZOeh2lykmu6ykMSJuUMkCBxYto8mQlfVVrFtmd/8zWNswSCPq0iKjiP7iAkIKjtz3euLVHgcPxSAUxkiEMxjg4Moa0eIyRinuNI5/+S9+Fpdbfuaf/mMWVhYxlFw4/Rjf/9d+gje97a3c99pX0+t3GF68zJGFJd7+Ne8kl9DxwZg3HSRYC8V+yYF+j8OnDpF8jkCC954H3v4Obr3l62msRSNovMM2FYI6pAx5jzEBWajqmvFkiNSa/+1v/w3Gw30ub19jd2+bJi8p8pxiNGG/ymnKhqacUjYZ3jbMhpu6yrl2+QrXLl8lkQ5ZljzxkY8E/vE738npp77E+/7uj7F50fPFaIrSDmsFXmpkFIrB2hpcY0myGFtaqrKgjiVr6yvsbm9TVQ1V1bQT5Q0xQOBjt/Yd0IozQkvP41uhk5rvq5QRr371vfzpn36SyWQKZEzGU4QU3HH7CX7oh76DXrdDt9uhqur52Pbxjz/EDJb8D7/066F1Ou+0hqJ1Viy/+S0PAPDgpz6PdQ6F5Zu/9T382Z99mtF4iveeJMtYXFlitLOH0hqhBNZ5nPX0Oj3uuftl/NQ/++ct900ilcTFFVVRMDi0gr12mo3tkhLLwuqAtJOxt1dxcKnPodXD2Ds1/tV3svCxz/D4U0/xn/7bdf7Kd30HDz74ZxTRiO3yMkm8RqEaNkbXGW2UdBc1wiu66QAhQ/QpOiVTaRvE4Cirbf7sU5cw03cjlWK/cnzk01c5dCKiSWMyX1Eig2CKMMYGZNK+2NJMa6xxgV+JoZ7V/LZkNLqIMxOOn7iFW0/dQT7OefjBT9HpCqrRNDRSlQrJJs4SJTFxInF1HHiSQgUlektxEe3iwpgaKWPqckhVF7ziFV/FLXceZ2n5Gh/9fYfWEqUkkVNMjaHb69PrLvP2t72bo3clfPQPfpfHP/cocbQMdkK948EruioG6ZGxJ/JjpkYSRX0qJGXT0I8WmOZD1gddNuoJk+0ClaRBoFeVCOmJ0oi6rHDWIqsGa0Qr+nM4XxHHGu8VkRrgZYMQHluGJKwrG9e5/c7b2N+bIsUC1jV4Z5lMdxDs4YBaSq6cfxKXbyG7fYrSo1LJ9niDw0dPEntN1hQoKSmmBgxY3xAnEV5F1EaxurrOaXF2Hg+cJbp1BFDINlULQKoILQ6Q9iM2N64xrh1xpjlwdJntK3ssLURUdUirq2vD9Y1LJP1dalvTy2A4ViwMNOcuXODXfu1TWJtx/OSAA8duYW+0R7NXc3b7NGe+dJqtvTGDpRhna+riOnvXQ2EcJQ0be5c4dOQ44/GY8aTEudBFEEIhvOW5p8+wMwpF697wAmm+hMsq9jc0B4/EqASwmjhepHKT+aJRqE7w4VWi9Riddb9+AIFh7dg29z9wgL3da6wsL4NvSCKFVKAj0DYl6mjy8YRff//nqN3XAv2bKymSVHLk+BQdh9a6cx4tPeW05NqVq1SVYWX1AFGWEcWa7c2r5CMD3ImUMd5H6CjQtV75mj533ncKESkund5iPNkl0REWw1u+5iTrR2O0AXSEL0oKB7s7lo/98Xm8DQKsd3zDEguLMc89eQljGkCG8cEpslhR5F8WfRL4ShBKGeKhinJMXm7jUAhrkL6mkTWTJkzq3nu0D5O7lwJrcybJhIVkldxXmNqjhEfKPbqJp5OWxCJFxAWVtXSijKIckmUdYplirCQVpuVjKqy3SDzeh6rdC2iaChVpIhVgeq3CRBuUvhHWCKLFNRaiNXQvxwnPzvY+WI2O4OqlbVQKgyilu7LC1qXn+eTpCUIcZLi3jvcO6RW/81//mOWFEbvNNZazHpqIsajweyXdZI0kjRkXEzqNZtJxFBslifAYs4+XCXsb1yHqU0630VlEVGhymyOilOWeYVhKRtOCW1bWKKRid3OI1YbEhRWtcJ7ClCxHC4wSy4GldfKiotgdI2NFt5NgrSWvIZICkYKpYxYy6CYxE9ch7kk6UhEtrjDcu8YjasTiYAnBEme292iunSdTET2xRNEx5BsXObJ4nO6xiodGJd3+QY6+/BiDxT7dRcnlyNEZOlYPLqDylOPvKCn1IoOmIF9WLF7d4e7bXk7vtSusrq5y6MARPvmxz3Hl0n/kW77+TQxHO6RZhBER3jVUVYFEEcsEiWL9wCq72xtsTTUrayl1M8bYkiTt0NQaRYenHwmFJkCWpQx3dtnOQ4xmqhK+dOE6fZdxcEkgo5oLFyWHDgkgpdjO+f0Pf5Ktx55mtR8TZxkTZXE+FLreeCabOSs9C+YAz5ZT+j4OVknO0qskk0RS1dDzkr/6v/4wv/yrv85DT14m6jmO37aM7hwjSeH7f+SHOXTLMVLh6CuFs4GXtqRakZowmNoRxzGVrTly/6u5UA9I4nOYtqhJ+wfwsSBOIpQBKX2bptQO/rScNRWel7IsSJKEWDtiKfGRIooUTREWfNI4tvMpyuRExvPk03u8/79W4BxeCL7um/4GB5dqlPXsFTmHjy/xwAMPUDe7bJc5X3XyXfgrQ4prAu1gcdlRV3DffWOK+jwb56+g0110R9Ms9TixsMLx46uMnSQynle9+uV0E8VgsMLG1lWeePwp3vrWt/H0M09y5cJlytxw7uxp9s2EyThnvDclSntkWZ8oPoytX0dZVfT7PTQp8qaxT0iPVCVg+Yn//X/i//g//wkbG5t867e+h4MH1jh2rE+SKPqLitUDWZhQJHjjaWzwz9Q6Yjoas7K0gK1q3vXO1wHw3/7L+1nQU4Sy3PGyO7n7npfzPd/zLfR6XZx3DIcjPvKRj2BMibWOunZY53nLmx/g8194hLyoQIXIuzhW9AZH2dp4los7Jfg1IrtHZj19lYGC9fVV9rYN0/EuSawRA8WJd70FIQRfePo0H/yDT3DqxHEml3d4672v4Jlnv4iuBXa7Yri3D+tLGDMlz2Kqekq3m1IU2zQ1KJUwmeZ477nr0Cv44lZoM2sZ8Tu/9m947WtXOHngIJvb14njmLqu5ygyBL5voChJBAZnG4x3AX3VrfmyCPSkFI+IMr77fd/D5595hgde9wAXLj7P9pUrrC2sUEmLkjEgcZbAY7YOvA6dKk/r5iLmhWySJAiRIiXsXB/y1je/lWzQ5fEzT9HpHgzRht5TVTWN3GnFnAIhEooowUY9sm4P8OhuhZ3mdFRC6ROmfoqWApNHFMqTJg5vhygSnJSUquFlX/Uq9vcrVo8e4dQtp/jMZz5LknVReEzd0NQNEhlEeZ4QdxjLkHvuBd3uAKmDyntcVRxeO8DW1U2ee/4sx++8A5+lbJzZRukVMA4RJUQ6Ih9NEMLROIGp91lcPo7RNb1uQpIkHDl8gv7gIF44KldSNA5x8TG60RLXNxsaUdBfGtBNuljjkVT4GrwhRGxaF7oxWtAUDXEcY/wEA1w8nxPHGeVkj+l4xHB/if39XW657QRXLw0C10/A5Z1dfJrQVJK9/U1KG6N9RWP3INqn0xPsbRVcvbBB0o2ZFptYUyM1LCzHdBYNpoy5cO46ZlyyP1SoqGaaF3iXYk1FEmfsTaYtSh6oFn5ac+bxZxGxZm3pIPmoohl5si4sL60zzYftfDNCS4UxDo9FqITaGrRULeI5QyAlHsWVs0fYuZpQV8fROgReBEpuy9FtuxpKZjTmHcCNqN12VKIqPS+cfunPZ9sRALZvpl2yNv/bLB+8qcN897kHSz734NNzzWjYwmL513/5NGF9LlpePwT6iW8XzWEh/rGPTIhiiVT2BhrpwVuJlQ0vZeH/j7Yvu6C03rQk3gIVT4gbg/EOp2Jin6ATh5Mh9zOyjtIYFBFdlTDxNVW5j7cJUoKrLL72jHauUY08iV+koMEKS7QsiFRMZRqM8zgXclOdc6hII73EOdOuTIOZbmkMygV/wqYuQyvMhcGoKiqm5S5pZwEpFoN5cGrBSc6dPc1iv8fWzoi3vfH1HF06wH41YddPOLASTIGbcQekwXl4xavvIqm3WGKJrAtXz11lfVFy9NUDDi9mCJ+yv2EZmQib5zS1p6wzeoNlynIXbIawFTo2+EygSFgUi+TlDs4fJEkqDomUQjhi71noJkjZZ6AFSXedzeFFumoZqQXrWrG3cxXpOwxW+vS6KcNhQVNW9PoRRW3p+AV6yYRRlePSHq6And0Jo2Qfey4Gb1Fpl+v2BYQZglih24fNRpKP9zl08CiLETx7/hIkkgXV57I37G2M8LHHpZ5OnSCaihxPHWviGoy1ZEmGs1PGDTTuAwghWVvpcXj9Trav7vHGdwlEWtJlHe9KvLHE8YA4XqQ2Bi8cFscL1zfQUUoU7bO3ux0M2elTTxqyLMIWIdFAyADfrx9Y5JWvPkDpoOendHTEIxd2uPtUl1pLXC257VhFpFdA1Gxd3uXdX7XC4Ju/k7/xI7/EcCqoag/e4oVECMsumuLRZ6jHW5zZ20XsKo7eewJByrSe4CrwEkSd0Bto7n/1G3joE3/KwoF7mfAyDq4v8vO//sfs7e2g98CicNKQxp4sjrCVpaMUTmkWfcUffORT2KjDp3/td3nquQmVOY5EUVvDxvXLRFJRC0cXTeUtsQpJ6cY4Yh0FcvrMHN47iqJAD5aRTQNaEAmJlgohgmWOSFNQkkxIbLqE0DvMEidWTt7C8VsypIRl4fH5mEGygEBzMpWoGio7IVINn06u4b0g0YKX3z/glffeS9rRXD17moce/xI7Fzf43u//evTKOtNCIusCRMzC+nH6vRWOyoY3fPu3s7e7xfHXv4xuHJOphFhDJ5GM93d4+vFHeeLRz/OJP/sEX3zsCzT+JEppFhb7TCYFjz3yJf7h//MT/OZv/C5V5bnlyCpFM6HTSbF2RBI3vOF1r+HZ504zLKdY55gWBVWbu2yMRSuJtDVeKpoqp9+PWF7tUeUFx285DsBTj34B4SFKO/zUP/5nTOoOIPihv/6djMY7/PYHPxTiVU2FUp66MVgneOub38yHP/z7lKVp06E8dVPw2Be3WV0+zN7wYRK7zWhzwtrKSb7pvd/OZLiBM46qqXj+4nPoeEA39mztXYXDx1i5OObCc88y3NzglpO3cfXZDf7B//aj/NTP/Be+5m1vRInrjCeSYdGw1FvBO0WR15BJJpMJUiuipYyqcfze518gju8OY6hxRHrIa776nZw/9whq1AURUEipQm6xtQEpbmrTFqECZESsFaW31FYh25abU8E799777uF3fusPue/+B3jqC4/zNe/+Wv79z/88i6mEaY2ICXxlrYOjgzfEUUhbiqII4cOiKZ8GKoQWOhieG0O32+XixfP0lw8iooTGKKIopq5qVCQh6kApqFuF+pOPP8Vkv8PVi8+iEDB12CgDEbEgM6zOsE1OL7N4NIW3xE6HRCEhkD5if2vIgaVbGO9v89jV54j7XXQU0ev1SJKYxcVF4igjijLSXspwf0xd1xhjGAwGyChQVfKiZDDosTeeYILyhqTfxUiJzrJwP9kK4SXjyQTpCbnpKuKWkyeImg3K2lDaMTpR5PlVtjeepsyb4O/rGvZHUyJ5FaET4kZx7doWRf8y4+k2nWSZo4c7we/YWjqdDouLiywsDIJoKlZkcY+qnvKFL36BZ5+6gjQrTPdijp/qc+niZ7jrltfy/LM5pqlx1nDx3A7b1zZxzuDpEOsaEWmqpEYdGXB1d5NmPCRekZhmgkhzUhGB8jghqZ2hLGOMVyyu9dHZmEHvCC+cm3Dlyjn2hyOGo23S5BRTKYGaKI3pHr6HQ4uniKTg3pe/k9GoJDc7dPUKD33uY0ynkKgEbxuqpmnTjnxLnUpp6gWypI9HU1UKWJzXQlXpgZS65iWF3Is3IdJgRfaXbnNZ4Zdbjt20hffd/O5Z2zwEh/j5WB63CWRCCkxjcW0xvLu9h9YhRtWYwF/Hg0oKbJUF1O7L3L7sgnIWySW8wDWSmhgx68/jcDJ8lLYKjyLRQbVUeojRNHgSHVoYjWvwtSMBUt+hIid2AucsSvYwYp9IdMMkWRuEBIcJA1dj5kbTENSpxjmyRCOkp6jD63SqqPIpuxtniaKE569HJCsJC4srZPpW0q5jtb9CNdojiWI+/8hpPq8fYTIuiXSHUydPYuliXIWzoKVkvDVly2wyqfcR8YiFA477H7iVu192imLaoJQiTW7BOIsxNToKfBdnGwa9B7h+fRPhBGmWUBRTJpMJQiiKfAHrHcaEVbhrPEmSohAkccz21i7D0RXWkz51BXGs6fZiikZhXB2I8TuaA0ckvd4ao+keSoFONHXVx7tFstRhjEOKJfJJh6Z2JEkWTHWdoMpXsNbS6y2g44SdWBAniolziCQiTfqUzjEd51hVcnjtAMI70jhDISmwlGXNaLRHqhxp6pmMSuKm4Oj6OmVdYsuC00+fo9uXSP8A+7slebFFrzdAe4EtA7LR6/dxiHAtI0ecWJaXDjMc7eG9o9NR1FVASnQWzR8r5+D0ruMdd9zCc49e5AMfPUN3ZYX+suPh33yEN7zpXu5/0ysZFzWNjNndrylPHmP97vv5g9/6EvvTGkFQPnpb45mC97zw5FOc3HmIy2uvpafWqZdKnn/0SVaXOhxb7jBYSKmdJZcFopSoSvPqe+9jbzpCbD3J1XMJZ7s94gxOn7tMv3uIwUKHvf1NpF8g7mh2qyFvuPVWfvOjH+Fjn32C73/N3XQe/hWqJyom+n3IQY+k22V3vEeyHUyjJ2jwDYgGJTt4b9tnJAgmpA9ihbIsqYebYYI2HmeCMXucJmS9LNBDXLBg2d6Q1K2/HcCFM89RTHyLGgUunLWWTqfXcpqD9cXWRrB3EiK0uYeTgms7Izafvc5wb5dI93jD17yKc1cn+Gs5Wdoj5MVvs725F7ifrQggiiKa1rjXudA6TXoxKwcPcvDgHXz1Dz7At3zv3+Rzn3iK/+v/+iJdp7GjCUjPufMX+PXf+BC/8Is/g3Oep774JX7zl9/PB//bb/AzP/3PGU8nXLhwgVhJOk2D9I6ut3RshTXwV7/3r3L+7As8+vDn8U2Nx5G6DvlOwf0PvJKHHvx0O2gHEZN1hggZihER8IxEwJIKYon/99//Ej/2N38UqRRpt8M9d9/BP/+pn2KxF7WctDABnIy36OplTrzyOFV5hHMXrrKzf4GHP/0Z1k/cgl7okq6s8uYTR9jbbzi3t8vRgyc4+ZqYr3nve/jtf/6v2C9rFk4e4Opjz/DGB76P244/xrf/wN/F7w2p6imFrSnzmlgKjGmw3jPoL2KxOGnpLvZ4/rl/zcNf8EgERVNz5fTzbHzpAqurq9STCUVRhPGwvc+8txjjkUpgkeAt1pUIHxGpkFJkPHNB2HjU8IY3fAPjpuCFzXN0Dy5wbXtEt79OMdpDKUUxmiLTDuVkTKRSGuXAT3DG4mVIkpJCI0UoIvOJwZjAWSyLnCNH7uXYoYNsi5LtC6EVjyQgV02D0oqqKvFlzrlHHuLcI1dBSVQUYbxCmmCzVeoK34YQ1Ba8b5DO0uBAabSMMcUYbJc7XnaUxx7bYnV1jWvXNugNesRRSj6tmEw3USoKyTN1ePaKIvAcJR7TVKSRZtDvcn3nKvU0J4u6LHaWefyhJxhv55w4cA9+HjrgEd7iXEBoo1jjrSAadFDK0rVdJIEaMuj2MN4gZPtaucjVq9e5fOkKuvDIakQlOuR1zFvf841IXRHHMWkUI1rBk0MSxFaOWiviziLf8777+aV/9wucee4CTz72MS6de4Jb77yDg7fegn/YIHVEFGm6q0scP7mOb71KtfRBiS8EG498FE2OzjrUvqCyHid1SK+xYJwhVRrUCKMEa7fchdnaY2F1gf0vPoo0sLp+gLxoqPPgzSuEQirFn/6Rw/t4XnBJ0cXTRUeKve03UJY5M39T33qc+ht2GzinUHJhPrf890C6/5E+8ivT3HylxeSshBQ3OYS8+JsFrlXkh+Oz1gY6ifdY23ouYyim+0DgyKooAA1aD9Eyx7Xj/pe7fdkF5WwTUqKjaN5imx3ATJEqfWiBzHKnhQ98KBUp8rEJPCkhEAoaZ1HWhAHNBdf3qqnpDJJg8kvgbsxUlkVRtO0NP+dOFkV5Q/1pPJHSwXBVMld5G2OobUW9W1Hs5yjfJ13tsHDwGFcnBbF0eFHT1BH9foTSlrwYcsfdh9nZVtSNBetZOjhm7VCG0AkqPsDCco8jxw6zuz3CuhilFHujgl6vh1IpZW1CRBAJ+9MKEXfC/kuHUJajR1baCdpTNhPSNEXJiDROwrHGMaaqOXDHIoPBAk1TU9cltQlcQ+cFUjmiJKLMS7rdLkLAcLRHnAbxUicb4JwnH48DNyYNULsSgqqq5qKDuqyYJdE0VcVC/xTj4YhOkjKZTJjuQm0N1nSCP5kHKTRxnDIcDmkmHiE9zi2B1zSmoCrXqHNJf5BR1YEXK9QxLl56gQcfepjVQxnf9C1fh3WGnekEoSSDwYALm9fpdDJ84okGMVNTMsynyKQh0j2KyCPSBuXBJHbOLwTLlefP8uu/WXBl7yp2YYBDUY16eF3zn37lt9kY9bl0fUQnjijriqzXxVQjLj/3BK4q8Xhs7RCixptAUp7uXKJoRhy0NeN8wqTY49DRNd729W9kdC3n6cs79JMlFhcbep0+hpL1Az0iPJEvKEehUNqZXmdqupw+fZ1HnrmMd4I0iwFLPVF84uo2z79wllfeeYJPvf/foOQFjt13gmfP1BS7+0zzgp7u0BQjXJRQiWAirkUT2uXGtL6T7b3fctq895hRjUoUaZagE8/zF85w4NBBBD1MWVGrMBRMJxHWLjHzejXllCI38KIiwuOKSeB4vbDF3vYOawfuxdl4rmLf39vg4qUtyqoAPJ1+ytb2dTySKIrZl8FawxiDHmuyNG4tPxJcrVpbmqCwzjJFYkomF8/z6FNnKJ2j3+3QJEc4fMtJ3O4ezfVdWFQgJR/+8B/x4Q//EZHKWB8IEuX4k9//I/7wQ7+HXuhR5wWiMkgl+cm/8/fDvezCtPlrv/gfCfFrYW7RSCppqYbbfORDH0DPxEoyWEJZ4/BtWwwP//GXfp1BNEVi8d7yv/zgD8/RrKqo+L73fluw21EzdXJwynidU8jdHKsynBTcf/triLsZRgrEtQlslRTVNfYHKR2tuVMLir2CRy88STe2rMkuT7/wHKOLl3jFy+7hH/6vf4f7lo/yud/5GLoT019eZHV1mf6BFfY2t1hZXgFj2Z9MqLwliyK4PqFA4bzDGsvS0hJv+9rvx4oSZSq8C5ZDUmjyPLTIlYpo7IySFFTtzji09gH9bj1ghQwT38Jqyq/+5i9w5z2v4VVveDs7O1s4uUOSyhCVuDnm/jd8FdQlxXhMIWt8pNif7KMziK0EGYAEaxuiNKGuLEJE4MDUDbvjKVevbCIGikNH7kSIq6EZbw2qtf8Kc5MkilOSZIGs18U5A2bGH57Na7TzTZjvRnkDQmKMAxl4d7ffdSd/9vGPM5mOWOv30L5h6+p5VldXWV1exNg6jCfeM7U1veWUpcXj9Ab9uWBnsNCj2+0ymjZsXLzIo5//AspOWRwsc/utx9jeGGFdlySOw4KRRQwxQkqqynPtmkHoiijUDmGuFg3eg8PO50ytdzBWIu0S5Vjw6Y8/MTtSrpx7EFtLhBTIuahZoqNA8ZFKorsaZxxpFBEnt3HLqRV0LEFEWJvxyMMXEPIYpg5xl/t7U4wriNNgt4VKUNITJRovu1gnmExGNMUEkDeBRbSLU4HWMZNpwe7+lHxas+332d7ewTYN03xKmsXIpMP+qPXCbp1hbsB2HutnrV5u8oaZjVazlKybyjRhAx/cSOTsZLzkfX++CPzzKOPcSe4m553+Itx+F60Y7oZXjccjxU2FaDsICWAy8bxwuv10AcdPWWJd0BmssnTwCGvrmsV+Pwj/vGNxEWxV4GvDZLRHXhqKQlDVsL9/AdPs0l9YoKtL6rqgqg2VMzz3wlWaeopS+wjfCoX+/9HylmrGUcvQURBQiFZ5ba1BaDW39PCtyWw4kaH1Vo8Mkeyg1ex14b0oQhXqgsfZjGQ9rapWoRQKH2tpszVdW/gIhJCkaYppgkEuBEPbsixJlKTT6YZVqVTIqqKuLESWc88/xQl1ivUTr2Bzd4jYu4qMa7xPwqTmUyZDEDYJA6iMkALuu/82Vg5tgwLrLXXdMBztolVCv5+ECb2yuGaMIgreXlWLINY1dd1gsRjX4L3DDicMJ+MgQrEpUliyLMO7gjSN2c/3WVhYwJqa6/u7wSoBGxCnakiUdFs0CqauwJhrpGnant+mNSAuMY0lTgKSV9dDADqdlLLK24lA4HQWig9jSbIuO1VJ0u2yeGANPRqTrllQNc7XGOuJoz7eKdIspm56ZFFCUVREkSJJwiCvlMLUNd4FpHU0GpGmKXV9COdrvFMM8zG2ieh2+tR1TT2tWOktIVVAuEXtSVWEXuzhTAQ+xdoiWBv4BmygPcy2JGk4umioiwkjn1DVBlONGCSLKP8cTXUBJcc4J4i8JN82lOMhVTMKy2Jm6lVLMKEViAiiqM+dy8f5k+evEMcQZTHVXsE/+Nt/h8efPEt/rUcUSwTLKG3xVcna4gEGB5ZYOHgXq0s94m7NoYNHOXr8CCqVnH/hEt3UU1SezuGIq+cexUyuEcsRL5x7iMV1T75f4jVoLbBVxcUvPs/d77ydYmsPdEQdA3WDkC2iZ5mL0pSQ+FZMUlY5se6wt7ONdRWDQY98Mma4N2J1ZR2rQ8tHON+eA8BDU02xRY1SEVVpQlukrkmzDhtXtuh0UhYPLXD52gWcPREKCCGgnlKNq/mVEU4ynYzQOqJpBXWzhWBhLXXWD3YcxmMNJEngA6dxQhzH2KyHjCzZkqaDYjLZYfP8Dnu7ml6/j81zyr19HGreorHWUo5LXKyRWhDFGjcpUXh0pHDW45REeodSGttYZJLQPXwA29Q0VUFxfUjsHM10gnCext6k+PQOpTWFurGK9xDGpihC6LDanxX2pq6JsgTnQjEqnKdpDAg498xpmmmFrGosDcaUlKYgSTO8VHR7iwwWlmkmBUlpccuCptvjvuXjnHnwoxwabfPdp+5l7XWvINYJ9U6OG9U8/mefJzIK3evSLHdYX0iZmAZST28xQiqLFpq008dLweteewfnTms2N3bY3xvxJ3/yMCuDMa+4M6Pb04xGHk9AN2ZociQjqqpVLjvodDpMpwVRFNPUrnXo8RhvGG5OedMbH+BVr32ARx//HEVRsJBqxns5K0sHyM0OX/WOd3D5zNN0Oh2W1vpIGhK9Qm0atIDBYIAXnqoqOHj4COcvXuHihasUVUFvocPzzzzFGx94A3tmh88/+HGcu5WskzGdTvE2ULfC8x0Q99vuvI1+vx9En+gXWaoopeZzkhBB9ZqmHV44e47r16/T6fT43EMPhtZwr8/O/oSiFijd57Z77icvi9b5QOGlYAETjKWBYSlpjKdxluvTEcbsstxdQHeWkUnGtMgZFyV/+Lu/C+oI/d53hcUI4OxbKabxTEsG7p0I67DtAyduRqxmzyTQ2HAten3fpuGEdutMrRwq0hvj6WxhaQHrBM0oIPF1ERbxUqh5lbO7FQR0SjFXiEsVImSruV9oDY2lqQ2JkmRJiuiI4HrhoHEGa2Wrf+gSZ1G7gKm5cPEyTQ1DUbO8vMqlc2exzlLXFYPOkDjep66XsMbgZg+jmCF4N0R21tr22IKH7KyYnBVPouVD9voWfECr5+dE+JvOjr+ptBS8yBd71sV+yba4At/2PaCi1n7oJq/hkNYTwAAhPUrptpbyXL/i2NkMIJuUkkN3bdDpjDl4wDAaXgfRhewwyBrnJNuNRXqPkI47X38IooJpZfD5EM/t9BYGjLchFxpD4J96O+Tcv/13eCokwTqoNYT8srevQOUdDtoYA66en4hZWyrRul3Szzz03Lz4C2kGijhNiHSCaULmZtbpkegM5yuSSGKMxRiHUhHQhBxjL1v1bxxsKeq6LTwleZ4jRdtqb21EvPftRGVpmgbjI4ppST/KEL2IopiQiIjrz5+lt3qS46dOcv7hiwgR4UWDEJok6TId5midIohw1iCkZLIHy4NFvPA4LBpHEmmsbRjtBkPuOOlQlxXSSpyJiEVMQhc7HLOYLIHReCRxmtE0DR29yuL6EtN6SmND8Z1EMdPplKXlNZqqQIgEiSFWXUAgk5JEV6RZRGPD+cryCXEUOBtZklKWwSsuVjEykqB8EG10Al+nqCsOLxxBCEHTVAihWquZYPIcRTHj8ZgXdnOE0EQdg7fdkExgLC6SmLqicBVp0mPHT3A2RqoQo5imHZzNqZr9IDKpGjpZn/0tg/fBzDhSAucqvK8Yb4XFSl7mxDqiaUJbaLG/yDDPaZxHKUFRj3GuppseoSgatl6I8e7YfND4+Me+wLWd56lrw87OR4PH6ECT70w5cHCFJ574EtNakAiNjmMGSz0quY9zBc4bnLNYJ7A2FOpJGuO05KlHHmT76ee44//zt9g4d5ZireFTn/okiG3e8XUnkLLH+qF1ummfvc2LbG1s0jAFhmw+9wRbVYrqRzyqOjjd4avf+Q30eqvsbF4hixP8TsO03OPO227lud98PydKGF1XeAWyTYyKvCd/5gzp249S6oIeDaYWCG8xNpoPUAJQbREjpURYRRTJMLj3+tRVQJ11pFhY7uAisGUNrSn0bCYSAiLliJRFC4+RhiSKGRUlwgkGXc14vMNOnqP8ISQuFHMeVD0hbipEqwYWWtGVorX1CsivsEFY4eoaS4gi1TqmNlVAYW3DpAiRdaVQ9GyMSjW97gK97gJbusJbw3SSQ6zxrpkPa56QnlJraFxJNbI0vgxFdltwShFyriOhMcKRCEWiMvauBtNh4R2VrehFPawU+DpYxcy2xhlqb7DezlEOCK6lwlqEa5Nb2vxu23ZgfNvFkUIgRRjf3CBGdhSxXsHXNa4pGeAp6gIhNbVSXC9GIA1L/ZSJN0wuXGJ69gq6KyjTRURuufQHDyK0Zm3tAJeVJl1eod/t40xNVJdMtxsa1ZAsLEFzkLXbTjEqt3nuqafJugu4ndBKTtOEyWRKuX+Oiatx6k4mOfMxP/jc6TkyOcu211pTFKFtKqXERxpRAgS/zrQT8aVHv4gQCQsrPe5/+V384s//OxSG/d0L3HbrOioOlkbf+I3voZYlJp8APbyKaZrwPWknA6Aop9x57wFe+drX8qHf/SB1PSZx6/zRhz/Jra88yubFK3SyV+CBTqePEEco8rQtEqDXO8m162POnLlOpCNs08yVvb4VL8zag975dl4J9lXerzPcbYKH3yTMd9YJtFrG1JaP/9EjiECGQOsIQejUxXES8A8vETpCCDVPH9qT+2RZxMk73oj0QWASJym1XeCF0yK8R4BzNxbSQoDUGqXbe1C8BCcTL/mLZD5WzGC5kDIk5v9+yew//2NWqIZvkjgPwot5cRl8MmGW7a5VRKQleImXAokB75BxxuHlEywdP4jyGZsb19i6vsXO+edxjUU4T20KJtN9EBGJWqbX7ZLLhmoyZWdni9UDq5y/eAEpFZNiwsEjD5KXS7z+jW/FzLPmZyikCGLPTPD4gw+yN94kijR1USG0xNtwXXWk0VJRlo7Xv/UbGCwexFpLpBRORPPFVDjvL/6OmSMErXXPzIVAtOlPSsHyKngFxlmCZa9nFhLgnAOhQ3yscfiZNRcW52WLYEqsNSzbAmrB1u6Ijo5pxiljcw2fCsqmpCwn2NoT6R6Hjh5ic3eCTvqkaolMRdhRzaf/6OOcK/dIhIACpLuMcwalArXROIOS6itqxn/5Le/ZSbNBvSda5GOWP9CYZn4Cb95mqzspYmIV473AGNuqlYIVUdgLgRQaayzWhFaKcaGtKpTCW3dTsRiq9DhKWz9KMUfqtFTEOqI2JQcOHOKtb38Ply9f4cnHn+Hi5RfoLXRxE4szNddeeIZ7738lW90utdU0foy1DeDwYoqxBTM/KI8n7sYQQ1U7nNUkWUpjDd7HWG9I4gREQEIgmKPqKGY8rYg7y0yqClvVSBVRj/fpdzs4B+fPX8SImjhOGY/HpGnGeDSlm2bzSMtIxQxtjXUS7y11M6VupkRxGIykMXinEL5VP7YrMa11WJE5O7eTUSq0rabdnO3dHfI858jxk2xtbwIOlEMnkrKcoOJwo8edQNZOopjFfoeN60NSlVKPJam2NHKLqqpYXDjE1saIrBOFleNCD2N3kdWAQpf0F7vUZgzK4GKFs5I4ykiSmKKoWFhYx1pL1o3pD8I9NlhO6KiUKGnQScxgsICOupS5J8LwmU+WKCVxzrN35Sk++MhHkPEqHQ3CKZKki1BjNp9SPPSHf4TUEXU1pLu0zN/6u3+Pa/sjTFG05sFB3RiiD6GpSrRL6N//1awsddnfuEZtKvqdhF5i2d+bMli6jdU77mDlxC0MRzvsSEn3tvvQqiEqDPT26Z1KOXzobqal58zZi1w6f4FEG2IdUZmaODlAkjuSjuDc+S0iv4LaaRh6RV4LpAx2KsZX2P0N1pQLk5+OkEqjXRGQMdkOYloExJxQVCa+xpUFcdJBaugsdWmcDUhYMSaJOiil0X42obR0karEVnlIc9GaSTEF56jr0LZa7SwBS2xfT+b0CQDhG6QPasNECWrbAMHeRXiPjhOssQTHPUEsHJO9bTqdDhJwtiZRAUnyWAbWY0kY1xZT7TMZLrG33QHXwzUlkY756rXX8ukdza4LNh2ZPsVf/85Vmvw5jr/xPq5eG2IGEZ0Grjx1lmvTHbaeO88DR25jpy5ZvLwB13ZQwrDnDdY49CtfzQ//51/il//VP+T4b3yYJVbnY9tnfc0ki8knyzzl3xlQKAGvTP6EJX+FpioxOqJyMYtLa+g0Y3+0Ryw1op1EmqZBe8HWmUvoWDF0hiyK0V6gZVgkGFeS25ppUSCBUsegUjANVWyZ6giZLSH2S6blBMqa8da1kFZWNlgB2eoySZLg4iWWjh5AxUOcv0D52JPcdtutHOuv8tFP/CH5eIFb9k5wSkcUcYEtSvqJ4chejs8NzoSIurKwSJW0IQAWJ9Lg8ShlQIeMxzmBMTWN20NKjfEWqhiu5IwmzyGXlvjYb32Og0M4KNbAFBzOD8Nj28gXtrl49gIijsiUpqw2KW2Jtg4hFPteYI0LbVghmWjBq8tbmI5HoZc4vcDO+TO8fOk+qnI/jGvS03ArT/mF8Bqgqd6Csx7hPaYO7W2pRBvXGuYlHd1cqM2KOPGi4AHXIl0zD8DZXOi5GbKatVPFfDqdzZfOQmXDv6cFjNuF4SwMwXuPqRXW3jS/3tSBFTCfi1/akZ1/100Fz/zXPqByIV7Rt//+818xa716L+ZtWcEMiJu1ah1C6NbSCZZXPWsHQUpDFrVKex+oYVprLlx4jqdOP0E9qnHFlFhEeA2madpYVEkSy9BJsBW4ikjDpApgSe4Nx44cZXdvGBZpHvRCxfETCXlVhaCGtnC21uOFoN+TnPvSPkVzreWZe6zwWAydJKWqp6goIRaSQ0cEKwcSqrIMSCFNq5oGJ2bRrfoGGusl7iawDcDLWdE5S5dS8/b3jOkopAz3jQ/m/VIJULIVx3ikF8GqSwBYrGnoppIvfObP2DddOssRSWeR/c3LuFqQJl1UHFE2hvvufw1HDg+YVBOKUrG21nDt6h5J0sXUE7oipxdHWNWhbFK6vZrpeIJvUdz/oeroL9i+Yg6llGI+cYTqGXAeqWbyev/nikrnHFVZsdBZmnMvnfN4Fx5SpRR1ZVAqAfy8SJRShJPfmh33sx6NNa26MLS3lVJUZUnccivDijJ4o9XWsL6+zvPnL3Lf6x/g6IUVHvnC50B2aVSNq8aocoIXsD/eIcvitg3gyNIB3klMYzDWBW7i/pDOQslovB8yMDcmeBqMqelny+ztb1NVgTTeNA2RDvvnGkea9MirkizL6HUHJFFKvrvPeDgKvEUXuJeyNpjckTjoZRl5nbPUGdDvDUKCjHEsLK7iGtCRQMiGshpjrMBbz4EDh4JhcVMRRdG8CO8kPcbj8bzVmOchwumu23VQ4Nmau46fCMTtKIJWNa+iUJBWpovxQxo7RAiLXY1YXT5GFvdZW1sj0RlxlpImXbQOaLJW8TzFRmXxPHVhdo9EUYxUoQUoRbhms32WorV8mrUYTYFUGU2lqF2FtRVRDNeuXkCIh1sPMsHXvuc9XLywy2R7i8m+Z5gP2dq9Tl1mKGwQDgBKWr7xm7+VxgrirM/KWkZ/0Avt4dKByOh0FpBS0unG9LpLiFuOIa41ZL2Ihz77Ka4+f5mF/jJ+cUI/63BkZZ2tzesMpyl547nleEhgMMsrxIMlhB6g7FVef/dh7rvnHj79hc9y4dqQWklW+yXXzp7hV376/+E2mbHT1fRdjaoPAeHaOhxja2giTyQEZQNeaaLKYSJHaOe0VAEh5obTSim0D5nSWexxxuB1hCAiURHdbInpaIz0M1/H2cQDEhOQRyHwztDvd9nf30fKGIulMeG7aq+x/kY7yGBxBAGPlRLZdg0AhJdY24T4ShnU62nj6cYdfGOxOJRwSDRJEp5Jg0DakoFMsGWNT/fpxyDphhwjYSiVws4mch8mtZ4z2KmiSm/hnlcfROgamy5y2/2GS0/8Mb//zBmOHTjBsthi98JFaq8YdQQrIuHqcAu7d4Xd55/lvXe+i58vfo+NZHTTuAbHSslgeRmxFVDRMKlbIhxSeowt0V7QjDZwrksvaUiEwHpHIzxZL8bVTcgNF5bDOqY2ZeCk1Q4VRdiqYilWdKKaYlLj8pjGX0dligl96iIn0RMi3SVuJBExPhYMRYWMPaunjrFx5RrJdo2R1xhuXuHeV97Ld1y5TjO8innwD0nTjB/qd6jtDqPohTDZdTzpcoZ3BvPkeWSL1oXcbI2wAu8FIR6RMOG1QEGYLIC4walP3WipzsqVahN/bTPMV53Ze/qwv4P8wz8I6IzzUDH/7JfObWKeMQ3UbY24mM2LMCEyvDuPiy/c1LEUXKwP8sPi63BCkmUdkvQm79N5y7gtBlsU6UYxOUP15hXhvFScW8jcdJh+Tp5r3z1/a6u+9bM26/yRY/bb8H4//67F5Vkb8qb9az/7De+A/pJFCwnWzXzI5gJdIUPhHcYFO0eZ5+EgMghIrRCAmhd083N9E+omlWrbxq0llAzFj7ENWdrj6uUr9AcdTtzao2k0YhZbKzySDq4qSLKUPTNgYb9P0/cUI4GwDlmF/bTWYKwBHMJbtBQcPLCGdQ472Wu9aT1bW1sUZc3y2hr1qKAzSFlY7GP2HFKreca8NQ6LZ2GQgrCBu6oihLAhJUyGeU7qQF+pjWmlLGFcDQt6gWkKhJRtdKXEmQYvggAKFXLchdAgb1y/cK6DpsTdRJm50Qm60fpubI3WCc6EO0AK1YZcgsC1Sn/Hlu/zrX/zu/nob3yQvemQhTgi1znXrmwRJ4qqKakKRaL7nL73ArndJ+llbG51iAYHiBdqLl99nBdOXyHKMorSc+stxzlyap0XxpOWH62oneOmPf5Lt6+4oHRt/vHsRACgXnzy/qKi0nsfzKptSV1XNE2NwKGkpDY1SmtMUyPVzIaixroaayRJFLXt7opy3vIOCExoJwSxQJZGFFVNGme4OqcqG5wQPP3sM+STKe/5xq/n2uYWly/voGJFPh1x/sxZjp44xZFbLNe3NplOJ0RJRFN6VlcOoqMIIT3Cw7HVWzix5qkGNUomyBVNmqaBt6iCdUYcx8RxCs6TJAlNVeCcY1KNGE72yfOc3f0RvXiZtcOHOHH8JL1eDyEC10QoBV4SRcmcC4drEVgtEEIhkFSVRUuFVA7BDe+2cpq35z6gkcYHPtr+3oRTJ7o3EEsZCnel22hLqYmiBG8dqh14QM4jt/AFSZxivaKqwr5YV1GWU6SCulQ04yg88G6E8CmNCYW3NxHVNPAnXVMjhCKNE/Imx5oRkY6xYmYg6+ekbFzgyWqtqcyUJO5SW4PUnlgvMMmnTPI9AkE/FC+KJX7lA59GNCPM9hb7oz2+eOYqG8N9qnyPKEsRaKbDEUJnbO/t0x90KXOoGgsYmjpkATdNMKp+8osjzOQir3zDlKPHTlGPHJFYYvVOMJ+9Sq9zgJIu00nFgtKcWNFYkdHRmirNUTrnxMHb8d7x+JeeZnmxyx//2adYWTpK1NFE3jIeDxmcPMj3/d0fJd/ZZzTcxe9sc366DE+HhZbzcPy+O0mkZqsaI62kM63JpaXJR3MhXJS0gxeypX3UOK/Iqxw3dCyvL5NPa5x3KAlx5JDC4J3FzshHAb5on2cXyiTrKHKLdwZBEG6EidAh2tfc9MCHaEoZeJx1ZYL5r9Y4b/EtUh5y2wXOTYJqv50EjDFcvHiene0hxjiSNGOpl3Lw1G10shX2Lp5nOGmom4UwuQqNWVuBjZnPIKhI06s97tASndtfS3ntBV5551GG1ZTe+iE++6GrLJYQdxvi7YyLwxEucwgPprDsK8/Xftd3cO2hR/jQz/5bDkWSU1E0P8SzrsIkFfVkO5iDCwHO0+QN00QhOksMlhc4dOgQm5cuUE/GoBxWCKxocM7SeGi8RceSVEYUeY7ScSiqOwl1U+Jji48giyLSfkpuFKkSxN7jmg6JdDRRReMiItUh8h4/zeknKSPXcOgNr2LvMx5/ZpOjJ29D+IbqhcucWz9LdcSG4t5uI4RCao1ZvzGF6CgUQnVdzdG3F2kPbnRD57SxGWrVMsvwDOcI35e1/fc0D+3v5vSCL/Pz/EtmxGR6nTtPP8AzrKH0Tbnh7YLIz7/jpuNpP8Pd9O8wzPp54SGEmEexhteHMXT2Qb7dmVkBOitghW8XQeIGbWJer0oxP5ca8JGb79MM6Vw7IHjtGzqkadzqCUA4gQsVSfsMhxarmhX7KlDMAi3mRrzxTNUON84J3EA/rStxVqHVIKCGusT7QE0TZKChbjwrKxmRiMO87i1atcdoHCrukCQZiUigEUgnUS5CGrC+wpgG5w1ad0A5qtIwGU1IoxShHXWek8QxZVWhCHPjaDKeo9eLgyUa64MXrwg2gzYGa0J0aNM0c06sQ4AKtKGqrNA6zKPCObqdDr1Oj0YGBxmHRCY3/CNdm3Tjvccbi20sXggQ7sY95drfS4+Xwdt2tr20ZS6AXtol0gmlLzHGtu1n14qkgkjMG0e9lxB3j/P6r/9mXnj+HOVkzM7mFRbWF7F1KOBV6rhw+mP89P/5x+2DkIZVl+wTRTmuHtNPOxgd3BMwh+imKVpImnn9dmNR8eVsX3FB+SI4ty0cb44LeumJmrXGhXBtIoBDy1BtO2eQkcY7G4YIIULxJmgjFUXgoEgfzIabhlhH86VqkiRMJ5O5qrysK7SOKKuKLOtQFAWf+NQnufvue7j85OPYJOLu1z7ApQu/TywF3ku29/exSYZt6rn6DxTOKl553+t54vExxoQH/WV3P8Btd7RKbcLE4nwd2uTtJBb4WTJYXHhPFCnwNgzW0uNEhRdQFjWj0QRr4cqV6whipvkl4ji0fqWUdDo9+v0+3nvSNEapCCUj9vZ3WF9fZlQU4ARKJXg9CUhommJMHexcFBjnKcucuCtoKEO73jmyLAPpmeZFUHsD3W43eJA5R1VVLCwskBcFWZZhTYMZj4miiCSNyasGpTRx0mN3d58shcFilyKvsVYidEMUKYqiQogc4WN2t3ZJ0gik5NmzT3HixAlQmnGdI1VKkgRhU9YNN7WUIcLNOUevs4yxOWmSYLyjtgVCK5yLQ7tDK6QSPPnYw3zT1/wqNloi6SwRJcvoxNHTDYvdAVv71+n1O6QyZ2PzKgbB/vYW+xsjtjbC4+CtRIgdEAUIuPD0k/yrf/u32N2b8OSzW6yuZ0zHe2ycOcuR+1/Ny+98GQeWVhgcWOD48buQzjHdL9jerdjyFoSiVIamilg/8nq2d55i6dgC5XSbKMowTQ8/LugdWmNt6TCRllTDy9jJlORazZkrnfnkduLOw8TNFitC4NMgIHDa0RdHGY8nbGxtUdc1ZdW06GTwwxssKg4fWWdra4erF6+ytnYgoAY2ZzKahBa0tTgn50jQzROx0gLXOKIoIc8DAiKsb1fwQGNueoNgtJcQ5NO6nWhnY4QPMZIyZDE3jQ1es5GgqjwLCwt86YnHOf3cucC1trIdpjZBxCTJFbTWnLz9PnorpyiqcI+kSPYO3o+79DTst8Vto7lwpmRwUHH6Nz7M1oVn+MTSIZKox513HOb5p/ZZWbsHaU7ywrUvcSaPWFMpUxHhlGOzUVyqjyNqy7TJSKN1hsUNDlUZeSIfU7pBwIucwStJlHq6WYfb73oVW9tD8kKydus9bF++yP6100S9OCiUhaQeTVFxAqVFuoZce5QL2djKhHGgaSqc1zgRCgWlDcZppqSk8QTvSpzrIJsGk02xQNZR1HXJwajHE7/9+8Sqw5F772C4M+TR/Suc9BNecRSoBZEKKJL39kXcvNlNMGvhzm+HlxSTsx/9xUiG58UV6Je53fxyPytQX/LJ84n4f1xcipfsb90tObm2Q714kHe9WxNF4bmfYTFCSoTz8/Sf2XdJBEIqnHV4H+YKT/unazlwUVApa6WCN26r4nYtj5aGVrzUFn6tgbYjoFFWgPS27RM4rHdIHehfwkukM4SIVIF1Dqk8a+stqldb8BaJCo+iu1ml7sAZEHH4TCRaCmpTIWn3ydjZ0xyQ55vmdenDn1ZoiAyWfTwSSYJzEUJLjKvoJBGNKdAyiCadd6EVTEvxkD6cF+XxkacwOXVhqfJ9tA3fq2OFNz4Y6GtBrzfANpCXDZ3MESvdzokpvqnpdFPSLGY43CGtbjgL4EIhB5IoUqgWCPNOECcZZetsEnLgG+JY4xoHatbFCnO88AIlVEAlrQ1JOiLc8UJ64jgiSXrtgj54OYZa4oYTzqxr6qx80XkN7XgbxlMhSLtREPEW5bxTW9Ylezs5Ak8USabCsXXtBT71kXM4q1g7NGD96HFe9crXcW1rk6Yc411BUzbkU0vtCnaH1xjuj/FmipkYxiNF78gd1PsThsUkdE3EiLquCVTaQP0S8oYK/cvZ/n8oKEG0RPJZpmtQMloEcs4LmD2MM/d621iSrsTXnsYUQfkbSUpbI0QUHkhvEDZByIYoSijLMjzmQhC33nRxHFPWFUpGAZXsdKjreo5mBbTN0uBRxjLc3OEt3/hGHnj76xnueg4v1Xyu9zHG4ymuP0DjGQ/30RJ6SZe0t8Tm1jVOnDzIsWNr1NUus9GqKArKXJBPpkRR9CLREf6GICggsVULcbev0bQrwmBr0O/0WewfnIe2t70iyiqft6StCVZJo9EI5ySmcWRZl0cefYhXvfLVrKysBG5NkwfxjRJMxkMAdrZD+z1NUxYWFhgVZVtIdhmNRpw7d47VtQNIHVqiztacPz8FoK5rxuMxWRbOrSQQdJvWBiRNM6TQjMdTlA5+kAuLWbg2VUEca6z1ZGmHqgrXEq0YjfZJYk2adtjf3+fShcskaURdt3FfWYZzkHXCClbrOFhLyQhfW1xrOeWBPC/w3nP+bDUf9AHe9JbX8fXf8F5+6qd+ik/8yW+Tpl0m44L+AP7jL/9b/suv/CZ/+sHn+LZv/R6e/NzDXL96hlsOH+V73ve9/PKvPB2UwZUFYqIowxrHK+5Y53c+9Bk+/B9+k2//se8jX09ZWT6EPlwjUVy9ZlnoVizWguNrJ4m1JT7qwArKwtB4h0RhEkVz5wIXd07y+U9/lr39MzSyRxxJtK158GNnUMlVNA5X75BFFfs7knJ6D7PZ8MrlbcZLE5TUeF/gGkfTWDauX+Ts2bOMx2Nsi1JILXA2TGQL/QErK2u8/k33E/dyLl85y4H14yEtSmuMCa2aGVF8tom23V67cAx1VWGaGi88Vrg5hcKpQPKZ4TWPffHQfJ/DNROIl0BFcy6ZD5OnlJI8L5hOX4MQD4SWu7xRkHgEdeWpS8dTXxQobXG2g8dTVYbHP1HQNCehFeqNfcoHzkvURUf1yU28Xw1dD7eDZxvFraQq4uHnJ9jmOE59Jy8Ij/QeXzpkJ+VD/2UfXxkq8Z2cM4LmpmNwNcjqpk6vUAgnkFOJqEounz7DmeEul+yELNGsLXRZkh185XFaUwuLkD6I4mJNLlw4g94SxSC8wQuJE+DQmMaivMc5iZcWJSc0hGJGCgeqoCosqdRMmxIrHGNR0U8yXCmo9is+tnOWqRe8YrAAjABPYw3/51MG52cKbj2vEFX7V3NTHOdf1gN7afnoYY5G/+VbUN76tqCzrl0ctJPcHKG86e+z+3Tesv4L6lfhQ2b6bN//l2+TvPGBTsgJv3lf54DJDLlUL3oewgerm35+A+EM45AH9Hz+836m5p0BK76dF2fi1Zk9lpyrj5WKWguwiJlYQylFZRpUC2SE17d71LagvQvtYaEFxroWTQ3PmFIKJ0KbePZdQgjUS8AfxU2AkLtR/MzOZ+iCa5RK2uOVRC0tSqs0tJh9ilARUSSwJkYpPVfVa6EwwuCQpPR5xR33UVQ5070hZSkZbVxkPMqJoz7dTkLdNAiVIGxNXuYsd1MqUxDrYBE13g92eHZYIJyn1x8QZ4CoiJJuWMj6OHReRImuo5D+Y11bHAYLQxTYuglFp1RYoUmiDssLEdOoRTnZo5v2GU92QZU0rmBvb4/1tSOk0TrnL23S687qFsni4vK826LEjENq6XUHZHHGaDTBexMKVC1QMsLVE7aHO/NCNIoUK4tdRksCpaZorVBKcunqk6hYUlYTnn22pKklxaTGK0UUg/MNSqYk8YBOd0DSGZAu9Eh1n+SWCGOGJKlC2ATrAWHQUUaka7q9hN3dgmDHZ/hLH/ibtq+goJyt3mywKHEO58P/Zw9KKIwC39FZN3+P1hqtmZNxgxo5tFx1ktIYg2zbrEop0lRRFaG13dQWZ8E7i9bRjeKhncyste0gUKNURt2Mcb4hjgdYa3nFvXfxyCOPcuKee8mv5CysaxbXjjDe30b6hrXVI+wVQ5rc4+uCUuS8+lWv4B//k3/I1Su0FzbwRjvdlG4nPDQhfixqUR3XRt2F1vvFy5dYWV6jKeu5EGY8HbGw0Gc0GgGSujLUtWE43KPTTWlqS2MqptMpg8EAgDhK2dvbC2KXxZTpNGdhcZFbT93BU089M+cbOufmxrHGmFBgExDcbreLunadpqxIkqS9HjFxt8e4yDHOkqYpTVXjW0GPUgmLK92AbrU8mjS7kdUbjl1y4OhB6jpwTUwdrIoWO4Mg4oljJpOQmz4tGqwt0TqjKWqK8RAQNL4hn0zwQmAMRLKct7vD6kxSlmVo42pB3TRUdTAEFkLgLDx/OsG5G0KJRx55lMbUvP71r+fxx5+kKi3LKwsU5T7f+z0/zitfdT8LqzG/+oF/T75fgI74qq++m43tXaq6aVWrGuc0QiiUVvSXltk8+wx5d8TW1RfornWYNtvE/YRpXrNz7XmeP2sQ8eNMiwpsRTfTCBWTJAtIAYNOxsL6gEhMiHLPxsYlfKpwl6+w50uW+h3+97/3w1S14aGHPscLz1lOP/M02xuCsgqCFrzn6Sd3kHqDosjRWhJpTVEU7O9cJssykjgj6obF1uw+1VHI1b1w4QIvnH+er/v6t3Ho8FFGo3163S440LKDsxbvXjwkzJTipo0/NbULBs34diAKiJaUs1bczTP5jb+LGVeMm+Z7wfx5ljKI9abTPFiRSHHThC25UVSGBS2e1udwxjVqDZPb8xS4bQIdRSSZIh105wWHUgozGzfaRaFy4OoGaxqoGrRxREmMShJkIig7HSbjCaYs5sekZ8XPbEWPB2tJsOxmkk/tPA0yJokUV4oJL0x3ONlZ5pYoI63DM4tQCB32NfYho1oqEdSzSlLWFRDRGE9EjIpTnA/BCW7W3hQKZIz3GlnXlKYizWKqOqehYWAlXkgeuXKaPAoG43aat1co7HdlNEPnOXj4CBfOX6at19oukW+LF3/TTPA/QiVfsnlQypN1MhYWFrh+/fqfe0m4HwIw4Vr+q1I3ouYWBgusra9z9uzzOOfJsozDhw/Px4vxeMze7m4QOLRKYykFP/ezP8dtt9/On37kD/jVf/Mz81tycVmzo8SLFqNzpS030KObfzcrDGeF38zz8Ma/4+BPCa1biZj7wM7+DJxwgRD6Bp1EBONwCLoCR3BFcbNWOMEmTwsdeHvOBcDft9xPFwQhoW1+o3idFYvWWrzxQbHbonQzIcmsgwjMAZJQrMsW3ZuplWfXMiBtwjq0CNxCa2u0mB23J040sb7R6Qi8+ZmKPIhY08STxQnFyDPNLdNdR+23KMow9kuVY5uKvPA04wLhJWk3RRKxeW0bKWL2h5tU3pJGMV4rqlrQSxZIdJ9uOiFKOuE+Ng6hInSc0pNgTBm6o1Ji8dgmtM6dCItwKWJiLblw/kl2JiPG4wmulhhbs7W70XYPO3Nz+qr6zDylSbRm+zciSiVZPCu+Hdv7Y7rdPv1uD9tUdLsZtHSJpmlA+xYZFRRFhSdc7+2NjNH4TWG5oiSv/qr7iNQWAsNCP8Y3nmpimZYljXFMixKtU5rGYKqKcremzCvyeIe6sEitGI9yRpOmNQZyPPDa16EWDuKNRNge1tQotfgVNRe+goJyxvyIEMRh8Pag5Gw1Fgo/6z1O3GiLC+/xwlOUAdWbJWJUTYP3IlgD0a4McUwmE4pCYJsg83fGECUR1prgNNkKN2YPMe174yihbqrgwWYgThRFWXH+/GXe9p5v4rc+8GvUV0q+9ye+lz/93X3q2tDsbHNub4LqepbWF0iTHr3eIt/7fd/DI48+xrPPjphOs/mxPPH4Uwz31byIq00zL7JmiSHG1GxubdDppMzMZEMbOhRZ1jbtAx+QocaWGBsShKIkoj9YAQiiGSFZWe0jxACcYGFxhbo27O+PWF1dn/vAWWupW1Pr2Y08s22qq5rpdBxalWVBWVRz+L0sS2hVwXEEk8nMrDjwS+I4pixr6qZBibAy3traotfrBF5jFYyOjbPgg1gqkjceqOAzGlZo+XhKr9dFazmH8oPNRlDVSx3PZw9Jy++UgQMYRWFwnSXABFSnwTvBeHwA71fm90JZTNjZ2qWpcr75G7+Oz3zms2xsbBFHPfCKLzz8aZJU4ozn5JEjfO3XvIW77jnII4/tE8WBkO2cR2mFamMMRdTjfX/rJ3jF6Y8j9iQHl26nP3DU+ZR6oSFPr2OmGbHr0xQRXsSUucVUhuH0PE2UsXGt4vwLFYIG10npZQNW5DLlgkOaKZu156EnN7nwwjWKiePyhiXqrtNZEsjtODBZvKe7kKLjlLQnUSJYKY3GG8Rx4P40zY17cqbyb+oGT0XaiYCIP/3jT3PX3bdy9JYD7XXyxDqIsPB2XvF5D0VV0XjAa4yFxgiUznA+QYowURprWVjSLCw2jPZv2Oq0T+e86rh5kvZAy3dv206acTEOXyzan4XeCzKEN4OQYSE7h2ZuICvOC+RsUm7PlTMNk+t7FFiEkgjVihRkQAPaf+CFR7XZ58J5tNL4SFJYixtP20VahbH2RYPrjCsoWnqGs5a+2GHQzfmj8Qbdg6uo2lI0NQc6K5R1w9l8l1SucFu2gIwkIskQyFAQ2NClmVmFGOFQrgKliNCkUUztCmKl6IjQbrVNWFQLWkVrTxFZj5mWJHEPFUf4qWVXGM4mFalf4S3veg2vzreQ40fnHntVY/BKtcd000JACP71v/7X/MzP/AvOnn2Bm0vI2fLgpRzFb/mWb2E8HvPRj370xvkSQQCCaBcINxET52OBtbzhDW/gZ/7Fv+D7/tpf49y5c4DgB37gB3j3u9+NMYZ/9I/+EU899fR8jLx27Rog+NEf/Zu8/vWvRwjBRz7yEX71V/8b3nt+4wO/zbGjRzi0uvqi4tda1w43s6LqRtF4M5J6M8Vrts0KsJnI7MZnWrRWOGeYsS397H/tfa51Ni/cZsc++75ZUon3vrUlmlHL1Pz3QgiwM3GsnL93VkRKwvg7n6+FIG49osNCqhWsSf+iQlIIQaQk1pt5Aenc7Hm9cfxaxUTRDT9oKQEZgCLnHEliqfMJWXIMIUO73zYGrQMKWzuJcwVSZvSzmPHwMvujDcZ7Y+pakXnPQqdD0zRYW5NpF9TXE4eqYHP/KuPJXmhnu5pOBGW+S5Ik6GbC3mTCpCypfAj4KPJRoMaNxlhRoqc7OFeiVHj2TdWEboszaKUwtUbEBUI7rm+cIZvWCNMh1glpplhdHNDr9VhY7JPnQa8ghQ4emk2Yy4LuICLPc6IoCsXnZEqSJAz6HUASScXgwEHwnrpqcFbSX14gjmuqqsI4T7K2ivOCoihJ6HFOBXqEAB57+BhCHJtfmThK2qGzFV3h2xCVIKbSUiAzCyjiXqBBDKKG3qJvhbOSKxcVRdVgzLHgtuPsTYk6X972FftQNk2N862P3E0/n92YsxYY3IQE4NDS0emGtqgTsi16gqmu9cF2wdvggZSmXZrC4n1or4YLpOfQf5KEJJkZ4lkUBU1t0bFAOkUsU+q6RGpNvzdgeWWRsw9f5sf/zg9x5PgpNq+Pef3b38rLX3WE4wcPcdvJY5Q2R4oEJUPWqsCzurKMEMW8zeqVxhBMidOsg3Ih5cYYM+ceah1x6ODx+crZGo+pA4+jmFqs1S2q2VBVIW1ka2OLqirmgplZ4RQsltpIKdmgdTxH7GYD3TwdBU9VVcGeo1XBv2iw0aEI66RhQKvrel44Nk1DHKlAJVCKSCfkeR64HE0dTGYllGXJeDxCiKWAHrepKUopFBECsC7461VleJikNwhhyHopxtfhIfYO05j5Sk/IgG4aEwrkNE2hjRWThMLW+VZ4lSXz7NumaSjrxRe30tQBnFrh0vUxQkgOn3obW+MvMbx8BfDoaIliqvBeYeURHnlqwp99+ouYWrC3G+E9LTppECLHOssTT+wwOPgMuh5wcfs6/++//mkWFw6itGawukCiY7p90Fmf/sKAbizo9zKiNEFnq3SVJEmnqLhLWTUUo5xyWnLm4lnGOzknT91FPdnjVz/8n5mM96mrMUJWCOVoylWa6g5o23x5DrHLUHRwUlJMa6bTAbFaozGeNAmJFjM0BSnbCDwo8wlaRXhveerJKWmWUlUR3d4CZdtinORpUM8SXBZ6iyukvTKkPOkIZy1VXZNlwXYoVrqdKODrvsOxtx1CCERbMMzU1sJ7kIKmtqhWfOCamqYu0VKQdnv8/of/EGfCwlMqQZLGNNaTT3OUNKA1/TRFOMPeaIQUR7Hm7egk3HsHVr7E1Q2BMz0EkMmDvKr3DKtyjHCOwuaICKppjmwcKRGGmoYaK2Jq73AyFK0WgVcqqGbxxJHAxqHwnG0Wj4s0zlh0A95ULHemPOpy6rVFBoVjfXkRtdLl+pnL4CVJmnG9mbCSxJhhQe4N2kEkg/G1cB6BD4pcGRiqRSta9N6jROuC4GUbrxeeYevDM99IR6LiIMqQEm0sIonYGu4SLXY4Mlhg7cAB5LUSJqLN2XY3FX6CI4cPkKSBy729vQVAFMUcO3YsLCCM4dq1azhnOXXqVobDIf1+D+/h8uXLfPCDH0RrzdGjR+ddlI2NjTnaffBgWMjkec7Ozk4YD32YAL/7u76Lp556qvV89Jw6dYo3velNvO997+P+++/nx3/8x/nrf/2Hb5qZBHfceQf33PNy3ve+9xFFEe9///v5/d//A4bDfZ544knW1tb+XFFoTENZ5jc6PO3C5GbU7kUqZ8+LCr/5z28qCL33VHXwI46imGCIJZAt509KSWNLoF3AaNmON22rW8o26cQjpZ4jfELcoFYF4/CZKh28d/NWu/cCJwRSi3mBN5+HjQm0klgh/QzBnRWsQZQshED5dH4sUv75ore2JUpF+PazTbv+bGywAhS1xLkYY4MCGq8RSuKFoTYW4TvEaoAtY9I4I0s13q0g64QpClPt0zR14CJaj3MR3XTAdnmdmpJpXjBYXmEyGpN2MobD4PAyno5ZW1ujLs7za7/6L1FREHOWxQhvbOgsRhZXGpJY0DSepqwC8qsiJCFkIIqjMF5ZTxIJel1JL+2SRB2MLYibAcWkYmwajFFMxlOkEtR1yGEv6pzppCBLOsGL2Qi8TNFobAX9KKUsyzCuVIJer0sjGqqqoZ5W2DrFunCdi6lBSk8n6dNL0rCwnlFRVECAjXVIIWmaGghpgUKERXVTN0ih0Eq3qn2BsTVKhra51hqhPFEUxpEir8LftaJpHMoFgfANh4a/fPuKOJSivTln3nZCBF7UnLgsZyhA+Lm3Qf3kPZimRItQxed53raSA1JmWw5VEinq2lDmFc5pirycy+1nD75Qcv7weywzPzGlPdZotPZ4LEKmFFXJ5SvnyMcV7/3m7+Qbvu89/NJ//iDVtOCrXncfx4+vc/nqiI1PXqL2U5xzlGVJt9tBCM+1a4JpvtYiNp4zzz3P/m5AEGarDyEEtXGodgV6owXv22KrRVNFPW8lzIrPGbqYpik6CqvBpmlI06wlx4r5AC5VBJTzRIqZvU9RF9R1Tb/TbfkxHhVJkGKOmoYBTgUjeKWQhEI8TVMcoSBs6pLeIKy6kjQl63YCXG9tK3pKqJqGw8dOzle0Wdado41Ij7dBRRdFCTNlodY6tAR1BLjWxkEFXmTblrq5JaS1DglH7bFHSTgGb12LvAloLS2apqFpcqQMXDCA50+vcv4F8P7AjXtG3Mnaetue9K5FxRx7O4bd7VkyDtT1jQEWMiDc408/oXn+sWfoRyk1jih5F5PdsN+714OZd/Ch81g7bT9vPwj6RIuASYWQe0Q6CtdFRujoKMI7nnrcUjYDqvIBrKkCV7RFBrzXWJPNn8Gd669BStPyooIbAsZiHCAERXFz6gPA7NyGVWlV35hYv/SYbicekJFGtYQ572bv9Dx7fsjy1KO1RSIoq8DpXVtfB8A0VbhWyU1GvjdNQEKCFpLUpzjnkXGMISAbqiMweUOSZWSpo+E6Ks2JdESShIVkpC0vf/khqrzg6s42w+mUlZUlYjfFVT2saVCtUKKmQCnPLC5ECUNHbCKrixiliYWGRpClEY0sqao9vHAkiSatCyIVUduQXKMl1MrRKE8cKabao7wLecrt5hxIF4VCMBIo55l4z6XIknrNW9/xdo7ffpTf+50P8uq3vI6HH/wcSSEZa0spYDnOUNIGz0mgpEEQo7wDG9AdLzxoSZwm8/FFMrN/ca2gI9i2iUiRWolGUgsRVPgOxrFk19Z0fJdru1c4d+4ct3aTsKBvC48ZUvnOd76Tt73t7TjnePrpp/mFX/gFAN773vdy5513EkURP/3TP01d16yvr/ETP/H3EAJGozE/+7M/y8LiAt/y3m/Be88HPvAB/uk//aecOXOG++67jw9/+MM8/vjjXLp0Gecdx44eZWdnF2sdhw8f4pu+6Zv44Ac/yLd+67cy41G+9a1v5cEHHyQvCi5duoTWmtXVFabTsOA9fvwYa6urpGlCFMUhpauuKYoc6xxRHBEWYy+ey9I0od/vzuc1YI50v0hk6m6M5bPt5vH9pWO+VCHxLYpa/qWbtYsV3guSpEUobYjLUVIiW16lN37OzQ9c+xlgE/ZJq9CpkfKmffQeKdX8ebM3+R0C7VwTjjKorD2+tZ6xzQ3RrG1fK+WN1vfNRfZsX4RQOOPbsTuooZNZS1cLvLLsTfZJuh2q2qF1ihKSvNhHRhLlt0AqvJuSdSBLUhpTk2URXu+hmwRrWv4pKdOJwZqSA6td8sk17HgLN9lAu4Ysajj+8lswLriOJJ0MV9c4UYWgE1dzdGWB5cECK4tL7A+3uLa1xcVzBdaHOVKhqeuCOA4G/d7naN2nyA3ra0e4574Bw/0x04lB1iEauCgq8jxneXmBbrelicleS9EbEB+ISZKM4TA4mwgXCuM4jiiKgt5Sh163S1GMwXsiFaPSuAWnGjSOpi6IIoV0FjuZstxRLPaX2R8FAWBw1VCkOg6LDBnuJddSKG44FrTzpgr3jVYy+Lci2hTDNprXNCAcSSxQSmMaOa9RZvPBl7N9RcbmHjBljbGTebtitklpQYp2RRrQMS0VSaxI04hId+n1U+q6nseq2XZVE1JwXJu2IBGEyvhGCzcgoioKK71ZsSVEaJkLHMY4dKSoTRgk66YhSRKuXdviQx/6EO98y+t56NEX+Mj7P8mJtYzx1Yt84skvQubo6AwnQsxiWZZMi9CmrpoMWMH5gPDk4zGTuAwXwFmsCWq6qqoQUhPH4XRKJRCE1pxORPAMLKHX69M0lt5c0GOxth1EBERaY4sC4xVZbzFcIB0sC6yDSIU2cpIk80JtrSV/62S2CtZzsVTUWpwIIdA6ZHDj/Pzns+i7EA1mb8pmD91A2xaTs88L1zkMRq79rigKA/jMSDuS4TPmr4/ChCXczBEgXHMpYlyr4MY7dBuz2RjHJDdUxrStmYBMplpiuYG6hlvSM60cM784gOHwxoQftrA/LbbAn2Psz3QG3uJ9ftNnK4QI19faHqUz1FVo83mj52KAmz9IvORjAw9tFlEaHu4GyGnmvGNEKHStNS0nKuUv3Xx7BN616lEfuHjt99u24FcztM3adnUb3BZoF3DGmPb4Ba62WFG3x36jybV5/lmund/ACo1ovUmHoz3OpymegFDiLUomc/rEzeiOlEFNao1GoOb3lRDBrDe0jTwHDqyTZCnFeI9e1iHPcwaDAXfceYI0kZw6cRtf+PwXub61zfb2NitLa1wrArLjG0OapSjlg3VHu+9OSurVRbpHOhw/cJyLZ84jao8pC7pygbou8bWjzEuK2FHXFrzCCc/IlCjn0EJTYMEoKi+w/sZFjoWiaQxOh3b3YpLw/GRE2u9zdHmd65ev8PL77uaF58+xsL7Ou972Vn7nw3+IjSLKSYOSGUJrnAfjCd0a6vAzJRFa09jQisOBsqB0irU+8AylDihXuzB3xiCVCrwr68h0SifKOFeP2LYV77r/NTz73HN0+gnXr1/iFqle1DY9deutvOtd7+IHf/B/ZjqdcPDggTAhtr//yZ/8Sd72trfx3d/93fzsz/0s586d4yd/8ic5d+4cb3jDG/j2b/92fvEXfzGcmzhmf38fCCDCD/3QX6fb6/I//fAP88gjj/CpT32KsqyIIo0xwcblzjvv5Jf+w3/g277t2+bn+MiRI3zmM5+Z/3tre5vV1VV2d89w/vwFrLPs7O5y+vRpPvSh30Frzc/93M9RliFLWsmWyvASdq9SirhFT+e8QSCacxtfXEzNiyrsSwqsFxdcxpi5EAMIqWrWo3VQf9dVSBESetapcnNwIDgfiDbE44avpfe2taQCZ2cdqZkI6cb+Ce+J4jAez+zfQurcze389ljb8zIriLXWqE6GsyUzCoecnbv54sXR72ZMp2OkkvT7Xap6ymi0y3C4R5IkbO/scOyAR5iL0AxxNnzHclcymQ5JokVUWhMJizU9FrM+qtHoaMR0b8rUV3inyKIF8tLQmAJrDcI7PvPpz9NVE0bFiE6nQ9JJuLp5mW63g/GO6fVLrGQLDJY7SCE5dugEB1b7mLKmKQxL/VWkcnzenEYlKb6eBEuyOMNT4SxEaYz4/3L239G2bfddJ/iZYeW1w4k3vaxo6SlZtrFk7LLBgI0DMpgyVBN60DZQLhgDRkFVUTVIxSig26OLatNNt2lCmWAbupqmCHYZ29jGQZKTLMmyFV6679377r3nnrDTyjP0H3Ptfc69kszrXtIb95x9dlhrr7Xm/M3v7xukxZqernZslpKHrxuKokTgeXCnYW9vSqIkzboln5REOsMaSRznRDNDW9f0bc3B3nQs0A33H9xnqgpms9n4ffZEo7I+zzKEUEQ6JY46rDWU5RMUWYLwgDMI6/nK52P6QWGM5/6Dh2TZDI8MaLiwTMoU5xyz6ZR6UzG0HXmZYYzFKwVSYcxA33msNSRZikAymIHVesnR8S0iHVrwk7JkvdoQJ5L/X7Jy3nBBGcUKPFy/PifLL4uaPM9DZFoUhvE0TUE4iqIgT7PRlzFmGCIODo+o26DoDcIRGzKCu44oGvkcxtF1A8G+JKirkiRmGMxo6jlSSMf2966KRtIMNWVS4iwgarrO8iVvfx//h+/8Tn7xo5/k+//J32F9+gIf+qPPczgtSKYlXgy07UAkQKCYFCWbTUWRTmhVyEqNlMYj2D+6wc1b40puLMa2xVmcRrtBJaxCAS/ROsL7YPXhRYSUW2Nv0NGo0JQq5NRqvVOvb1eqWysd5014zAYVdpZlO+GFcRYnLhHSx1syANaGQXv7d2stBpBS0RmDlin1KNwJz4HgQ+lxvdm1uIW4JHU742j7BqEkrgmrHSMC+dgYE5DKNiSOqG0BxXZ/glJfSBBeMNgR6QzjJlqrHRIZRSqsypwCcTkQIjyH13u07jBXqR7bj/HbXx4XD/jP+/FRBOLRa1+I0P7cZr2GNKUrf//8dx2nr3A8QoQkBKUUgxmQzu1EKhDaVNt93ponP74PV3fOjV5mW2QkpHoEWoW1gb+aJAnOWpTWDENP27SIEREOxX6gcYS6312qPbeIDB6lzrHdSwjhiXUaVJAklNIhXFDXZ1Iw9D1KVmG1O1bSYkvLAJySDG70oBypDkIohBEk3gSe70LRb6rgViBDvvyTT97i5P4D4kTw3LNv49O//mt84Ku+mp95cIJpG5JoStV7kBKhNG6oyOIZTR8WDlLBIDUPHix56f4ncU2D22yIvUBohVcSLxw2c2grSSJNOwzIzrBvFMIrhjhhwKGNxccKf2XIbI3BSU8iBLGUNHLgJBPs5RkXmwXnZw/54OqD3Hr2Tbz3vV/KPMlJsox+qOic4VwPWCUQNjTeUwRWRTg74JEY02LHMUAYhxQe3xscfpfR7o1BeI8SEh3l9Hag8uCikJ7V+55F34FUPPvUM3z7t/1ePvKrHyVaL1C1ZGu87T28773v4yMf+SibzQYhgtBlOi0B+Pmf/3lu336Fn/3Zn+X97/9S7r3+OtevX+fP/tk/y97eHkIIqqp6lH4ybv/+3//73c//7J/9M+7du4cx2zzlUNB+93d/N9/3fd/3yOvlyGs9ODjgmaefRmtNlmXjhGx3d9l8Frw+f8+HPkSaJPztv/23+aVf+iXu3r3L+dk5Sio+bxuRG+9CzNz2djM7XuSlXVDY0UtvwS0XePvdXeUiKl1ghqum3x6tJd4FH2bhk/He9eiRRw+h6O57A6Ibkb9LQRwCtAqL8SyOdp8lZbDz2Y7pAZXqUdLsqGHe+x2qbu1AmgVDojCubtEpiKLQsbI23hWd28I68FvDPifSkUwKwOFMRyoV6cE+R/t7eG959qk5eXo0ClaOcTbCGY/yFjkNi9o7J69SlhPiaU+RrNEZvPl4n+7pgnrjeeX25/B2gdMRDzYrsqzg7W9+E2kKvfU8/6Y3MwwDX/H+r+Dnfu7nuHZ0TBxrptMpe/tHeAxSRAGakj1VFfLX4yLmwbnC9AaLQekYSQTC0DQ9cTTBuQZrLEmcoZVgeX6PtlpxME+5WC3ZmxYYUyG8Yn9+jXicLyd7BV3XICVMjvaI4xhjDPv7+/RDw7VrexjT450KMY7OIfFIEa6PLZdd6f1d981JzXw6IxlpNbHSTCYCLVPe3kywrmewls06aBl01KFUzLS0bDY9WVKgZExdt0gJ5xcn1GrgmadvAZqqrVEyp207jq9pqupljEg4fuKI1eoerTtHqYw4e5wX/8W3N1xQFnmB9563ve3N7O/f3LVKoigK3D8T0EI1HnyRZeHxbmBTd1TdmulsTt8Hj8Q3v/WtvPbqC7SNIYpi3JjDWxQFWZawGRWMwzCAkFjv8YNly9nrtiu98cK3vieOirHIkkQ+xcmBxfIh/+yH/hc++pF7VGe/wB/8Ix/ifb/9d9KfxhwVksiBzkI7zAmL8/2OQPzgDvzix+/tKoXn3v4EN2/Jsc09EMcx0UiiHky/I5Zv7XW2rvjhxhQj92RsV/khmD57GKyHKGLwnnaweCcJfnCStgktbpygarvRn1Gx3nQIraj74GspvBgLRrErHrct5NDKCIPwFhlVMhSsxgcz2LpvSZIkKJ1VEERtByqlFEMbjndHxvYyUM59UKb6sW0WSTnC6wpvBoTUAWHxQVhhXBjspJZsvUiF9AiCLdR2RSwEOOsw5rItF1TLw85sXQjB/r7nre+IuHfHBJFA6O1e/rt97NEqcwsfjg85jOlYL1/ksph7M0KEyXQyEygZbI+c98ReImRwOAjH5WGMPbzcgmWIH9FP5wPJHenJnCXBj7wsRa0kVqsrIoVLuw6P3VlPKb2kKH8mnFsHbdvRth06joiThK7rON6b8WVf/mW07X1+7Ef+HQj4mv/kt3L68AG//usvMZ3uYdxA37cIL8iLhEiD9dGILgbuljE9koe7ydiYFhUpNs0qKBrjKc44mqYh0preqLCoGblYfhSNYB1SQuQtTR0Wkw4RUnXimLrpAiqzeIDtWtIopq4qDo8PqJsVdV3zFV/+1Tg8q77hxTsvcvTUIScnJyRMgym0jhj6junRhP5s2BXcxkMnFPPogFtKIicWe82MPoCCJIrBtkhh+LUXPsMq1bzrne/E3D5h9drrRFrj2p4o1lgVFnJ7xWx3hnsNNo1J2gGpLLfNEutDJnjXrpHTks/du8v7nn8fmphPv/A5nrp1jU+/8GlskfNQWbI4Qw0RK6fojMMKRz/0ZHFG2w3IOMIpSWMHnPNoY0bD+HAPAqFItx4tJYPQqEShOsu+iEi9QjUDx9N9/s0P/zD/9kd+jOe/9K1cS2LMyuzi6RjRdKUkaRLT9T3TaUnbtmMyjidJUrx3SKmI45jv+q7v4mMf+xj/4B/8A5555hn+8l/+y5ffTd8zn88BaLt2Fwm6W/SOHLDtauxtb3sbf+Nv/A0A9vf3+Vt/62/xZ/7Mn+Gll17Ce88rr9zmmWeepiwKHj58iNIaG7yu+Pqv/3o++9nPBl513/PxT3yCt7/97bx+7x5t14Qs9cc25/1YTMKj5s3bNnDEbl2+C10Ix7BFH5UaAzZsQPbjJMaxoWlrJpMi2OP5kG6yXoX2p1bb5KqxmLMW5zxpkXI0naLsHlrHAazQCdEYGqJU+FzrDHYwOw781qt3i3AKul0hulMOu+DZCGDNpS/iNjrSx2EcsoPdGZ9778FbBIJIRiRRgogVTV/vKE9CJEipaTYdWZYRaQ0+YrWUSFXSdkvwltXiHKh55aXPMS0VOo5ZnPQgJV/yzLMczDP6VUOWRBzMZ5yfP4uUMJvNiKKMLM5wxjIvi92CrjeWYbC85T/9/ZRliVKKs4sLFm1HVS3J8wlaKpxvKPemdNYS5RHDg8DVLyc51SgC9NaTpjld51CpQsuEalVzcDDjK778rdy/e588Lnjv88+xrjZ4J8jLgqIo2dR1GAOTmNXKUZQpZnCsVhVxkrLeLEiSlNl0j/Pzc5TqKPIEaz3Xj2/QN/WuqyOEomtqLi4uAm1CC5r1BSQRWklQEWdnmjQxONtjup66q9k72OfsdIFoEtJoYOgtQmmUF7i+QkWeTKcc7x2ikph1VROlEuUd5+fnLFdrbjx5nU23YZIYTl+/i1IRs6IEIbg4XX7e/fPFtjeOUKYp3nn2n3gLN28+ZjGAY3AJyShO2aIurXMMDFhrWS03+HQP7H288eSTFKmnJFGFlQPxyA8xvmYwGk+HHUAlPV2vkfIyO3O7sjRDMN60zmIGQRQPY854hDU1eRFxcvoyn/mXvwHSM5nu8+M/+lH+zb/6MEplRFFCmuaBYzgVJHFGkmRjoRjTVhPOzp8INw/wC798m+v3QztHq9CmyLKMIkvQyhNHaYhWnM5J0zy0ZKuKzWYToHsXBh7rtzfsFSTRXbY4BASvv7GocGO8nRACVLAAcd4i3JWVqRTjYC9QKr4cFGCHduJBXYmE2ra4nXPEo8G8Vo8q/3ZcISkY3BV17ZX/iZEkHNSpQQSEUGMrOqCxToRBWEo5DuJjPSc0W6hMIcA63JXCTIjAUfECuqEPqjUEHhfU2JHg9/6hmPOTKBRsX2Dzu+Jyt/uPbALByYOBn/iRBgEMA+A9Og7fxdf9roi3vGnK6bnhIx/+D7z+6mtEUUaRzRAKrDUYY1mNk4ZSEdZdpsEgBZFQ1G0QXr3/y95PtVjRtS2L5ZLzs4ek5SwYmvc9eRqPLUxL29V0XY91A1G0BGcJ3FRF49eo2KGjgIJnmeDd732GO3c/zG//bb+DT30qcHaeejLjmTc9w8Pzz3Bx8Tlm5QTTr0FAFJV4b4mTwBs2JviixSNX2owqYmcVTlq8cWRRjmss3npUHAVBWZThhgGlBEoHzrTddhGMpWNsJY4qU7zHm47I9KRxQm/7kBhHhLGG/aN9fvtv/1p+5md+HiNSfvmXP8aXv/8rODw85KWXXkBbjYjSgBibHh3FfOPX/Q7+/t//4ZEP6Km6lg9/9uPI+AxsgyCEAAAkUYxpG+SY8NPnGXMxpXn1dRIzIOYZsRfEUqPlWGQMltOm3V030yhD1AOD1Din6EURct739vnP//Sf4nMvvsDZ2QW9t/zAD/wgH/rQ7+WkfJ3IarTKsW6gtx1pHJGhiQbF4Fvywz164UmmMZHWJEZwo5ijrUeOSNm2BSmQ5OkkjLvOYEVLKwWx08TVgBWKRoDREX3dM5tI3vmmd9D+wk/txhTnwj38K7/yK3zbt30bTzx5C+eCUvXh6Sn44O5wfHzMjRs3SdOUJEkpipKLiwsAvumbvumRe2q9XlOWJWmacvPGTV6/+3o45zuKjdgNAh74U3/qT7Farei6ju/93u/lf/xbf4tXXnkFpTX/9X/1X/Ev/sW/4Mknn8QYw+npKfP5nNlshvfBq/fd7343UkrSJOVL3/c+/vW/+lcI4J/+43/MfD5Ha80HfutX8Zf/9HdxfvqQNIIhHfCixTuwNmR2I/pR4ABCyoAq+SAGbNsWJwacJNjMeIk1AzrR1PWGoekxQzCHX9cSqSzWG7wTDKajsWB72AKm1ni0TrHOk/UpJ0uD7wTT6T6TfI9pMWUwYREtiJAyQhFzcHCAVoq2DRSdLMt2FlLICX0/EEmFjgStCM/RUcLQO0S8DSLYEoAsyjMu0iUhm5pRaEpw5bAdxoZzF6t0TH0RrNZrhDDhOb7l9PSUqj1jcX6BswbXOTIVMckFe/OYdzx9k8G0nJ4/BCx9N3CQHTGN5kyfvMXh/gGbwdDYhDiWdF1PZwUqSzm9OOHO4gLrA3UgVkHYm01LlrdfIIoS8jynbdYkSUzrVrQ14xwd433M0DY82KxwIkfYDGHXoYsxzjNOOGRv0IVmcDVJlqKjnDw7YG96gNawX6Zjx3Wgr3tcbzk9PUWlMb0ZWG1akA3rTct0dkjf15iVYTY9orE9Tx4cMCnLkOY3GKTwHBzsc7HY0A5hHHvmLW+i7xzrVfCOdnjMYOmbAZVJ1uuaRAVhVZRMuX33grjImExKYtsxmJr1wxa/f0icQWUsv/rii1y7dZOHJw8Yug6MYVqk5EXCYVHS9EvKWYIjYb1cguuZTffojSHZv0wH+o9tb7igvDhb4ZzjX/2/PgbyHG+GcVIPE5xjNJjetghG8UNg7wqwMaZXPPPkMSf3JL/yC6+iZEokM6zpg7pShKJmcRGjpMZ5gW17EBFDb0bEbVRhAXYIRYqxdscdcdbRe0EUH9IPwdLj/e//EqJEB7GQ1oAkzWKUDko7KUAQoaRiG/YulWZDRBJvETPJ8dEeR0dutzJUhAQCY1qsVRjb0PYDZxeLUR11xRJCbXl1ASUT4/J8Gx6/RdXEF+Dl7Abh3bYVWlySp7fPZfcZ7B57vA3lRVihC3f5ukffn0c+Hx5VNj6umLxKUN8WoV9sf+AykWL8pB0FYMdL/AIkYKXUroO9ez8hcCIIPG7cevSYHz+GL9SKu7qfHkkcbXlSAXEcu0scHgniuOJtbz3m+Xf8Purqgqpa8y//5b/i7p173LgR8rZnc82DB3dxhtGFICJJEmId03VtUO16xYc/fBstFXkaE0URKqq4uLjNpCiQUlJVYX/DBObxKrRvLeBd8CDcXoOu7YmSlL5peevb38pLn/kcL99+lTe/6a286z3vBmDvxiGf+dQnuHXjGptVyKKO4zSo5JueIs8xfXt57nwoiAcnxus4QogavMMLQ9NviKMUoRR1W4FwNMM63DdeYrtAZZE+mC8HtDqId4wIEnDnXEjJUpr10BOhkdLh/BDoDl7zyst3ef6d7+Wzn32B1WrBm9/2Zp577jk++cmPIZXDMRAnMbP5jDTJg0Hwrr0Z7qGyKHD2nKNbT7C/d8gnP/kp8rykqWo+8MEP8NY3v5mmaThfnnL/9du8+OKL1FWIUFMxdL0FQrZ9GiuuH+/vrp+Luy+jEcziguuTI5btQBRbNpuan/jxn2e1XpMVkv39KUobdGSpfY31PTf2DkmdIFHQ1xVWgNDgVcrJ2VmgDESSVV8zn85waUIrwsLZi9C5MabHmA1u8zAQ/63CNAqTW5zoSTxEWUGfSJzvSSPF13zdN4AOaI6a67FtK9g/PODll1/mB3/wB/mrf/WvYK3lF37hF/m+7/t/ACF68dVXb6MjPbo9rPkn/+Qf8xf/4l/iW7/1W/nwhz+Mc2608QkL0jt37tC27djiNhhr+Y4/8Af4zKc/zc/93M/z+r17IzFE7ArX3b1LKOhefOEFPvLRj/KDP/iD9H3P3/ybf5PACdf86T/9p/lzf/7Pc+fuXZ577jn+0fd/PwA/+qM/ygsvvogA/vPv/m7qquHmPMWuznZMyt7e42wpqOoFSZxRljMsBB9UGRSwSqldcQkxi+X5riuXJvlOMNnUA4MNnoNJDEMfujZxlCBVEhbuImW9OSdSMUWRjara4C9b5JNR6W4oZ1Oc8azW5/TthqrahKJvFDFCy50HW/GRQYuY+Xyftu2RQmPdwOHBDZZnFc4K9vcPx8CIAa2C/d4wWKQYC7MkLOCyNMcMA2Bo69AVirQIYg0XB4swFYSWUoagiyQOCXVZnrBYnHL68D55dEapBjw92V5OqjU3jo/C/e4Sskzw1M1reAnFpGS1DjzGmzducnZ2xp37D/B+oG4Mg7Wkecb9Vz6DxWC9waCIdcRiEzjWD87vBteXocHVFdpnWOtYVxukKqi6lkVTcX6x4cmnD2iqlq5rUPsS0UsG0+1iiLMkwdsI00vSbM75sqUxgvXQszw9QUce7yVFltN1DW3XjG1uQV1XDM6ghGd/7wZJZjC2ousumE2POH94wo2jJzhdr1m1jjJzYUErJRf1PYahIUkysljxa7/+KWbzfZI8Q6qIvetPsjhfUuQ5OvKsFyucVIF3r+H+ay+R2ymdE0yymN5Ijp55itPTc05ef8jgPV4LPnfnBZb1koP9OYvlKb0+RHVZWGgoyWazQqsQfuJ8z+L1JQ7FYDafN3d+sU08Xhx80SfK/9LDJew/wkuhuBNi5HGNwoQrP2+fb51jNpuBd7RNG+KfRiUagpAT6xlv4pa2adgaGgdFt9995rZoDUbJl8WFH03Wtzyw0FqGa8fXUTratcK2x7F9D2D0sHuca/fo9u3/WcbNJ9SOT7NFCSAwbraw9dXHd8dvh/Fn9Ujxtd2uWgFdLShhLHjE5c+BCycv/+YvVdKPH98j77EtHLc8Onf5WVeLqy/02i9WYD7+/C9WuMkrX+zjQKL3fqfev7oJ/+hzHv9OH3n9G1SiXf3sq4Xwyd0l//Kf/yoAw+DBP08UTwHBh/5TzZNPiZCfPpLcb9445hc+8rP80A/9EEMXLIyuXbvGM888wyuvvIIxhs3m8kYUMiDbddPuUDslgvl8kmf4PqD7gzUjN3J0sttahOBHGoPdcWur1RopBJFOcHbg7W9/Kx/9xV/gLW97Kx/4qq/mo7/4C+R5ytNPP8nP/fRP8f73fzk/+7M/TxJnrNcVwgd0oyxL6maMME2CJZP3dkeb8H5U2eNxLqRcbCkcxoQAAmHY2V7pMQq1b1riUQHq9CUfK/C2wr16aSwt6PsGqcIYcPOJp/j0p36DN7/tLUynJd5I3vy2N/HLv/yLlOWcT/zKr/Le930Dq9U3YIzF2IFv+j2SH/pHP856rcdh4iZa/Vt+34e+gvP1hrt37/KZz3yGyWRGVVX8X//O/529vT3aPnhtKqW4OD/lzqsvo6UgSxKGriUvUtygyMuM2dHe7py+fu8OUaT5h3/37/PqZ18mmk3Bdtx64ik+97kXOTw+5nf/7t/NS6+8zNvf/g6KouCf/uMfoDp7yJGKENYSRwltZ3FJWMiuug3vfs97eOYtb6I1A+XeJPDbYk2IqpUhRSoOC4o0moAPySoqMkzTnNbDtVtP8Pf+b3+P//DvfzLE3XjP9YNr/PHv/pP8wA/8AE/de5HvfI/acRH/6u09Xnl4SjDuvryXlBIjZ/jzxxb3Bu+5OI4vx+Nt5wNwI6XmPzbe+HEluW3NbwvP7bj16Pxw+T4Cwf7BHsvFmuvTBNWsdkfxF/7o17N4+5swpuX4+DqrTQ3CkcXZ6NsbXdKqVMTFxYL9/X2ENzv/XSEEeV6yWCyIkoQkSYnigJzFUUaWTrDDgLUdWZ7uPGK3wRNaByeTIGxsg1COGu8EbdUH946iHL/DdORwtzRNaDELFE3To2QULGocOBMoQVmcUNVriqIg0hlaZSRxgZQDRT4jTSZIqXccfq2DiCqNUooigDNmCG4ciFH85jRe9WH+Q9ENBj2m+sSJGj2GHVplON9gbMPQGRCOzWbFMFh05LG2o1pfkKURTb0hVnp0i4A0m2P6LohZ/JZvHRLkIHSxDg4OMP3Acr0in5SoJEUpxb1798aOo6VuK/pBYLwhSjTWa4qJ4Fd+6dP863/xSxwc3qBqz+nHMAvEgBnCvFaWJYtVw1d91VfxFR+4jlae1bImzTM6oyjLkvV6zWw2YbNZjXnwsFxeIJWjyA4ZTIuQlljPwAm6fsXQNfgEnEnox+S6vek1yjxFSIcSOVV1lywtWa5X9KbGIpjNriHQ9EMHCiIZ0bcdXdeQTxL29vZYLWuytKTul9h+9BwdaoQcULFAioj1xhA7T54XFPmE1WK9uw71CLShhtHre6SXDSC05b/9s3/vC7f/HtveMEJ5cHiwQ9i26rJd3NXVbWwP71oa4xjgXEAX66oiThKSNButVrYGrZ6tj+NmE9SfWjIWiiMlbitZuFJU7jYfBhQ3EpC38Vzb4tJtLWOCSmJEYi733YSY+Ctj0vZv288JpOit+MN7NxYn2zQENw7EZseJeAS5Uxq5fecrA+Hl7n/xwXk7YErE2M67fA/4fPTw6t+uvsfn/X6l4H+8IN1u0o/t7fFv9rH32r5+Zzp+5bOFCKRx7xzmCxzvFp0VQmD74Tf9HhRXzrkQjxT+gZr4+df71Ulp25JXjz4BN351Qsorl+xofSW2/CWF8xIdSZwPXqnnq1N0nNI2BmcEWZpQrdf8+q/9Gk0TbLGyLKNpmtEfVGL7gcEaBqDMcoaho2kMgxsQ43XbNA1FWQKSwVl2CRPW0Y78YKFCIWbxmMGQJjm9HzASnnjuGd7yJe/g3r17PHHzFqZvufPKy6xWG4oi2KQMfU9ZZMGyKk+w3iBFwnq1oe1qijLwmLt2uBIgMFIr9Bjz5gXW9nhjA5dPSbo+JFD4xtJ1NVEcM6z6sbgL508qFcyRtw4DBGcIM7bdJnkRIkOjmCTPee97v5TXXrtNXqZ8+Od/iTTNSFMN0nDziWdY/prH2AElNF3dUZYT1usmNEZk4Gv+6x/7MTKlaKqK/UnO+eKMb/7W30OZZ3z2s59FRTpMmiIhyVKeftNzJFk6FmoOKQxCOdrBsDKXV15xeIs3Pfk0X/nBl3n5M3+fiZLUfc7FomX/8DrvfP495JNDfvqn/znnFx3PP/885+en/Lk//1+TOMn3/K3/ibqqiJRiaCsYDAfPPcfXfsd3hCxvGdFsGpI0oq43JDq0JJt2g9t4tMq5MIZyEsRzk/iI00WNdQN5UvG1X/0V3L/9cd7y7FM8fHhGlk54cPdjfNl7b/Jb3ncN7n4EIUax2NAxtpS4WpUJdoPvlXv3C96iX3Db0moe7xqIKz9fDryfP66Hse/KmH7F1sdzuV9fKM/bc7nQDsZMl0rvLMnp0pKiuMYwDEyLPVBgupZr127Q9z1lGQyfvYPZk3OapkPKjL359eBzLEPxOJvuY33gWTcbh45juq7j4nRJHMekaUzTrLGDJk1Dik5AbDdkWUa1WaJUaItHcUzd1HhnmE4KnAtCyK4Pgsm6bciKBOuCfVcc68AnbEc3jEjTDw4ZlagoJOosVqe7xV6SFNw78SiZkKXpSG0Q9EPoME4nB8SblOVyxWq1ZD7fI01KhmEgSSO8zQMHVMWUkznWheunbSze9/SuQ0vQcYTzlijZwxrP/uFRGBt6SRxL5PFAXW/w8xCpa+0QWs7KUlVr0lHUokawaTIXo+F3z7pzLM8roijG+4Su7kMRRMS6OidONB6DN5Iyy0nihHpjeeu1p/kNfZ8oimjbMEYgNW3fEmuBlhorwIuQ6R17zTufexsvfPrjXMsSnJEcIGjPLnh2b05b9RRRgVCSuq45OniCHosQEksaOqzW4lRPkpX07Zw4WdK3CjELsYl2WNINHd4a5pMJaTqlqoOeYTYrMcbQddVIb3G4QdOLijyPmU5KnIPV2QVSStrNKfPpDZzyGLOidYY8niIJ1AwlYkTmSZIcMwims33SLEIpwcnJabCqMsGDUgpJrCdgHdNpxhvd3nBBmWXlo2ONh5D1uLv1efwJV4cGpcaCwvkdEVx6ueuKh7bjdgUb4ty2qlc/pvIgHkUQrxpBbB+XUoZ27pW/OB8I69bu3nYsOK+81rP7vEe38PvRNcHBdbDbglFJFNuB0u/QHMbjuYqmhTQRh5NhAv1CKOQXQvauPi55FJm75D+5z0Mrr77uUfXy+LP94i3qq0iiEIItfnSVU7kt6Lz32HHQ3pnoPlb42e3nXDlPu31028/wo4E0nz8pfJHZ6+p+qyuTyRdGOwIa8shnP/4853a7veV+7orosYAKiLfEWcGwtjz97HP88T/5Rzl58Co/85O/SNd17O3NOTkJhu9NUwUDYG/RQgbUb1zodG0dVJjAZrPh+OCQb/7mb+b4+Jjv+Z7vwWN3k4BzjjTOSKMIoUdS/Uh3iLOUzhl6b5ke7PGBr/mtVFXFJ3/1k5ycnCDxPPnULfb3D+l7s2uVn1+cgfcsVxcg4OBwxjvf8wxKKR6enPLaa3cpigkAWkZY73YuAZEMKL0zhigOZvvCDxTTCcYInnvrW7n1xBP88i//Mu981/N86Zd+KbdfepmHJ2e8+uqrGGe5f/8+Dk+a56PAwOMZdguTxfKU973nbRSZ4slbN1iv19y4FXhzL77wMkIIrl3b41OfDAO4Fw5jKxBBoDCYAe8daZ7gvaXuPDrNeLhY8Du/8Rv5Y3/iT3L3/n32b97AS4HseoR01F1L3yhM3TL0C7TQ9J2nsUuEinaBBRDswuyiCnyjoxldV6E95KLky778qynLko/+9I/ygfc/yzd8wwd46aWXeN+XPM3Fyavcfvllvva3v5vJNEZ0Pb/6sU8xjeY8++Zr/OwP/d1d0lTT1XzZV74PSw9KMstnnL76gDzPiTLJ3iSjedgyL0v0xjMIQREpVvc/zVPXjvhrf/FPI/E0tqL1NfQRB/vvQf/yx2nvCbaj4ff84a/jv/mXH6FpHaenW+Nswd7ccHb+AKlkiEV2Hq2iK12YMQVm2y0SI3LpYTINqSLbsSMszMJk5b3fBSMEdfElT3zLSd6igKGg6oOv5s5TFawLqWnbe1Wp0bvR+VF04/nD3/Wd/Ov/z//GH3rTdd58/pP4sStjpSaOU5p6RPRyRVv1WBs8AMXYBhVCkCQJQz+w2TRMp3O61rBe1/RDSxQFT98oCQu0pukRXc+kCC4o6/WazcYGxwuj6Duz48VNp1PqusLaga4dI3wJPo9SaKTQdP3o5GF78jzBD5rBCZTKSJQiT4KryjAMTOcJ2IhJLvF2oBka1usKqRzd0OL9QN+uA1IpOow3eDx37tyjLEu897R9Q98ZVqsVk2lBZxRni3uhYGnD4sYYE9ro90Ro73tBMZ2E8U0YtJySpjlZlmBNjO0FSRTOTZyVrKsOnCXPc7SMcT4dTd49yisme0+FhVPfhTlm7CymGlAO03eUE4KYUziatiIWniht0W2DNS1ZBsXBHsMw0FQbtISu17x+9wKQ5HnBxaYHHFEUOjxBpa8Z2gEcSK84v79B2ZiD2R7LRYuSA3iD7XvSJEQkqkiRTAq8t8yiKW1bIUTEatNSFhKhE4YevNScPdDk5SHObWjb4JphfUhGWi3WWNEHoaoTVJvRrzrWaO1ou5o0SnDCUdVrVJTQd44kyRFSMFjHg9PXmU/2aPqWYrJHke0FAaZpA4JtPX01YGlJspzqtKJpGqRwTKYZVbdBRzFni1Pm8328ktw7OfmCc/AX2t5wQfmt3yFxTozmmVwiJ4z8NtuzdVR/dLJ2u0GimJV84md+jfnBAU+8+Vm6psdphbhi+ZIXGa+8cMq//Tf/K7NZhB3A4XCDGHl0wZbGj/F4zo4KWi9I4oiu79FK4bFY5+hbx2//HX+AJ597O6vVCqnk2F73+JEbHkQ3alekuCsFFkCsBIc3FFeAwPBcQTBv/wJIIVwWWd57lPzNC8irA+rVYk+OnMF+tNV5vMCSniDSuRJ3eXXbPqaFBO+3ns+X+xFO5vgdPHp8V/99vDW1a195jxQSd6UY/0II5nYW+IKf4fyOV7l97u49tt/FYxzPqxm8V4/z81DQL1ZAPlZo/2aoS9gnjfUNnjE5aExoeurpt/DBD36Q1XLgJ37iJxic2dkrbVX/N27e5OL07BGz+W2aUd00vPPd7+Iv/qW/wt27d5nP53z9N/4uPvrhnwfnSNOY48MjXnzxNt57EqWxeBgtg/q+pygmqLahWmz41V/8ZT734gu8+W1vZf/4iHe9+51UVUUel2zWQaE5DAN/7I/9MQ6vHYIIXM2nnnqOZ555JiCkRcFf/ot/hRc+81myLKfeBOGcH6/RSAXuURLFXCw33Lp1i//2r/6FIB4ynv2jY7Ks4Bt/37cTxynGGN75jvfsEq5msxnf+73fy0/++x9HjosxZzRpNEWTYZyla+Hw4Anu3nnAz/zMzzGZTPnqr/lKmqbhzu17FOke145uYG1LMJb3nJ1r2tbiEWMiUEKzKVFiH4fG+4H9g6fR6jm+5//0g0gdBW9bZzG1wPiOyaRgsThHCIfSEEWKs9Mzfsu7v5TX79/BDpdZ3u94x1t4/fw1Yp/xjd/4+xC94WD/gDTNGAZLXmQcHu1xfO2AJIl48rrinc+9n84N3Dp6kk3TkGcFkzzn+Xd9LbHyIDxlMQGhGIwLhbxUtH1D07UkUcx7PvBOnDMMvaBvLbMSVCQxg0EMgNcMruG1exbJQBwlDKbEUxInmocPLUenCZMtIAmcPdR81//+v+C1ewP/9ocz/Hhevuqrfp1/92P/hOnePut1xdBZDvaPMMaxWq3QWoYFhZJjHKrk+s2nmE73OTq+xvve916atuL6cUircYQCUmvN7du3geA1ub0fsizh4mLN4vyCw6MpWofC7979+yxX55xdrFAqQWpB39fs7e0hZcxyuWR/PiNOpqw3p7hBs2nOOV885NaTT3Jyeo83i0uvWGvA9JauHsjzlNX5AqlinBHggi9uswmdBTeEMSBLcppqA8JTb9ZoLamazWh5FvyU+75nMpmwXnU4K8miiMFYhO9JspihbVFSo6Tm4YMTsiJDolE6QgqNED15lgCSoTOkURkK367F9BFg2dKqIp2yWXdkeYJwEauLlijusX3IiE5TjdYTkjjwCr33rOoL6soxneRcLCvSLCaf5HQmJM9JIen9knQCKha0ZoOVG5wQeAVeBXCoGxYBoaWn6zvWp5ayzBG+pG3OKcoYe95hO4EiQgJ5NsUogXOGLNJ0r3fMJvs4Q7CQEwIV55ghWAdev34Ta4dd+IcxjkSWaOcRyEC9EZoyC+IsH3v2ygSExYqOroM0VxRzR1X3JLMZD05WSDVGQPce44O4M9YpURqRxgnWO3Q0MDjNZO8pjBVUrsMnOWkRo4qB1WpDZILfNBic6RDSU0lDXW/Y2zvixvWbXJydkOkJ00JTNfcp4p7IbmiHiqNiivOSfmjAd/RDjx8MyIGmMkTxhPnsEPTA2fkD0jyjHXocAmMTVk1HHGsuTh/QNAM6Slk39xm64Fd5cBCSg4ytSZMYZ0Xw70yDWDPNshCu4gzWDix9ixscWaq5qAYeLu6hdULbVl98cnxse8MF5ZPPbJXCEqRCimB3aYxBoIj0ZWvj6kS+nZAHIJtJyo8vefK5Y24+F7N42JDMcugNwkkG07I3k1Rrh+cOeZYzKD2ScT06GjN9uUQCDQ5vR98oLfF0aB3TdQ1xLOjajmm55E1vjjk9FyRJPLbrLGZnfyB2CFTwfd4Wy1t0UNC7YAOz3batwC1/1JgQK3i1gLnkGrlHkD/vfSjAHmkPXxZRYtfSYedRGEm9K6webyuDQDg7do0ea6NfQSqFEOjHXiukeOR8fTFE8CpC+Tgi+vg5F4995tXXX/2MR34X2/ff7vyV7wuP9OLzishHNrFti139DvxuweD9F3/tFwSHr3KxJHgZ+IpaxBgTovm0jqnbjpdfu8sHPvhBfuqnfoosy66Y9geT45OTE2aTOVES8s/rusaOBsjOOb7lW76Fe6cPWdYVRsCHvv3386Fv//3gLYkOTgL37r7On/tzf47F4gIpFdPpFIUg0hGm61EqYr1YcrE+4/z8nMXZIe99/5fxw//mRzg7O+Obv/53sVgsMIMjSVK+5mt/GyrSrDY1KtI0dc/HP/UbaK2ZTSZ815/4Tv6Hv/bf8/DBPbI0RngBMkL4IIYriglN13Nw7SZ/8b//6+STkvbhQybTCcuLmvuvX1AUBauLc7z3PMSA98Q64uHFgv/dH/nDfOj3fAuTMud7v/d/4hOf/A2EUDTtBik0dbXkhc99hv2DkijqmEwNXVPzqU9+kqeenvCmZ9+MIqQ9eIIR9C99ZGC98hizXeBN2Nv7EFHkMcOAEKCk4qM/Z/HsE4/eqjrSBM9bN56T49H0f3steX7ipyOcnz2yIPrMSwEFQ+whlEIhsC5UaWH95sHvj4vvASH28daH5wrw5OBlWJBRIEdqQEDI7UjNEYBFEOOJ2fLEnYsuRycRCmrvVXgPBJ7JSBG6XCzJceEpRcxvERHfdO3yaP7Dz6b8/E+pcYXNeFyWV+/eBano2wGJRClH1zdEOohNmqYlijReSNqm5uu+/ncwnezz8it32dQ9Zvxw4yw4j9YxQ9cT6wglJGma0rcdUqsx2GL0dtQqIPfHx1SbhqODY7q2Jo1rmq4n1TlpnNE0XfC4RdEPLUJm1NUCa+Kdz6L1DtMLRBrQ00AhGejrNfNyn6peIlzDbJJjvWOxuCDKc44PypAQ1nVjl6Ijz4L9zjQKdmzRbDqi1p62bZns7eGcoGstUaZG4U5CmkVMZzmbdc1gQQhFmU3w0oc44yEgsoyxwThDFqe0vcF7i5CGuqnG792y2ayJ45g8i5Eu2A8J53BDj/eOxWJFHOXB2mazQUSWfqiwLmQ3N22FjoKoSJMiREysY3pXMZvcZDAd3o8WbWKKEBKVDihRoiVYAm/8eO9GiHWUgihWuKFlfxaxqYL1TVoWLBcL5tMZm2WFMhGJBtcNTPKUrl6wXFc0XRuSz6IQljH0jk987pe4ef1oFMF0dHWLTwRplDJ0lvOzDW0zUBZzyqxgNpmyGc44PNyn6WrS7ACpo3GhENHdO+fByQV5ntP3w+gFKUZhZEw/DNjOkRUp0NE0kBdPsdp44tQTDylGOJwa2Cs0ph+IlGdTrcnTIIiyokXrgV5EfPbOA+rNKX17nzSZkKY5R5M9jg5v0HU9m80GKTXx5JC2bXju3W/i9bt3EMIS5xG9tXStJUmmHBznxErRmIGqGeh6g0oEAkNlFOl+gfMK4kMePnxAazxn5w+YZCU6nSIE5HPNarWibQX780MWiwU9AfioW4sUMcZXY+CFx/qwCNDRF592H9/ecEGpfIwSgN4WTYGgL6Un1o7ebQusMMhxpaDy3hOJnETFDHWLchLtI+bZjNZ4YhHhNCgdo3SI9NtyKKRIcRakNmM6BJiRSGtt8FDUKg6DXt+FInPnUxjaX+fn5yNhWWAkl36MY5VnbI/g0phcjCtZeaV1rbQAby4LpyttXYQYhTKXkVdXt20G59XvQ4tLhXd4T3tpLv1F2tZXi72riKZzl1DD1Tb2I+9zpbiErTHuWBg6N05eX7ywvPqe2+JwK7DYbo+/9pHWvhg/w30+f3RbvT3+Or89fkBdQW+3+/NI8fzYd779u3jsdVf39Sp/dEus3h3ryA8OPCxBEpfBpgpBHFtkiDAI6EHT8eSTT/It3/qt/PN//s85OjqiqWsGb+nHttlitaQd2p0RfhrFrNdrfuc3fgPved97+ezLt8nyHOc9D87ORsU0u9bi3qzkr/wPf41mteHBvbv84+//JyEFRErwQRF8cnHGu9/9Ds5OT8jznLsvvcTywSnX948oioJPfOIT9H3Pc295Mx7Jiy+/SpaXwYJASvaPr4ENx5gkEf/j//lv84/+53/Ij/+7H0PFHmcMwksEmsWiQcUp/9kf+eO8cueCuw8+S1mWSHFBFCki5blYbpDSIWQoYoQQXDQrokhx7+QBkRI8md3g+S97H+nsjGvXj9hs1sxnB0Q6J0k1k5nm67/5G9HikE11wfPv/wqmxSFJkvHaq5/A+ZsoHYU0CdpH7hspIU4iIh2Q3rA0CdwWMdpXRbHismErxwVTKATCo+Ff4zqU1p93oUVa4wm53jiPswNSKoQL58YTYhHD60I0XbgUQ6cE6RGMSSj+Ml/Z+bCvUow0HufHzkooouXIMDI2tH3lyCcUo52EtQ498oa3yUlCgvLbWLvP38LaNES0OUJHaLm8wFiIohQ7dGjtGYYOM4TrMnAkJednS778y7+cLC346f/ws8ymh5TTfdarinKScXFxwWw2o6nWzPb2AnUiVugoFMRd05LGCQaB9w2zWUFbDziryYvAn5vPjjk9fUisNN4MxFkwbbaipywnJKmnqVfk+YRq2dFWUehWOcPQGXxyOe7EUcJeOSdNMuZlgVA91gRrsqNnn2NSzkK7WGp63e+8hZNYkkRjepB1u9Q2pbZdr9DN8N7TtOuAyBf74Rg3hmplKYqSqqrpTE9vq8D3s444jun6IIiw1rJaLVFj9nOUBrHc4INvr8SxXq5Hm65AJWuaBus0eZ4GbuWwYXOyQEYabST90GNtj9Yxq8XJLjRDoCiKCZuqoTMda7HEmhAb7BzE8UBVVZTFDCXO6fsWpQTCS86GDiEjoihiddHT2wEvgt3QZrOhLGJAcLK8QNgYrRVZntB1DX3fBVeVNMX1Fp3E9Is1rR9QKiL2ivuv3ydJQ4BKXbW0J82Og4rUbOqKZXMWFoNCInXGpz7zaYQcSKOSrmvC+Dk/5t6DlzCDCy4SjPcTEOmY3oRAiCxO8Azo1POZFz/NgwcPEV5Tb1YoGWN9RaQ0fdcEL2atKSZzUCC8JeWASS6xtmc6OcIaQAyIyNE1AoPnXu2oKtByju07mmVDFCXc/djLJCjOF+ccXJugtKHeNOTJlMP5HKRhIhLmc4GQodNkmp75tSeomoq634A74PjGnLPlhmf2FTIyVJseJaYMgyEv9rGu4/TePeaTMsx/SuNzR297BsK92ZseYyxKKKJH/JV/8+0NF5RN316aWo9m1+FfjR+LOjVy6bbu/YLLibvr1+xPb4BOSeI8WF4Ii/QehwqQ/LjK896jownImrbdgI9QKg7EXWvwwqH0mC89clKsqfEurPzDfgqUCHD02elpGHBH4Q1wpYCQqDHu7jI3ezuIO6SSY/EWeJ2PFifbgsqPhtCB7wNXiyM7TkzRIwibcw43XHKLwmS2/b7kaCoePkuNKS0wIsI7uFOAD5F7zo8JOf6y1b4tGIORMHjsDnUJFjYmcIq8wws5tlyuFml29z7Wmh0tYHt84WLcghp6F+FlzKMCHSEE1gVzdeQWzRwRSxVa8cJdqoS3+yhlEKbgPU44UGNxbtl9b24s8rfxUJd8zvAdBWNgj/GXKOmWC6i1wjkTTLx1OGKE3L0WAcKL8JlAEof3DibNQYhlTEdRxtw7O+ebv+1DnJ0/5Kd//N8HNFNr4ljR9TVC7JFEDYrQ2qmqBV/z1V/Ld/+JP8Irr9whieOQ20zw9BQiRF5JIYjSlE3Xk80m7B3u8c73vYcf/fGf5sH911AMeBsR64j1csNnP/MKv+Wrfgvr9ZIHD+7x9ne/ha/8yq/kIz/zUTbVAovng1/1NTQd9L2mbtcMQ0+HoKs7uqalrZc09Ybl6YKua3jimWfo24ch0UqXSAlZ5pFK8EP/8/cwmeZcuzkhijIQmps3b5JlCUqPdBgn0Eow389Qsme9bLlx7SZFGbM4vc1Xvn+fL//Kb8UaTZnNabuaerMhjmMQjtPzB8RpzeH+MV5UdO0aHSWU01OOjg84fQje2d0CYHfyCJwhoRXO+4BOIVBKjkBYQPc9wTrMOU8cxWMRNsZVIsZFazQqoq9s3mHHmFmJQGpFMhZxW7R9ey6ts2ilAb1bwFlnMSbct1KEyMyQ+T5SSqxFyIA+urH17d322g/IpnYKqTQQkC5rDUKEaD43GojvnMn8SBkSn4/Ke7Zoa9AaewdP3Io5X54gpcfqDZ0BrSCJc6qmDRniWnF6vuCdz7+bp555Kz/2Yz/Cm970Nu69fp9Ue5zpwEUsl0uuXbvGumkxxtC2LWmSh3Y5IdUFoDk7DU4DbUOWZHRdQ6ITIqmQqQMKynKgWiVor8iijrY2pIljaC2RlagkYWM7irIj0h7TDDTdBc6xK/xiJVm3DbNiDh4W5w3WeoqiQPqIFz73OdIso+ssRV7SmYBCNhuB1gPrakXbNaHQqTdMipKyLLEmxtg1vWnpzIAUEVVzThRrlHToPGbZrPDCMjDQG0cWF0itybKcWHk8FhE5cEuECEbw7bohSVJ6VdHVkrxUyCHFe4WzDc5AFCVEQlBXPednKxCGPM8ZahPMz2MwDgZ6iqKka2usHdCx5v6D1yjLCRpF1/fEuaauKlarjjQtiFNNYxb0bYfSEttJ4ignEoa2WeGcoR/WmCHEWTZNw3qzYTKZIUXE0IXkNGMbsjyiadYk8YRIFgg0g2lJInAuIs9zhARpIBEp1nguLsJ3vXcckl0enp2hFERptIt7VkohZIxQHqSj9xVRMQk0H1HR22DI37UDSZpS1Q1ax8go5nB+yDDYoMR2MUUSMQxrfv4XfpS2vyBL5kSxIoknJEnwJI10TJHnRDrZJcwpmaMjgdJj/LSK8SZCmDBDaSlBOQ6OZwjh8aPHbd93IYqxFxw89Rzee4ztSSeOuq65s+pp2p7BKowxpGlMW/cY01NXL7Ban3NwsM/QnSOxCOnpmpa66lFacHHxSdpuzbXrTyKwpGlMP7jgmdy2zPem5DKhSGJWqxXkOYvlmtYZbP/GIco3XFB+5Qe+LBBcm4a2bWnblqZpwn9tHfKsUbsCKFJiRGOCoCZJMqp6QdcvyHI95ppa4lRhbQ0iDpOCCvYD1g4jAipw1u2KWR3poEJSUTBWVorBBtHD4BxeujEfOyGKgvqqadtHjiXYBLiRV2KQQj+Ctm1RuKvcyKst2zARBQXg5cB82dp9FA3bcgCD6fj2/f2V97PWXrZst5MRfpfw4K0NqQ4yDPhbVbz3gNh+nhvb8PKKYXkotqS6bFGHIniL4AWzeCGCZcD2uGHLK91yIv1lKsMW4fB+LCa3iLVFqm2utBmLrpD+ErJns/F7YJemst13IQVCbZXiijgOxO8w8MejqKkZCfgCp1zI/9XxSCUQbBOcnQxxgmIsMe1YVEdC7pDrq217KXUwshcdWw5waBpyhVtqkFJhXVj0KKEw1iKRaJVghsC3e/XVV/lj3/nHec+7nudH/u2/5qWXXmLoJIIJUq4xxuJEg/WKb/uD38Hv+/bv4ldffBEvFUkTjeblocjFCzpj6PoW7y1Dm+A5o61q9vb2+bqv/Wq+/x/+P5nsH+AjS3WxZD7NWF/c4+Mf/SUms2PK4jpDZPjhH/lpHtx+hTzPmRQZP/Hv/i0/85M/gYw0e3szTh7eZ38yY7PZcPPWEVLVHB5NOHzGc3Z+j6/6wBPsXXs/3g+cn6+4desWVX0K3hLpkjTJabqWopjgnSJJ0uBVZ3oC38uHzOxujdYZeabxqqHuamQmqUzF+b0NKmo541XMoNCyQMUNaW5RSUuRzqirFUIoDmY36fuep249yR/6QydcLBWKOaenjv/3/wIhQtozmVi+4w/CwX6DlA7rB/JkQtutA0KmY9puQxTFpFGGB4ypSeIYOzpOdG2P9RBJT9sY0uSyA9F3nqLMQBgiFTGYDYNXRDrejRlt27Kpx9bWaKEUkKge5+1Y9CmcMSilSUSMVAJjOqQUxGlKXQXeptaBN2aGHkbx4v5sjhOGwRiyPEGiuFicgg8FStOekWUlQydQakB7Q2d6jl9t0S+MqCaCb/p6x/ufWEKSgpfEsWD/aMVf+iun5PmEbjMEoYg2tH2DUoG+MQwDX/qe9/DEU8/xoz/6I3gfuHS96cnLgr7v6dqBvdk+3o6L9r7fCU36od1xa5MkoZxOMKYnz3O6bsA4SZmVGDuOXc4yn+9jugA8xHHM0BuWyyVlWVKtlxTxJNjO9JqzxTkAzz57jDl/FSmDkMJYD0pzfrEkTiPW7YbpdI5xHVUtSJIZeE+aBhqGNRDpAkSPxzKfTfFiStc1TIpjuq7j3v27HB9dRyiPRgfLJTcQ5zFVVbFqGvJsjlaCujPEWhGlGd3QI+3Aqm9RSQw4TBdM1M/Pz9nb2yOKASx9bUiiKWKwaKFwTmKMJE40fWewriMvsvB7H6gwREFgtKlrkJrJZMLFsqbMMhBRCIxQKatND70MhV8vWG16qsqyvFgwDAN1U9H1FklGUeQMQ/AZnc1Lrl8/ZujmDLZmPp8BKfP5Mf3QMZlMLjn4UrI3m7BYLKirjjgqsYMnFwXOD5jekk/y0XopxCkGpFjSdQNt1+PxFGUI7ygnE5RSKDUKVLWiqg1JVuBtyvlpw2QSUdct9+/WRLFkU23YP7jBagVCOtJYIfyAHTqUEpTzoNLfbDru3LlDMWu4WL6GIB3n3XDdBQeAcVHqfUDRpwdIkaBkMFqXyoGzhEAWyOOQ5930HdaGwrssppTlFJBEidwJ/4wxIY4TRmP5iMG1DMayOWtG4VqMSo6Yx4fBBSTWSAVm6IgmlsOZIs0ijm49TdvVDFXDMLTEScr5xYYoCgWrsYooUphIoUROJATX5gmd6Tg9X/JGtzdcUP7yRz8WOBt5TpZlTPM5x/vXgwedEDgf2gKbpqZtW+q6ZmgbmibEYQmpUXHEZm05eXDBs0c3cKZGyUAoFtrSdaH1G8d650EVZRFmEAy9G72VBgSB85WmKR6DiraogN/x16IoWH6E1sFq1yLe8gqvtnelEAgtdoWEc48qfEPLWD6yqhePwcCfh16wbRMH/p61l6jdFqjcejCHYnVbvF4WrVdbw35EFQJSOdof4ZBjS86NhWaI3RK7z4eAniC3yOq2aNomHV2iFaGgCh6D4orIZ3sc4bhBXeHLyhFNGQaLUjEh0UjuMmq3nnYBDdwWzQF9lVIH1JSAMEVZQEiNNWjNrsDz3pJu/QxdyAVXcXyJRQlB5BMYhU+WkEW+89r0YHw78uPMbnFghoFIJwxDx2B6Ao0joFjhexZ4B/fuOAYTcoqNCS0pCCrfLT/Tewde8+sf/xRpcsStJ76OxcUeq9WCoe9o+4brR7fouo7FsuUzH9vwX/yb/46ja0fsH2fcv/2Ag4N5aMcIR5YFCxtnDdYPLJcr4kSTxMFweTIp+fqv/1bAMSnnnK9P2N+bkSYhBOCzn30ZZyryRPOed7yF+H1v5+Bgn+m0YFNt6PuWSEcksca5pymKnLKYjQhfKISkjpjuvxWDZWhvUDUXdE3Ba7cTBjPHe0NZ7LFed+BjyjLBuRDTGGmNlIrlcs3+fJ/arBk6T5ZHCJGBFZSTUHBEMkIOKbbzOCdI4hwlwDQdr5+sKYqbnC41m+ocQcziDPJiwoN7A5PJE8xKR7Vpxrz4bedBkKaK48OBw8MKITxlmbNZr3hqktEPlkmZ4XyYqMo8RkrBpuqwpqEsJwihqOseISVOOLy/zF6GYC4+DMHLLUmDhUR7viZNc4ZhoOsajucR1m89eftRfBImyLquKYoMb4PfYRJJqn6gaSqiOAi4uuacp25N0To8vzU9cRRoQHGsWS5fI04TkixkDBeJ5qmnZyyXC4RY4b1EylB8WevpzAVH2QS9rNitn/DsHTvk0wLnK2aTG+gYluuG9aqlnCri2DFgkSJFJ6GdqZTive95P0pF/PRP/gRdU/Hsm74E6yAtclbVhr3DA9brNZPyBt4E0Y1zjkgpEMF7c5s8EjKMDdssY+Md7WYTFMjWBCs5DRfnK4qy5N69e+ztT5jvlZydndFFLWjL0CvyQvPKSw85Pz3hv/nvvotf/Tt/l21uuPeeWAvSSIYibLDMyxyBp+trlBiQIqZpKpJUgQ9K6yQBb/VOlS6lJC1CYRPrhOlkwmqzxDsFhMQyYztM37NeVeT5lHhc8KskoWo2SCmZbFvPQiC8RUtFVM6ItWIiU7Jsez117B0GT8LFahkU5lnC+aIliTTCS6QMY7C3jiILLU2vLcOgSOOUWRmKqziLyZIIqSLq1jH0gqzM6IaWqulZLRu6fsCZnqpqcDZkSOd74xhPysHBAV7UDIPD9DOMGbCD4OzUIITi3NYkSUTT1JdxlS7mYdSOkcWSTlakaUy13tAPHdYqbt++N1IhAoCSZSG9zliLRqEizbPPvmkn5GrbYOHU1g1tZ7Cm5N6dJX23JstyFucdJycXxDqh7yv29g6QUjOb7QXwK4oDnclLolhzcb5CEDGf7fPyy7c5Op7RDxVpojC0lGWBcQbPwGRSMJ0VIzhT4K0hSxOsrVGqR/gQQyo049htOVnex3lB1w3gBav6HskyC4u03oXc8Tgeu5YChwzeo0WOM2A6Nwa8aKzxdCbQLqJI0zpHLCO8ylBC4pSgGRTWZhg3JcoNZRIxmJ5MhQjPYl9RO0O7qrndNiQ6QmDIsxilNYv159P4vtj2xjmUSmOM5eJiwcOHpztRyha5StMkeG4VOXleMJ8dkkbxyPcQtMOKn/6Zn2MyKSjKjMHUdP2adbVAeM3oF02SBlJuHMdIGbyxjAkO9UIEIcQwBI6FHZWuW58/reOdNYrWmsHUZFmGsT2D6RBjYWcdO0QQYLBdyCC/wg3cZkVfbVNvW8DArvV0uQU+J3wBfp8PljFXuV3h30tB0FVl9xZBexQVBbgsFEVQn+yQNgG7tvvu5Er1SOHsH1GaX14k4TO3bf1HuZtXuYaXyOu20LzcsygOx+W9H9tq4XiV8ME03vsgmvIe5+TlZxOU+l4EE3prwmC0bUtvB27vLAKFFAJr7JjHPP5rB6S3Ib6SYCeCUvidJyl4JNaEgty54EspxrzywNfcipoCf83amj7UmPzEv9uex+35CbnlQuy+CrwDYwY8fhQXxOC/gjiOkNIQCWj7NKRhTDz37gXe1eJcsjj3CPlW7t4TY8tS7NqiWgerFaX0yP31Y762QKpnxv1X4J8GIXHOjolPb2frxXqJXEusc+D2rpzP0UfUBKXqJbYej5y+MFiG95gg5Wy82uZbNuK42AgtWevsI9egs9mICo8JVyJk3G/pHFKWI9IesW1BBwgdlMywdjJe9gI4DmEIgpEyEQHD+D4xfV+xXGxPSlgc5mnC3syRJiX9UDObJHg/0NYdserpzYY4yliv1zhnme+V9L3h4mJJ1w5keYLWgq7psAxIkeyOrWlX5FlJ2w4kWiBlSu8HYuVpmhaUoMexqTYkcYh6VVqg4xjnHMc3j+nbDmsNcRaKFIdFaoH1Lpjce0NvOuqmp2ka4lRgB8d0skfT1jxx65hNXbHeXJBkJQOee6cPqOoLjo+PkT4N348ewv0xzImiGeVsn/rKyLKu7tP3T+I8vPzqZ7j51BG/8EufwjiDjAymT/BW05sOISKuHT/B0dE1Hjx4wGuv3d5Z/OztX6O3FhVF3D95wBNPhbjEosjo+gY5igC2Y6xzwQJl2yUqsoBW9tZRV2ERuNls0FqyXFdcu37AetUglSVPg2VVlgVj67YObeG6Ocd0BXnpidUT3LxV8k9/7TO8/x1bJ2AokwgiifEgdaCvaK1pfTa2NBuKWYExlsnskLrZUBQxi+WGerUgT1KklKxWK+Z7U/CeYTAwWPbmM9bLDc50aB9GoIP5Xoj6VWpEt2CaZ3Rdx/HxEefn50RRNC4yCqqqxhnH8eHx6M865eTkhG69RGvNNInxdDhvuHVwyNnFncBRjBKqejmOpxFxHOPcgGlb0mTC4tzRtYIsK6kGgtDVRyjhsH1EGucUScTN4yR09/qGLEvwXlBXLUk62QE0XgqqjUBF0PcLuqHfXQcAxvRYOxAlMU0d5ui46Fg1FtlpQtTrQNNWZGmBcwJhJFE+5o7bgTiOWdcdi9cfUBQF3giiNOHV1z9yuRhpO4bBBrpaBIuLjqEP10e16TAdJElO3xlUHFxi7r4ertk4Srm4uCBJUiaTCWaQaCUY+oEkFcjI8eDkDvO9YNae5SVSBpW0lJKzocKMiwxjgsvBJm4Q9KON1IBpA/XAY+n9fdbrdUgByuKg/xjnXGstprG7XHbr/dj1FRgXss2zKGY6nSJkggS0jvBdEE91rceLiHZowUv6tkbpscNqFXGU02HZtN1YgEajl7YErclnBclkwNng09o3LV549q8d8Ua3N1xQChn4h0oHCHWLYO3+swE6XS5XdGYY25UhMUMrxf7+lGlZMgxrbt26zju/7LewWlQYN2CGkH25Xq9p2lWITRIa6EZkMFwE4SK7KrpR4wojFJdKKaJ46wc5tsi1Hlcy5vIkyWB6vCsCg+9QQLOk2PEtpQzCnnCDXCksd0XW1aJSP/J32D5nW7h9IQubLf9SILksaHcFJZcFo+PSpHebh321AEVGl7zGnQWSfeQ5jyOeVx/TOryeK/sOAmcvi0k9+rxd5Whu/2aM2hWHkgjsVa9KML7eqZoZC2g7bFsBbkzksOM5ZYdYaC139ADvLd6NsWhaIMbCRI2F9BZZlj4ILISK2KUXYen7MEEJD0L6MQ+3JU1TIlniR+GDEJ716sXf5GYI5U2YKy6L60umwyWHruvZTWKr9daP8wuTnLfvdxVJv/Kmv8n+iCv78fh7X6l6t58zFpJsS0Jx5ViuvF488prLPz3y8xc5ms+/1v//2cYggy8gIdnSMa5yJnfPHffPWkPfLem7HjO0FEWGxIXEDi0Y+hqlHUNfIVVK13Xcvn3Gwf61kB5RxORFjMegI4d3+SN0lkn5dFDx96+xXFVkecf+/h7L5QrvLQcHB2HS0wlZWnB0dI3PvPBr5GVJEgcxT48lzhI2fYuxPXvzA+oGyjKnqhrK2YQ0imnqDZNZTp6nGAN9Z5nv7dH1FdY7jq5f4/xihe1ARY6Dw2M2mwEdWZI45/y8Ik4t9dqQRT1puzUyDzzd6fSYiyTBiB5lBqbTIz75a3dRkcB4x3JdM5vGzCZHpHqCGRS/8esvslzfJ80kkY4xJmZ/75jPvvhZinxC1wVqFMbQNA2M8Zti9PAFwDqWyyV5Vu6oLWmaIU0QTq7XIe1li0J1XU3TNBgX5oS2HVitF3TtgI48UTTl9u1P0KxL3vnuKZul5nMvfBp8hpTd5XWJYD7dQ/qQRZ+mCZtNQ5pNAEeWZyRZihShdSwSQaw0e9OgVu/bgWFomRQxbbWmnE5Ik4hZcUjbVWRxmOSTJMcjqboKZ9ZYl9GakJQTKU0Zp3TrmlRGDFVHmWq6akW7Xgd/Sx9iBK31QQDmItIoxZnQFSzyGUIoyms3aWuBjmLKJ2/hHLRN4LSfnt1jcu0J1uuahxfnxMozyZMdElwUGXUdolfXqzXGh9Suum6ZTPforeP09JQsjbm4dxedpNRtRRSn4AQ60ngdrGeyLOH09DQsfpJwTzXdgJSaLMvYrJYU+YyqDciz7VpynaCsBK8ZRE0/hJZy024wNnRAZ/PJrtCPYoeXY6crCot1GUsKnSMpmM+DiCjVJV0XYmpNPwASpKPrOo6ul6OoUVPXY3qZgzzNmM5LrAlJQHmeUlVrdCLQ4gAjKgKtb7Tc8Y6mqamqTUCZdfDlXC7XSAxJEubidbdCOA2xpmpaojhmVdU4txnrlwEtFc619GP60XbBNYygysXmNllcEj2M0DoEOaRpSp7nu9SpSAfnCLyi7RqSJArgm5F0XUQiNYM15HkJKEzb7uonrxR+FFA26544TrHeUzXdGx6x33BBuTVYhqsI1qXSeFsAxIkmSTOkZESYDE3XcHpqiHTKtDzilVde4+7FkjQuiJOE+WxCMSm5desWk8mb6VvP3/+7/xdoGuI4ReuQNJBlGcKHVg8ERCyE2Jsx3svu2icB2QqTStu2IBxKi9G2I/BbjAmClG37eotKbtu7OI+WCqG2nMVHJ7arKmLntr9vn/d4NvY2fuxRK53t+2j5qGjnqqAFQkEpAnw2PkESPMm2Jcpl73pbKF7lbG633c+h2guIjwc7Cmm2z7HWsu3ibzmZ28iwHQd1uFRJIwzOBxP6qylB2+9RiQhvAC/H50sQQfgQRRJjepIovfK9DURKI3BjrKXaLQ62CF5QfjsGMzBJS3oTJkkvLVIYwKOEx9oh3HSZpa7XwXerH4jimKKIkdIRi9eJlMPhiCJHFGkGwyPV0yMF1u5EbX/fnhdx5Zl++3+2VZt/pMC7/HlnDP3I+XqskLrytpdF56PPuyxEH9/Tx4uyy8f9Fpq9gg5u4dr/WGG426VHa9HL51+pOMVY+F7dk911zlVBDbvfHzmeK+//uFPA5e+CEb7n+jXF009p2nYgz2OapmHoJVLCfL4X1KK2IkuCXYwcFLmeoeMYhsB17PuWtm1IJxlCKborg6s3G2Ji5Ig4OiRnqwV7+3s0m3qcbFoGO5CQ8/LtV8h1jBrCfdvUNUkUsb44p5xM8AM8PL3PcrmkamriOGZvfkAcRUgJfVuTZBMm5ZTFcoUQgt6EdvnZ6/dZryrSNKecpEit6G1LNzg21RIdKerWEOVTmmFg01kSEYSAxjg+9qsLRL5C49HRMffu9Ny7d5+iPGZSTinTiFhpOtNx797dcVEfsb93BDjW9Ya9oylmVBuXs33M0HF+esYTN24ihKAzobOEC2NN3/cjT6xH65YkzsYFXrYbQ5w3LFcXIaI0S/FtS5ZldMMKoSWRS5BCMZvlNF3NL/7iL3Ht+iGTIkPqiNlexId/7lc53L+BEK+M1zgYqdi0A7iQzd33A04lJMrgnKTatDghWW/uMQxB8ZpkE/bnJXVbE8cZSRr4e1VVIWXMbDrj9PSUobccXTvGW6irjm7o2Z/uB4P4ROPHDkvXDag8Yr2qmM5Kehk4jPN8nyeuPUnbBiudIi2IophiMsPYQHkYTMfQ1SRxSde06GhOXwjSPCKIBQ16XMRPn3wWrTUHM8+bn32avh9ou2EEO4JaeC/LmU6nVIPFmJY0i1hXm9BtcZ5pYciLmGn+TBDgGIlxjk17gbGCs7OGITH01vDUtVtAiJV98PAEIQSr1YooickP3krTdAjbI7wmS8fELKlRMiIbCz7lBYlQKCfJsgKl44AoUgSHBtMSJ8H3c2hDPOpqsSSOetJMMc0Luq5GS4tHItTAwVHGZhmhZDjnfpxvJ2Xo2KzXFcINYFOqqiFLS7xRgVtra6rNQzrTBs2G1rQX7ciHbrDOkRcZXWPwLnhWl9k8ZH0r8N4gpUP6jiz2GNeQRhFSRtR1w7QIxvJe5aSBMAsE9whtJSoKxX9rDNZ4hO3Iihjja06XZ0ynJc5bquoBeVYG2losQSf0VoDXbNYtZRohhGJTLVkul7sO8+HBMXUdkF6tYgZrAlKqY4YvkEL3xbY3XFA6G1qqV1Epz2WLNM9DNR04iMNYOIRWuYoVWaRYbdpA0i0zptdmVJsNvek5O2t5/f7dgGomCtOFjOFItwghaeqGJIkDdy0KF2cSZ9R1fZlQI0JrXCp12U4TCiVEyE1WCq0Ug7nMs4bgMSmlxLph915XuTbbgumR1vdjkxkEsfLVAnPXZnZXJuVty3krwLli1h28xq6+95jw8gg3MyhD8RK/VTerbWqJG43f5WXbdNxFOyqlt+dKCIH0wcR214Z0miv0TZxXeBVeo1RApLdiHCEDBcFBUIYSVs/WWrABMTTWsBXzeOFxdqsCV6FljSONgmJNeEeiPVJajOlGjpJnGNZ0XUOWpTRtzd5sThJJ1uslWkuKLCVNY1arGskSTItUDqk8kRpRZuHofc/QxgEtpyWOgEgCFZFIWC8rnnpKcuNmxL17DQjP4ZGh7y+Lu614CL/7Ksfz464gfOxa9aGYDzm51lqkUEGJP1pIeR9a+BBa25eF2bYQF7tzsa2ztm320MIARCjerQ1o6yWCOCaFXEVPr3iuBkT4Mrt+uxjZOhl4Zy9Rz92+ud21ub0OwkRw+bnb+98TFPt+tKcJDgFbN4MrtA3C9S2EDNc+W4/FLYq0XfdcFtshnStwiLeowva8uB0iL7lxveC//LM5w9Awme7jGfAW9g8OOF+eo9MCYTx+gE3ToSPP8c2bgS8GpJOEzWodWlUCbKOYzVNSlV65HQeaqma/nGNsgxQZuWjRgycTGm09aZzidIwSPrT+jOP84oI4TXb2MFJo2qYhThKEEKj5NHj4ZQVaRwz9gPOWspwgRcJyuWQyKTk7uwhix02N85bZpKAsSwQRq4vzMM74URjpBF3VY2JHGmVE2uysw5yz3HntFb76ma/i07/+YaJ4n/XmlPsnt5nMbrA4v8DT06xNUMVKh4ockZJ0nUHKGGM88/kcocfsb+eYlSXrxZL06Tdx794DDg5m9GbMZ+5NuD+l3BWPQnqUDIjgZh2ESMYYLi7OiOKUsizJspR6E9I9vBMsLipu3LzGyel9miZ4Cx5fm9NWA/fuv86t6zd59aWadx5m430brvh127OoNnTdgFae9XqJ0jltfYKSCZFOKbqUtlvTd01oPa4vGOz1wEUdNkwmU7ou5GrXVcPFugrXJDF3TlZsNhuO9g9wQrOqW65dfyYUelqPXskdXdeRz6dY6cn3p/QmoRWedVXjTVhU111LKmLO7p9g+45rRzdo2mVoqZ6uiXSYL/Ga8wdnKBmRJAlCKzrTk6YZTd8TxxHrZs3F+ZIbN54Yi6glWZEG6kFXkcQRSRTQy2lW4tMcYwxlkofMcTMwyW/R9RVtt6aYP01VL4j2FJBRlIG6kechWvb5Nz+L84bzxYIkSVBOEiUxZ2cX9L1hs64QOiLSCZtNzeAk2fw4RBkW+yAV1oS5x/SGVHQkcYI0lvk0AzKiaJ+mqTmahTb9Zt1QFvukUuC8oe9i4jwjpuVglo9iuZBUpiTEUUoURVw/PGCwK5RMOb4m6VpLb1xYiBpFOcmhadFWUhYlfZYFmsaYKCaEoMwc/dDSdR3etaRxhsNjXUOWa7RXOwAuioI1Yj7Zo+8MWI+VhiQJNJWu65CjY4PpLJGMiZJQnAaBqsGZEKPZtT7w1zuBM93O9aUWNd570jQmygwbYyiykm5wFPsR3oS6Jy569o7mtIs1XdfhnGFTnzOfHHFyesEb3d5wQRkn2yLL7NCvgDT5wO0aLWeUkEjvEFLjvcUpgXAOKQqGfo2Uoz2MkWg5IUkkAkecJjgXQuY7aYHAs9Haj2jigBDp5WQ4TqaBCxBeC2LXDrfWgjcoGYVc1YszkrjA+hBrpaLA4dBRhJIRkVDj5HyJ+m35e87ZHXq33a4WGrvCDggCnK1oJhSU21b2I4bgXBqjb8Ux279toe4weV8+L4hutjxIsTsPoIJwwHmk8Og0tK+tD6/dmggDCH9Z8F4q0kORbQazU5hteRzeixEF7nfcn633Wjz6sfV9jx0CeqkiRZqHPNBAxHYMQ8tsGs5dliU0VYWQjixPRj7OgI7H860gzVKapiaSBuPWtMYxnRzStAuM7dms1hwc7LFad9SN4uThfYpJSlVVCKGIdEIchUEq0TFd0yHzngenq52oIo5StE5Yb0IBEbmM3/8Hjrl7Z4MTMAx98PHD03UNSsF0Oqeuu6B67bowGORBvd52TSCXb9bMpnOapmWxXDPfmyGVI9Mz+qFFak+kCuruFK0S0iSnayweR993QfCkJLFKkFJjhoEsTRhMR9dvyPMJQ+/GwnpBVQXXhcPDY6aTPawTNG1FOUnpu4q+HZhN5mzaFbFO8S7wjnCGth3AQjmZYvFBBYrH2h7nDWaAKEmRymPagbyMsTbEtUVKIohI4hLnBySefsyDLvISZ4PaXuvgaVlVFXlegBdoHa6bKBWcnS/QUYZzlqpuaIeWqlqRZDl9N4QFpqlJRLASiSLBfHaMtT2taRl6yXJ9H00covAShfM9Tz+d0rpz2qGl95IsLVhuLuidQ2rJ/bN7mAEcDWZwtPWGBydnpHkoEso8DvyupiHLCnpfc/eFu6TxpYVGWc6ZFgXL9QVpnJIUnjg+YHFxgdAaYoUVjmEwKOEZvEXGmsPrx5h+2LkZ3Lh+naapQgcgE8CEvblH6mR0rIjI85Sub+mrlsP5FOcN+Y1rdF3PrWvHLBZnLJanWNPRthfMpnPOz885OMwwxtHUA889+wRVPdA3gfNstlQWrXjh117gd6gbvO9dX4OTF3zk519jaCLUXkttG7KoIJ5GoHq8i2krgwO6YTXafTkO9p9hubwgilVQN8cJkdJUVUOgF0VUVWhh996QJQmofseVNsZQtUvi+RFPPfUUt2/fJkkinFMgoa5rHCq4XXjJfG9Clk6JdGgBSuF5xzvfxoP7p6RpGMuq2rLa1OztT6Dyu/GubmseXNwf3TMMWZFycnKfYjqjtwNOO6zt8E6CypmUBUp7Vt0FUmiKvRnny4cBaW2bILxZrymzGfP9A7p6oLEDq241CpDgdHmBtKGgSNOUTV3hhKNuKxyhvXr33gat4Ohghrc9aRS8G6v1BWUxpfYtr588oJgKlus1aVIwSEPfVGitSYuMofN0g2FTBWRX6YS+N6w2CyZZzhM3bqFVuPakkwjnaasaNxgiHaypzhfr4Amda5wdEAi62hLPJBfVa2givFPBhSOdUSYl1tXYIaIbHN2iCegzfYjWdaFDNViDbSyFiinLlFtHR3R94AcfzQ4Y3JQkCdnRD8/OSZMMIQR1XQMSrRzGeQ7KOXGWM5/PWS4vmKQxw9AxLyaIY4UZVCjaYkM3GNpu7H9YR5RGTLIpg+lHV40evMMYS54f4Rno+gVdFxJ7ersCkbK46FGxJksT+t7uOppaB+57FCusAS1TjvYPaNsG54O6PhYxiS5QLsQ3p2nw5Y3jFG8dSgSxVtMJYh0hEQgTFul5WYRFGuzs0bTW1E2DHvlXQxu+51hkmLZF6DHyuYdhMLiuJs0UprGs2jVFPqNb90RaM/SWF37jFfam+xSzEGKhdcrN6/t0XcdT12680TLxjReUkdiqdNUO/VIynDStYryQoAJC49SY8ypCdKBWGpUp5MhjiLJ8TKLYtsw1kfB4r9EywmuD8DogYXbMEhYh2mrrCWmGUFBuiy8/htTDJWqEiEBovBx437vez42nDrh3b4kxA6vNBV3rqOoNTVUzGIuSl9YyWgc0NtGBRyP96LHot6aqAj/aGSmCbc3jrewoUgyuJ0okzgdrkGBc7JEuQooE5zqieMD5CR6LVoGoHEcZpvdo2eNpUTJn62YsxNi6Fwo3OJTQ9KJHSoXDYwaL6S1KgMSh4+C6b114jjUKhcf6mixLgtCpqZikGlJNFEU07QLvwwWP6FGxJFItFxdn5GVCFCWs16d4HEUW03RnJGlE29Y0bbBU6PqOw719Ot3RdxXGWfpBIkWE8Iqm0Zyd3SGKJWYVVPuRzrm7umBd3aMsZgiVYV2DqPowoThHkiT0Zol1nqZy7B8cEcWCPM2C+EoI6rpCKsPDxQO0lkz8MUkUk6SKwRg21QrvFGmeIZWkaitAcvOJmMkkZOxmeULfjWijT9CRRSnNelURpROiKKLrB7KJ5uy+ZTaboOU+XesoskOc7wJKS8Kmq9E6FAJ1veFmEZSGXdcxKffYbCqm0+NROdyPTgUa23suFifcevIJIjkP/DFbEUc5R8cz4jhD65TX75xxeDCnH2omRURRlHizh7eGYiJZLnOStKRulujI8OCB5+3P71NVG5RIOF+c8pa3X8f5HjNAngVl46ZakET7DGaDlIEg3vULpEjxUrK4uCCOW7JsnyTOcc5gbYuUmjQrkFJycnLCc88/wWKxpK4rVJZT5HtsqjPm18DZDmc9cRk4rp/73DnPPDOj74NNVZToQFtxCWkW49wC23ryBPTckhUTehOixJIoJc1icBmrumZoHVHiGLygMRJTNxgD3g9jDGaEjGpmhyWRmpOmQT18etFyfG0PpT2L9SYUeypnenC4GxO9F6wHh5rMGFzGw02LNRc0TcN0WnK+bpEymHKbPqAKVb0AH1EUBaofmEwmvPT6Gfv7+6z6hkmWg9dju9+xWi2Jo4K7d045Otrn9MESna5QGpq6RYqUqnUslmcUuaLqejwKhGI620fJEmLB4SShcwaDQ2YtfX9JLhFScnxzyj/6R/8b3/YHfhsHx/u88MrH2D84wNmKbtNSmxa8BmURTgdkRIfFZi8NQqbsHe7zyp1XiZIIvCLSKdZYNqszDg4OOD95QFGWDG0X7h0Z0jnwkq4NtkSxyrn3+kP+zb/8dxwdHY2JYp75fBra+GaKcRU6Szk975EeTi5e5OjogEwkLC7ujx2AjHYYOFst6UxCUoDfXBI9Vk1NZQ65c+cOh8cHrE9OSdOU5cNT2q6mnBbcPzlFqoQkSoiiiCjWPHE4Z3Fxhn94TppERJGibZtQUB8cce9sgUkCoPH66Qnnm5Tj4+v0fU+1qkjiCTqCKSaM83GCc5p+GIhjhx9WtK1n9sRNvM1CS9d5pkcHnJw+JFIx5V4BONANMlEsFoEO5j2cLU5J04wiDwI6r2POmjb4ZBYT7p6eUL/ecvPmnEgIiuk+m80DhIBiOqHvK9IsKOfruqbvBto28Om8MLQbix0cq+o8oOPVmumsROLJ8yl103Mwm2C6niKdjYJZRTdYlsslAz1ZUdB2DVkW4/zAfG/KdDplsV6hSFCRJoo9c5eTx/shBjN/CEBTdwxWE2c1TddQ1WC4IIr3caqibwac8zRNF/ivTUCfy1mJsQPNWtEKx9HxBCk1OrJsNjX9sEKrmIsHnybREXGcUOYR0xEF7U3PMJM0vWdxUZGXOUpJ6sqDFfTDgGk05Swsniu3wTmIdcosjUdKl8D1gmgEl4RXNHVDnGq0VqRJSRon7O/PabuKEN0bdCBt2wadiI4wdkB4yOKMST4LjghDN3JhgxtJ3WxYL5aYoSeRkoO9A5I0RU8USqYBgCvAWU2aKPpJFZTjRBhnENYjrEFxCWq9kU08zgv8Ytvf/nv/1EvJmLns2BqJuq2hqH3Uw/BqsTcMA7rMOHvtDj/3kz/Nh77990Ecjyv3gDKqnXpb4S38H//6X0WqBwxdGHS6YWA2C15527ZB27Y4exVNC8fivEFrRdf1RDphsdjw3X/qL/CBD34pdSOY78+YTHMEMUJ44iSYcW9Rp4uLC+pNtSOFd10fBCd4kEEdFdoWo8BjPN5hCPYSSimMGdvaItAFtFIYW4f9RI6cy9DmtgYEFiHtiGiKIEoSYeUiUEH12fVjMRlSG4RWSBEypadJho5CuzuJYtI8p+k6HCEqbW+a0Pc1nh6pYDbdY7VcImSP9TXLZUM5DQKnrnX0Q0WcKNqmI8sKEEMowKsGpTzlJAn5oknJMHi03PqAiZEb5ciyIiCCzlLVFyHfOC+Zzw5p2m6HcBvTY/qBvu9JoqCylMpTVRXT6SHeK6yrd9/zFsXd8naTJGG1DFFiFxcXlGUJPqDNOpLEOqLuWibTkvPzkBCR5yUSNfJCHQfzaywXNU6saBrPfL6P9w1tJZnOY9b16zS1YW/6BA9PFpSTgvlsj/v3T5lOpzi9wVmBFDGR1qNRs6NaBxWobyAvS04e3kcpxeHhIcMwsFgs0EJysD8lTVOUkjhv2N+fc/v27ZA6EwXT8km5j2PD+WnLdJ6Aj6k2LQeHc+6e1OwdxJyfLyiLOU29oMxzht6TZ5rVZolSMYO1rFYbimJCMUm5uDgnUjOMN1R1SLaxbkBKgbEtQlrKYs7D09eZz67RVA4vVwgUg+nQKifNJMv1BVGUYI2jH/3c0jRGyZS2NUg1jJ6BDusMVbVkPjvCWsHp2WsU0wxPh5YTJsUx5xcPQBiSuKRtew72nmK9XrJpL1BKMJ0chuxwa9jfu0Ycp5ye3Q9qaiUwAwx2NYq4QirRZJqyWlYIEZEVBUUxJYkLPAN1XTGfXRuReI0xUFVr9vZmWKNZru+T59muHQ+wXC5gpAtsqjXTyYyz04cYE8yEhQrOBn3XBPRkPg82LHVQ1motODt/wNB7Ep2SZQX7e4dEseATn/g416/f4qknn+HTn/k17t17wM0bT1GbBUMPt24+i7UDg6kwthtt1RIm0X5AXGgxNhRu5WTOdLLPnbv3aRvDrSf3uXX7DtPf+IkdjeGT5Vv56z/6cYp8wpd92Zfytre+kzQtELIfE8syhmFgVZ3S1g2np+dUmwW9afECBgtxUlBt+p3rRhIFFGYymXB4uE/f9+RpQlmWFEUR/CYnOYvFAqmgKAqyLKPvDP/wH34/H/zgB3nXu97FyemDHRqUxJKLxQl5XlJvgnvBxeIhSZrSd4K6r+jacK05bFDWthHfcq3n+fOPsWWTfPrweX4lS0GKnSVTkiQ7a7Fu6BmcRcmEYbDEKqRpORn47sNgx+fDer0e1dRhAZ8kyehOYtAqJssK2rYj0gnCG8oyRYyxhof71wI6PTQIadmf3yBP8mB1NoZsCCGo24pAdtfMZjN+9eO/wtHREYfHR7vu0WazoW3qcQzvGaxjaA2CoBCOpKKzG1bLjoO9Euks73rn8yhlaZqKPJtwmJdcLM+J1FaLMCB8mJclCqeDuKauayaTGdY4mroKHoa2px3C/IO3SA+TyWTs4gWwpWvDHCm0omrq0a3Ds66rwFnuwjG3XQCf6npDURS7kIPeGbphCS4lTuDibMF0coRWDet1w9BbJpNJyJXHBiGnDHZQveno+p7pdJ/VaoW38WgLFRPFgs06zM9xFFFXLVv7u7ptKMuSTVNj2maXEtX2HUmUobXm4uwU4QTHhyGnflLOWNdN4IuiSNKI2WzCpu1YLzZMp9OdULiuOkAymOBRPAwhgjTPS+qqp+t69vYOAg+5HQI1ZOzCDn0A5dIsxrge5US4t4aemzdvjn6aG6TS430c03UDMnYYZ4ijIoAEXYNWGYwJPNZCrDV+bM9/3z/4X98QkfKN2wZFo4+fF0BQEYWUmWCj4WWH1tv4QvDW7VIglNJEQuJ7gySs2OMoDlYRPnhP2qHCugFE8IQKsXYx3gXYWYiwGt7aTWzbrmkWoHs32j5sBTpd1wJbFbOnqWqqquLk/pr791/HCYMUCVoKdCRJ0py8SJlNphwfHpM/le9i8qQMOeNt17Fer9nUFavViqHradrAgxEucJWkAjGaum89GUMhpNmGsTjrgeBjZXuQKg2JNSZGK4WTw4gUGao2+HdFMmIyKSiLlNm8YL05px02WBomBzDYMxItOHtwn+PrT1JVdxDKUxYpy/v3uL8Ik3PXdVTrU+b1BHwghmstef3e62RrS1nMsENKkmpOz87JsxnrVQfCEMcpk6IMKQiLgaO9p+m6jmmRU20WxGkU/ENR5PMyGHkbSxQnzMoDlIrouoGmqSnLgvU6tLDTNKWuWspJTp5nnJ8Gj7XJNGZT3afMrtE3gcvyxBNPsFwucdJTFBPqZsNqsURqxXK5IMvSwCeyQVw1nZbcvXuXKEvQImZS7JHGCXJcCNR1y3w+Zag2FJGibgv2CsE0j8nSOVXaMC8POCoOSVLF3ddf4q3PzrFDhDMNT+6nCDFwdHwMPkGgENJxcXFGWc6QZcl6vWB+fARSUK8jnn322WAmqwXH+09z7/W7ZLqk3bQhu7hp6SrNvLyFMY6DvQNkWrO42PDqa3fIJ45+XQaieLWm8Rkybnn1tSXW17x+cps82qNpHXECq8ozn+3z4OQes/mcvkuZTBV37zwkm0Qsq1Mac4ofKRJdX+HcQJ5PmBZzzpavEWf/X9r+pMfWLT/vxH6re/t3N9GcOOfcNhsmk0xKFKVSoVCugWHYgAEDhr9GfQ554qE98swjD23AQ6vggVVwyS4JFCUlRTKb25022t29/eo8WPvEJWGgQAOsnZObicgbsbu1/s3z/B7Faf49LgrmDmIoyEuHsxPjnFz3IU4sbsI6y7Hz1LalaQUPj+8BaJoVwStubm4QQrA77JFSULVnEXqUSKl59+GB7cWWw/GOcT6iVcXJHgjGYNiS54bdKTEcnQ88HHrKbCLPW07HGZM7TqcTjnRIGuPITJvcknlK4rh9+MD9/g4hC0wmmIaefp45Ho9p5V1vGKcTb+/SRTPPCyYLz6BhSLgxITRalVh/oCgKtptLZjcRncfPHmvtc0M5PRyI0bPMghAlNzcvmEJAmowFwe7hke+++w4pI9fX10gp+f6Hb9ls1qxWLd9+8xZVa3yI/Pqvfo0ymkyrZ3za4XCiNqlimqfIenVBlmUMw3tMqfBhxjnF797+O/75ovizZy5sanovLq7Jc/i3/+bf89/9639FURQUeUWIlsy01HVJ22zZXqzZbFZcXq6pqnROLs6S5yVapnVuIHE0y7qmqqrzVmHm8PiAUibpJLsB731aKy4Lyxzoso6yLPnjX/2C//Af/oLr60usT1nR3keqrCbPrjEmQ6qeeRmxURIWz2wdEUlRVUQRESGyzDNRpCnijz61SD/0yLbkE3f38vIyTQOtRYjUpFbKsMyO6D4FTAQMIUXsSYMfOrploipLpu6IlBpKlaQb58S3Ii9xPiWWSCmJUnEYBYudKEzBm4+PFEWBkJ55GfjN929w1qN1gdFlWleGNFhIRbI7FwAzhyXwm7fvmOfxOT1FScnQT7TrFe6sq1dC8uGwx04zZaX47LOvCFnN4fDIf/vv/oKb6xdoE1EHy190b+mHhCa6uXpNZhTL3FFkOeMpgbYvLy+Ti3vvyPMcO1u224pxsuhYIs4MwO1mzd3TE7e3t7RVjVKauhU8vPvAqm3Js4LTYU9dlFxVDcfjEaktxMB2c8k0zWwutoQA3iUDmewmMiJDf2AME+v1luF0YJx35EVD1SjGeU+eJS1nXVd4UoqUsyOr+opNu8EoyenUUxQwTY6iaMmzGqNTtPO6bVJRGiNNXbJab+n7nsV2hOgYhxloKM4u6q+/eIFREjsd0n1tcq4uN0iVtiB5bs7F44n8q58SfcIqGZNzOg5UVUVV50yjx3uLW1Jk5DAMrFZrQkgb0v3jgNICpXVKYlIGKRXTOQ5TZgW7xycIkd3TnrIsKYsqGRCD5/pyjTY5h+Mj/TQwnHpQOk11p4lo+pSqpAuisMzjgtZ/V+73P/T4exeUeI2QJnF7+JRxm8wf3nu8EoiYvsSfvjghBJxPFW/Z1EmIbBeqqsHGADGkFbCMhCV1s1qVTLbnU5Sf0pIwOuLZHSwE58zu/9+J6I+EefujvuGcFStkoKozNtsGlGRx0zmf1hJi0ts8PQ28e/eOT0gggCJLeq9V3VCWJXlV8uLihi9ff0VmkkBfibQGnKaJrus4HA5M08Lp1DOMI5/0jkqlFYlUn+LW0rpDIlniiLUaj8Ozo8xz1u01l19c8uJlTaY9p/7E77/5G354c2S3/4i1J6SwbLYtm6uv6Y47Dsd3/G555HKzxUVY9tBWko9398zZE8fjjnE6kZdf4p1M3V8e+fKLL5jtDmcjVZkzdB1lltOUFTE3SKHQ5xSXukyOtLFfyPOMse9YVRuEjOS54Xg8Mh5HjM5p6zSS7457rJsoS4NfHH5SZKrB2yOZVhRly/E48/R4R9u2lEXNfndPUrnNfP7yFXbxdKeOQiXuXLQBg6afTmid8erqFbvdDuUED/d7rq6uiAtoL1jrCm0D6/aS7tinHNt54k9/8cdkWUZ/HFgWR1vMaJVhlOHw+MTlxUuMikRpqIsL2p+nLNTCFPgwc3f7ls9ffsW4pEZrGDqst5T1BqlKiqJkmgXHY8c0Ww77nqfHLhWUzrHZSGKQHMYDy7JwetOzWm14+/Ed0zgThORvvv09x+Hdmck3UFXX/HD/5xTZBh9GhqEj4inK8yqFjKwsmFyHNA2Ll7z98D1COr57d8/l9nMe948EYbl7eM809Zg8GZxSIkh5lmjk2LlkGPZooXHRoVRG21yyzILF3p0TTyKnrsfkWTIaCUFVCaq6IHjB9uIV0XNuBKczMy5D64R6WSZDXtVIlQrBomjO5rmK1UWKtnva37NeX3L3+AN52eCXnMddx9X1mv3hgapMek5T1iy+x0qLUjnOw+l0T2EqciT7/ZH4FBEqIcPuP7yjKFOB00+W4/7ENM3M07cUpUJoi1Kau4893qfs4U+PqioYhokYBc2mJMaRu/t09ix2YpoGRIjkRYqNVUJiraEsS/rhwMeHxzMezVIVmoCDqgSRcXKK7njg9vaWq8uXlHmBakr6w54QLau2wFqXHOouQrTc/Ow1Jiup6hK7DMlooiKr6w3dyTKNmmkZQRuETizTT0tgVSw0lzOuk1xctgiZ40OSIoSQsCaPTwc+3r5FfpsmGkIopDDEmLYTRVFQ5mn6mBfnxk6KVJw1LVdXV1xtr2jbksvtCy4vkjTpF79QieAhBG5JCJc/+sUf8i/+xb9gWRb+8A//iFPfMS0OEQZ2uwN978gyTd8f6I8ndJbO4qAWcBqtBMFbiDFJKnA/uukAFxasnfmUrjLu0rSM4JFaYWeLtcdzeEZqCkyRzkGlDESNECrJLZaFqxcbpmnitO+p6zXr9QVdNyT25yQwIcMtgWgmrBNoCcswMY0zUz+gjESqkBrSKCjyimVxTNNIWWQ0jSH4BR9A64y2WeN9JMsK5kkmbfo4YF2kakpi9OhM461DnI2bupB4oflPv/krgtXkRZJ1/f6HW3TmsbNkcjPaBPK85K+/vU3a+9izLI5oc/I60rY13nv6fqTISoRI/5uIjt39nizTzzByqQTLmNzQmTboPNLWDV0AOz/yxeefMywLY99RtTX+DP8dhmSE2j31nE4dmanQWlLWAiVWqGrCyMuEHrIzdtpS5AqpJff397TtKmFvzp6IdVYTYmQiT1I0a2kbw2pbQTQcj3uUlkgV8czUZYv1HcV52vz09Jb1dou0NV2/S6lj7RrvRvwSyE2BtSOuqnDLgsgMVVnjLWg0mcyw85wc4CKy2m4INk3/rr66wFpLnhdsyoC1C1m+QUtxRk/lSBGZ54mfffkqDdCIuADLEogI1Fnb7f3CdFWft3eaYFOKlDGGorikHw4cuz1NXaBVxuubF4TomN2E0QVOrNPWcErBD/OUtKH/4AVlFJFw/k+a+qViTskE+MhV4gUmeEZCj0ghUEKQZwZtIv10SmsIEchNjgv2HCEGwcQzqNqTG4GQpLxMU2AyxTikyybG8DyKTlPK5HBL0OdPU0r7bDhZlrQWO51OCCEYlxmTp4mIdxEfw49xfEpTVRfE6P9OTJ+1lv2xY3dIDigf7LN7WwhBWda07Zos01xebnn9+VVyqRv97Nbq+ifG0fL4+MgyTmnMHUZMHri9f4PP7lnXV+hM8ea7X/NP/8l/zqps8PY7/q//l/+Gf/qrf8a0zByGI4fTkavrLcKWXK1bshihh1pU/PSX/4R+mejniVwZlEor8Vfbr7H+yLosyUyRxt+FoOuOTIdAZi643L7CnR3xda54dfOa3/7mG17cVIzjzHG/p222ZLk8x8wdsLOiO45Mw4mmrbCLYhx66rrFLo7gZ4oiuRrzfM0wTJRNyneObqHJG6RTFNJT1BdkRrHb3dOuVuRtmVy9QfIf/u1/z5/+6Z+xrtLf3/c9WuaM0fOP/vBXfP/dG5QT3GyuaZuGn7z+CV2XusmffPkFbVmgpEFKzVcvv+Dh4QG1Sl9qD2w3L/jw8QeaJsMHOB1P1NWK9x++B7Gw2t7w9u4Ba2dUpnAWslzz+HjLDx+PZ13QQAyCYRipmoJhPCZcUYBCp0tT5orDcHwGGX+8e8eLFy8IVtJPjr7veDwcmaaBEB2o5ITOhULR8MXLDQRDmAQqm7DLTNsabh8GVusXhBBYr0qk1IgA797/wNXFl1iOKAzORsbxRFG17I9vaFcbyvICH3rGaUAg8WFCGccwPWLdBOLItFja1SXBGUIcEUqAk4iYzD95kXTR4zjRbpKxpR9ntDCs2g3epul9USq6fsdiA1WVDvymXrE7nvAuUjdrPBPL4kEadk8nlPYYE9nvH6ibktl2TO6J2S78zW+/pa1f8vbtAxHH1dU1Hz6+pSi33D984OLiAh8mFpdoAX3n0DJDa4FRSTIyDB2nw8IP391ydb1hWnqkzPFo3GRRMqNsNTGctUfnhzIlpogsdmTsHUoLjvvdc+Ss1gYfHFmQdKckA1BScxp6Frtg+5GmWaVV8mkhU5KoApmpGZYj49QhhODjww8URcG8dEgvUCbj6ZQmGHdP90Q82/UFT++eCCGiVUGwgaLMmOxAjIHVuqGbTwQJmYmMx/FHTFOMDMNCXa/YHUaKEsrigmP/RD/0/Jf/xf+c+/s7+u6JVVvxl3/1ay4vL8mKkqlfiFHQ1g0+WGbbM8wz/RRhz3NCyicjoI/hLKMpz2dCAt+vV1uiFKyqmqapaFclr25e8hd//ue8f/+Ri4sr/uAPf8764iV/8EfN84r37v6e+/tH7u4f6IeOXBVU+QohA+PYU+aCrhsA+3dwVVobDqcjWVZQasN+/0hZls+JZpCmlDEmSkhRpEzpfpmY5wN11bJ4R54nU954SOQJmQXQC7ooabR5ft7WxlR4ZHVy0XqYvcUIKIqCvEymEqVy6qqlGydmv3C5WXNxscHOPevVJRHBu3cfWOxCWdZEu+DmiTFYuu7I0NtUgEiZdJ/GIGPA2oSXEhLyXLBEmG1aj+d5S4ierJTEKYJYcIvD+jlJuOREiIF2vWZxBx4eOwDsbHH5iLWWDx8HtJE07RqPJ0bBoTud08QieZalpmEUPB5HothRFAX/6e07Ap71ZkXfn1DjzM3NNcN4OEvpUjrfvNymCbIo6bo9ebbis1dXLPMtTdGilE9G3yjIspLZBpw7sl637I+7VGSuamIW+PjhkRACN9dfczwOOOtJPqAJo2uuL78+45kUWhmEibTNK0IImDAgZEaR1zRZBXlGedGQyYIQF+IyIFRK1hqnhawtUSbjdDqQNTlX8gWcJQR2ms90F49UAednlim9nsJUWC/xi2ecRpxbUk3RaKZlSUzO04nJpn9+erhPRWmRtgHl2YGe1SVVWyFJ29L1Zc2m9891UpYXLPOA8zl51nIYe9ZtzTxYvJv57Prledv7D11QxsSrU0onZ9z5iyKjRApYLKkL0p8g0wEU2BCQQrCOGk4jx7sdpS6YY8QLico03TLSnJlowboE7XYe9ckIpNK4f5pGsix7dup++pvSeiE5q5Nr0TBN41kPEymKnKEfCT6hAJQURCmY5gUlQ+ocSV/AYZl/XKvPn9A8ydijjKYuC0LI/pYBKL0+/bLn7mng2ze/R50d70II6rKirmva9QWbbc2rV5KLiwsO+zv+/N/9P4jhxOc/MdTuNfvdPa8uL/jFn/1Txm7mw+//DX/6T/6Ef/L1KwoDp67n7uM7TtPA8fDElzc3xDKBcxGRY9/xOBzpl4GirBnHI+t2xTyMiEywLMksEVxgmidW64Ivvvwptx/3tHWFlALrI5vNNd6O7O4HqmxLWAwiDLy6uabIG0II7HaWi/YzjseO1y9e0s8PdMeUp2uMwc0LZVHwtH/g+vqS6CRv379Hq4LryyusGwnWUeqa/jgi5czl9pK6KrgoJaXRNFdptWLthP/sJcvQUVU1TZlRZYqXL1+zOzylXOfXryjLMjHwmibpPkOaYuwPT2RFzof3t2RlhSVwmJLuKc4C6wOn42+pasPDBw1y4tTt8E6wucgZusi++0jEUtSGMDpCMExWcv36hjdvv6cfHtLvLWr6PpKbgnHcsSxH7BKJZc3udGS93hLwNG2bir/LK+53ey4vAkWRcEzOTdS1Yj5rRLtuwJqB2VqGU0bTGl59/geM9pGLzYa+c7x4Kcn0GjsfGYZbtuvXjMPAZhPp+zegPUpV/OwnPwO98PT0yGwfEWNA0uBsYB6SvmmylkpqnJvIMoudLLm5wc2GEATaBOZp5PHpwKqVZLkgItkfnmg3NfvDHXW1TTigENkdPrJZrQHJMgmKYsPx+JHL7WumaUmaKDHhfY730M9HyqKiyjfcPaQVKMGTaUnZCO7ue5p6i5IjZWn5m//07zFmS7uBj7ffk2ct3WkmxsAPP3yfssVz2B8HwmIoC0XwC01ZMY4dm22DyhWX2SXWJhyNPLsfTaaZ3UJkQYgK68bnM/GTVrRpDWPvaNoVVe3OODPDsgyAYHYJ2xFjZPZHhBfpAs8zTt0jIXqqrCT4LGXOTyNESVtumeeZopSM44CKOTqHeUo54nmhub55QVlkfHh/z6a9woeZvpsZholu7Cgrg7WWb37/hqZdIyUcpwkpz/ijc5V1Og5EFCEqVqsG6wS7Q8q3btdbfv2X/w4pA4fDCUGBswop0rTs/v6Wly9fEGPk3W9uaevmjB5JU0tCPDdWAZNrrrYZOlM4b89bJkc33OO9Zf+gkybUzlxcpGnm737z1/RTz7/6b/8lSl7y2WevWK3SBKooMhAJqq2iZDxMHOZ7fBiZxiEhltYVPj9jsM5T2XEacIXCjTORBWME43hiCUkzHhbPMByfUVR+lzSTMoB1M9OwQynFYfHP505Z5RTFimkZeXh6SJ9bEvPQ2jQtHea0sZJR0q4bhmHi0B3Qi6auS4o8Z3QLyEhdp+Lz1E3Mk6UbdhijWNxZn5klcsapn1FasN1ecbkun2Mxp3nAxYgWHoUmzJFlGtG5oaoypikl3MWQInKPpweabMvueDynyBgCFkQC0yvdoYRmHsZzTnhC8WmlELmirDLG0WKMZOj7c/yxBALzNBGDPht9HDrLE09RCKSC3e6AEBEfFe9Pb1m1iUwSrae1krrNQGvsqcOFHtt73t4NzPPIOAQCPWV+gV14Lp7dYrm8vGS2CzFG3OOeuXtIWuO25c27WyI2peOYwDjMrJsLfv0ffg+Es1YxaRKnMwpovblAG8Ew7CnzgqLIEeEeIdQznqc+kz/qskIuQJioig19b1GxAzSznqnqDD/PHE/p3D2edihTUdVrHo4JQTWe06KqquLj/gRdkk60bUnUFW52RLug8wwbPF4pemtx40iWaepcM/uFeI4cHo8T680lKZNrYrYLi0t4o6Gfaduc4+GRXKcc8ugnlP77+Wz+/yooV6Y4s/PAs6BkBOERpMi2y9wng0OEafzEPipApHgnOx74z/5ZS7P6ZyzLW6JpiNEwjZCVOWFM7MPMZAx2wC6Bqi1YrCf6CefS6sRa+zwGVipNKRN3yzxrNud5Pn+YHfLsRJ+GU5qWaYGde/I6Q2YSI3XSbkZFjIEYPFpKlEnom2X2zxzLT1iUT113Sqg5Y3PcxOXlNc7ptPYnoYGE33P/4Q0P9ye+++Evef/+d1xcbLi8uKCuKy4vXrD7MPBxnLm+2nL/ZJnHHmNyrr/+jL959wPN9pqn3ZHH3Z7gPW6ZMSKwOz1RlRmFNgz0LHJ5diur4FmtGh7u79KFsTth7UhRppWldRrnWm7vjuyOT9zd3dE2OW1zxbu3dzw9JPNKXSd47/uPH1mtEpri8uIF3WlMK8erCx53TxSNQghDXW2Y+gmT5eyedmht+P77N/z0y68QouPFdU5wj6jQ8Ic/+RXzMjA3R4ZDTqkUlcn5+rMv2O2fEMDYd1zdXLDetKzX67TSlhrvBXf371PXVhW8e/+RfjgxjUlqIHXGp6QkbXJ+//0PXF1fM44jP7z/Dog87kZWmy2gCWphf7I07ZZxgnKzYpxOLGS0lyvC/IRdDCwFhSmReaAbj7z74S25yfHqibvDoehfCgABAABJREFUE2bImaYFf7gnz3OKosb6jn5MF8zdwze4mDSNUz89x4x2/eGshUuf8+AjIuRoZSgzRdmuzyisnMPuhDS/oSou8b7HLo6qNTztvgEETdHQD3sCfWLomYIYFHkB3/7wl5Rlzu74PXUjQYycukOSYJhAFJ4s0xjVEIw8i/FrhnFHlidodVQ1QeQoEyhqzTyPVKWmKK/px5llDoTwyFdffcU8WKZR8Li7oyzWRF8iVcaXX/whx0Na76+zimN3i19K9qcD8zghgmIan7BLhxLw9PTEPAVs8By7A4iHtCKMkdxc0A09w2hZtVsm5zjt0wXvVCRXJTqDrMk47mfwCu8j+2NPZhoOh8AwndAqGamWxbFqchY7Iud0GRmTkRlDVf4IHRZCnZFZAmt7TscBCChliCGlPVVVdY57TSYBo2u0EcmIkimqqoKQmHTRCcgCBBjHPpEgcklQgWpdopRh3A+U54tCyiQH2h+PtFmDHScGd8S5SNMmc9uynJMwnMIvgqwqkDpneDqm0/+89S7yGunTSu7UWfrpRIyCX/3qn/P2/RsO3RPb7ZbHp48UeQ0krNrx9MS8WBY74QPY2THK+fm5J2NjTz9O5HlOf394nvw1q3S2fDrXjSm4uC4oyi3OLYQAjw9PbK5qtqJiGE9Ie+L2/Ue+/70nVw3TuLBaVxijuL3bQWnwiwBhMUrgo+L65QohbXqyInFXpYA8Twkh1s6gJN45TCaSyaUoEF4SQjKXLUsK0JhsSuBKdx/oIsOlSQuz84QpXeSRJAVLsZHphdZaI5TALkmPvh+ShtQUyUX+dDjyuEtZ0QlXlyRlp2GfitppoSgasqKk0jpp3mJke3VNliUCivWRZrNBEnnx8oZ5HvFueTYxxtlhssg0LrSrLadujxCRGBRlYVBC8/Lqc7oxmZScWwgxsq4K5mHGCUtWGoq2/DvDIykEaElcBKeh52J9wTSkKboy6bN6HPokGZEpdKRdt3SHIwaJRGKniVAkU9M4J6lJdAPD0wl9ivTDKRlO8xXGROwiyVRDvYkU5nWKJCySHjclskmGYcRkGc6lhKxMv0o4Q2Z2p3ecuifqYnsuoGu6/h6pwtk5bVMz0E0sy3KWWDxiTM7u8RbXNszTObCEwDI7gpNYO2NM+lzLs1TP+SVFJZoq0VlCQJJiFstqzbIslFUB0lAU2XPKVNMkxiWPj0zzQJGtz7jAR7SBusmSljxGyvU1V6uCbhxYloVpmtjv7rm82nI4dMgYuXy5ISKIOKRJ27T1pmG/P1LWl0TpaFY182QpcoMTnnH+2yGt/0AFpeTPmebpOdkgMykiL/EgYXQpm7iqKlzoklA4pAQJk2lmInVe8PrnNZP7HjtJcpmRK8EyWwafPpxNcYN76vDhQIwBEVUCBlfyea2dXNSpqCuLdLiljkk/44Sc80CaEuZ5zjDskGLG6MD7N2/YXDYoWeDdRFWVHI8JRPvy5UuWxXG6PyVnnNFIqchFxfGUMDt5lTrPaZoS9qE0BK8oM4GTks1qiwgRu4w0bYGdRvLqwK++/oLgQZy7z7q8JIZz6k9Bclk5gdGJtN8PMypL+q7PX0n+cZ70V8rI8yrdMk1JT5qbxJ6UBGJMh5Y0WUo7kQLpD2Smxft4PqyT2clFhw8D86zICss0Ovyi+cnXv0CoCYLn/v6er3/yy5RZnOfcPTwy+SPKeHbHE9JIsmyL2aTYrVW9eYbgz/OQxNYIfvL1HzOdBl5sLxiGnu74jnGybLeXFFcZb979htfmCz7eKg7HR7aX19TrK4YxMvY7joeOosw4DqmDfvHqJb/5m9+lBqKOnA4nhJD0ViKESdoyABnJsxUgseeUEu8tm21DZgx9P6BVjXUPTJPC+YnL8iVLl4GaOB2e0HmG131ypPsJETwqU9hxAump9JZpOFLkGdpEMqPZrF6ye1zQVOTFESFh1W7Pvz9FXHkHKssRemQ4P6/xtP/xUCRxW+lzcrPFxYHNpWEaa4I4Mp8cmap43H2LDJ9xdXHJ/cNbtusvGGdPXuZ0p5mq2PLhwzs2mw1jJ6mrDVW25XgcWNUrenuLOesDnQ0EInVbsNiedr3ieLolL2uqesUwPRBZyEvo+wN5VZ6jynpMYYh4bj/uMbpmHkaGscMugZsbg3MHroobimKd1sfCU2Wfs19mclMj/Mjrm1/y9s0jUqYV4bpdM88LbaP5cP8WhOV0GshMmSYksqHIJ9xSMfcF1u/QWmF0Q1O1BEbCULDfdbx+9Tnfv/uWuk255mUlCMFRiAopJUUuyQvD2FtMbghCQFQomeP8iHM/dutaS3wQBCuo6zJpyYMHD+M4o7OM/T4l3kSviVEh84bD+EAUsDgPMcPOjkkuBOfJJkVVttRqhRtSY+7O2mulLdZNCDRKmfPaLGUKT8uCFBmKDJ2XjMOEdw7vPZttjdokpuVw6livL1g1AjF9OFfGkOmc168vOT68pSobXn7RoHTNZrNhf/qBn/3yhsLUfPnlmrFz/Po//paf//zn/OSnr+n6I9MUCF7zqz/5KYtN90Tb1jw+PnJ5fUmIqfH+/IufPrNr7RnAnBWWslYYoxhtxxwCdV1h54WyTWvx9bYhxC1OKLQOOCuYe8WHDx+5um4Zhp6vXqzw7oibJRKPXQL740iMgXE6r+3OyVXBS07HKa0Fswyhk/HDuZCMLUsy4iit2O/688ZDgUkYvDSxFggUWmpcBG0UUcLkkzN+dDPWW8yZgDH3Pfqc1iaH5dysgcoU+1PSa3p6umFIBbbOzvpUKPKKaVoYx0fGeaKqqsRR1jq9jiHgXTzLCHK8t9zv72jbGrssxJgmqUWpGKYeoT3WDZgsAp5xmAmigzww2siwTMi8pl7X5GbNNFmktcQxkEIuHMOQ8GYhRoJwhLCgZEVRVCzOoXSGkTJlS0uBUiYxLbUgiMBpORHySJRnCZ3OqITELxZiTvc0EdWS8sZDSVm0ZMIwnhw712GXIxcXG+alI/qByFuWIU1Gi8w8bzKfjbHaJHMUgqISmKzh88/W6bM0zwz9gpZpWHU8ps2IQDP0U/o8eocPKe6z3bygKCV9N9O0BT6kCWxRFZSi5HDcQ/S4sOBtmlyO44iODqMFxiiGYWE5WYpxR0QhThkyTs+DECVzng4HmmZ1bro81u2wNjU3dV3RDQkUr5RhHB74UNXP98YnMsChh/WqwXvL8WFHfwo0TcNsZ6Q0nPqB02lk1cwYNbFetyAVT8ckCZEq//uWiX//gnIcd1RFjswFershYuiHgbIwbFcV/ZSxP3XUmzUqyxMbce7xYSEzJdXiUyKKkjRaMsiZYdgjRaQ0cJEXjEtE2DusU1g/MrmMTCvGLtCudCqU3EiInhAi3kdUrhjHkapcnV/IH4vOtAZPk1Q3H6mzbzjaW/70Fxegnjg9dQjdk6sV1cWIbQSrNkHC9/kT1n1kXV5R6BYtA002EOIjyzzT1FtWWYbUARcduQyEWdBsrrh5fcWpPzIsCh80q1cvkV6QV4rgE5OLOBJFQgS40DPtPfIsJbDLQJZpFj/ho0OonKqpUXPg4f50NhydNTI6pReEzmLKjKAVITqEULjRpuJTCsLScDgmvVNeVJRVjqAHwNqGpsySAH0dcXPCUEx2QWrFZ58ZsP7Z9BRJh8KyzM9MTmtPHI97yuzA7e17lO5o1hXHk+Wrr35Ctx8x2UKea7xYiMrTjxNtU/HmzV+x3X7Gq5ufcffhjqyYCUA2JjTF/mlHP5wIRpGLkm4YmLqR/bGnHwcGOxLGIxcXr1mWQJFXWO8ZfGJROj8yDpH7u4RMEkBmUjdu59Q5DvaJqmrO+JsLfFRMfs9Wb1jcB6bFkGc1SnmqpsTO6TD9k5//io8fb7H+wNefv6LrJlbtF4zjxHBaGPodJtMoXaClR+kcO2d4P3J9vWGaBt68/YbNukIKgR3HNN3XkqIoGMcRk4GQimHaEZkQqsYuHj95NusVh8OOVflzirpKMXaqZhh66rrGupm6zmiqLciZaeoQypDLy+T6zBRGGl6sP+N0WFi1SSA++5lcKRQlzgtW1RdM84mqeo0VMNoDZV5jcsvQLewOR4pCE+MRES2vrn7G8WlPWWna6jNCaTnsO8Zecv/wLZuVBdlxOniK8vcs8567jwLrBnwYUWKFNpKu66iKDsc903xk035NqxtWF5FhOgGSEHtq9RpXPDEvnkyuGaYdg0td/meff8luf49WgQ8fv6fKGvCCTEe604gxmrLKmEbLNHqcS9PFeQwokxHjjCktyyxQ8ccjM/jINO8pSs2psyyT4PrihmmaCUsyEuZKY6eZxSW25373LUZXGF0T4kLXP9A0K5bFIZUiy0oOp2PirZqCGATdeCA/N5N2CRSlRYYSoqIs2/PP9FRlSdtmWOuJTqF1RrlZMc0dzkpCyInCsT8OjEv3tz0q7LsT93ZC1gWD6BHW8uXVL3n34cD98Y7VOmcIluhmisuc9U3D/e3I9mXGpsi5ez9iZeDVF2ukvERoxX7/xFeXL9hu1xyPHVpnTNOE1pIYDeOsCD4lcUFIkGWfihNnJeuLmrnT7I8n8nIi6gE7panRYT/ym3+/5xd//AWrRjCcRoqsIrsqGSaJagXTm5n50VKtssSdrX4MPt0dTjyqtNLM/JwQY1Kl9adOjF/gzBZW2GVCSghRsgiP0hE7OaQ2aCUQUmIXC14m/Fs8a20DjOeCL+nyS6Q632X+7Efw6byeZo8UBdZ7lMxwQTwbhxbXnWVWCqEMp358LiidP6Q0Fp8y6p8DNILjbp9S3mRIWn8fDUYvRO9SilJI7v8sU2hdooucolA0ZUXTloSQaCTj2CGlwcUBY1oEEmeX5KqIASk0ImbEoOl7j9RT4h26iNYCLKgypesJqZgXjxCcXeILWhuyrCDI5FmIgcRElmniN3enZOYVM9Jo6mKFdzPWpYjE2U7oLFCXa7quZ/aeU58ma3VdE4JDqhQOYoNjOWrKvGXoLUJalmVC6wwXA8sysX5xRXc4EuyCNobvfvie7XabsrQz6E8906A49R33909kZZ5MjRbqJmezvUqg/qcTr1/d4EOH0h0ytvjgEEJSloY8V2c3vkQpgRAF0zAijSLTEoFmmQe60wGTZZy6I3lWUK1KDsMToMlVMiy50DFNe6xN2K6q3OCc43iy3N59oCxa/DmY5hOUH5IspWpqohpx0fLtm7dcXq2RBZzGWzarL/7hC8q22pAbgYoBF0nB6kVFU2ZMhxNjmNm0OXP3xLAfeHW9Yd+PNEWJwTOfV8WZ1szOUQhNVq05PO1o8zVS50S5kFXpsg7eUOaScdoTIkxTAv1+MsJo/WMOqRCCcUwcqyzL8CG5pz5FiwkhGKIlb3KurrdkpqKfj7x6dcPD/haRZRgcZa5Y1bB7fKIyAl1VqDzg/I6Xm4Lbuz3rqmSSkk0jWeyMtUckDqVr6toyjw/85q/+E+vmmt3+nrIsebgP9JPl6voVi4f15pK+T0xGrTJytUYXE1IuyGAoi5coEYgC5ikyzV3iEwbPelOzLAuRQAgjEkemI6Za0S8Dtp8pixYdUxcdgsX6iTIvKXOZDik30O/3VHnqgL1P2ef+rD2RCg7HHVJrsGDnlEyUjEhnM5IDkzconQ4y4Ss+u/kSHxZ+8VNx7pqTe95Ixa47cDw8ElXk6TAwDoF1tWEcPHX5gseH+2etkQuedrNmf3ji1CX38/piy4e7D/z+3TdsNxvKPKPd5Kyva4apZ9lX+FGSRYHJLaumpDWJCbbYE5O3fPnl5wx9chnXbUPfjYzjhNYGreqUKKEidV0xHAYu2itEWLhsvsDZmc36gt1uj1giOM+mrpiHnkwp2uxzLtstyj1QKENvDzRFwYuffsk4jlxefoE2MPRLigGMG7rpnjKLvLp+wWZdMw7zGaqf8bR/ZJwtpjhHdy2Kog5Mc8Pj/khRKubR8cPtD0gZKYoLPn58j9YFeZ6TZYrTKZk6ykqiM4saC4JzXF1dcjw94W3kYvsyTSfcQl0rysrQPz5SlgLrBVWx5rDvKarUheflQjcGvvr8Jygj6foDwVm26xecjh11K2iqtCrfxDVSOZZRs8wTl9cVb9/s8WHm/cffULULwxh4+/bE1fUa6wcWlyGVZ5gHXO8pi0jEsW5rqrzGhx1Pt55225CbJuUgbypktOjlhrF/RJeWUheEELm5vKB7HJFKUZcvsP7IMO/QYUN/8OS1ou+O2CUjzyrsPHPYPaVVtY242YEOEBrsMoD5sVsfTxNFqfBjpDAtWbXw9HifnLpCMPQHTCYwWUZV1ol1qFuUqLCTRWmDloHumBq7sjIcDgeyXBN9OEtpkqEhBM8wdIQgsW6hzCuUVImzh6dZG4Lr+fDhgFYV87wQxJHlYUJgsOf4OoKgaapkcPxbBVbEo02RNh0hae4KfYEXf0HTBKI7T9/m5FK+uq75/q+eeHrIWW8NYw+yMOxPB6ZpPgP6LXmhuX9IzVqMI2VeMJ/Nk3lWI7VJd8I8o5REixWLPaFlxmx7dF5Ql9U5RcsQM0OIke+/m/jVP/3HrDY947wny1uOpx2tScizcfIgPTqrn9nB8Cl4Mcmw2jIHlda/hAwvFEpFlsUlTeWQ5AKf8o6VTIVvbrLn6L5lcTgW8EkCpVUOQZFlOeM4Qki/2+LITMLaCeHx7sdENG0S6UFKybwk7f/iLCE48sIgQuI+u7jglnT/VXVx3tL5FKsKtG3NNDqUkuS5YZomqqKkLEt2ux15lVOUOVleUmZZwvs5maQhbmEYenTmaJua06lnWsZz0eFZX5YpI1quOZ0GlhkyEuVgnD1aaaTIEOLIYh0yeASJgCBchCCwURCDTjr+Z9NsoiEQ0gSzKAqmaaGpW/Iiw52nsUpIdrtDmvZHx2KH1GijEne0KNGyIssTU/R4PGDUjzzglKhnkDJgzIq+mxjGHfEcUbj4BRVSLTGPE0Z3rNfbJHOYB66vr1mWmSAHxuWTIblEa5UK/ZhIHkUeGWZQWpKZnLJesdv11E2O0Zd0p3TXhnPUs3PhHGlJMu0JELLgdJiZM4hxQcZUbOoYqaoCFMnYuCxJqlcLwuIoc0k/u8TErRu6/v4cnJG0tkOfBkTpc5uGQ0pIprlnnNz53tDc3h3wwlDmaUX/5s39P3xBWcUC5QUQ0EYxWsfsYRmnpCmZI7M4c6+MZt+fmKM4a3eWxKEyOd08M9uANhqdF4z+yLu7PU21pr1YkWeSXA+01QXEtNbi3OzFc2zip6Iyy7Kz9ia9QEKmTO9PiTWfDpEQAsuho/vwAdmHFCMZFlgGXtUXeCVYdhOSQGZ7XmcFdbPhYb/j8HjLZCduleQwdszRkWnDt7ffgQ+o82i9P9ySqZLLzQVOjhQbhxgt7eWGN2/ecH11jcw+4o6et9+9QWrL6TFn3b7k6ents9mlaTN8W/O4X9huvyYr1kidobKcUieWVVW0eH/WrAqJJOnNc5mhM4huAiQyCnJVEhfJ0C8URZXcn1lFLgoWO6GEYlXVCJMKy3FMh0hZ5kmzR+JlpeSWPOU8R49UAjdPgDvjNDIWd0IpwdPugNaKMisos5wYI9cXn7FuLmnaAikl45iKJzdPICJLnBNM3jqcX6iqPCEODk/MS0AfJoID6RW1LukPHYOa6fsj2uRUdUu5yXl8fKQ0G6IVjN2RcZi5vHjJ5Soj2JruONL1HW/fPlCUGh99wgYdBvIC5qlns67o3RFtWna7PZeXW1bthsfHJ9p2ndZZ40QQGhsi9WpFU2v2pzuWuDAPIz4uqMwwuxMqj9zfvUermrpVPNztUSbiQ2S1zlnVLzkejzR1Q9939PNIVZQEMnxwaC1RWck0Fph45PMXrzkeBlQ+siprtJEM84GLi6sE/+6feFn/BGv08+s9j4FVc03bbDl2tzi3sFpt6Ic9SnvmxVJkG07HkTyrESHj1fWWYehoyhZhIsoqHp7ekmcNj48HpHI07RbBiRAWvv76p3z77e+p2oUsu6bRr1KHbR6wi6Ab9vzyj1Je9u3tiZc3n2HdwDTtEfGKi4t75kmmOLlwS5YbVtUrhPDUxQW7w54sJ8WqLZFl0QRSMsTkLA8Pd6zzxEPsjtCfHEKUlG2fUnxwxEHSlBdEaRFiwnuJFhoRLbmRlKbCmDT5KbI00Qg43ByRqkpF5flRloJ1c0GIDu8XHncjbXORNGDjiFYChENLw9hPHA8dVZnxuL/l6vIFTVVw6vb4xbO92HA8PZCXGWVe0bs+MfCahuhT81mUhuAF3fSIEjDPA6NNq1ezJPKAMjXWelSmkXiUqdC64LA/UdUlSmikBJOfjYPn51KUGcYosizhT7784muGoeP7b3/gl7/6Eh86ht6y3W4ZR0m0DSbz7PcLP/vFDR/f/pZuChRV4n1+yiqXRqOUoO9HYgBTFigtz/BmWNzCNI/nJjUy+ZFSB4iCfj6xXa0oFs1ke4JOeLZ5aCiyGxx75hCZnccUgtwXzH1AaUteZSwm4oVKjfV5Yvjp+RqVIWTELjOZSeERUmaYLD7DokPQGKPPDOQE7DdGnfPLky7uUyRxiALvBCiJnReWJeBdWi2eTieqqkimSJO0lVmecpzzQjMMUwqv8DN5bljscC64FeAZx4GqLs7FVkLwOJcmZ3lRIMfxGabetJosM6w3LYfDLhWw2iO0R5rA4kYWB1KKhMFrC4zOCVNkXa/Y7/d8fLynKlITo3TGYT8Qg+bxaUrMaB+Zp2QOSXeQoKglp9MBo6FuW2wYMbJMRBYj6Q8jRWlQsjw3Gx53Nh/lZYofTNKNkRAlx+MRjlDmmhA1WgrqJqeuC0ByPCap3TxblE6G03Ha09YrTJHz5WaD956hG/gUhDFPCzoHHxxESVamSbGSGVksE8LMepROKL9hSng5kyed++wWJjv96J6XUNcliw/PbvZxOSU8YMw4dSNKGkII7HuBRICMqPN2cXERFyTh7A9JTMqn5NKu0xYqxojODD5anHA4K7DTQBQBQUZZNbhzznnfWaJSbNZbuq7De8jzguPQU2Y5fX+gahvCEsCncJh+HM5g/oJ4Zn03q5bf/e4tyJHCJH7oP3hBeTvcJW3RmNxHJs+YlgXnLet1yxIl89Gi8hFtIsfBUtc1h3mPNBWPj7dkWmKMwgfPcNyhdUbMPNt1S5gPfLx/IjxMdH1NuzYcjmnlK0XE2cSglCoSQ8B7MEYmZ5i3GJOeyqciSAjxIw5ISrquT1/ccWEcHFcvG8ZhpD9N3O5vKYVmu10zqgg6sh+eaC9b3LSnbdfUQeGEpNUaH2aqSqe8TTKyrGJQhr4fsd2BTZ0Tl45MRB4+fM9Xn20RsqQ7Lbx+cclhvyO6SFkYnL3ni9c1YYpYt3D/8I7bD4Grm684df+RTV7RDZ7je3jx4obuNKRs4TmwWm3Ph5/CNIbMrME1BEJCnyiPd4qyWpOFCWfDeZo7Jl2KjEx24PC4ozA1qHSBHbpDAuWapF8yRmFKg50tAUGMGSImkXfAczjNVGVyJAtlaJvkTl2WgNYSrXKEd2Q6FZJJd5OYlkjNMPQUOueyveI8RMERyZuIzJ9Yr9eUwmFDyvANAewwIaVF7T9SlIplcey6HWQNb94deHFzBSZAPvFx9wPH3z7SrtZIqcnLGmE9aMXYd3Tes335BQ9P3zMNimtZk1eRbvDoPOfD3S1/fJGSbY7HI1mWsd5cJIfyPDNNT9xcrPDeslqV7PdPXG8vGPoFuyg2myu6+RGKPd0ITjzh3RrnO552FrucmBbPxWaFULBt27PZTLLMAb9YzKrAy47L9dfJQKMMdlmx2AkIZEZQVoYQ1lRxRdc/EkOBIAfR4eLI0o+I81f+6nrN7e1HjK7YrF+xLAvDcASXojcfd+8Yp+ToLQqTvnfRU1U1Q+eo65pueMQuD7x+/SXzPDOOPUItLJNBq4Gnu5GiTHqlGBTXF5+laDf9BT/5ajiHEORcrP4EKS1Xl2sW/4B3kpur/ykhDHSnESUrrq7XrI4pDOFwuqMuv+B0OnF5/YpxcNwe3vD6sz9icSeCyznuHbd3P6B14OPHE3YuKRt11pwVjH1H07YpR1kNEEqWMbFsy7Jhv9tTVSsAssxQNEnHGOP6+Uwsioz98S1GGbKsZbMFKTTaKLQTaHHOE9YS1Zgzz21gtblECEkUM8pILq5SUdq0LXVTJO7euUCQUoLUTItjf9qhM4XJU3EzLynVSGuNCyk1xoiKKCeE8uSmZBgmFjezvmjQqoCYTCham7OeMM3sFrvgi5Isr4iyR4mX7Ib3NBvL//v/9de8/nzN519e0B0quuMjzWqhXEueHo+4MDL71MxXVZ1wQvOYtPXzgBGGqDwmzxmXnjC5tD4OKXFEZymuLkaPbgogMk89WWUYpo6r9pr+9olYJID+/smyuWjJqxPzBCEYssoiZo8dIlhJVUb6YIkyonRC2f1I3YRlnrFIdJ6+D9ZO1IUhxJmyzpKcymjmacY5f84bjzhb4X3CCI3TRFVrpAxAnhzGy2OaZpJ08MPQnZtzi1RgdJ58CLNDyHiezCqkyCjyDB9HlEiDmxDAEzFFQVZUuLPec+inMx9TcDp153Wpoh9OKdEtWqbbLjmzVRoUhOBQKmk5lyXgPPTjwOPuiRg01s4s9oT3guAjRmVn6Pz8rONUUj+HjKRHIMsNITg+ftyd0+tyhsHTrnOsD8gISkfaTUtwE/OS+JJVVSBkTgguFYPOEc5TuhAc9arFqIws02zXNW1bo5VkWnYcDz1VnabEUkJSJwRMpogRhmEk11lKbAseEVNdEAUso2AZj+k90vEc5mCfE/e8SHWLXyw+QjeMaCm4e7inaRpEzLEOQgwMfsCoT0SXJINQEryzTFO6axeXNJ3zHM5JST5hlQSp8TMaVCQIy2QXslIzTwM+aIqioC6qFCMsA9YlVnWWZ0SZIiADBudnpCSRaqTk0J1QiHOBPuN9wPqept3QDbukt7XubBotKOozwzVEqjbn0O0g5qxXJf1+IdL9wxeUi1jYPe24efmCu7s7xJiE6P14ZLIndFYxLguFzFmZEiUEfp5RYWLuBnQZCdGzO1/IUsLYT9RVRXc6oHSFUIq8KDj2A8fDRLO6YBiPiRIvIlICInUbS3QJU2RS/rb3Z1i4/BEp9ClBQmvJaViIpuT6q0uG3vOhe6IwBUJpLm42NEYhhGKcFsZxIa9TOsdPf/lTXly+4PffvKVatZzGjqpQrLcXjP2As6Q4p01LXhmGYWAcJ/aPTzSN5vXrFxAzJrsjixPvvvlI22zIy4IQBXlR87B7oigFLz57SXN1xTCeKCtBDDXzAVSY2XLC3T/xantJDCO6NNy9/Z4YUzLFadAMXWAcIpeXl/z+29/w9c+/RpBzEgV1swaZknHKukUZRQySomxotMFYyzgPiSeZ52RtyzAM2HkguIg0gSLPmWeLdQ6tG1SWzC/bVcnil6Tr8x4RI2Vunh2AEUuUOc5ZhAzpZ4JLjFIJ623L4BxOjOeDT7H0J9qmZJM5zPyEVW1iMp71odW6xU4LL1/9YXJJSs2nDO/9xRNaCtbiC/IiramWJenSFpsu3mmesW5mffM5XddRI7n56k/59ttvufvut7TNmuHhkWbtqaVmv7unKjXLMnF4ejyjG1p0HiiV4PY+ubif9ie899zevWW9WSVEVP+AdycyvaIfRnR4jSk8NzdfcffwDdaMXF9+Rt/3Kfd2ipy6gV/+4g95//4tVjpOp5E8z+gOw/kwNMiw0Bap+CzqBr9oNmtN1++4uPySsZsQauDm5hXdMfK4+4GihOBz7m9PXG6/pOuf6IcdVV1jXMMcZ0Y/sL18xYeP3xDoaWNLJV6k1z2/RISRy8s1QjoinoeHB0QssG5ku/oJkYnNakumHlCyIs8NT7tbnp4OSCnIyxOEyDQtlHmGjQcWdyL6jPX6BQjH73/3Oy4uG6I4cHxoOXV7Pr7fsb4omZeev/z4bzl2t2jdJAOMTFQHpR2XFy/p+j2blea7b9+iYsRUgnkuWJzDuY4sb5hmS5Yr0IZcNQQvEVIy24mLq0umMZ0x0zLi/YRRDdrY5zPx2D0So8aqwDB8oK1vMNnMtBxZXdQsS+SiXhNCoCwa3r59jykEi5sQWJxf0JVmnBPDEGEZZ0d2/p4pCT467CKIWHSmiLEmhBEfHaaAXEm6U09ZFWSmxM4zSgeM0cx2xOTmzNVdEhpqGZEGfPzb5VWiVfgoUVqT64Jx0Oz6H3jxukLpnDffPXF8klzfZLStpKgdFzc1d7cHDoeeKDI0Of3UcTzuQcZnCsZ8zqYfpglvLWWZg084nZS7PJ6j+wKLHShMmqrvDgMXTU2m07SzqEuEv6GpezaXDqnB2REpJEpojFkQ+Yq5C8TjDu9SDvynogh+nFBqk1B0RJASdJm4lmWlzySR5DYuylQAEtOAYp4tRd6kZqjIcX4mcpaAmaSl1CIhp5Qyz3p+ay3L4hin7nxPQWGK58lp8EmL76Mjzw3jOCZNpCpQWc7pOCXHvjQsi8Uu6Xmk0I6Icz4lk4XA8dg/p7RZmxiuSiluPx4BcBb6UyAvRJpILwFjCpqqwjmPtR68oKwyYjicA0cyhmHAyE8ovaRBDxZyVdM256hG6ZDyE8g/Mk8zRVERPSA8UkDwHh8iIqbiUaLQ0hACZPknJKGlKKpUKPc99w8f2WxW5CZjmQNllTMvHaZQTOME0iCDwLkRpVIDtkwLQkiQgmmazhgnjSnS536eHcYkz0E3DGhdYP1EPyRUz6dJ4mgddbvCZBofVWJkzykNqa7rZ9a0FBGTS0LQTPOSYltD+uzrTOGcR5y/c25eUhHoAsbIRAYJltxoxrknp2Q+zJyOSTtrsuT69zi0Njw9PbDeXvP4eMe8DFxcXDBMM22oyYuETVxsmnyqzCC1YlgWUJJpnsi1AanohpE8S6ECXd9hbERnGjv7hOBbX7LMh3/4gvJitaatVyiVcbG9OWsPIH+RNGwyTrSvCpYhoIzEu4W8bJmWmottxrHvwAfqVuMXT5FX5Aa8dSg0yuQE3ZM1C3ltGe0d63yNPZ1wVlMUJq0o7N8Cji/+RwFyTMdFwi0kM07Sq0RMZhA24GdJ8AkBpGSOloqh6zCF5tjPKKHJTcmL9RqVa/qp5+27Dzw+nmjLAqMMp8NMffGaZQocDyPrpmH0I2EQNFm6lIbxRFPktFXB7fsnNuuXoCqkCVTlOo24tScSycuSlXeMx4nDk0DLhnV5Sd/vOB6+o2kr1kWNNdcMw8Djvj8X5AumSpqc47jj8/Yl+/6JdeVxwwOvNw6zPCEw7HZHnqTF2chqdcHThwBSczz1rC+uaZsNR5dRndMtbh+fuLi4SJFMIlBUFfa04zh3eG9Zr7dIIZlnm/LNhcPZ5H4MIZ6nPAk2n0THDr84gp1BRoosS+DtM0N0mtI0VUhDdB7nA3W9JgiNKavk1A+OfjghlUJIj/UzJtcsywk7BJRYMc89bVNzsW2wiwNKiBqtDaa9fkZKxejZGsniZrLMoF4LbC9QOvJffvmPub37gb4/8eqLX6U4K5Fh/cLT7oH1RlNVM6fTgRD1+YIIbMsMkwVKFTidOq4vt9R1RaEyhqGnVGsePr5ntX7Jvv/IVbXh3Zu/pj8qtK7w/Qe6bgIy6vaCly9+yrfff2ScOso2ZyNLhi6izIKRhrlbqFtJVeUMJ8/JC7757i/5/LOfoM2K//7/8+dcbq9o6jXf/O4vMGrFl1+9pK0rDocTVZnWVVJmFNkWO8Op+8D19SWHw0LZKL748g+Yhoi3jhgW6vKKaUwMxKenPdZKnLfcvHjJw+MtX3z+E5wf8LbicfeWafRkeUfV3ND1O/JK05SfczzdUhQNjjcc+zVV8YJVe8M0L5y6mWnu+fKnF+kzIS4pisjhdORnf/SSftjzB1/+MX/zm1/zz17+T3j//j37XU/brGlXNf1hpG4yVA5D79he/grrBkYf2D0N7HczVzcX+GCp25a720eUqglxRKuMcZxxfmSxDmXOn904ATmOO/yPXHPyvMA5i13SahO5Y5jShbE4S9U0DMOAlpLOnWjbimEyiBCZ7URZJai5854sN8zzSBQeZy1Ewexm8rwEYVAqUtUGYsp1jjFju7nm6emB6xdJ6nA8PSJEpCgqZuvYblv2+z3ep4vR+RmVCaRKkxTCJ02hIIpkmEB7Cv2S3emBIPbc3zu0Lvj5H/yEb3/3yO39N/zBzy8JrqSsM5R2PD305FXg6f0JTwSVLmoXEjHALj7xX+eZtlmz2OmMXDpDfJHMZ2pHoVv604iUmqvtK3KZk+ct2+0Fgx/J4+f88g8yfvbHnt9/8xtOx1SUIjxDb0Aodvcj20YiRVoNB1ciCPztR5pkDef1pWJeRnywHHYuafa0Yr87obVOpg5PKlyEZ7HDeW1tzneMYF46lMpwi0MYRQzq/BtTxF/btkmj65LbPOknPYgz/D1KfEzA6U+Rf8bkeAdiCSkVR2eQENVMo0vED5WMO04GvPVY75AyTbcAtD5r30NEConJFCJEgvPMA4zdkRBSOMh+15FnLZnSIBzdcU9V5vR9jw+CVb1hmZPeUgiFXyJaZgQL+32fihq/hxiJCwzzxGbbsPQT3lnywpy3eoHoPJFwNhJpCm1wNrBMaX0vQuB0eGQYJrabFcPQM3QnYpRIoSnKDFAp0lhIQOHdmD7n1rPb7ajyCoTARsfiHNp7ovB4zkl6Kk03lYhkpUQgEaZKfGuhEErQDX2KxjSS05i2KkWZI7VE5RlBqlSAyxypNePQEaNF6oxjNzwD9rOsTDg16zDGoM+Sk4jHncNVpFT0w0KIilM3pHX1/oALnpuba5SSeDwf7h+fV9n1eoWcFEFKqlWLXwKjm8irEhMVMcC49OQyR5scozeU1YaqSKzMruuQMX07tpdXLFOPDTORgJIl+/2evPgfIXrxw/4HTFZw++aRMq/YNDWCwLwM1G3F0+GJbvaU1YrT7hEh4L67R5sGNwQ06RATEaSRTGJidDNFnmMKTd8LxnmhiCNKl8QgmRZPVrRImRFCR56Z8yQy8SqT09sjMEgZnydin4w6QkQQ5yxvu/Dxbs/rl6+YZ0sMKQYyqzKCgCK0aCkRISQtoirZjz1N0VAWOd6PHA47qkbS9Y9IzHkFDyaL7LsTIXQJgVB5dNVymASirDnYHW5IAv91vWLuRz6+mXF+YX/8houLCxrjObw7sLg7TCmpii2vLn/GqTswzIqu2yFlGt0Hk1I55jl1UnEa+Jtvf89PvvocgcMHyXW9oR8t09zx9c9vWMaOcZipKsPjw47vv3vDl1/9lId3v+fWOuqmQdQlxbplebjj/i511ZuLK0KWMciUlb3fP7FaX/DZ65/jQzL16GDIRCRGRQwRo1P+KwSOh4F205JnOavVmnlZcCF11DorKNuEU5BapA7uzNcTUmDthEIwTzMyy8nKxPRDS2IQPB0m1qsLhmEiA4wq6M4JIk1TgYgE7wgR5GwTJsYYTJazLAu5qPBT6uqFKbF+4v72SF18Tl3OgESSIgFtlDTtT1M0l/dASAUAjr4/Ib0jRk9eaMzhEednTs7yYf+BGAMvVpcEfUW/DMxW8sP7W6Z+Icsq5lFycRkYFkvwIaFD+pQbH4JjMxvePO2pV4bN6jMW59id3vH2znN1pen7kbzWiFjwl7/+K7recTo4bj8cIVrsrMmLD3zz7W/4+idf0J3GNH2DlLO7qsmzhtVqRVgs0xzojyOH4xNNnXPYn3j52Q3f/OZt0gSayDh1WOvITMPT3e8Ye8XHN/+JyDl/11ZM7iNVLZCyJ8+vOOweMDJyeX3FYRcI7oJcb1mtVnz/w0cOpwEbbolIno6Ou9sHslxT1Yahiwzjd7Tthv/4V++RytP3a7rBotWabo48vHnAhwlxdDzej9hFUtXpErh739GsYdWuebjv6PsjeX5kGhO9IMsFSoezKarA2oSP0TLDW0dVrui6jiwXz2ficTcQvKFp1vTHU5qAiGQ2CFYzdWBUS9cdWZaZssxx9oTQBiknTv0TSimapuDmZs3TfmBdXtJUlxyPHYfDLWVZMvSW68s1VROJUlBWn3Pcae5ve16++JymqVBqxntBPzhiiGiVDD5NWzJNM1WZ44PkdDqB8GhdwnKe2CXpH3VVYMSWsGgedrdUlwtmqsn4BU/7H/iDf+T43V/3fPM3kc9+XpIVlqY2PD32rC5XRI7sdx3OT/iYPZsGqqph6HtCdExzOE+g/LmoSk1lXuUIIZFRUdZV0mkHS1XU1Ks1T8db5n6imx7wHLAiox96+j412GWxRoSCxfdI4VFxQ6YTF7TMa3L/abmfiknvAnmVdHBRBfCRsqhYpCJEiYiCLC9xzvH4tD8XeAa8pl8WTKaxS0CKmghppS0UWpOaZcXzZLgsC6ydz87xH7F3QkRml1zvzi+ARFiDdWOKHHY2DR/KLE32okWbEjDoOksSJpKJrzscadsWH4ezyzfdtcDZxKrxfqEqC5xxBA91nZiGh8MjbhnJVQqwQObEoHCzYPCWPCsS43fq0GJFllVYmxLogk2YG5jY7z8QVDyTCEyKWu7TVkrFnDAJorRJ6uYDzi3YZUbkObkuiDFgVI4RCo8jAG1ZoYVmu7ogOMu8TIiY0e96yirHh+QWF9Ii5cI8e7RQVEVBIMUw57khLzNmNyLP7Fil1BnzRTLkSANYpEhcybJKbFSVKYZ5YPFJ8yukYJgHsjxHaMX9/oHgIqvVKiG9pGKeZzQe6xK7cg4OSaBoM6JLhiSVCaYljZmnocdIc240UuiB1pLRzUxhoSgKunk4szUFURoyWRKE536XJHmnYcb7mTovUAiWeWSZzglPfuZ47CnLluFMkpnnZP46nY4YY8iy7JwXXyRNea0TgztaQvgfARuUm4jzM5dXW4q8Sg424XEG7qcjUUqq4oJxXFCyoS1WhGLBhgmpDdPoCc4hcfgYcWFIAmW58P7+lqZaMy4zORXEHB8KlskglcfGPSqkzsT7RNpXKumpgvcYk+HdkoCd51W3UjIJ6qNL7nARePvmA3/6x9fM48S6ukSqGc+MKRvyRePjTG87ejFiIkxuJtoFpSb2e4+QiiyTdMOAFDl1vubUp0p+HB9p2pKu65HC0A9Hxn6gMiX9aaLarIhhopsCoDDGYkpFMA2r64wwlAzjB9aXLeNg2Z/uOHU7MpWlVWtcKExB1ZQ8Hfa82t4QVUgFlhEpSzRYxn7Pod/x6ovP8VKBCnz3/e9xLKyaNbuHA01V86s/+xnr9RpVLRAlxoHRAeF3FJyoi5p8nTPaHUuXUEIXhWa19tze/zW3/R3eG1abC5xzfDws3NzcYJdAVuTkefm8yrK7nnb9Bcdjd4bopq7eDRNaKmT0KKfQIuXbOhmRWqGkhpjYpj4OzJNNa5Ylwcq3dc0y7alMRGUlbjGUZQKJL84SvEaqiFAnJmp0ZhimGXGOq3JzQpn46BlObzBqhVRpIjWeZkzukHIgWo9RGXaZiOd0EKEyjNbkeZkmGDFP4ObhRL36jE+RnSGk1VOwHWWb4Lyn/pb9Y5+meNPb5BT9m5LD/oGvvn6JVg+UuaTMFRftmml/x7e//YZxjNhoUw6sgMVNjPOCtZ5V/TmL21M3JcucAYHF9jR1wRdffMZu90jfOb793T2L7TFZ4phmWZGYgNM9h+Mdb98pMlMkRqlLLv1Mw/sPTxh5gcz2DOMRoiTPa5SvePvwLciZwrzkcHjkt7/7AaManFtYbwse79+x2rQM4xEp7wheJiyW7WmaJ5yVHMZbBAWIkmk+EcVfM08BkwFI3OJxsUPKj2htiMHwsZ2x1nA4/SV5pqjKkt3jxHq9pSwajqcd+12KaSsqwdTnHJ86hHRIZfA2kmfq7Dj1BG8JLjUF87wgjMIUhrZpwM9cX9RIYZ7PRBEFUgpiXChywzwNlHWKvVxmR2+TXrjIDcu0QPQIKZE+Ep2nLRtQilyVLIOnyTY0VY1WiroquLn6KYjAaT/QrDQmnxmOLesyJ85Hrn55lSY34yNtWYBsOBzfc/Nik9BBsWIYd1xcVWTGMA6KVVvQDwfo3fMFSYTtqklRmKdk2qragS+/vuL2u5ZuB9urkazQ/Bf/1Q3/+v/+lm9/u/Crf/Kaqi457BTbG4+bQYiaq6trTt0Oo0lrZTx1m+JXx2FBSM4FTioGjBHkRdLDS7+A8mgjqDMFbqbIKw77if/sP/uviCrjX/3r/zN/+euUFX39Mm0h7j8q5hnaVUHuA8PBs7kOXFy29H2HVQMi+1RggdICbZLzN6DQssV5aBqJkDFF5eUahaDUZ4d4jOgiS5rgmAYbSuZY76gyTYwuTYOVwItIUZaEmKI9E4MxorRBIMhzwzzPlHlGIK2cnXMYWZDlEm3OhYlWCBFZ7ADCJ0xPSKEZoJnPxZExiVOoZI3ORSrIZfIs2HlGC41SitPOIlWEqPA6UTUy1WC0pizWdN2e6sx2LrK0FZymiSJvkoFFm3PjrKirDHx6LwV1Wr2HhWn0GKOYpgUlLNYuKJUh0Gid5AzeLmTGUOc5602LiJLHu0eqckWMln6eWeaZy4sXfPx4d/7GBdq2pWlK6toQ/UieFyiZg1T0w55uGggxUOYlxy7JgxY3J1SPTuawPGshCKSckUYS5pjyzpVnGgakgru7DmXOvg0JwzSlv9tDnmccHx6ec+p9XDh09xRFhqLAlJJhOKKMQhUZudIgLeMyIWJaXc8jIEkbV5kTQtKtBjzj1LNaVwTnMSUIHThN+3PWvELKglPXEWXKqJ/thAsBZTwTETeOzNNAYWqW7gBxQUs4HZ/o+pGrqwtAM82BaR4IMWNeUuMxjZ52U5GXOdZDXmp+lMb8AxaUvRiRRuPsxNDvk8ZAwzyeUFpS6pbxsENqhczg/nRkniJ1tWKzrdB+ZBgS+NT7gc16TbAaqTzZSvP93Qde3qxpC/jNqaMP91zm1/Rzyjct8shipwTvzWT6kBcSKR3eJ5FzXdeEc5wTnC9ypxFC4+WMrldcvqoIHrQJfLibyKqI6x5ompqLqxtel19xOg0cTntEnuMzwVJkNO3INKfx+7rZcnd3h8yeKIs13gdqW3LRXHHRrkE4qnLDd9//jsl2bL+4ZJo7IHJxvebd2weMkqzXisvmmt3jDMVCtl4xRcmH4y3/+T//M24/fESTc+g8QmbUm4ZVWwGBqR+4vlpx9/CBzfWKofdEJq5fv+RG/QHff/97yirjcrMluh6xBIwKyDwy2FtUc4Er1rz66Wd04xHXK6ZlQEgwzTUyb9n3I7HQvHx1xcfbe0Zr0brk85dXLNPIw7t7YiiY+gkevmc//o6qqTjcB6pmy26/Z7295tWrz/nh2z/n9auvELJAaoV1EyFG0A3eZUTlcGGh2BYocuY+YJeRooBlmcgqSZPVDJ3AhRmsQCibWGrCM5wmlKywY4/UC0qnKW7XDWSZQnjJgqcoc2KE8RwvZn1y7bXN5iyVMMToWK1TznSe58nI1HXkek3AE1iS2SKrmOYZlKbKQZIc2VlW4L1FaEVTFXhrkbplWUZe5td8/ZOf4aOGKHl4/EhRpoP9k4h+Hk+oEPhq8Bzv74lZ4E//0ef04x6E4/Fhxzx5HruRECHPK8ZTx2JHXhRrZiuYFo+UFd3TDG++5bOVJPoAYUQJiZo03mmcH1NRJECR4cOC1g7vA0K7xNCULUvXIdUef4LTMRlStI748IY/urzi8nJLNw4cncX5gBJrpFzwvaQn4A4Dy0IS1ncnsrpEmJLdtw/UueFL0RJ1QGUzSxAYWUHhmcOCMRuG4SNRFOSZwVlwHuLjDIy4UJLZAuUVixSYfiZ2ExFNZl4zHSLSdBjrWIIlqAwxRnIt6d2M1oIwlul5O7gyG2ZfsEwTDYrgMpwYEf2Mlj9qKOvqAu9n7h8PFMWa47BDnDxCzxhdcuocZaUI44yMORk50+RwdkGbEj0olsWi1EyMfYJD247FJ5KGzxTBeq7qLcFPWNej45E3//bIersmy1aI08JloXg87dBFhZ5zRPeE0pCrlhAs3Q8HhuEt3geaesVK5fzxZoPyD8QQklzot2/5p7/8FYf+kY/3R6rLPe3vr5EfVuTNB1a6YTyMFMOK17/6jL/4Nw/89DbjdQHfdZY/dBu2YublTrKSR/opABXLcaHIc4KfKMuaU+8hRrRRKCWZ5ymhraLCWUdEUBQa7zLyfKTOt7w+9eiy5n/15Z/w3/w//yV/YjM2rwp2Tye6v0lO3ZfNiCmgUZrOeLqdZ3WUDKNEx4wXxfhcUAK8tJGf9wIlAzEEwCYdnFYQz4XfnBwfOpN4ApnRxADORbxPmsrgR0LwZLZgCpq3tWQ+B0dMU2Ia5mV5noIn1u88WwSSqi3xYUrrc5OT5xXzPD6fOVoIpNDn5LEmFTQhRX/6JVDXOcfulCL2LioO3ZFKJVqI0hnTPBLORky3xLPUKzKPFpNrrD/h48I0D0AFUwARGadTKiLO03YhBVIIyrLCu8jQj2it0AqqVRou1VXL4+OOuinQVTpXywq0MkglSFnqmjyTCHKKsmYYD6mQzVqUkHz2p695un2grquzDnDgiy++4Nvf/xatFetVw6GzDOOBVXuZ+KUicUmX2bPYDJ03PD09MU0Li5QsznPz6gWzT43ki4sVJhMcTnuUzGnajOPJUlZbpB5ZpgHrMvaHA0om06DSAYQECsZhwTtwLgMRMDqSmYw8W3E8zMSQ8s8JNcs0sN0YvHWM/YAUNbqakBiUTpK8umjIjKTvD+hshSVQmJyigWmMKKkRRNbVluPpkShC0p9mFikz7CDIjE/oKyTSTHg7U5QaIZLWtigLAo5lUqy2LV46Tt2OsthQrl/ggyPEESGTMUgtLcOy4Fwgy/6Wxufv8RAx/v2qz//d//F/EY3acLF9xfc//BZlxrT+iSZNdqTE6FTERZdzPAwQJ6pakWeaYwdVuT13cYGrqxeMp5myWAEpeWR70bDMO/7DrzX/+//D/4ntOmMKICMsYcLonOjiMzLnU6yZcw5B9jzGTuggnqGwVV1wvF34X/8v/2f81//1P+ftt+/wYqFubjBZeuOiPxK8pqm33N5/pKpzrl5c44NmnkCXJ2TcJPCpHen6RxCOTBZsLwtsH5NeSmcc9wd++ctfcux67u8euL6+pq01wZ91awrGqaMwGcGr5IKOkBUl7z/e0fcnthfrNIaOOU8Pe3QesCG51grTIILAZIJdt6NZJSRTDJrZLuSF5PHxnqvrC/puJgbJq+srpMlY3Mz3794QBdy8eM3sPP3xxN4dz4fdDAT6aUQbg9Q5WVaxu/+AlKkTDiHQljmFNhyeDhSqpC0Mq6YGL7FzwM8Tl+uasR+pixalKx4enkAYyrbF5BndMDFNC9uLK8r2FTYuDEMgzyqG6ZHj8Y6Li9dU5YbMCOzi0ZlB0jDMyakXZGr1MlNjRJXWUiI5WWcL69X2PLZPtD0pYVrS+sm7QNd1KY3EhTOCypxNXva5KRESmtpAzFmsBeEhGp72XYqfOx55fNyd8241eZ74l1VRUtUlhcmoV+X5ELHM88Ljw4G+H9MKWQvGWeG9S9ywsuB/83/7V6zvngghceakMkghCdEDnhhTRymlJMa0wkrJM0mfFUPAnZ+DDx4lFBCTuzhGpEwGAuf8eWXlcC6x/jgrnTgb4YRIQvQQEgtWnPekWhl8SE7+ECOEs+wkJnGeVALrwvnijef/XyQKATFACAgpEjkgBCKRENLPefdpJRqRSqYfjzFNGmTC3QgEMZ7PnHhmwyn97B5O08PkZE4pUulvEzK9BtEHkApBksUIoQjep5UlSd+klEJIgZIyuWX93zk+09+T/g2AP79W8izF+RQfHZ+fi4iCSEKecP77P2HQIKWNBR+e3SMhJEqFIK3CYpBE3Fl/lSIERXIrpgQY59N7BIQARkvyrEgIH6HIyxIhNLfjR/7qy7fnvz3yi+9vyKaGaUwrzLxQSJlj7USIE1Im0HeM6XX33mGXQF6UzNNCXiimyWK0SWgW/2NSGUAM4YwfiSCSwzt9fsXzPxMFUkmCT4VlDJ48r8l0QYiezfqCj3dvcc6iNXinWObkhDcFQOJb2iU1BklfGNL6uJn45qf753fu699t0FOWzpBPr/enq/D82svzd+XTfxdSnItPfnxeMZKSFxXBB35oIv/bf6wYFo9d0pq8LHO68YAxmugjMSY8izKSaRqfiSRS/kgl+WTkybIi/Y6QXicfZrTOUuJaTH+fMhIf5lTY6gx1jn3MsowYAnXVJrZjVTNNU5puynQWSamoimTKSCaTNkkQcsM4p6ncfE5Iy/MSo8bEerYBPHSnAe/heBj46ic/w7MnBn3++yeikNSN4bjfU1Ut/bywzI7oPU1TpUm6j+dkpQYvs2e26na7PUPPNdl5UhumBa0l8wRGl7SrnOPpgPeWdpWxfbHh7u6Oly9fk5kqJY/NA8J4mrZMa3Z/hJjRtjXOj0yTxdoJlD8HOYwIlUD03bHns9dfcep79vtHjCoRGD5+uMeH9D0vqxxwDOMpURXGDGMM63aVSBh+z9hLyrJEUhPilLaZp1QwbzcZH2/fcjxM7E+Wm1drBBKpBE8PPVKUXFwUjGNPJDXUgoyrqyv6vqcf9iBH7Jz4q8vsUBoEgSKv0ap4NhAFne4rKSNScSYG1Ekn6x0X9ZoYFUYFptGfQwgU/92//OZHrc//wOPvPaHUpDf3/fvfUeSapnmFtcngUBSCqRMMY7qQt6uaV9cvWGzP7mFHqdZsP2/RqqQf9jg3kWeO9auSp6cHpLK83P4h796/4R//yef81X96oOs6bq62jH3AYVFaIRUpXeAc2p5+/9k15T4dsp7gP10miVUZPCzMfHy6x6wqVNtydd1wOkb2047v3/4GKRuUzLgxBV/96o/4q9/+mru3f8NqvSUEuNlecTh2KQKtNDTZJcMwcP/UM8TARVVgakX0ipsvvuSH27e0zZaXX33FMB7oT8k5qIRks1mxahoIghAEpWnQZFxcXFDUF5R1wTCPRG85Hp8wDVRtBaoiRE1AJ8aYzti8/IzffvMfKasVF9tXdPOOp2FPXq6wIrJ52bI7HPnu4TuCSBeXFQttu+aHD7/HO8HV5ppNLBE+8PDuPU3T8MXVS47DhJ/h9PRAnQeaVYsQiqf9jmN/os8UPrOoKvDmJNhmmmAlwgEiMC4DQUgWXROXA6ERSBlZ1ETX9TgXKArD8fSWfnjH4izzKLm7u2O1Ts5W379H6Yo8X2PygqxqWW++QmiBUBkiRoJX9IOj7z5yPB6Zl4m8rJjGFBM6uwGja/LccDg+cv94T/CCp8cjFxdXWJt+zlrLer1mtWqfobvd6ZQcmr7DW41UEPyMkiXOxh8vCTmfD/eULx+jQInkspSk+L6UkBApqhIpUqSnkAHnJjKZ2GYuBFbdTPP+jiWET24zQlieC6kI6fJJsc/nVZJAyhzvHT54pDIoOCeuJH1aCP5vSQ6SxkoLgZD6zHRNazLnfMICdUf0uTj152LT+XTBpqWbQwDL4pMjUit8TCiOT5ex1kmE76N/LgqN0YT4qfjziODPgOC0wpMixR9y1jvFM2ZDSYnROd6H58JWnA16SqaCLZ6LE3G+9GNIhZ3SGkIyKDyXo4ofi4MQUx6xkKkY1pxfaY+zgag+aXv/rrlDRoE8A6rFmZEbQ0Sdi9yk5RZnXeOn9BV9RvZ8+nelwsp7RwwJbfRjgUwyVEiJiopgBEZ9KjLOes2QLmUlJObczCitUTLDZBnKZHiXnLUI2O93HObDj5RvBOvVBav1iuPhSN/1CCTzOKG0wygDKGJ0ybwT0wTVLhFn02cphHQm+xAwaIQIn8zj6deoT81QRCLPrwvngvr8UhNTcce5gI9JCyqkJJOGcRrPmrgMa1MTmBWSGD3LFM9GFIeUIuVrR86/AJKZ5cdH6m3SLxeS5AaG5/tESXlubj4VjoEYUjEdQkhPjPS+cn5/QfBlF7m6Hfi+lSgRsfNEbgRa5Ixd0tRluU48R+tQIuXKZ3nGtCSdXgJffwJ/Jw3dOM94m/KXvUtGuQRgN8QoyfKMIBwiOpxLmxWjFd5GlnlECvAhhX2kHPJ4/lyqM5IvnnF0jsP+hNQq4XOcoD5ni0ulKGtNWeZE7yjLnK0tAWiaL5BGczq8QFDQdR1z73n58iVFqbh8uaUuVzzujuz2jzzcP9EtC0pqoo7ILOO0zDztH8nyRNP44f0HCpM9vyfBJ7On1oLDoUOguLpen99bwdvdxPTX3yEEfPf2wHZ7gckyHp8eqKoCx4KKiptXG6ZxpCgHFttR5A0Xlyu6wx4VPEVdM48RhWazfsH9x5Es31CVBbmOZKbBWs3Lly85dU/0/Ymqqvjuux/4ydc/R6pI3x3ZbBoOB8Fm9Qcsy5LeLxUYx5FMtUjbcbgfOT31vH75E242IPLA/f1HmjpB1ePa8Pr1Fzw+vSUrDOPQs3vqWK9e0vWPZFnGq801VQP73Yl5SXdDXddkJplQl0lwPAoWt+PlRYtxDZnegBxRa0cIEFcb5nnk2Fk22wqhjqy2KVqSUP89KsT0+HsXlNEHVs0WKSWnYw8+QwuVxKqh4HqzQiqY55F56Xn7rqNpL/nlP/rnWNdx//GRvj9xsb3mcJrpuo71q59ydV2w3++5O9xxu7tniBfc7Z/IVw0LjpjleDeTyZRVHWKawgitcGeXoJSKxVuk0T9OB+InFqVJhafOOHnLbgnc9jAXgb53GLPm6z/6M+7vHun6I//+t3/Bd3ffsd1c8vbde9p5oGlzxjc2dUWZoK4L3rz7a1zoydQV/bKidz3LNDLPnqpsEsn/+MBsF9pV9f+l7T/jLcvS8k7wv9bafh9/bfiItJWZ5b2D8gWM8KIpQDDCCI1AAqmEEZJomQZkkJpBjaTWCLUQAkkII3xBUVBQwhRlyTLpIk1kRFx/77Hbu7XmwzoRCb/pmakPrfMpM+LGcXefs9/9vM/zf2gbgRAapTwOVzOKImM88u2JjIjQiRhlA/KiQAvbnBC4HmWVIX3FYKPH6VlB3RaEfUV/3Gd6NsePJf3BCOl0VNWU4Sji7CRjepgQOjs8+vhzhHHI+QsbDIYBaZZArAg8n41hn+V8QVVmhP0xVbHi2n2XkMIw6PXp9UccHJ2xs7mL72uqvEApxWY0YTwaMF/OkL411e9euMBiMbNcsa5BKcFpWjMa7lB7muduP8e1a9c4OToh9EKkcRgOehRVhvINXevTGAevB1tyyO7meRwsQ+1g/5RSTIlGO+TViuce+xCLectka5Obzx+sfbOGqBcSRQGj0Yima9Cy43RxxOOPf5ow7JPmOa4b4Ac9lHSJBkNmyRTXVaAkQeCRNymro6lt+fD9NdetxQtiqjtqk7AnxuG4T11lGMcgVLj22DT0er27LRhtq2lrC9t2HIu4uoPdsCcOO7xJBXXTULc12WpOtw4tmHX5sNH6LqFACksrAIPpzF0FUSDWKqBNPWLs8LiejpBKgjEWlK8kBoPUeu0FE1Z5xdDpmrppcD1v/XcKrW3yVDkOIKxKtFaNpbFX0wKNFOD4Ll3T4XoendZ4joPr2BOdDQjY98FxHegkaMvcc5TEDQM7oGAsLLoDhGNViapeDy4SuQYG2+FN0HV2gFHKon+0FGuPnh0IHCnRmLUKJFDKsUEKKWgbW6GnzVr9XG84DBLMHYyJPZEjXrhQl9J6ynSn1wOJWA+R9n2+M6RYde+OGteB0WsxTK5VTL0eGhUYbV8L4u59IqEz3Ro7AnVrKRZ+4NvEq+8TBhG+F9lWGMfh5OyExeKU3d1zmK6w9yfBlYq2q23KW9wR5QxVXYEncRwP5VRsbW2TZwlZnqzDf1apx1glUQB+4FKVDQJ7jAgBZWlrcVkrx1Yp7dbvidVDW93dPVa1ufM74YUhc+2FtyEX1w7jrmtxRGiqsgZ8PM8es/b91ZRFRRC6QIcUYn3htVZ817e7F2RK4SirJt/pPraP+YKqKuWfVo/VOvRpB+c7z9fc6YmW0qqXwmEcepwENqQnnQglXdqmQeHQdetQ31q1xjiEcQhoAm+9DfhT6ieAKz1a1eG7nmVjVhWOH9IL7cVHVRcoFK6jcFQErj1PNnVLFA2oK7smr+saEASB7QGv65ayqCnaYk1HsZD1MIyp6pZkVSCUpKwamsbyMo9kvU5V22rYpjZ0CIRY0bUaIVua1n4ODB3L0tC1GRubA5LVPlp3dLqi6TpoWqLIQzmghaBuNZGvGA77GNNjOp2BgaZu8DwPx/NojWVhh/0BQgiWeW4fF9e2DCnr0Vwezzg8ma/7t63Q5fsWrP7MjROU09kATqfpDwPCoMd8ccSo16dpW5oayrJm0LfDsRfYRL3rSOrK4LkBj3/2EKFq2rbBaInjBByf3KbVCa7wGQxL8lVLtT3C9w1B6LG1vY0rHcqy5sIlw4c//CFG/YvUjULogqZcUeWSKi/XlY6KZfoZ0D6Op+j3+8TRLp1JkapivkhIshhn5qFcgxIxvdjHGEXXwHAYUPkt40mfpovw5JiN7RLHhaODisn4AogKIaGtNwl6Ekf2WC6m3HfffVx/6hPkefm5jomf+0DpxXZF5nkjNjbGRLHFUKRpSl03xBuWBzdPTmm6FULB6f4B09URRigUteU1dS0IW8v1+NNP4rn2w7gqS6KNHs/cfp5WOUjXpzPagrn9EXWd4nnOWpnsQLtIYZ9+27TrtZddlymlQAi6NX5BCEXoCo5un7J3+5RVekpnLP1+terwnMsM+xFGHzK6Z0jdpOTlnPGoB6JhdnpC26QEQYivYZGW9AZ9lBqRrjrcsKOuKqRRdE1KrmtGwwlFmdK1BSdHS0Yb26xWMxzlEYVjpCeYpzOapsFRMePhLrPj22jdMRgMkQiS1RlFuWQy7nF4CkmSkGQr2qOCLF8SeEN87wr9KCTJl4QjB9+JuefSFq4Hy2zO7m5MUdXsPb/PufOb9PsxJ4cnPHl4i8lGTBQHlIWmFVOKtGNzdA4lHLq8pKlyru4OOT45Q9QB/SDk6Hif3iCmKnJ8L0QFAX4EyWxJW6aEkcfueIwQirLMOTnZQ8QZD913lU7Dhd1zoCSL1ZxaleTKfpl5UcTBwTF+CJevXCTJNatFSqdL5u2KLXmZZ58/wciA2fKYj3zsk/aLXHhoUwIRabZkc3MTjEvbCKJ4iOu6bGxs0LUOXhBijKaqcprWkGYLAi+krTvKKsVxHIsJifr0wyFN01HXDVE4RFCxfW4L5QrqJmU2s0y3IAptS0XnUJYlSIcg8KjLkrpu16EEUMqgjWWNAei7YNk+0+l0rSy5OI4kigfrda5FfbytukUt7EmQ1mDW6sKd9bXRHaYx2PPi2vO0Xp8JDKayfy6VRElb8Ya+s7YTNFW7Xh2/MCyZqrNr8fXe1nGcu7FRIYQdotFo1SFdh6ZpcaVNnHZdi9YNVA5lXeKsV53W26WQRtkTd2WHQ20so5Z6PfQiENIWIBjX+peMNminxRi7+nUdj7pubPGBUWvFyN4kkrqtLai4s8OLaVlDzWvbymEkbdfiuhYSfGelfGd1b1rbMW2M7Sq2QyPc3X+yHoaEXL/X3RoSbgeBr3ndg2T5yobCjCKI+iilcZUkTa1vtd8PqKoaY1zypGA46mOkBVM3VUVT20BDW5cY3XHp4jXu3b3Kzu4DbGxv8buf+DiuM+Tz3voWDqsVaW7oyBlGI37gB/4uzgWHf/gPvovjk2O7Mm0THH/AT/+nf8Y9leAR54Wv/+ff/TZmr3wt/+m//Ds++sc3+Y4v/Qs88MB9uK7Drb1P8dgTH+P2rQOm0xlR5ON6CsdzefKx25RpnwsXPHbOe3z4D27ykgcfIAhadGfLDlpdY4RECAetGzpt6DpN3TZ0usX3HPKiQAiJqzRd19pkdK340i/6Yg4ObzPsb/Prv/mzDEeK17ziz/P49Y8xz57GtNZDvLsT8+hHD/G9HV766k0+86nrdOUYJ9DMThO+5N4dLov53df7qxcdbvWGGMw6INTYWVk31HW1Zjp2OI5L17WwVlVdx/IWHce9e3FxYdXy9be6uxFy223t0rQWNdR1Nok+GHgo1aMoE0bDCatVShgGFpje1etmN3n3YlQ56q7CrYTAc13qVuMJlyCI6JoWR0m8eECe24AhxkUqMJ0iDCK6tgN8C7kuSzA1ruvTtta74Xq2OlBgE/eDwYi6rnFdu91L84S8yPB9F893aTqbavc8j7ysUEqsK4jtwNkU9hPorQf709MzpLQ1rm1jiH0XMFRFgaMkyarEC6y1Jo6GZE1Ls2626Y9H1tLg2fO4pkMKhcCnqRya2qDWIZ+2re2FjQeg8ZWH1i2u8BDCv0sUgIrWNERRSNsaHNchzTIW8yVR7FMUAnsNW4FbsEzWYdblGcKxlq+u6yia2irLboQx0OoCz6/ptKZrJMerY4Zph++4PHfzSTzHJ4wkRycJQRAwGPYQIuOhlz1CHGywWtrh/cLOLmXuslzk+LGibFIcX5ImDcKBsgxp6sqWG+Q1eQ6O39DUlmayNWmYzZP1RV6IFw5YrZbEcUxVN2RNzYVL21T1CjeomS1TulbQ6wc4XkNduXSyYnd3m6PDE3a2L+OoP2U+/v9z+5wHylW2oCiO8dyIrnXQtSGMAtoup6pTZskMx+2I45A29WkrjZEd0+QG2lR43oDIH5GvbtsDuFMI4VDVIYaaostIsylVYzg8MkihcYSDJwVd2YJw6FqrFOhOIro/dUI1wvpbjMGsfSVWGxDrtYWkH0ump1P2bx0y3MzpGkHWLhBOxSqDRWKTZJEb0uqO0XjDtrZojR9OaJo5nuvbtF67gaEhz+ZI6VNk4MQNeVly7vwOy+WSg7MZg3GPKA7wGkGZ5vT8IU1bo9vMpuSRIBwkcHh2Hc/xcZTPfFEhEXjKIfBC0kVB5bk0dUboB0xnK86f30UanyzJ2NocoIXDfDklWU7Z2biGH9gr/fFkh/zgFmEMjmso84TJxoh4EDMY9pjOZoimoG6X9PpD0rykLQ2uI4lDjzxdoZucjWsXqMuUje3YwnOXp4xHm+SzBVqDcB0GozHDvl29z06X7O5ewJjINhYYieNamLvje4RdS1EmDEYj5rMl0mT4oaaqW/aPT9YNIz3C3oBerNFmRl2njEYD5kXJ29/1ausZKSoW84SyajCMMcZQ5B3RpE+WZSyXGWVpFYC6Kwkjl9PTU1zHp9cbcHySMBgMGPV3SdOUvGgQXYUzGJGlJWjBqB/TthnLZYvWHXVX4MjYQm/LzPpUuaNowmqRrn1Q0LalPd6FIPBjBgO1VhkdisJWzp0/fxFfSaRrV6YXvRTXcdfbSIHxHHAd3vL2t/Od3/GdfP/3fz+PPfbZtU/QrlGVkohOUzeNVVbWnw3XdVBSWs/i2mdn7iiZQiCVIBCeFcKksN4dade7d06Qd8gJrFfFWps7m1IcYVeyjulompJWGFAO0pd81f/0dXzln/8a/tq3fxOHh4f4fmBX2hIb1Gg7tLENFcLYsIYtIwDHddGtHSa77k7q3z43x5F0Gozr4jvQdetXJe54JiEw1h9H2yCw99cZgWolYj3c+jK0vsPOJo+lCddzqaCuGoxjLSlSebR1gxAtWr8wUCplofl6zQZsdY3neHTacHbRpz+8wqVL19jfO2B/esaF81fJspIsnyOVYN4ZDBYbFMfnePrGIfPFnm1SKVuuXXqQF7/olcTBmNe/+vO5dO5+5kVD2XTsHz/Pe//m3+Dd7/pCXv3It3BwtIcXBkRSMU0rPl4e8Z73fBPJvRc5wBrsjXZJs5ZbE497w2uwfOKuRzDZ7KEevMbjvk/1yMOM3/1qPrV/Sp5NiS+8iIdf8hqe/IVfZ/NlFQfHz/LUU88QhT1mI5dFV3PcwsPjiGd7HV1XsBm5pJn1Y8Y923mtTYcfRfheaNuwuhqhsENza/uGPU/huJKDtGbn2mW8172Ugz+Yc6tu+WST8gv/+r+xvflifv/vv4eNC/fy5BPXGQ4ERCHhm6/yq7/4KXj1a2hessVnP6a5fDHg0YOUl2rDRfWCSfI4MPyJWDLoj0izBV7o07QVUji4vYiDgwN2d3bWq2cXjLRKlQd4mrq2XkKlXEynrS8WaS/M1vQRIQ29frSmPaxriSX04x5oQxxGdJ39rnWc/trqYLcyas3MDXyfNE1xsKE6B4EfBhSZreYsC6vCR72QsiypmoR+v0/d1nSd/T5pG+stVcq2+txR3G3gxKybkyRGK7uO9RyUa6s4pYqBEK01RZ4i1p+rbL7EDwLbuOIowrBvMTtOZxVQaY/T0LUXmdUyRcmApLaEDWEsOFxrTVtZD3PaZuR5Q7Iq1oxcie9bf2enNZ5vQ8FSQttWeL7tbV+trDoahD5KcHeTIMQdT6rCdUIWixWeb4gCl6q0q/+61UjtEcUuTa2pmhQ/jOnaHFCEQY+msQUHvh9Z/6luKesczw3I69ySMtqCutUE/hhjKkbjAUpZmH8YhkjPJa8bGpFy+2gP14nWfs2KwWDGbH5G6DtcuXKJ+XyKIraVxwYcFSG8lnSR0VYFiJqwiqkqWKwSLl4eUFRnSL3N/sECIW0ozHP7pCtN09ZIGjzPp9creP6ZWwR+iFIRTZqgHMP8eGkvrEwfQ8JpsI/uFEIrPPd/wMrb8Vxix4JEhZY2xFCuGE+GNJXC8Yfk+Yosb4h6MW3bkNc1vrOBoyKWqyllq8F0LJMFUWiTsG0jrE/HHSJVZwe0coHj2Csq6diDdjDaIM9T2rpFGQcjFFIoNB2OA1q4KGW/7NtWW5O8kbSNNegb10ULh49+7Bm+6i/cz+Ft63kTuiOrbuJL69VIVgukDCjdmixJ0aZGqg7HGbBYLGmakp2t+6lLw+bGBdJ0tTbzDmlIOdybEvoBG+M+q9WKWllCftuk5IVGOYa8sBVaVWk/fFm6sEqXO6BtNatFQtPmRL6H7/tUZYnn50RqRNWm3Hfffegu5ODoBkoaTk5cnMBhZ+synjNlc9euX6dnGUWVsrXlMz3LmJ6ckaxsA8P58xc5TBdkRYnn+QRc4Oz4iGE/ZDCK8BwPTwXEQZ+t8SWaKgMcev6AZZJw4dw9nM0X1Bp6vR79eA0pTxNmtUNWVqTlihZBOBxT1jmmKmlai7loyoYgCJjtH6J1S+EoPMeiJnxPMpkMWa1qqnpBWs+ozRZPP/csQSjpqhA3WXB8fMxgMGB6VtN2OXEcoJRLb9QjS07BqRBuQRC4LJIFZVHR1j1cEbO9sct8dkrgdWAW3Lr1PJubm4ShpOsWtO0K5XQM+hM6s0deJggT4fgOruzo9RxcR+D5LlLZpGNRVMRhjO9aP+JkMsF1le2lhbvpzbxICQKfXj/EdBrf96x/tMotDuJgSr3uaDbGULe2drJbqwpNba9IhcSmsYW0KiUvgP0N3Xpdq4EOR7o4roPr2AH0jhpnsL7BFzx/kqbR1itobOBDYD1lYm067GS39vitV38SXOGtPZQdxqh16MR+vXhej8AfIySEQUjT2GOww9C17Xr1KlBSIJQ11rVdtUZ/CRwp7qqnndEURYUQVml9+9u+gLe/491s7+wCcHpyzB/8wYf4nQ/8BkWR461Tu01T8eCLXsxXfPn/xLVr99Lr95nPZzz+2Gf4pV/6rxwc7SGEuvs4Qur1GtMO2UoopGyRa17c/fe/iC//8j/Pgw8+TBCEzOcznnrqCX7sx/5X2rajF8VEQZ8srcnzku2NTU4Op1y+PObihRexWKw4d36LxWJBFPboRUO2Rhe5NNnm0qUXsblxD/tHc/71v/13fMlXvAFv4wK//9lHCQKXyxev8sxTj2K05qve8zV44YimvU0gHUbbu3z847+BFxvuve8hFoucoB9TlxWhO+b48BOcTGcE91+F5frL3RiU1BydHPL0U3u89jVvYjzaJc1rBkOf1Uzw7DN7/Mmnf4OdnUt82Zd8HQ89sMezz1/Hu33A/u1PoE3J0X7A+Qs7zGYrjFCEgYcSHlXZAI5d1xcd05NDWm3bfIw0lFVO6IY2QJHW9PoRe7dmvPPzv4KutsfxE088wVf9+a/lda/5fL7jO7+D2zefJc1bfDZx8RA6pNevecObHuCjf/gML3nFPaySG7zzXV/Jxz/8n+3jr9fg9qDVKAFFuaJpSlwPPFdSFhVKSrY2NzHankPUeq09GPaAjq3tDY6ODvC8gLbsCP0IR1VwN1Rm8H0H1w9oW73mB0rqoiPuNSjl0rYWJ2WMti0uUZ+qqtmcbECnyfOcXuRzdjplPBis+5Y9hsMxy+UStxetawtrWqNp6hLPc/ADRdMUa4unxnU8HKkIAh8pHcpSY1mYBq1bRGOHS3vThKFP01QM+j3KdT+1QBBFfUQHnifJygrfV3iuwAmjNUwdAj8iDDyaVtNpLCbLDXCUIvKERZF1AoWgF9rnn5cFnhfgeT6OkqggthsOz8dxHPKsxAt8UHZT0LY1fiiRRtOZlrLuUA5EUZ+6Lmnahrpq6PeHGKFoW3sBW5QZcRyCaSyJRluvquvaLvNOLy2CKIayKPCCHkWWIxwL/R/0R3QYVquCOI7BOEilcdBgaiQuoT+g7WqCwKGpHVpjbTtpCh22FMSpB3Stg5ItaTZHKYf0aEng+SSJ4DOP7VmGs7Ftb9b2EYBTrZX0HN/3mU0TG2Tq9ZmdzDBOjXIKEA3W8qRx3I69wz0Gcc9iqOqGNLG4LuUsyfOzuyzVKOwhVE7THdOLN1FrZbwfSzD15zomfu4D5WJhzcuKBs+1DS1RELGcZRaT0pT0QisN357eotWKzngs0znKnRM5Aa5b44gQgWCW1WyMLuM4ltZe+w2r5RlnSjJbGIT0EEh6/QGn+YzLl+7l5OSQ5WpOV3eW6aUUVZ2v06FYpVIYpGG99rO9z8JYvXIwDLn+2AFNe46qOWXYP0/TDdCkdFphVIkTGZLVGfWy5dzuZRbLUwZ9W6no+dZHUVRnrFYFi4XtKzViyah/lbaTuG4fpEOSpXh+YDmEngAZUlZLel6PQA1QSLzAwmHrpGF7MADjUZUlw8kGTe3T6ZplljMabdBULVk9pe9PeO76HK+3RLgBmxvnSZKEND1CmF20hrPTfdI0pypqhOkYDcbURcmgN+G+lz/I8eHzKFUzGPQIAoeyWlDXhmtXLhBHY7RuyfIlVZcSSpe6Ljk6PaA/6OGGIZN4lyyvqdH4PR8ZaFoEnTHEcZ9bezcpy5Qg9nCCiGyWY5EGhsEwoqpbZBBQGYHqhUSBYNDf4vQ0IcTBCVza1sH3FfuHz1O2BZoZD7wsQGiB1H3ytGP3ygTfGyDY4bHPfpTLl88xmyZMxps8/vgJm+MBy4WgSAp2tjbpTEsUu2Tpiq47BrVi3B8wHo9xL+/Q7/eZzmfcaV+6feuIKLameeU7oDtOp0c0bY44vk2W1/R6kYXHtg5lURNFfeJoRNca9vbSNQvOJsctr3KtclYpxnRcvnyZy5fP01UVRVVydKTpLRJ7sjV30rD2y/6FFKoF9rOuzxNotFn7lswd75XGGEHTGITQdKKl6STleui8Mwza86tZ5xe6u14+IV5IHgv5gqdLSYlSEsdxUVLgKJtatsligSM9MBK9DkeA/Sw6Pggk4Ni+aG2flwk1nbmTypZ23Wy69euzr7/tKqRep2AxuK4HwvBtf/W9vPGNb+GPP/wH/O4HP4Dnebz8Fa/i67/hm3ng/gf5N//qR+4OES97+MV8z/f+zxwfH/Hb7/910jThwqXLvOVt7+Y1r30D3/c938HZ2QkK13q0hVonug3alLjSQQrJW97yDgD+8rd9J889+zS/+su/wGq5YrIx4UUPvRjPc6mqnNBvWMxPWNx8grgfs1qlbG1tcuO5Z3j44YeZzQ44PTlgMthm68o1dkYP8PDrX8Uize0Q2g349Q/8LL/2vl/kr733O2nrimtXL5GbBNOVZKubfP3XvZlrl7dJF1MubG3S5lAVJc8+9xhvfNNLOH/uIsIRSK1xIxgOtsiSFZPxJhubO3DEOhhjfXVPHZyCrnjwRfest0EgCdje8WjKGRcuBgx6FzidrkhTwcUL17h6+WFe/pLP44nHP850fkCZC1w1wnSQZ5rRcIvxJKTfH3J2ekxRzgmCEJA0XU1d13jSguSroqBoWlbzhGyhee1r3swHPvAB0izBdXy++2/+bf7DT/wkv/xrP8n9D9xDk0ryeoErJ9DC5sZF7nvRLT7z6C32by8YDgf84X//NL2+Il3liA17sWKEIAwiYt8jLXK2tnYBTVHkuJ6k0yW93nC92vYtyqsqiVwfLVrqOuf87hZNrUkp8UWAMeW6jajG8wR5niNKlyjq4ToK34ONyYgkXVAWBY7jMR6PaNpizYTNkLhURWqbxnybzJVCY3SHkoKqzikyF991yPJkbecZU9TV3T53aVxat8UPrH1GKUWeZ1RVhevAcDikqi1kXiHx3B6OBqQGpREanMi34T06urbF90KSRWLZjX0H1TW0usENXfI8YzDoWTtbXVLUxh47aMrG4Ll2xBBC0xsGtE1Hlta4bkjcC+l0xSAeWkKKr7C5NQcpDEWVE/V9pGu3LUVVE8bWnuC5MUr5dF2FdLRtO2obfM/D9ywasNUdYWQv9o0WIB2k7CgzQxALtHZoG4PRdq0f+R15LhFOhnAaojikKnI81cMYB2NSQneAr1z8NcNTiQAjFPHQY7ma4ToBTeVgaBBS4geaqk5ouxjHCWmrmsjv0XYVjmrwvZCyNGgtELgop8XzJkjRorAXB0IWKBms6QcV6QIctyGOLQaqLCF0RlTNGV0XEIcT6jZBSHu+M8piEo0xNGWNF9iLca8dsbMzpmkzqqpBihDXxOtLiwxXeBgi8ux/AIfy4596AiEt8LfVkjByGU9CHK+zvsZ8XR1mFMtFjuf4eJ5EN9qebOsZmIrRILYdvkGf524+hZGausvAcdFNTsAGqzNN6NY40iVZroj7Ec8+/ThV1bC1uU2el3cT3oO+/UDpPENom5R0XZ87YGkhJMq1qUNDzjKVHB/2CMcNWZ3Q4VAULdJJUdrHNA3xYJvjgxOee/5Zzu1eIs0leVoRhBaM2nZwOp3T740oqpoodjlZfJa6O+OeK6+mzRuuP/1Zzu8+yGq+YOeCbdxwPJemFeguoyoXjAZjihyEsmu7PM/oRI0vXFyvR6h8hDyjbECbiqYtMX6CM/ZYlXN01TFPVpzfvQfCiFpW9Ps91Fr5uXTOrmw8J2Bjsk3gueTLms3BZYKeYTo/YzK6QGeuUNZTIt9nsTzDV5J+YP0tydkc08HGcMjZYkGUKVq3JnAh9BWegrpsWJ0tiAdDUp0TxX0uXLnKfHm2ri1rSIqcwWBCkiTkWUMYOXhuSFFoshV07QyBQ9MtODmt0LgWKaFakCc4DDg5yWyNomMbFyI2WWZnhEHLa9/w+Uznt+m1MWEkuXhhkyxf4DiwdSlnNOhTNTnnzp1D6xEnJ2dcvj9GOi5KBghCfMdlM9hgMh5Q5JqtzR2qZkGeNSRL2+Eb9w3KM7ziZa/iicefZjabsZhnNC3U7ZKijlgmEQBBEKA7g+/HGNNRtT5K9EhXKRcvb5JkJzz62E2efOoivb5LVSg6A/c3LRiD6zjoTiAbgaPUXQVeSjtcybsBkXVe2Jg/qxwK1ulp64HRd0Mqd4IHdpC7GwKR1h+pTYcwdi0u7jRuGIFSkjvR4K6zWI1a2IDLnWW6VUbtg9tVNDQNtrJQGLq2RuvyT4UdhFUotbaKqdZYK4i46wWVKkIbRVFWdBqk6BhPxrzxjW/hwx/+I/6X/+UH6NoapSQ//Z9+mn/+v/5vvPq1b6D8UUmarHAcyRd+4ZeiO83f+9vfS5otcR2XTnfs3b7Nt/zlv8ob3/h5/OZv/CpN29C1Gtex7Vsg0MZF65ZzF67yLX/5rwHwod/9bX78//UvuVueAJhf+FmapsZRLkXe0uv7JKkk8gyDaEQUxrRDh3jg8o7P/7/x0oc+n09++gbvf98HeM973sT141vo1qBkwFCWfPITH+I1r7qHe64+xLPPP0nkCkJ3yNmyonVPeP2bHkK55yi7DE+1mNAlPUvwJjlf++6vxhURRaPxvR51u2RxOkfGLd/w9V/J7o2GgjueWJB4UKz4C1//Lh5+8WWm8z1GvR5NrVFhx/P7x2wNXszLX/cG+lEfx4e6rFhkCUfHh9x+7Dpv/eJ3sDXZYLl8iq4oONg/4uT0iOu3Esq6s75bMUA5LX48QLkKz4kJfQ3Gp9ENW77HcjXlFS95BM9VfPxjv8sjD7+cN73x9Tx740m+92//De655wJ5mlHkNVI6PHvyNDvnRxyf7bO12efytZjpSYWjfFzPsHM+YHnaYMbmzuFGVVSE0QY7F3Y5m8+QwsdDovx14Ktr7O/fOAz6QxbNCQIPXw7YmHj0+33OpocMhiHxMdZaoiRKBZzb7fNMesTGZo+TkxOgB0g6nePQIU2BY3y6pkLQoVCEwXrNnTdsbEyoi5I3vu6N7O8/xzPPPMOrX/lKbt26xcHRKY7j0Ov1rF1DGDSarMgRKIy2HdIWTVSBzJEqsJWPKifLE/q9CW2rEQqcwADK4m6aDPDuXtxJKUnSnMKpGQ7HlGVJsmgsn9eLyFc5g3HEcnnMcLAFbYx0rUdRSknkK8pige/E6171COlonGGBF1rv93i8QVcb6iajXMwYDYZo49CaGse1739dGhzHZRAqyho67ZDlLf2Bh2klUmrqeopnJni+IC/mCEASUuUtXVdgm80q/MjFDxRoCHxJI2vicISSPoaKzZ0+/d550tUpvhsROrZowfMGCDEmNymG1rbQeD38yPpeq7rFVwF5WaK1xnV9HCRoF0/27XBYVpRVS93otQ3Ko+kqpCPXwSG9ZgPbpr+2KzDUGNPheQYpHVzpI1yBlIEND7UlSrmURYXjhkgVk6VnKMeeV6vSEHoKoyK0bujFPlpD5Dn0pIvULbE3xokFaZoilEdR5AT+Flpbzrjr/g9oykmSBCl8us5gRENTwXKaEAQ9HEeSFQmuZ09Sda0xXkS6KunHwRpLoKjbjkW+oG1blnlKVQniuE/Vdug2xVEdxunTNJrAC2maO9VBVoHsxX3bPBF4ZPmKKArQuiLwBK6IqKoKsQbkIsU6lWf5esYIhsMRxyc5v/SLv8+7/twrWKUteT4lLxqu3nOONK0oypRBz2eRlJSt4ZmbNxmOQhwRopWPFh6OJzl/ecJqNaPOV5gSTNlj9+IFi0GSEi9sWSaP4wUurrrI1tYup9MjVsUKaaxvZpHkzBc5999/P9PpIU3XEfVGTE/3UDKg6wpGo20a3VGWJeONLZo2Z5HOEXhAS13POTh+nuGgz/HsiCT3mYxGNEayKmzBu+cWjHsTsjLDOA1l57A4EQyGWyT5EozL5mSH2WxKspqjowHSbBJHG3RdQtzzqAvNZi9mY6g5S5Zc3LjKcXZKXXVUZYa32ccJfW7dOqE37PP8wS0WyxnDkUcYhrS1YD6fW3M0LUm+xHMDEC11q5kdVSi3Rkkf5QuqLGW8scPzt57gnmsPWqK/bmnyiP7gHJqUNM0I/fPMZ/usZp9lMhni+Sl5pjh3foeyipiMzpGXc/YPn2JjY5Msa2gbzebGFmk25/RkSdtknJwcsrt5BSUj9m7NGcQTy1HsWhazkiTJ6MWaoBfgeX3+8A8+zUte/AoQzzEYlayWKWU5siqbUbYyrzxhMOwxGvrM51OO9veRErSWfOzjT1q2WyAYb6b4RY+z0wTNgHOdT9vZjmfXDXA8D+k768AKeJ7L137d1/COt7+T8XjM/v4+P/dzP8uHPvShta9Y3119X716jfe85z088sgjRFHEyckJv/d7v8cv/MLP3+XvKUfxgz/4Q2xvb/N93/d9fNM3fiOveOUrcV2Xxx57jH/zb/53Dg4OoLFp224dtviSL/0S3vKWt3DhwgXatuPw8IDf+Z3f4Tfe9z4MhmpdLVbVJV/25V/J2972NobDIXt7e/zkT/wEf/LoJ623S1oP4mg4IPADTk5OKcvirjorlR1uXde3KqcxRIHFlSwXc+IwAOGtWaOS5WKx9n22NgQkJWEYUTcNRWW9Ua22W47p2Zn9fluHC5t2zR8VAkc5CClwlUvXunzZV3z13ZD3z/yXn6JpLafNVjVWKCHQngMaHrnyMJcvX+HR8KNsnh+hhcfu7n1sj/4C/cEl8pWiKjt+7Ef/FWeL23z33/k+Dk6XOEGL6wWcncy5//4xb3rze2yYAsBxUQ5MT29hupI4OmfX8k1L00r8fsR0/yaB6rj+7D4vf+T1ZHVK10oib8BimbGc3WYa9TjHLi8EjKydor8dcP7qeVynh+9sUNdndJ2gK2M2Noe8/nVv42QaoduAIOqDkfSFpB8v+Ppv+gI+8fht1Ob9XH7pO3n9K+/j8m4Edc3BjacpVprp8ozjoxnP7R9yOstZnB1RlQW1VhRlizQ1p4VD1Wne+I6389sf+jBZGTPZPM8yn/Nd3/33uHLPOZQboluD51v24oUL9zEYDSnLjPn8JpPJhMV0CVKT5SsuX92l3HvmhXU34Lsxgo6mmRNHGonDdJoghSGrxRoRJdF0LBYrjBR4oYcrNLdvzPHDKZPJhLpsWawOLSO1thDoUf8+HjhvQ2O6uUjXGja3JixXZ4y2LuE4DqtlSl1ZDBIAwiWIB7i9hqqZIYRHlmX0hh5X77nMeLTBcnXGzrlrlHXNbDajKg2SABdLW/ECSeTHdE2Nkp7dVrR9PK9hEG8S9hrKMkeuuZeuq3Bdj83NbaQ07B08y3LRMhwO8X2fuint+cuzw7J07bGtlI8ULqtliTAhrtsnzTOkUHTGJs+lUijHoVsjlhxPkZcJURhiWmhygzEtruPiey7j4cRip6hwZIfuXIoqR6oaqawQ07QpSvXx3A5HGtp6xWrZcO7cOYqqwXFd+7kIBygErmM3hI7TQypBlmUYDY1u7oYeQbNYnSIdYQUw03JyckLgeiyKmbUkeT3A+uGDwCKrokhQVyWykRaiISvCMEZ5tjGtqV/oZXccu9FVvotyKvve1g1BENHWFqjvSgdHOQSBoM6rdVe8h3LWXM61QIZ0CUKXJMnWIa71Bqrr8IOQJJnjeQ6OYykYUhQ4XofvO2gtyVMNomHYG9OKlLYGSGkaz9oWPI84tK1WVW0d4p7T/1zHxM99oBzF56m6nLqu0W2I1lDXKbpNqOsO6UiShR0ATQer1Yp+32G1OKHIG+aLFaPRCNf18Pw+s9M5fuyR6GOELHHNDm1VkNEx6W2yWrZsDHvM50t6YUzndvYqqrQIAM/xWMxWgLEQ1qJgMBghZUNdZyihEEYQeHa6FkJSFQ39qMfRfsn7fvWP2dw6T0fO0fEet27P8f2AusnQ3TFRELC9PSHPE/ZWcxQdw1GHdEGbgqhnyLKOIo0xvuV0nR53zOc5ly9f5r5738Dx6ZNEPrR1RFEIfHfCfHmbMPJRKiZZFeRlw5PPPM8oGpHXSxbFlLa1SbCyWpF1HePNAbrsmK32LLS5c3CURipNMPBZJftEapuaFEfDyfyEOA65eXaCkh5e20N4C3TrE0URtcg4nS5YNkMcXwI1epXihj1aN2Bep0xLmwbs92Nmq5qmM2T5DVZmC6EET50+TlI7IFzOijNIK3a2rxL0FGk2o2oW9AeSZKnpahfcgl4gOZme4qiYIFRM53MQDUq6hL0xbVsTBeeYJ3v0+hOKXBPGfequpc48XNljuDlhMumxf7RHf6zxtIcQPWLvAqfTZ8nzmlHfx3Qew94lbh9+HMfpcWH3IaTbkmcNy1lNmZfMV0dcve8Ky3lD4L6UC5cmfPyPP8VwFKCiE3rxmNOTlo6Ky/eeoywqXNdlvlowWx1y+9Ca6ueLM7a3t1klM8LQJ088Ol1w6doWRV5z++iz9IMJ9953kZ3ti+zvPYfWmroUjDcijEiYuD0mwz6rJCc8K9GdbfqRUmDalqos78LWv+EbvoEgCHj/+38Do+Ht73gH3/M934ujHD74wd+1vkgEr33t6/i7f/fvcHBwyC/+4i+SJgkPPfwwX/M1X8OVK5f5oR/6IVzXvetVDIKAf/SP/hHPPP00P/uz/5XtrW2+8Iu+iH/w9/8B3/Pd323tJMqCsP/O3/mfefjhF/OZT3+aX/j5n6duKi5evMyb3/Rmfv+/f8ietH2rjH7Xd303Xdvyq7/yK/i+zxd90Rfx/X/v7/Fd7/3rzGbTdYBJ8k3f9E286c2fzz/7Jz/Ek089seZW3lnj29Qr6xBJWaYcHx/x9ne8i8PDIx5//LN4rsfLXv5yXv+GN/K+9/0qvX4PKYZoDY8/8TgPPPgivv07vov3/bpdeV+8eImv+7qvZ29vj4/88UdQro9wHAtVdzxYn3SatqPTmpe+7BXs7+8BcN/9D/EXvuEbOX/+Ak3T8OlPP8pP/B//lqOTQwQwPTzmZG/OS17/CrTqoxny0D3vYu/wFvO9BSCJ3Y5nrz/Gl37N1yKjmLp5HiVHCMewmM7YPe8z2domLzN7gU2DrxzSxW18z2E8eQAtNEpqhInQGE6P91ge3GJ34yHc3gBnautOPafPydmTZIs96o2XWzTMepi0r1PyW7/3Pl7+0nt4+1u/iNNpbRtAXIei6jg4eYzJ5AJRb5cib/HwCfyI+fPP8IpXvJT3/o0/x9//wf/Ap59+lsPDiA/93rNcO3+e173qKteuXeJ0NufhN76S19FHOg1ZUbJarYh7ObqF+bTmuRs3yNKUo+Nj0sNTnn3iOufPR0w2Y/7Lz/wSJ9M9XvbIPeRZg3AFvusgDOR5yWKxYjIa0jUTfM9hOM4p8op0lQAuV7fGwPSuL9kLOsp6StvFbEzOkaSnnDtvOYwyqyxvsRdyeDClLLTlNHohWpRsTkZ0Jmc2O7PbsM6swf226vDGszc43bYA6tFoxNZOj+nZip63ixQNnlLEkcSkhij0rZcwawk8UF5EV7t0neH604+TJNboerA/pd/v0zaKpq1wpY8TCtKsIQpDfF/j+TA9mzMcDikzmEx2qLo5ZdkQ9wxxOMGVvu2yjhRh5KF1BbogCAa8/MWv4myxz2Aw4Nkbz9MLPKpKU9cNxtiiEIWPUpplcsJka4ckSRiN+9RNTpqmKHe4JjDYFLQNrkHT1fb5V5UN5zUVmI6WFKkcjKoxwgHjUBXSEjA8GxQMBxF5niNlaDFQpqRap6wvDrZo10QHYTraukMLRRjH6G7NzxUh8/l0HTIyBL5D1bTQCnwvQqmGycYGUirm8yVh6FOVFg8VxjFKxCTpjCCMqOociUPbaDxfok3NbDpDOYbA7+GHAWWZ0bYtQTii0R1FXdpaTdch8AJ0p3GVQ13aznbPVwjdIWVHt4bh20HYhiM91/oenXVzj+e5hH6EENKya2kIgh510zHoX7bg+a7D6BZnU1GWJXVlUCrA9wVltbDDqTEgFY7vIYWPWrcsNbqhyJd0rcVxOdGfZbj+XzJQzpYLhGwpqyVtV6JEZKuEVEgrczb6fbrIQpxdZRCiYHPcY3HmkFYtw41zOC6UdUE2WxHg0xdD0qVD0/k0ssB1BcY0dG3GsD8mTRM8x6PIM6TSVFVC1xr8NrB8PCFptQW8AjYdpSwgumnWeBZtjbyBF9LqBkNHtio4d3FEnhbsnIsY9q9SVh512SBbB0eEBLLP7WfnVKXBd12EaEmXKxxlIbNNl9LvW/CrGUqqDurmFD/0eOz6Y/T2BrgKfMegupyTkyVZnmBUTqdr2sIhDnuEPR/RKDrjkuUptV6gG0lerNjduo+qKbi9d8A4jmnqFf3emMoYVsWZRa5kho2NCcuFNTw3aEI3JisFFy8/xOHxEW4ccZTkZNkpwhE4XkNJwvz2M9x37yN03ZIiXxH4PfobF5gupkg35yw5pFQbHJ2eEUYugRqS4qG0Q1ML4mHE3sEBXjjEGFgWS5AORkHgbxH3RqzKGxCCIzaYzhKE8gnjPk1b4sc9lqsT4p6kUxlFVVPnR6hAc+P2c7z44dcQ1JKTs2M8z+A5PtJPWBZzPF+RpitceUwc2UTdaDJmc3sNuOaMpDQMhhPicBPJgM4knJw8y9WrL2VnZ4dZuk2nc7o25cWPPEiarviyr3o3Wb5ACM3tW0f0xhFhb8RTz3wa3w+5ePEiu4Mefm+IcDOmRzPSPCGuIRqEzJPrXLn6ctLlhLQ4ZLI5QHhjsqnDYEOwf7TPi19+P4fHz+MS4fg12WrA9sUtpmcLpklHFLkoBa6yiWTTVSjPR3f2OB8MBvzVb/9WiqKkrhv+2y/+LP/23/4Ef+lbv5U/+vDvUxQFgR/y1//6X+f69af529/3t+jWLMDf+sD7ef7Gc3zLX/pWXvXKV/LpT3/WKilaMxwO+eVf/iV++Zd+ibazRQGnZ6d88zd/C9fuvZdHH/0UAF/1VX+ehx9+MT/zM/+Vn/zJn0SIOx7mF0DQtnnDjiyLxYJ/+A//4XpwlXz8E5/gR37kR/j8t7yNn/7pn7KfZUfRtJYHmBYVy6RYl+l0a6QRdwHfd5brP/SDP8h3fOdf55u++Vvufk81TcO/+/Ef5wMf+IBFE0nbx/sLP//zxHHMW97yNt74xjff/flPfvKT/It/8aOUpcbgIoSLkYa6swELy7oE1/cYjyfrthd473f9LX7t136Fxx97jEuXr/Ce93wNP/BD/5Rv/qZvIE0ydi6/js9785ewdX6Xd37Bu3n4xQ9x37XXkrcron5EHEd88vc/zCtfs8lr3/QqTmZzIi9CG0PVNEyn+8xOZ/jRBo2p8F3Fsq3RncsTn/0YR8fP8LrX7NLpEqVAdDVSKZ5/+lFUXrK9fY20Sa0NAgFGkRcLYqdk0L+GbF6wLwDIvZtcOJtydfYK4mdukpxMmYwGgKE+OWT6O7/Jc4+/hC/64iucnh7TdC1NBSqpuPmp9/HDf/JBnG6DNzCiERl5mKIPlvzKr/x7glBwz/2vYO+3Ok6Wc7ZHW2xtj5jNC87vXmR+OsULNJONy3zhy9/Mp/VTfOgPfp8veeR+2maH9PHrfNG5Efd9/ks5OlhSqBVNV9kwiRaMdwfotqKtK6Q3wg9bvGtjirwljnqczW9yJfZRixewQdu7Q156aQtXDLh2z0VGI4+yaHBUwGAwwFBxeLTH08/cJAxDpmdzms7BVCOuXNtlY3yevFhSVAtGN5b0nr1JVdvB5uWveBgpOqIoJIgEUrhkaU3bdmxsbNDUsLU1oG2gKDKKcsXWxCZ+kYJGC/Jszkte/AhaG87OTmm7kn5vQpq2eF5EXecEoc/WlgLdUdUCQwvNgAsXJ+wdPo7rOVRpzqVzu0ynUzyl6U9iZlOLqqorW+Qwm07JkpxbNzM2NjaYFxk9b8RyuaTrNM66f9yVCmRJ02rCMKQuK/pxj6op6feGhGEIWtO2DWGoMNgtSZ7n1E2GcmKM0KAaHGlIkox+v0+WWxKG4yqaIkXKCG1qxuOB5XKi8b2YqrQ+7yyrCL0+bs82z/V6IWUBTavYnewihG2tSxJrvYNq3fwkrKfQaLxWUtd2rT/o+eRZgutEOI5AqRYlDI7TI44iiiIjiAxdUxEGyvJ6jVqTMFouX92hbQzaNNR1iZI1440RXVcxiB1OZ0uiKKCuS9AObV2uN1kdppU4jqDpbAhTqQClBEp69Hp9ksR6ZQPPpyxLWxutBI3WONKlrRvcwNaYhspFSYXnKZpGs0oSIhUgZIeSkii04SxtXHRb0LUOrge6dZEutM26dUwrpAgwskM5gjxffa5j4uc+UDZFQ2/Uo9eL6PUti6lMG3TnEHgOussJAkng9PEcF9/3yVcdug65/9oF0mSGBlo9Ik0qqtqy3WLXJxz2COIecRxSZtZHEIU9To47us5Q15K6LYlC709N38K2gTjrdLeJaVtty999H7AVgWHoW8YYmk5LlHIoK5/kVCKMJD0u6fUitKqw+TMHJHiBy3Z/hBw61E22XpsbAn+A6CJaNWAyGNOaDE1Kdgpe5FGnEt1JThZnCG0YDiSuM6U8cen3e0TDgEk8QQ1iRoMxl6/skuUlZwc5lzfu4dKVMaenUxaLI3RZI9qGPJ0SiJC26eG69vVvb9xLmq0N7s0I16sp0xXD8TbT4zlPXn+W02nBM89eZ7I5pm0gGpRs727QdSMO9jtGo/M8+ukboA2uKhiNhtS1BW1HsaIfD0myFaOBT6sTilKCtHWW6JhaFAhnxfysZrRxgaZVCFEgHR+k4PbxDUbjDfJ6wc7oAuPNCxwc7NF0Ia2tnSDsbRD3hxwczGjMHEcuiHsRWbHixq2nUEoxn9acvzgmK1qOz07p90PyrCUMdjma7uPMOnqDOb7Xo5yHxHGDFilNWeA7W6RdCU7C7vZVHnhwwGx+wvOffIKLF64ynZaMhtv84e//CTvnPc6mC1vHKAqODwqSVc0qmdEbhpyczSjqnPEkphdP8JyAyWbLaOLRNjF1rbl27UFG8Ta90KXRthruRQ9e49r5V3Pz4CNovWSV5bStZJUVDIcSjcvN2Qk4FeNLfbLHE/wgsF80dUMYRVTS4Hr24/r+3/x1e8Ek7tTLVfzGb/wqf/EvfgsPPfQQH/nIh3n1q1/NeDzmP/yHf4frSQLlWai5lPzhH/0B3/KXvpWXvuxlPPb4Y9CJNZS945d+8b+t0T4SRykee+yzAFy6dIFPf+pRpJS89a1vJUkSfu7nfgZH2YFNONZi0nV6rSqau2DmX/7lX8YYbT2xxvD444+R5znbOzsUZQUIhDH8k3/8j/mn//gfr5EfL6xjhRB38kfAnXYSSNOM/f19rl+/zqc+9WmCwOMd73gHf+lbv5UsS/md3/mgvQsBnuNyfHzCpz/1KT760Y+QpBkPPPAAX/zFX8x3fud38kM/+IM0TXM393THDmDWbSnDwQiwwzzAz/zMz/Cf//N/otOg/+CPOT4+4bu/+7v5pm/+f/BLv/hLPPKGd3GQz/iFf/9rPPn0Y3z3974XZw0O71qJwCNJMl76shdx+co5qqoiFBFplTDoDZhOb3PPtdfSG56jritC6aCxVXd1m/CKl72F0fgcebtEdAovdNk/OES0Ka982euZjHepmtKGBGgp64oqW+IxYHPzKs3e9bvf7QLB5A9+h29c1gz3fxf5m/+dIWtVWCr6ecpfyyp6s+cInvvf2GnbNXNTUFcVr8qWuH6A64XQaYyAddMpXVOh6w71J59Yp1RLmvZJWycqFYYWg7rzrnMoJFvAe5Sge/Sj66ClPQ6cT123GCdplSuznoedtcLettbL6rkuQrD2BtuygTJoeX6buyvvrG5ZZjXjEVQ652SWkCWaz37mI1y+dC87uxtEQcg9V+9lZ2eDs+kxt28dsL11BakqTo5v8cpXvtJySqtnCMMjfN9FoxlOety7PSaMNc/ffJo42ObKvefJ8gWDaMDR4ZS2NSwWGWEY8uCDD9K2mqKc07b2fbj/vitsb2+zTE4Yb5ynLGug5fK1LZIkIY4vcnyyj0DT7w9pmzHz2Yrz5wN2z00YDB+iF28g9JCw1zGfjxiMfDxPMZtZ9E9e2EavwcCir6oqYnZ0Rrdm1A6iiPlqSVvWRFFA1hQMJiHLRWb5q6pDULO9sclqWYCOcf0SYRRbWzss5itc1yP2ezS9mjRJ0Z3LcBSzWpxgtKCtBFEQI2SLcjoUPcoyJ4430C34gSVb+GpA6ELdzpBCEvcUHR1hKBGisev1rkOqhlandOT4nstkMqAoV/ixi9EOnudYLmbT4Di2/hFKBv0Q3SmK2sF1Q4LQgc6uqre3d9CdvFvNu1ie0bYK3w/I8gWhH4DnkmUJwoVBPLQBxjUVY2u0CYCMJFIa2lYhhCXlSCkp8ophr49SirJKieOQNM3JshVB4BEGHm3b4geKKBwQhJIkSehaiOMIRE2yWuG5MY6jqcoVrifxXCyz0gmQriQIFUV2BkhczwY2o0GPNCnww5CiyGhry+32XJ+mrqi7HN8df65j4uc+UOZJy3I2ZTTeYu/pBdKp2D03Zjmf4zljlLPJYjZnfnaMNA4bkz6+6tE1BuWcEoUh8/mcqqosJqNvweJhuI3jSOrSpa5ymrpjtZoyP1tRVjkCh6ouCb2QoqnIypTxaMPypYKQeXJG07YUmf2FK+lidE1RFDiOg+4sMqDXi5COWJvmJcvFDMyKwAtIlid2DeIFKKfD9/oks2Pi0EdJnyiO6PUdapPQHyiE15BmNfv7N+n3h3Ro2tKCwD03xg+g14+QZoLQkqZYIYwiSTuSrKYdBQxCxf5yzvQsxQDJasXFyzucHEEQbhMHMcqFg6M52bIiWdouzjwDoRTPXH+G7e1NJsMBTz2xx3DDJUtynn8mY7VakOcFN59/lCiKODs5A2MIQ5/j20t8z7BcdBy7h5TlAldEeD04nU+RoiaKPcqiZWd3izD2iKIBTdXhhisWVUAv7LHIjpiobcrGAydF0FFkOb43BFFxOtvHCJftrQHPXs84iT5L7G0x6E/46MceA5WSJClN1xH4Y9K0Jerl9HoDunaPttHk6YKmTWgqn9NpRZqcYYzGdz1czxAFhvlqynK5Ry8eoynYGj6ACk5w3IKdyRVm8wolcvqTPp/57JM0TUeSzMnznKOjFXUFm5spSZqzup6wWt6k6kpc1WOxSPAD6FrJ7KwmDDdYrVI8F+oyY7U8Ydjv4bgB09OE3lATzDdJpwtQc6q6YHGmmM2P+MP//hmUW3Hl8gM8d+OUqsiYTZP1lX3OwfHzTEYXGY0C/Mau8cSaDbmqU2qlbGsM8PQzz1A11p8o1on0559/HsA2o2jBxQuXAXjve7+H9773e/5PP9ODQZ+yzHCUrYKbzaZkeQIIHMel7jTz+QKAOO7RGRv2OnfuHDdu3LDwZ1h3eK9RRJ1er2peAKUfHh7eRa/YhLkhSRKGwyGuo9YsxDvFBC8UFIBVIo0xYO4oapY2MZlM+H/+6I/y/ve/n5/8Dz8Bwt7/Bz/4u/zzf/7P+Cvf9u187OOfIMsyBILv/Bt/g4ceeoi/8lf+MlXVoJTkj/7ojzg6OuTbvu3bedvb38773/9++5jGci/XMWcQgrrN/8x798EP/jZd22CMIAxCPv7Rj9F1HY888gj/6b/8F27sH3H58iaPPfYZNiYh1+65j6NFZpUYpUiXKVmWsLXxRoTpI43GiIog9imznA9/+A9597v+HIE/IEunZHVGHMdcf+YpfN/hyrWXIJQgX6wYxBPaznB2eEZZZUyXQy4KByVqtHZQvkNTlRw89xl2+ltsbp2jPXiGF3pj4NmNI8zE4DhL/jS8XQDtsEVrjeNkdwf9u3V4WmPMuglnXVt55z7tf3ZrAPg6DIa4e6zcxQz8mdsLz+n/7GY+l58wd1TyP/vzYl2YYwwcThOeyVNcZ8rzz29w7eo5wkDg97e4uXfKMisZjwb0gj639x7j0qVz3HPtYZ67+RTbWxfxoiWPPfknBM6I3bKyIRdpL0Rm04T4ni1OjmdcPH8/SXaE5/YZDu7DdSwZIgwCyqIlimIuXNjl5u3nmS8qeuGEroMoipjPz+j1RyAaTk6eYXNzG01BfxBycnLC+fPnWSynIASbW9u0jY9UgosXXkRTx7h+zWpxRFEJ7rl/m662/2482qasC5K0YXN7m/5wA4GD6/UQD2Ts7e1z8eJl9vcO6Y1ChsMhi8UCL3DwfZ9HHnglxyeHhBFkxYyuTRj0PU6OZ3huzIXdc5yenrKcLdnY2GDUi2i1j+8qlIE49ojcIVI6xNGAIAhI8hVNW9jwiwtZmuO6IZ6SFr6Ox2SyBZxnvjihMxmL+YqNzfOkSUPRtGyMJUXdkK9KJIIgiC3iR9gAb1NLqqpgMBgw6A1omgZJh+sNKbLSkkvaCkVEJ2xSPwxDZmcFUTgkyeYI2eCqHnVbUOoUoy3ayJgCRchkOKZpbHWmXpesrMoVSjqEfohRLcK3VIWqqmg7QxjcYYMa+v0+VVXhuRrpO/T79gLWdTocV+I6AWWV0esNEMZ6I/2gT1kFLOZLdncuURQpZZUiQpfFPEWYjqooEKbDcwVJuiCOdhnuuJydruhMQd3USKemrkq2ts6RJhnjSYyQVtD5XG+f80C5sy3JM0Fbz3FkxewsYXa4ZDzcxe25lE7J1sYuYTBFdzlNmYHrsrnRp+sStAkIwpCtrQ08L6AqNfNZRtdYEGyWzjk+2aOqWwYjnzpXTKenDHtjgsCjKlcMByGOGyOFQ9vZL7NebwC09AOP2WzGZDKhaRo2t0YYYzg7OyMMfYb9gKxKaFtBEHjEvS1Ak6UtwgiMlkjh0rWapMxpmoZu2MeYgtnSGpU3dwYc7C+4dXSTrd0J2rScTo8QUuM6DgqF7/XpRx5eYMhSgXag6hob+9eSthU8f2tB6KZrnIxGeRbTcvPjT+IoHz+wyWktShw3oO40olP4aUXXQlloOpNyeppQZs/alcxSUTcJVZlYQ27nEgYxWV4jZM2w71PkmtNnpwh5tK6isvDauimZ1wtWiw6QuFJQdy1H04SsXOG5Pbb62yjV0N+KmO4/z6osGbj7tMah1+/oeBzdOST5dUznsVisiHohH/7ICU4gKeszAmffJsfyhk6srIolDVV1RhA4TJMG0c2YjGNcpShMBQ6slkvq01u4niBfOLiiz/a5iNnZAUIJvMBwNrtNsqrJNmvK1JCmGeNhRZkIvGjFeLTLdHZkURrSJ477hFFF21XsH84YDAZoU7B1IWaysUmWFYyXVm1uG9tc4Hke5dLjuWcP8QNoasiWLYaWpikweod8lZCleyjH4Pseq1WG0Fvs7PapKpdPPfpZ4mhMq5cMRzFdI5jOZ2z2tzg9vU26XOLMbU2abXcw1E1D5bRUVbU+UYq7lg7bHKHuNl/cAVXfQf38+I//G55++vrdLyxgncqGk9MT6rpCK1uXZzuy7dBWVXYtU61r24zR1GVxtx1E6440XdlAjXgBQWQfR9r70vYPHeXguO5dXJG2PB47nNyprTN6Haq5U4P3wjAgxRqyvmbLGgxf+AVfyHA45A//8A/tEIpeN2HA7//+H/Ct3/oQV69e5dFHP8nW1jZve9vb+NVf+ZX1RafAdIbONHzodz/Et33bt/PIIy/m/b/5/vVjgFkrlGCbd7K0pCzLdV80nB6fEvgh4/GE+WLB/uEBSZIQRTGO49KPe9SppqlrvuZrvwEv9smPlwwHBkFHMivYPzzi2r0vtb7muqYVHb4fc7h3gzAYce3eh6jbgiRJME3LhcGIGzee4ROPPsWb3vy1FOUStKCsC4RWmKaixcGfXEQ4DvliwSAY4AU+Nw/2ufd8zLlLl6lNQzbZsL+TO6rsunbwjm/07uB45+/kn1aNX7jd6RxHrHvm705vArS+w8Zf/71tecJYvp423fp4+dP96C+MgHcpWetj146ja2j1HYyBMX/m2L4b437h//4sUQBD4whW20PGQURerDhZHKD2OjxfUJQtw/6Y2XJFmhfAlDAMmT/5PFL4tG3LdHbApUsjbu/f4tK5gEVihRKEsLWoGs6ODzk+TDk+zBhNPGqnI1kd0nUNw35IVU2JgpCnrt/gk598DCeQzOdzBvEC3YacLQ7Z2OwzGu5QVkuU63H9mWOi2Gd6NicMI5LkFKmg11PMl0ccHBzjuEOkOuZscZ1eb8DhwR6j4YS8MijH0NYuftxj/+AU1x3TmRGL5JA0PyUMY8pVi3R7PPHMPmEY4oUOaVkx3pogPUWZ5yA0k/Emvb5PkAaMxkMGgx5JOiNPLNLo5S9/gMVyymg0wnMkQlloeuTY4OJT15/g4sXzpOkKI1u2djfo6Bj0tzibTgmuDEhWGU0tKfIl8SimqI6pi5bRMKJpW6Jgg2WyoCwbHCdAyIa9529z+cp52ralKjTZckHT1MT9Hm1ZcunCmNl0ySxdcPHiLrPpgrZxcFTEffdd5nR2m8ODM5z+AGFq0lXO1tY2ULG1sWZtFxWjcR/lGFbLlOEopm1btsYXOD09JggVxjh3t6mu18cYez7QBLRtS9NU2LrQlI2tTfJ8PQ80HVEcEPdCdGeRXr1eRFmltsc+Cum0YDzpMx7GaNOwu72F50uWywVdp4miHeraNhMul0viuI/u7HzTi11u3rxBfzi26/orFyirFQKXskqZz+f4geLk9Iy4b2wDmhv9f3zu/7/dPueB8hWvukSSLulaga4iilShhEcYuayWM4x06JqO8W4foXzoRhRJhzEFjoio24Q49jm3O2AyGnN2tsC0CSent/HDPmWdsbEVsbm1RdlM8cSEjYWLrgVxHOG4HltbW8znc9Kk4vBwThj06ccho3EfYTqGE0VVFThty3AYUVUNk8muDRJ14PohnY5I8gTdCKRTMRhNaLuSrtEY04KpcT2PQRhjqPC8AKlAkzPPGrRRTDZ38QJJEDq2EkkojGPwXENe1OTzBCE1RvfQStOKmlAvQRp6/Q28wKVul7SqJa0bylXOMHJQbo+qOaNOXesV9UPOFscI6dM1LTIXtgLLC6lyxXQxx3dhmS1oOxfHFfiOT1lVtG1F3dY4jqRpKrwwAK8hHCrKsiOrE8rOB+0iZEdXV4xHI9pakWUZnms4PTpDyIhlm3BymiCbDlxBz1HUOEzrJSoScOpCa7E2rdE0bWmtBq20H7p5ggo9zhbHuK5gEG8h5SZlJdAypz9wKYoKx4Djt9SVopEtqNtUiUSYCNPF6MqejIxomc1O2Nj2MXpAEDtEkWbYH9F1hkoIxpMejjQEg4q6lixWK3x/066alkuMzMFxUI5GOFYxm85KlGx55voxg/6YJF0R9yRN5VuwsG8wJiDw+zRNjlx33yp/xni8y96tPQLPJ+47oPvINiZwDBiPdOnghwpftdRFSdiDpilZLXOqoiT15hhCkmXCZtVZL5+504MsUNIGywAuXbqC4A/t0Kg12nRcvmIVyb29W0gJB+vwSFEU/MmffPLOuXc9vEHX2qrAO6uXO5rPHc+iNYa/oBQCSGW9jPv7e1y+fJkwdGnW3eRy3XqhtaFpbZCmM3ZwLKqMPLcKn5TyLqDdGBtmuFPpaGeE9WCyHjKElCip7PMTrCHRsLG5eff+NAa5BlAjxDr9aF+D6/psb++sn7+0A6mwqC4QOJ79WYvxYA1kX/dsCYkxHW1r6wKfffZZHnnkEQBe9PDDHB0d0eiWs+kpvu8xGAw4Oj4GY7FnZydLPN/nyj2P0DQVkSOoih79oeLg6HEWizN2rw0sHqQ21MKg3JbnbzzNpQsPMNkZs1qeobVmNBqTZxlFUfKWz/9itrYvsbd/k+3xFqtkynhjh5s3HudkuuJN116EMYZhPEJ3NZ2WHB/uc3Wnz+a5y8zKjP6VyywffCnMbtNUNegczwlojH2vLZrK1tpWZYbrOrhevB7zbJpea81qMSXwXZQX0LbrgA/Wd2q6hiCAvNQY4SPo7HssHIxpqKuMOBrieoqqTtGd/Xdi3aHdrfErAqgae4EjhKTTHRJ5Z7x8QYkUtrdbKnuMG20LLoRYw8XXPz91XW6mJdv9HcoqRzklSZrSzjpqkXDj1k2GwyGRF9LUgo4lUc/aq0xTsVxNOTrZRQnD4cEneSB1bQ3oGtZ+cnzGvh4hVEmaHpFlW5RliR9A2wiqOiNdpgyGIUq55DWs5gVNLTmrZwTBgKpuSNKa2wePWTybryjLiqOjmvFog9WyBGErh6fTgvlqCqKj1QWzxRnjSZ/j2bPE3nnS3CPLSoxpaGrJY0/eRGtNlt/G9z36I5/5fI5GoYQtN5BS0tQdURShHMHzBwW+7xFIh4OjE6bTKf1+jOcFTKcL+nGN62kWy4Re5LFYFvR7I6q8xHguFy5sc1gcEo998qzikZe9nCBwEKeGqO+TZAWuiOlaxaVLW6xWS3TYwx+7dF1kA71KYrRjn1dvhxs3bnDl2mVOT8+Iw02EkxB6Lvfccw9HR0fs7R0RBjAaXaDrOgYDl6tXNlGm5dLVK7RdwvbuNllaE0Yjuq7gUm9A1MsJg5iNyVXLnw58hKjoGof9g4YHX3QB1+9RFBnL5JDzO7tkac2lSxfZPHMxxjA9m+N5HlEUYYwhTfN1lWVJmpb4WK/icLRLXTXrnxMEYcNydUavF3P+4jlu3dxDeIbQ0ziuIgob+qOIyaiP77u4skeWlRS5QxyOWKWHNKWi0xLHVfiBQxC4bG6cw+iA09OnuPe+y9RNRl0pen0XJa4wnU7Z3tzk3Pa5tUJqsXfz+ZxK/A+oXsxnHhvj+/B9xcH+KdFkgBCatsvxo5isKAiHLXmS4qqB/VoRNUJJTNcR+R5d6zFdzK3Zt5ZsbW2isfiAoBcihKJqUlwnZjQO8OMJujUM+kMEVqre2tmkyCv6wz20gN3zF2jqlmtX7uO5566zWE7xPA8pHZYr2y6Qpimh61JWgroyxH0PbSpQEa0WROE22jTk+QrHiej3+3RaUncaYxo09qTeamnN4NpjNstQykOqDiUVum0pWx/ld5SFpikbDCdIEdBRkxctfiA5O50RhjFS2XWZkgGhcsmLjsArUNJHd9Y3obyWgTuhKjROYJW1okwwjiEIAtqsxAhBWdW4wsUoTWMacCAMbbq1biq8KGI2te+FJATtEkUDqsK2jvhhixM6zFYJQgjiYUjTOIRuAGjczkFJD4UA0eC6HaMgomtcVllK2PMQwtA0Ha5URMJBKUVd1+RpSxQPaWoY9/p0umSRWKxL5EesZhmpVLierRob9PrUurKqX2cZh2HsI7Sm7hr6GwFN2eFHPdKioStzxpNNhv0BqVuhW5dJ30NrWM7mdB24UtJ2Dmm2RClFGPnkWUNdQ5rmjEYjkmRJkmRIaasYZ7N9JIquGXJ8fIjuJI4jGU0CosimGMPAw2jNIL6E5zrcc6+FwFdlR2dWzPMFulPMbu4jcEGUtF1DL+pRNgGhb+h0ihu45CsPKTtiz2ckPaQs1nVtjkXliBeSdl/yJV/Kr/7qL1MUBRpBr9/nS7/ky0mShM8+9hmMgY99/CPM5zPe856v5fd+73dJ0uWamWilPz9w8TyfPC/+jHqjlEWE1OuAwZ2BUsg7aqLi937vg3zLt/xlvu7r/iL/8T/+xDp5yd3B7I5IdNcFaWwSESwyyXEd7qyvO1NZTrt0GY/H9Ps9zs7OKIoSga15vBNGsrBlqzbdvn0LgHe+8x08/fR1KywKWzX5lre8hbZtufHcc2AMB/t7dF3HG97wBn7qp36KJE25Azx/5zvfCcAzzzxz1/sXhT47O+fI0ozFwqY7wzDiox/92N2B8i1veSv/8af+IxhDXTd8zde8Bykljz76KAaBcXzmixtI0bG5OUK3MVI2aBJ0HXJyPCOIQs5vXMQ0DngFpnBwdcPBySn9cUwvGrM6PMR1rYcqS1KKZMm5++8DOwdTtCW9qE+xnHF6tuTyA68mki7J2Slu4OO4Pm3esTHpc/GBb+BwnjDwN3EliK/4m+j5jOxkRn80pcxi0AMbHOwEgRNwePuAk/lHuHjxFWxdeMgOdk4IzpLZfkO+/1E2z+9wsorJXYc2WSKU4uR0waUNydbFhk99pGIpPFwHpHEQQUtU17z2FYrbxz7bu6/FaQ/I2WMnUBzs3caoEZictCq5cZSyORkhS6iaDOW3DDZi5rME3Sg812EwCZjOTwl9nywF3ULbGMCqU1UpCCPDolxxe2mIZYDsClzjIp2R7VfedQh6O6SrBiM7jo6O6A02WayOmSUe/XgbQcX27nmSWUoYSBxHM7u9sMe/tInoxTLlZr7k3PlNNjd38DyPs7MVQio8P6CrO4QwlLm0GwanIS/m+IFLYzSdk+P1BLUp2Nod4YcBq2XKZGeDJJ1TdAlOzzbgFGWG5wa4gY/WOb3YxZF90iyl0i1lc0SeGdq2RZuMjcGE0/kBjhPgeQ5FV1CfeRgsjk8bu20Z93dZNAuW2cyyZdFIsUVj5mTljFYrGi1ZzqYEfo/jaYYxLkHoMVsa0ArdlpbFebxge3+JETnXn7lB0+agYza2rVp58NjjxOEWQajsZuq5Fb6v6PUGLJbpmqnpMBjGzGdzoh5kU8nVe+3A9+CDr2a+OqWqHPojgXZa7n/4QV76ipdzdHCbrjNEvaH1j64OuPaibUJvwCw55PL5h5ie5khptw+r1Yp77nkzbSNwnQ2cYMHBwR5XrjyMbkMuX91juaoo6iW92OHS+UeQUhKHJXluB1vlFgxHOyT5EVLC1ugSxhSEzhbT+Q3SbAvlafLyDFf18L0RUmqmZ8f4wQZ1MyTwJmxONhhGiuUypSobAr9msZqCjhhE2xSZTf5vbO3iueCqIUE0pmlXGKGZJzMqkzE9mjPPSra2Nsh0w9HBIXEUIkVGMS1w5Iy68ojimijKqKqW7Y1zbGwO2N2e4Pmf85j4uQ+UMlUMJwPKtCI0MatZRVLUxANFmXXURUBbaARDhB8hZMfGRo+uaxE4+E6M5yuGoy2KtENSEcaSeDim03B2MrUDUFvRdc065u7SiwLSJCUIXQ6Pp+tOUkXYi2k6TZHXZHnNpz/zGRwXpOPSmI5gHQwJopCsyGmlouoKkrxAOIog9KibhjzP0MZQlEuUcvGDHk2nqdrCImAKTZLkQEtVNQRhTNsWlqvZrHBd13KitIcxObIwtEajdYtyDJ0pqOqSyI9puxaET9s6FIn1dQo0iBZXeZRVi+M4OMqQVwld1ayDDgaEojW2EQBhSNMVQRAglUFJj7KwSbi2q8E01FW+9rIJVssUiSTPS8riFMe1aTcpwvXVqEORVHieA1rR1BJEtV5RekBN4NeURcOot4XrWo9lmiyJex5SVSSpJgqHzGdL+qOQNM1oSmO9QNMZUrgkpiOMXHTT0nUdy3X1ZLLM6HRrO1HX7MKyLEFadIPpGpQjUEqQZwVCuCzmGUoJdNfxxJO30G1E1dTrPly7OvV9Hy/ss5gtoc3wfBcDVHVtB3JtCKKQqm5AeGxsBESxg5AtV++9yKDX5/DwGD+cUJeWr7ZcnTKbnzIZ71CkGWVesVrm9AchnudiuhapDLpxaGufrmvZ2RlQ1w1tC2mqiaM+6WpFWxvCyMLPvVGPNLHIitXBCoN6YZV3d29ob8vlkh/7l/87v/X+3wDg3e/+QnZ2dvmRH/lhm6w2UFYlP/zD/4h/8A9+iH//7/8jv/mb7+PwcJ9eb8ClS5d505vezA/8wN/nU596lPXmGbDwcrMO79x9bNarZ2m9b7/4Sz/Pa1/7Or7u676eBx54gE9+8uNUVc2VK9e4dOki3//9f/tPrSBBSeeuiimlpF3jj+54JY026Lbmm77pL/Kud30B3/s9f5PPfPbTVqVcB3SMxg5qa5/lB37rt/jSL/0y/tyf+2I2Nzf5xCc+ge/7vO1tb+fatWv83M/9HIvlgq7rWCwTfuWXf5mv+Mqv5Ed/9F/w/vf/JmmS8tDDD/HWt76Ng4MDfvM33nfXe3fPtXv4x//0h/ngBz/Iv/5X/3JtJxB84Lc/wOvf8HoAvvALv4DBYMBnPvMZvuqrvoov+7IvY29vj/f/1m9htMZxPG7evAnA5qZFSuV5wmA8oixz9vf3OX/uMv1+TJrlNLrDmJaqcjg4OOAdb30b8/l8XQ9nPw+z+ZSDgwNe9MjDhGFIHNtV26DX5/qTj5PnOZuORDmCZLokYkDoerhC8olP/AmPf/pTfN67v4CyOUPLPq3wGV14kMev/z5iJXjkkZcAAs9XLBcpXhxz+MwBz510vPTzXok7uoDsamQQgyhID5/grBUM1S79y5dYLqd0MsJ1+xSL56nCBjkacdztoTY2qKqGzng0dcaVOKRyBMdVxdlxyvmthxhuvo7T5DqJe4I0fXpen9nx8yj3CvFGnyJP2ezHSCdAssvIPSPuVZRFi3JaQrbZPXeV+WyJocUPR8S9ITduPW5JIeE2WzsvQZ4tmafPoYRmEAyQwiEKYpRzQnqUEPlj8lVDULtIal7+wBUOD06JQoMR29xz7TyrzSN6Yc+ukMvnkE8e47khWnfsbA3gvGBrawNJj0ZnXL58Dwd7CU3XUdYLXnT/ZVbLgrLqqDvBRriNVC66E7QtBL7DKpkS9a2aa+ioK2PbYAJv3bDSUNSlpSeoliB0WK0WhAHEvYgut+ze0WhMmqZgGqraMB5vsZgVtFjlU6AQyiBVR5MLtrfOcXR4Stjz8aVCmwZEhxLpOpXs0dQ58/mcXq9HXqzQnSQIFGU9w7SKONgA4TBfnBD1XPLCKnZl1+AHFk9zOjtisVqxublBUS2YzmqCwFYga9Oy2TlkWUIYhtx89Ib1cs5zLlw8x2qxxA9uEfoTqmqfsj3DcQImo3P88R8/zmSySS8MaFtbVahPlyjP9sqXOQSbLT1/h3ylGA17CGGrL3v+Dqv5CmTGfPEMvd6IQb/HjeeeoB+fI3BGNMUZG6NLtHplX3vrE/c8Dg4yNrdcikwjaYjXr+X4YM7Odo+bh58h8F3Ob1/gbH7E1njEyfEMZSST8Rbu5iWCkW/P02VNK2t2dq7i+Se4fkNTK3a2XRANVdkQxyPKsmJ6tqDf9xEiZ7FYMRhFdG1Hp21HexC6FOWCvYMZk9El5EDRFGCky2Q4om0EpsspMg29DQYTB20qNH0uX3mAs9n+//UDpR/1efyJA4JQrbEXgqZZsDM6z0plVHFq2UnKkuN93wXR4QYBeabRnaFpSp6/9QSDfsB40iMrDHUNfhjQH/sc7B+uk1qCoswQ0rBYnnD+3EWkAjeQnJ2dkWUp/X4fIV1u79+krjuisGfB3bQWpK2XFHlJYzQ4iqxs0Urg9q26tLqjzLiKvMnW9UQOJ/NTy8AKHWaHU1wvoChrMN26O9Wn0wWBP8B11Dr93VJXFjAqpYOQ3Xo9aFeqCN/6PtuW4SgmTXL6wwiJoGk62rZFSk0QxUjHo9EVQhmrcho7SBVZAUbh+y4GjafsYJanGZ4bIlTFMkmpq24N5bVA97Y0dI2DVraD1HMjOt2QpS26zdEa0rxBGlgsKsJggFN11G3GxmSTNCnwfImnHDxPsEpm1JXB90KiOCDNVihleYlFUeAHLloX1FVG4A/puoYgVGAUvd6QZbKwQYawT5FXCCCKbAe3hdh3lFVGvz9cNwh1NE2NwKcoC3yvT1vWFGWJ50b4vkvTgu7sh6ftCrSwfbVd06AVuD0XU2pc3yfLMhxHkmQ5XhjQdZo8Twk9H6kszBVjODtJcWVgB/yeg+g6PF/Tjy9S1y29fsC0g8Dt0Yt8ytpg2oYoGq6bnGqiUNDvT9AdeFLRH8QWPNzCqD8G2WC6jrbqcJVNEEtpwzKY/O5QZrTGyBcUyn/3f/xbXvzil/ClX/rljEZj9vf3+Cf/5Af53d/7nburYmMEH//Ex/mO7/grfPV7vpZ3vvNdDIcj0jTh4OCA//bffpZnnr1u07C6vftYXafvrtn12gN3Z/jrOo2Ugrpu+Fvf9z189Ve/h7e+5e184zf+Jeq6Zv9gj996/29QN9XaA3nnPltb54igbhqLAIL1sKoRUliV8u70amDNn8QYhBR0urWKpbHr2DRb8h3f8e18wzf833nta1/HK1/5Krqu5datW/zYj/0Lfu3Xfs3iToRdwf74j/84t2/f5gu+8Av56q9+D67rMp1Oed/7fp2f/umfvgsP79oWIW060/M89g8O2dnZtaqrkPzwP/tnAHzFl385b3j9G3jNa17Ncrnkt3/nt/mFX/glqrIEAXXdMD095uLFy7hOwOnZAZ5rmXDpasZyOefq1Wt2++EaKBVxT3F4+xApJefPnyfLbGVe13X0ej2uX7+O7/tsb2+TJSld11FVFUVRMJ8trY9rY2wpF1LajuWqIqtqVvMVm1cusznaZL44xvU9TKsQGj7zmY9x/txVFpdrlFeh1JgwDKmqgvnimPHwEhuTC5StDVwVVcZosMHsbJ/pdMq9928gPMlWP6Tph6QLTVPkLOZLPvNki/RC2rbCEQo3bGkXkqa5xRNPz+jUq3BDl6QoSfdqzl99EYvbJZcv+4h2j5H/MK1w0Soj2LpCujhDtxotFgRhQ+iOePjK/TTYkoW8qgkmCqlckjIlz1Z0tUIqTVIccTo7RGD9vJ4akeQJm1s1dA2qCxkPrYL/8IvuZ//waQb9bYoi5a1veAt+qFnMYed8wPFxye7Wi9BaM/AV8YdXCBSu5/PII/dy9f5t9vZvotyacTSk1RD1AzxfEsdbXH/mWTx/yHjrHKfzGWWa0LY1RZrR6ZzRKMI4DSenQAd1m1C7M5raBul8zxBFA3xPI5RH4HkU1YLd3fM0TcdquaLXs6+PDgI3wPEEulNEcYzAIQgtYk9KSdXYY3Jj4xyDoY8Q22xtbXBwcEDbRmRZwnB0Dh0saRtrZQqC4RreXeG5IY6jSBOBEi1VvaJtBIEXUWpN26b4oRWDmkaSVXO0aBmO+yjXYbIxYnFWMxx5FEVNUdYUTYHwNMrr2LmwSZ6UjMdjwkCQCIXrxFR1wmKZsn1uQpKecTpd0OuPWSwzZtMFjmrQbUfcH2Bkhzx1uHjhCovZCt93KfMZY1ch8MiyI0LnPJuTDc5mOed3B1S5j6cnDIMI31G0bYFC4bod89MZ6I4wjJiezhmMfJqyYtCLSJIpcRBTVRVbm5KiWnDp8gWrHq86zm/fi+O1CBNZoLuuyauGbGF/h6vbM5YnhzgMiGMfFXQ47og6L7hwcYsiMxRFxfndc6xWFnK+XC4JYomhpmlbXNdlc/Mis9mMOO6RFwnohsgPiUYevneOpsq5/95LGCPIs4quEcT9jlVSoITCESec2/zcFUrxp5WE/1+373zvu83hrZLzF6yZc7Va4PkCh5iiyJgn1iPQ78eUTclkY0gYuUwmE4q84cYzz9Ef91isppy7GOM7MYtFTSdS8rLBwWe1SjHCpak7u6YO7VWYQuH41huVZfkahKxQSrGYp7iuXWFKpQnjkCRJ0J2h7uxwqbW21W1FgZRYor2yXoeqKiirFN8LEdKsB6OStoEgiCjLnI2NMWmaU9clYTRAyI6mNnTaeiyF6Oww7QowLk1TrD9ohv83bX8Wa1u2p/lBv9HMfq5+N2efc+KcaG4X92ZblQ3OchbVuQxIVQZLpAQvFi6somgsAxYIAzJIIPHAMy+IB16QQVgggYHCLlN2FdVmZebNm3n7uBFxut2vbvZzjoaHseLckuChLCVbCimkiDix19p7jfkf3//7fl+ShmRWO/QUaULT1uGNF/5UsxaMuzqSWKtRcUQ3Hk6vMQ7K5tgiJPTdRJwE/pXg9EFOAuqotyMSDV4ydA15EXE87knTUJ/mpCWJU6TUVMcGj0UKHQYHPxInEXYKSVkdycAQ84Y8T2jqUHGW5ZIkDnWAWmuq+kikY/AxIgqBISkFbRfUU0HEMHbEETivgzcjjomjNLAS0xTrpvD3ennymFRYN5BmUYAJZwlt29INNqihIgfC0Nw2E1EMQmjcVDONNgzxSuEItgAhBNZ4pjGol+KkeE3TRCQV0xjUUs8Y+G6DpchKpmlg6FuEAB1Jrs4uqKuJKAkJ5mnyDH2NZyTVBWmaoLVmGAxm8OgI4kSEC4fV4XdpqFE6KH1t25KmeQhZNRNxooPiLOGT2vPf/G5zuql7/tP2mkn/U3BZwamiUJyGP3lqxglKmv+q99sH6VHI4D396rVba98rkSDeq5MQ6hm11uE/dyd0jv/KfymwwSke3uMTGugr3qM9VTsGbyc/z0eIEKewzuFPPjN34soE9TEA06U8+Ta/Wodbizol0J0NjTlfhYa++jOdC2t670LN6vtMh/j56w+rfAmo4MH9ygfow/etlMIah3WG5XJNHKcgPLv9jtvrG77+9W+ACN//P+0pFchTM1Zo4YIQLhJS8G/92/9z/t1/53/DX/hz/wLf/Pav8rB7R4Qmm+f88Ps/4u/93b/NX/mrv8Pzl0952N0hfM7zZxv+1r//H/LTH/2Yv/bX/hoPu/370FISx/zN/9u/hzGGf+lf/i8AcKiOuBPb8D/4m/9Prt+94a/8S3+V2WzG/nhExwmLxYIf/PH3+Yd/7+/zZ//Mb/ELv/rrbHcVUZlQFBnXP7vhJz/4D/nOL/w6o1ySZxPWxEjpuX79hrq+4erJhyw2FzgvScpQa9sfR27f/IAnT0sO+5KkkNTNIz97dcOqvOAn3/8ev/ALKz77fMfbrSWZF9hhRGaS7mFkGX1GFPWkq38Rn8cs04gsjSlnS/7o7/4HlLOIb377iqF9zdX5BXqMeP3qLU82T9lXXzC5Ca1nLDdrjlXLcrUhLgT1bkDoitfXtxTzgsGORPESpzzf//73mc0KZgvB3fUji/Ipx2PF1dUChCVPniDimqYaefnyJVEUgmdxZkj0itDvrqi7PaNriHTGOEHx09f8i//OK5yFJE343/2lDdUnV6FD2guiJKftW9puS5okqEjRtSNVM6F0wu3dA0aENXiaB8Zj0zQsZxckKqfvx1O6O2F/bKnqLdMIRR5W9bc3j6zmC+aLDMGIMwVtXzFMNc6EVhvvG/q+J840XTuw3syJY4e3MVEU0Y33XF1+SJ7N8GLkeAhUga/CcsPQ40zC/cNblEqYz3Nub8PKNUkUh324RA+jx8sjaRyT6hXHQ0+WJ1g67u7fsFxc4FGMrqKcL5DC0R4Hri7WKJ9y9cE5r1+/xUvFclVyOD6wXi5wJviknz65ou1qrBEslgUP91uOdcVyHXF9d8/5+kO8NBz2DUVWsH24paoa5rMVgR8Z0fVHnj/9hOVihvUH+g42q2fU7U+4Ov8Gi0XB8TCQ5ZKuGZHK4Z2mmFnMFPB5g2koizld151mkeBFffP5gQ8++Ajn6vcp8S9ff4HWOctVwa4+Mss3oeZQhKDOob4HObI/DJyff0A/dESiDKnrruds/RzPyN3dA7MyYZj24BX4hIuLc+q65vz8gurYhfrMk6AyTpYnVxcM/cR8Pme3fySNCp49e4HzDbdvGp4+X3PYPyBQZFlJPHlE1FMdLE8uLtFRaOn77d/+H/9/J/L+f3z9M4+ev/DxSz65lMR5xGgrsvxj4rTkeNyTZCn76khVHRjHnnFUFEXG2fkV0zSRFvArv/aLHKot67MFQhpubm7ouo4XHz7D3jlkNOKbHk/wkUTZjMGAMZ5usoixRRDRtSPz+ZIoyk80ip6qbvAWJIK+75nNZtRVQyQVwzAwDAOzxTyoWVO41Ss1Mk59GOZUjBChiL0sC6QM/MBpMuR5yTgavIDZYomWgsOhJ0kFwkQksTx55ka0EFjjwhpawzQ5uq5DZBJvW/rB0XdD+DP7ibwIK9E0Vjjn6IcOr8JwmaYpbpLYaaKuW7Jc431YfWVZaAwILSeCoZ9IshRnLNZNzOYp0zTx8uVL9vvQZx1nJeBCcCeLGUdDFMWMg8PZiGEaiZI0tASMHVXTMZunHJodWTZHq4S6rZmsxFmFlJbJOYwJfbRmgKKMOFbHUzd2SlGkTGbAOf1+gEVq+mlERprBDGjJiQ02UbcVcRJahKZpIssL+m4ANLNZjrE9XV8TJ2EYSrOCKHYMvUf4nOW8DEncLGYYW9pDRZIkGOPerwf7LrQWzGazgLsB8ixDJ6fgkwid3FrHlGVBkS/Z7490LRjbM9WhtqzptmiVkkULvAsd1m17ug0XBYIwoFvT03Rb6hNTTEgZlNwoxhlLpDKiWfq+KjFJc9rmMSB5fOh8Bd4Pa199+ffJW4mQYdByzvNzIVOc0tdBcVFChqzJKZgTLhIOJSXeuffDkpCB5xfCLfqUJg+r8KCkB7+k8w4pAmTceYc6hTQgnGdCKjiByM0UeoqFFCB//lrC++ZPQ2vwSn6lbPoTbiYorgodfdWYI3BeYKZgDwkp32AJsadh+H0q+PS+WOtQUiBlWKE7GxiIxhmE14yDIcty5stLJjPRDz1pmtA1LXEcVJxhHN+Db776mqYB7yxxkqOkYrL29E8Fj/c3eDexXC5PDx2BGQzW9Nzfbbm4uKAoU/a7KgTp+o7D4cD9/T3Pnj3De89+v2e+WKAjSdNU1HXNh598TFmW3N7ekkQxRMECsds/cnZ2xqxc0PYjcVLQDy3eW5rqwLOnF1w8OQ+NIzqhaY+Us4Tjfs+zpxdM00hHj7eOKBJ0bcs0GcpyHs4/BoTIqeqGXGnawyO3DzuMspRxhnCC//g//jv88p/6LYa2JY2TAIpOHFHaMHYTcezxJiWOjvzSL/zzvHmz5bHtSWNDOyRoHdHUB67vfkTcx/zV7/x1/u7f+T/xo3/4D3ix+iYvPzhnvoLO5VyuZhRFyfXtI8uVYL5q+eJnt6xmC5bzJe++fOSDs0t+9JMfc3HxIaNt+HN/6hc5bgeePT+j2tQcqx2zF8/pWsfVsyXWjTTTgC/nrPIIbyfONs/AS6yBff0Zb25aFsucLInx3nF4fMf+iy/o28BTncYJxkt++Iff5/LJC+5vW5LiwGgORGrJfdOQ5Qlg6fqWftiRJAW5ikMwSA80k0Oz4PBQ4f2Ooig4HjoyfUYiR272Ry6fPOPu9pHVYslmMadpaoQYQsCKPdaJQK5wjvrYIqXi6smG6jjSmhrTG9Iopu1bYnVGpi/Y3Y/s9CNlobm726L1njjRCAFVvWNWnJEkQbkehzwgeXAMvUQqTx7NUMmOvlc4Kzh0B6JY41WFnUYuLz4gz0pu7m6ZLTeMg0SIhrPNFVLA0NU83KQc9h1FWVJXI0qUHPeWRZEzL2foKGeRpLRti44VeZnRjwP9aJkVZ0jlkEpwdr6kSAPwfl4uQUS0bU2el8zmKdvtNrT4iBY7Fbx7u2W+DvWsr97+FM2CukoDBzKrSdOIvnPEIqXMJZmb0w57ynxOnq3Y7u84PA5cXK2ZpoHFckGazLi9uyPPU4yb6Ow9w9Tw8PqaIst59vRFeHYRE8cRm02JlhF+2nH25Bl1sydfKMahJ44Tfunbn1K1RxwFw3jA25Q8W5HnM6apx9gO6RXLxQodeermSHdsSZKM69fv6Nqe1Sbgin76oy/oGkeSasZOInVH1wjSpEObFbv2nqRbYg4jbbP9Zx0T/9kHyofmSFZG1KOi6fY8VBOR9ORzy/3be9oToPjYdLRtzdubRx4ODc4PKG3I4wWjOZAVC+q6ZrIG7zPevDbs9hUjNU3TYY04AWlBS4VSEaMxtPWBxWKDjDQPuy262qNOjCr0Vw9XGIaBuq5ZzNdEAsZRUJYlxg4kscY7Rz/0JGlEloUH+TQZwJDnOdPoTj6vhqLMMaNhv225eLqmqhrwhiTJiGNPZ8wpYRjScODoTYfWgYI/KxP6IbQGDKMm0inzeUC2qCyhaepTPZ3CuoEkjZBC4E4PY+cncIKiyIhjkHicE0xDT5QljGbCe4dWMUp6enMkSyKUjGlbS3Uc8V4RxfC4vQ3AcpUFtayrkElozVEiZ5mHJPNkB4wX5EVQQiNVhqGcHiHCMKR0yjQM6CgMGsfDgSxecjzuUNqTpQumwSBmnvmqZBygrY8kaYwXkr7vyfOcvp/wwpOmMRKN857JTdRdsE+M1lAddhRFRuQc9SGo1mYwNE1Hmo6h+cBpuskx+h3eO7rjjjxLyGcl1a5lPlvT1iGQI3z4vTLjSRmNEyRQH6ugTHlJEgX12nuLsR3rTYlwLVFchErMvkXIFUWaonXENCiE75gVJVrFQGCi1n1NkqYU+TlFIbFGoFTEZvX8VIdlebi/ZzbLsd6xXq4CmWC1Qeu7MNhxEtyEeJ+6/srb6H1IsX7FgVRKvfcq/hyzc6pL9OHvAzfQn5S9U6vQaTgNyBj3HjkUBshw2bH256if0EfuQIekrZYa6+0/pSACpz5xY6YTxsohvD8NgachVMv3qqfwAqU9ztlw0HuFcxZjLUqewjnv8S8OIR1gT+vwkExHehw2JLm9CwhJKdFSYZ04vR1hWJ7MiPOeOElIkpQkzRiGES9+nhIex57FYh7U1NN7/lVTDkDTBFVmNlucvjfey7nX716xXM2Zz+d0Q/s+U+WcpTrs329zqjowDJNE0VQtd3d3/PJf+Is87LbBSqLC6v32+oamPnJ1dcXhcHiv8Kdpyrs3rxjHkeLyMlhY6posz4l1hB1G2rZmvV4TJTH10BJJTaIFXdMjIsNydoFQJVJlKOGxZiCNNU1Vszu849vf/lWsHekai1PhkrPb3vHTn32BjL/O828VvH37ll/51V8nVppJWIwd+MlnbziOEXmSAApDg/OGrjvgvGJC4egY+5hYKiar2O9u+ehr32S2vuT3//Af8Z1P/zyL9RXf+/3/K6P6lOn1DuEPvFDnvH24ZZwqymmDjiRf++QTimjG6I785m9+He8l33j5NeLEIFWCFmtijty87kjiFd1hxI2ezeaMh/uRLM/R8imzoiCPCkbXEwGbi4i3b7YssivYdCSF4ea2Ic/XaHPPcnGBVrenvICg2bWkZYGpW+RUkboMaVKWxRnTdI0bR6apw5qW9bygqRvc1HKxzGnGhKo+cHlWcGhb4iiiO7TM8oRp2OKsZp6e4TrBR08/pu2O2MlysV7hfUNcnmFdR98bYglPnp1R1z12DBflSCpefLDCu5SL8zMmU2PGjG5wNP2BWGnevH3L+foqKKOTCNs3OWMcgmUjijXOhc96140kcRRCfComyWKqfU+c5xBZolhyOO5IkoSsmOO9Zb7IkSoiFTHHqsYkjqmbMFPD5DVJElHVB9pOkWUZZVnikDzutuwPHfOlYvtYsVrPGW3L5vyS47FCxR7pUpq2IkkTbvdbhAxBoceHA+dXF1S7hidXT9lsGpJoQT86bm9vWS4umS9W9NMIMmNyljdvfsbF+VPOUs2b69esFh8gI8njfsflVcnUOIw5sts3KBnh8KAgS2P++IefcXn5lDifMzaWRZlh/JYsjnjy8ZpYBxGimnpWizWT6VgvF8RxTFkI2u6OPJ9hpgkR9ZSzgqbvyIqUN292fOMbX2O3PWLtRJFrhm4kVhFxliKAcejI0ojH+5osySnzjGk0KGG5fXcd+s/bgbo+cnZ2xvW7R3bba1ZnBUUeUXU17Zs/JpI5V1dXf/ID5Rf7a870GV9+fkNTVUxDTJEVKD0RiYLbh5vTATtRzBI8lrf3jxRlSpYLvL/j/v6B2XxJpHL6psPbmMFs8eqId2EtqOKIqgpp4yRWXF+/Ik/yYGw/HpmMQ0iP0BoVacp5ye5x4Ozy8rRi8PTdSNN377l3QgQVT0eWNI3I8wT1fmhwTNMQBhXjUFrghSMr0vAg07A+W2DdSJJKnBUoaUGAjvypR1QifGhlKPPsfaBBqlBndDgc0DLBjBM6tkSRJI4zynlEVR1wToNQIdwjBJGMqA4H5vOStEjwXiAwOGtYLFdhdS8UXT9RlBnTFBqClJCnyH/oQhXCM1sWTKZl4VcnM7XFTI6hhyQ1rNZ5SJqXEToeaGrDTOeMfbgYzPIFw2Com4okLtBaME6PFFlG001MgwxNB8IEv2ScYU2LkJKbm3d88OIpzXGLmTqcC4pvJFXwZBYz9vs9aaSpmpY0yxjtRJoX4WfW9WwuNig80+RZzM/ohj0CxWZ5SdtVTH1QtqYx+D+01ljjqaaBLFJoldN3FqXDe/OVXSKOY8Z+YLVaMY4jOYsTi1GQF3MmM5CmEUKG35GQfO5QqsQajRBjANOaEAByo8E7GPqGslgQxYLFJsI6QSY3eL9nNJZ5NqfrR5SImM8XAQ/ESJ4HlXxoanSch98vrbHO4LwDoU6r66/wNyEgI5DvBy0pQwrbe3f6y79XOL8aEL9ae3NqFlFahwEUThcpj1QKJeVJqeS94hdQQuHP1lGEdx5j7HuAeRh8g1Jp3VeMwbA6lyqwXdSpOtFNAVskpSZ0IMdYOxHFCc6CmVzYmkuHowfx1ec1/DzC2txhrDkppeKEm3Hv/ZdCSfAeLwTGDSeUhmYaR0BxcX7F8Vix2215/sHL02sQSKHp+wGlwwPNnZiHP2dlfvV+jgj589DRV1/ew831l6xX87BOtD1CKLSWNNWew+GGT7/1iyghENjQ/qLC+jxNUxaLBbtdUKbGcWSxWHB7/Zbnz5+zXp9hTBjwmyaAxu/v77m6uuTi4ox2aEPIsOvQSjC2FZHSrM82tP2I06ATsMPEooj5g+/+I8ok45/7rT/PdndLkqQUSVjJvrt+jfcT8/mS5rilHxRJJHF5wsNuT5xIztYb2qFHJzF2lCQK7toWoaAoFyiRYo2nrQdG5+nrCuVDfW3V1wh9hrApzXEgUj2HwzWfvvwaVsdI4Xj95qecvfyIP3v+r/FHv/s9snmJmxa8fTSU85TF+TljK6gmwfE2ouk/YzG/YBgHvvziNUpptg/XeCxuEvTO0Q/1e4+olJL1es0wOva7hkjOeP7BOWkSkcUREsVyOadtJUINlEXK7Rc7rDfUzQ37x4m4CT4LJRMmM5GmKXEUgpGLRcbUCoyRCGoib7FWMpud0bR7+mNH107B2nOsMQ5eXp4HSobQ9PXAfDmja49I36Llis1iidaaNNEoQkFI21TMZjH9ZLk6/zpxOtEPFUm05PJFyqG6YewLzpaerHBcnH/A8bjn9bs93g0oObFaxvT9yOVmicBSZFFoH/Iebwa6vgWfMg4eZxu6rsNaQ3VsOT8/Z+gtu/vwXO/7jigWHPYW/Iw8LRmGFnGy/AxDh1IjZTlju7sPGxrbUyjAK6yHrm3Z7x9YLBYnESdFyYamT1gsVngh2T2OlHnMsdohKWibFpQh6cCYIAh0zenztz+CV3TDxOP2gdevvs+TqxVd63j15kfwfYezA2W+Ics1bVdzf7xjtV3x8HjDvNzz4uKMY93Tiy48D2SA0EdxyWF/x/3hmqaRSOWopy1nmwsmo0m8oukbDlWNUgX7+gEvWupm4PLsGeM4cLu95enTryOYsTyb84M//oKPXn5C12959faRJMnIS6hrQd+mTEOLp2Hcdyxnl1SnLfHZ2QVV3RNFirPNFXlZsD/ck2YJq03Kze0BNcHgBvpuJOs8eVnQjUciWdB19zTNgbP1OVpkvHu1+5MfKNvdI2+2HmMnhkYzTB3G37HbG4psjZQVxoFKMpq+x51WnFVjeNg24QFuFnTbGkxPEiXkeYeZRhQlUobVtLNDMAojsKNlvVizmC2o+wFrQo2WPj04hmGkrnqc1xzrw8/X10qCCX2e8/mc3S4kzJIkwtqJwVhiZoyjQSobapaQWDuAssRxHLqqZ/P3vstpMsEvl2Y4P5xqjkaSuMTYEaUCY6woY5ra0PdDeICT48zI+dMEM4VQgdaSujqiNKw2KXiNtTPsNLLb7ZDkKKHJc421Ax5FmiTEcRqGTm/J8gQdLSjKlKbuKeaetq1ZLjfcXW9JkgLrFa9e31EUKbEOh4WxLWmaUMwkUg/kRQCoTr2lqwRZmtN1Ry7OzqmbHXEyMI2Ki7ML+jZY4+Ik5djU5FahZM449kRxzpOLjzgedkitcdYzy1f0jUOieHp5yWAmjocGQViDjoNlNT+jaxp0Al1/REYaITTVsSXLCuzkadoaqTxFumBezpAiwkySLEqx1jONE8o6tIjojz2LxSKggdqGNPWAQeskGMKtYbFc4q2hnBdMZsDYibPVJfv9HstE2+8CUqPvqeuWWblkUc7pTc/N7SNaeRaznLo+AsHQPfYVPoYo1hi/Bx9hJgGMDOaGaWwRxBw7Q30MnfOpnpNlBdPUcne3Zz6fkyQJ+90OZ10YZIC/8kvPub6acziE4WCcOuw0UBRZWLsnGWaS75VD7wXH+p44CQNlUxvMqIgTWK1WGGfRkQ8VajohThTtsXpfCpAkyXvsU32sQnvD1DJfFDRNhXeGKEoYh6AGJklClhXsDw90Q8fzF894eLgLUOMooigyqnqPkkHdLfLFyafnGKYjRT5judygZMLZ2Zofffbd00pqGXxroseYKijfI0yjp+97pmkgyfLQ+S32eBP8fUqMpGmO89l7o/rZesPY1zzc7fm1X/nL/Jd+528w9Jbf/rN/hv/i7/wO/93//v+AL9+9wVvLs/OX/M3/6N/lb/+t/wv/5n/rf4pMCoahRSUCaebvz8T/1f/63+T51Yb//H/uv0dnG4yaiJ1gu+35f/zf/7d88MHXEELQDi3SS84WG3760+8xjEcuL6+wxoMwuEkh4pib23cUaUaWZXTjQJqmdENP13W8ffuWMi/w3nM8HsPrcwY3Gfb7PW1b8+mnnyKkxdgxnJ2zBe/u7vji88/Jyxnnl884DgY39FxuFlSHmt3ukW/86d9GRx5vO4RI6ceOaTCMY8uzZ88QQnI4NMRRHoIjCu62R/Ikpkhi+mGCSDBVE/M0pmkGkAHIPyIp8wKNB7Ngqh9Yny+puwPHo0NnA6nridKEtt9hKsfDwz02KbhY5uSF4HZ7w5PVEzabgn/8e9/jO996yWR76mrJoRJEacS290x9i/dnfHlnMVai5Cc4c4SygClDFwNrZairHdOgiaMdCMOPf3RPkk0BAC0bbnY1zaHnbPWcJIr58voz7JQSpWFrEUfBPhMninZQWJthf45yRUYx508ugl3ESnp9YJFKuhqKOMdHMlz2y4Sy0CyXEePgyWcFXTUytQ2tqYmkZHF2jncaKwUaydAMLJYZQzPR1ZYs0dTNQBzntPuGQ2Mp0i115cnSnN3xhsc7y2Z9ydOrJbvdIzfXW+r9l/TDESVnDGbEi5q6FkgiojTG9ILz8zNevwswducdZpywNgR7pBqIdcq8XPCwvaOuRubzkqbds1ov8LQY1zGfl6fnnqHrKy6eXPG439EPDZYOLdakZUxTd0gV83jcksYrpIvI0xXRwgUb3eRp25EsFdTNO66vq9CPPfXsdz8GMdH3YQvVDTXmVCcbxQKsIooz3l3fU+9Gzi/vT++35vXNH5Hql0TpyGRa0iLi2LyhGiKydE3f39PfVSTZguv7LTfXr3j2wXP6mx6lDWM/cHmW8mr/I3bHEMwZphFExOt3I4+P4axdNBHGtDg5sB86Pnv9iFc9L1++5A9/+DOyLMN7z676Ic+uvkUzDlzf3rLbV1w9PePzz+5x1JytnhCrnMddRVt35LmnyEpubx6Jkw4pg62vKFb0/cj9fUPRGbqpY7lcctg3oZZRL7m7+z5FseBYD9jRUuShO1wlE1muGY2ld9c8bq//5AfKwz45+Q5b4iTBoejqhKlXdN5gfOipLIqIrp1QPg7r6rGnTOfEcsbd7hWzRYxVFiktWi5QogI/0feWLA9rllj78KGdAspgGB1mkiw3Je1giJICZxXGjDw+7sjiBS5T9MOI9J44ETg/kMQF4wBS5EhpSaIlx+Ge+jixXIUVvZIp0IK0ZEVJmqYMQ0c60whPSBQCSZzjbQDsrlZLtFRU1cQ0jigZMYw93k20zYSbMrJI4UWGVkdWmxRnHlkun5GmJfvjLcLGzJYWJybSfKA5Kqo9rM4tUdxwLl7iqHi478gKy3Y3UM4ynB3wWO4fQz1hVYUkcjc2lDPFbGnJ8kt+9KMfkhY5SmZ0/QETw9inoSrSDXgLZbrh8DAicUiVkkYpCs2qjEgST3UEN6Zs1gm73YHFpmS3PdCPMc6lzOYF/WiZhp5EGm7e7hHCkebTKZGeUNWPLMsls9WG8fGRJEtpmo6hrlnMFlT1jmGYmBUx3icIA1KMpEwkzoLwjL4njZdEqsEOCcMAMjoipGE+23B0HXG5YrJbitkCZzKqug74pboni2YIpYi0Js8VTd2SJTldV+Nj0NKz3b99X90pXODlqShiuZzTNBXOTHhvUTookl6EgT1RYUWzXn2EZaBrQ9f5sQlrTe8ijG2xTpIXHud6ojwOAR4zhhYc80A3aLxxzPIZnauw3p3UPk87VeyPLRDhhaScz3AuAJ+nyeEGRzs8UJZz6sqwXCcs1zOGrieKJIuyQJzSnFImDG1FN47Ml5fU9ZHJ7tGZZrAjw9Bxfn5O1/Ycq3vmy5QkkUzHJePUIVRLXqxpK41UljixSKVYn2XMZmuyNOLiKuVuJri5r1HpGictRfacfrrHC7h/fMPXv/kEZzXeXLLb3/P6dQgNzG/W7HctxTKn7/f0bUReaKYhRjhP1x/RWpNlJUoposgjRYrWzxgng5cVXQuWjMkMRIkmigT3N1uEmPPX/2v/Mz795p/i9Rf3/OSHPwDp+LU//WfY1Uek9DirmVzLj3/4XbKZICmX1PUBJyAh4fb+9c/PxIcd3/nGL5Kkmr5RSD/gjeLx7ksme0McnTMOjvSr0BKWx9tblDiGoN+xwWcxVluUG3h8vOdskyFUjDQxvXMUi4TmWEPbMLtaEGclxlr8NDIpqPqBsWtZzAvybMk4WLQXIB1eaKpqT1akyDgKnb5DR5RKBtvxxZfviJgznyUc90MIF3iDEIp9tacfQ+iwaw9IBMb2iCRhqHZU2xs2m3NUFAcfaJziRc8wJTTHA7N5ifUKLSPiJKMbejIN7+qKD88XgfOLxLoJO3aUsaB7mEj0iNMzpIdxOqD7c5SydH3Ftt6TRg6pFfvHI2VuwDmG5hSw0kGNV0IghWcaQ7AtTwom6dC6CIn5eYmz4Pw54zjyyTcFUgeFfrI1bd2QlpZ6crR2wroIMLjKkWrBw/6W1WrFw/U94zSwHPR7H7WQEikVTd0HT3QUk6crlBbES5jG0N89jj0QzsKLq6sTGWKi7m5Zrhdk6RVaC4yzjKNByJZxMFxulvTDhBkcOtEMvePJk+fc3t5SpOdcnPWYfmToHbHIGPaKvMiRY8zuXctgY+bFmn7saBvPoXqF95b16hI5SZJU0u0d603Jfn/H1NWUxVOEvkBxTW8s1mjsCM4MSBIynREry/bhFUkWI8XI3d2e5WJDls+ppgZrI4oipz9KUrFExRH9FBNpTaZL4kVMVT8wK54QRQrhBId9TZ5FxIkEP2GmHpMkpHlGW1dIlTK1Ew5z2tpFiGlimjzjOCKEApcitceOFU44RtWwawRpmhLpAqU+YDCCeX6O9TXzMsG6A8PQMfgK6yIsgOvJZp5qF3O7vSOKBdYIsjgDdc/D/T19N3F9f481/vS5GXl8CMzb1WrBYAZE7PHuJ5ixYxxHrl/9EdM0ICNB241sFmvevL1lVz2SJjlKxfzk9c/AOvI8593dA7/xp/8MdbfncXfHrI8ZsjOiRDLYjL4beXi84cmzD+ndxLvqhyzljDxasN3dhJBYDNt9xzgNpNZy2PVkucb6GB+3OF+io4l+uEdRsJg9/ZMfKIfBEMeCthbEqaXrBrrRUs5DOw1eh5uo1CfVIqYoM6TMsW5kbBzPnz/HeYs1jjhOMd6xWCyQ2tG3I0k8Z7OYY4zhuOuwoiPPliffRU0/KKQrEZNmXpTsthWzQlAuDEkS/I5JDFVVkWae5XpCy5RZfsbDwwObdc6sfMbFkz3jAFfPctq25e52Ik0jpLTU9TVn5yucm3jz+hVZOkNKzTh6zs7O6LqOaTS0Y0tZlhyPR0bTBeO+sSRxTD801NORNCnxamCaJE8uPiIvPUne8eTDJVKeUdUPvHtzwLmMswvJ85eSLL/i5p3hR3/0jucfKdYXFmsT+rFCRxlepzCMLJcp+/0W5wc8MJsVeCZurt+eaqSeYHzL08sZu33C5jzl9rplt90iledsecl+fySJA84ojsJhIrxnsp76GJSosbccHwVDm9JVlraFNI/QzgaVsWspM01b1VycP2N/uCVPc4bWM7WGiAw7Ce7e3rLb7ZjNZszSnL7tGJqKsRuCz8+tEb4m1opIK9JZhtaaPC+Y5Qt0FNhkx27ETiPLZcAfTV3PalkgXUZSvGB/uMfQs5gtUULicsNhX8EYQhzHQ8fZ2QX7XY0ZNVFehoEahbE1eMk49iRxRCQj4lgHoLv3RFHM4bAnTTPSLAaXn1bKlrrbo5RnNi8xDmw1MU4hER3FJUWUoTTsdwfKxTkPj68pZgVCKYTPSAsYJ8foJoQSJ6g5COnJsiVedDRNT5oqhJZ4NzBNPWW5oK626CTw6tJ5TTt0RGJN3bXQNlxcPMEBWofX5nQ4QNr+AetH0gi0iikzxfrD51y/uWPqY1aLK4bpyKFr0PGEMyWx+gA/jazWnq4zFGWwDEz9jm9/5wllNsM58H1Ekmp+8uMvmfxEnGrwEZuzkk8+ueDy8ik3796RLzX39wG6e6wNWRJzrAYOh5757JxprDnsDeUsARFA74dDDaIhL1LiU7nAOIWAz37bsVyuOVb3JGnEvFzz7uYznj/5Ff5H/8P/JY/bnn/0u9/l069/wvf++HdJc/jo4w9OTT6SJI1o257Xb3/CL/3in0brYF/ph5FZXnBz8+79mdhNBzabK4QQ9EONThWRTvjy1U+o24bFfEM3HrFGgAgYr/v9HZdXL9FJifMNiojBThivuLt5zTf++d/iUNWB16ot0xBzd/capXo2Z0/ASkbXEuHI0g0P777g7uYnfDT/NlGSsn24RnuBl4EXuHu8JUsj1qsLHnePZMUaIRui6IybN6+ZzwCR0k011nqqqmE2K9je35ElKbPZLAQLhcBYi/ae3a6iKDI2Z2d0w4QxI94FD2/Xh5SptRMqksSRCudLHJh8kRJIkdI0E1o6vAl9xm3d0+xrXl5EgGAcLFPiMbFFuKA2TYNjuTgjz+ZcXErevXvDaj6jaxsApJXvFXbnAoc2+IUtWgVqhVTBQy1iCYTLoNYaL4K/OJpmZEt/sksRXodSeCxdF55zWlb0XYOKV4z9A+Mk8Ij3PuPbuz1vm7AxWC0KvLckaaByJNGa2WKBczFFmbJcwGQGNuslD7d3PHv6IVESfOGz2YymaXBm4PnlJhQZCIt3LYunq/CcS1Oaagu2xzkBg8M6SZGnjHVotzruerw9MPQTs2VG2x3ppx4tIopshveGrm3J0xV93aGJaHYD/eCYJ0/oakeUgh9lOAudZJxcULH6Aa1CY1dRZPSDAB2xWZ7hrMROHolkXs7pupYoThDW0rYtZRHOjrYdEcIT6RxjNFpFeGEQkcMycKwHxtExK5dYa6mrliRJqesjUkuGPoT5nLKhrzpNWK7XTKPleKzJo5TeGJRSFPmMpu7xTjBNI9ZC144MzZbJ9Bwf02D5iRcMrcGLCOcM1a4lyxKkzemakXFwmMkyxiOPu3fBrhNlHI6GPImZphC69U4AmrYZUUozmBprB4QI26O26YmzBGMHtFL4cc6h2pLnG5SIsNME1pIXMU11oJsMf/DH/4RMLfj2p5/QTDXN8I75bI2jYXJ3xGXOT778j8jynK7rSaIMo25BTIjI0x0ky1mJjqLwcywVSnr2uy2LdcZhd0deZJg+IS88Y2/+5AfKcOAm5HkYXJIkoZ8MzgU/VSxS3GQwzjMvwiH0+PhIliUI6bDWkJdzjscDWidMhhOQ2xJFEVmsOEVGmRUl3kjmizOEkEHCLedst3uyfMH2sUY6x/Onc4Yh4IratiePY6RuePYsI9HnLNYCL3ccd3uevYyYL3rMmFCWL7m9e40QA0Vh+Y1f/0Ue7o8M0wO//Cvf4HCoTmpOxvnmKW0t2e7CSt57Qd9O5HnB4+MjeZEADqVH4ihhtS5QYs95tqbte5IsBu841BVRnuJ6zTieVNBsyfFw5PJZxvosIo+fc319zXzp+fhbmuVmxt3tHmcUUZTSNTWzeajxGtqWb//iNzjsjiFUkLccj5bV4pvBtN/vWS2eMnYpTy46wPDy2QXRif0nheV8veHh4YEin4FwODNh2rAyLWYZm8UMN0V0rWc931A3Fb3ziGkg1gozDcwzRZrGnM8vAPBlxiyNiL3EuxjnIry1SKF4efVBOASEJJ8v6PuezWIZPDo2w6WOyU0UXw0lPigQGkWEYDQjL56v2W0PRCqiyBYYM+FoSHXJ0DTMC0iSlLZRoQJU9ag4xVt54vktMGYMbM0op5yl4CaKPGY2Bs7pk6sN4zgy9h3OaCIdE8UCYyYuLi5PLTyH4PNzLsDtwz2WabLUXc3xWJMVKeAZGkOZblDaMIwjZndksTyjsw1jO5HmOaY7NTXFGqVmKF0BLoReYsVqs2RzFjMMU4C9IyijnKbpSMo40AsmiXWaOMrZHu+YlZos3XDsDngX0Q1BlUc6hLX008DLlx8jmdDKM/SeN6/u0TolLTTGNUgtMUYyDh1SNsRlivOGqq3pGpimmqwwJNmcYZxzf1vx8YcvmK9K8mXC5OFi/R3+6Affpa0tZozZPlZ09VuEHIkSxy/+0qe0TQ9i5HDYBkyOAi+acInraya753zzIQ93FUk2kGYa61KsSzjUD2TJiqG3GG/Y7u5Yr9f03Z7b67f89m/+y/z2b/8VPv/sjqavmc1zJuP4/e9+j9/8rd/g7MmCu12DEgKtJa9fv+b+/ku+8fF/lbZtqJsjzgY27o9/+t33Z2JSSL729V8On+1YMrQjHs/nX/6QIl9QzpYcm3dEeo6QE8fdnnf3b/j4o/8UQqbU3SOJhTLPOdzu8K5muTgnn+dMdYudBrJiybs3rzhUd2TFnKGvg5IehTq09rBjXgZYfpTEKKWQQhOlOuDTppoiXbNYrBncEPylX4W0zMD5eYqKMvrpQBaHtdvY9/TDkc3mHK116AUWnP6/EdvHIzKCONEcjzXWBl5unoZLLgT/rjiRApy3RErjjUWrcO6P00heJEzTFAaP0TJ1Fefn3+DYTdStIY4sMhoYccQqZexC2tV4uHr+Ie04cXP9jm99/SO6rkWdKhu/GoDtGIohJheeMVpoklQxTSHMmMQZQiQnEoBFaonDhgtE32CtDUQCGXbZURQxTgP5fIG1cyKlWSxecrY7IuUB54NP2NiYvFyhtGR33CGEY9o34fOsHvjRFw3L+Yrz9Ya+HYh1QppmjN1IWs7QqsBLy8PDAxBS97v9A1kWoXxEmma0dcusmIWfzejZzDcMQ0+azLi/vydWKc61rJZr5NwxjiOzzYyx7TH1QCQjZvMVvWjph4ZuGJACVtmMtmsQVnC5vKTrJ1RsUFphiZn6js1qA7MV/dgwTj1pmjGZkaYd0HLGopiz3W5J44Q4ioilotrtmM2KEChNE/I0wztHluRMJlBRhskwTUeaVlEWC8piQdNWXJ5/yPX1NV3TMg5Q5ku0imn6A00VLDRSh/pOpRTWG/rHPXlenjzXgmm0tKZlWVwyyyR1faQoU6ahwxuLVw6lPUkclETTGawLNrIsT5FeYqQNYdRUIEaPihKMVWhdMk4NpjesludIH8LBSbQAL6nrlnye0/UNwyRxXqE1KC3RUYISEZtNxv3dA5Ee6UeHMxYzWpIkRcdrXGcRRqB1RVPtmfTId7+75+nF11hfSP74B9/H25RvfH1FXRs+eP4h1zev6GqDn2rq9oHzixVV26Hlkigew/N5MDy5nHH/+Iaha7m/E2QFWDeSZwsme6Q3/0zEoP9kA2WRC7r2SDkbqKuOyTq+/ek3efvmgWGaMKbGOYcUmjhaEubyiKm3DGZglhfUVQdOYKcJayes64LvURZMXjINTegm3d5xfhaSWzqeyPMNx51kPl/inOHpxYZsphntA9VxoqsVRZnw+PhIJBZ0zUi8GLl+13B2PsNOjrtbS930TP2B5cKT5iWzcs1qk3I4HPjmN79JP82YTMfF2cvTDTfi/PIZX3y2wzjJzc0d1sgQlrA+BEwiGIaGptrz6Td+GWMGNpfBt7na5GzOZtzc7MnyCOM7bt9aquMjZ5cRn3/ekiZXNM2Bv/93WpIoYD7iYofWmv3WUZYZx+ORSMxIVEy1mxh6SZat+OmPbyiynHrfkeY5bet4uHlHkkpwCduhBXEkjiOm1hInmnk2CyvyrmccerI4IdYa73q00kxKINDU2xphEmBCypihaVDe8PxJUGT2x4bV4hJPKKEXSLKsIFcSY1suNyu6dkRrxTi1OKsYx4HNfB0OQSzz9YamG8iLnK6vUEpR1wND25EkGc5bqv2OLMsQLkV4SV21CGnJ8pK7mx1RLEliy+Kq5MnFC6r6gSiKuLk70LQ9KhLkxQo/OrpxwFlPkWfkmafvO5xvwE+kacliMSPLwkNuuVrhfEjInl8saA4DRbE8NZdE7DJBXdecnW+o6wo3Rtzf7VksVzx9/ox3Nw9EiWS9mYFMmPqJNPfc3D0gZeiYPrzZkpdzhDSks5Svf+uSw+FI/9kj1vIeE9QNex52jjybMY0e5wzlLMc6iZARSibo2IKo0CpBuII4g6bdEeeXPB63aCVJ84y6bXBmhE6htOZ7f/R9ynxGlkQMQ0fbVcxnq1NITtL3DXm2wNsVo39g333JL//Sr2GmiD/+ox8hYjh0O/ZfTFw/TvziL3zMT969IlZn7A8TBs2bx5/Q9DuM6JDJmrMnK5arDOEzBAlSwu3tljRdM5pbjk1Fma9ohiOHwxEhY5aLkpvbz2lbSZFtsLJjHA5YX6D1jGGsiTOFlwotYupDxycvfo3v/Lnf5m//rb/HL/3ykXn5LBzQWnFzc4OTD3z6rb9M3wWV3nrPoljzs8++j9aSzeop4zCglEB4gTeCz774/fdn4mp9QV4sOR5rohjyNOfu+o5jfcNvfvufPwW6vhpGEn785o+Zxj0fPPsG/dQjlQ7lD1HK2+vXRBksFk94bO5RNiHROeM48vhwzZOLK+aLFePQ4b1imARp4njz+R+BGXn67GMOzZ5x7FEyYTErefPmDU2z4+nTF8F/LgzOOgo9o65r6uM1TxdPSDJFPYUwlRABtXZ394aPXn6bKIqomvpUw+mZhuGEZiuCJ3k0eBHWjMJZtts9wofX61xYaes4RTgZlGql8ULhEMRxBM5iEZjJksQaJYKqZFxG3Q4IXTOLc/bbB+qTz7fISx73e+brM7xQ/PAnn/Ps8gIrHeLUkCWlRCYBn5aePkfOuVOQKazGp/6IccFaArwfIKXUodBAayCoq0opRjOQph6lQuDSe0OSpnhXI4UmjRM8nvPzD4munuCd4OlVoGV0XaB/WDNi7RgQZEPNdv+WLO2p2p441tSHjqRrcX5gli/Bh2DgennFdnfHbBkTCUGWheDiLA8Ys74dyLN5aItTHu8CSxbnUdqzXCxwzpLMFUp7ut4wdCNCRsxmC87OEobekCUlaRQRxSIE5jy4RAOCbP0BTyK4u7tjtVqRni41QguOdUUWn/jOo2Fqe7x2xFKRaE0vDLFyTLZl97ClLMvwuZICYwY8nqFp8H4kT1d0x55yXrCanTN2sJ6fcazvWc5XVFWH0Q6cZD5bolREP4V8w1ec3biIqasjUgYV2tiArmvHiubYBCyfM3gv0CpCKYUximbYEcUaKS2x1kiVEavAHpbCIwuCBzFOGadAl5gmj0ATxwmaQKlQMsVOoQJ4ubhkGBqUzEhkivWCaWqJo5xYQqxitvcHsnjBbn/Dk4uX1HXNaPeo0SIpAipQTczVU4bpgPQSbyxf/uzH3LzJUDKlqq9pmwolc+qjomldGIptT1me09Tw7u6eZaEZx5iz9TnWtOhoTqwyVBxx9gLmsw13929xk6PtYrTK/uQHyq99/JSmHuiHiuWiIE1zjKkRriONYb6a0/cjdd0yDUciJWjaI0IIilmBd2HwMMajvGa33XJx9YTJdIzDCdkigj8m1QmPuy1FUTDZibrZE8cG4yKyqAAGHu4eGMee7a4n0pqm25PqDePQonRMXU1onbJ/UOz2BwzQVnPmi5KbuwfSZMGxvOdhm+BFz3b7OVJPdK0hiyeEiNgdRu4fP+fh/kgRZxz3D6TJDOMkzkpUJBmGgEt48vLbSDKcdaE/Wy8wo+Pt2wNFtqE5HLh5JfHeMlnD7qZgHAR9+4C90xyqgaI4spxvOD6mlHnJOAgwE6k6p68GhqkKQR+tkD7cVg8Px3DYdebE/JNEKqapDMJZ4kQyxQppJYkSjMPIhA03fTxPLs5O7R0dUz+wWs1wPvRNV8fhtNLtGIaBi8sVWZbQdR1Pzs6ojxPOT4zDRLmImTqPIEK4BGfCmkkrxdCHB5WUIWkN4KVCqJgkEUgRsZwX7I9hTThNEzpWKFGilcC6lskNWKsoyhmehDRPeP7yCUkUh4dWmtAMPV+82nG+eUIca5q+oesi4ikPzRJJStMNp4E6dMP3Q433jre3D8znc7TWOAmT8SRpzmKZImRElhcIJWjqHabqyIuEYp5QlBEffnIJbo81F6zWC6ZpYHmu6fqW5brEWUXTKcpZSjbPUDJmuz0Sa00Zx8Sp4lAP3N1UpKkny8wJ8g1KK/wEAkvXDiwWK7bbLdM0MQyOumo5Owt+wXI1IHyMGyy7akc5Vzh1YLZe4CZLWWYkuaQb+5NfVLJYzYlUhnE9Ko3J44KszPEuputaJq9oBxDRkSRLqaqKv/sPvkekSxwt2/sH8rxAyEfGeuTf/3/VPH12zv39P8KRkMYbPCPKR0SJ4O6hIc0WDOaAVgld0+P8gMBj2GEdZFnCrnoAl+FxeCv58st7ohgiNacdRkZzIEnB+Q5rBTiNNZClc+rjkeeX3+a/8df/bf4r/8q/ymh2fPTRf4fXr27QqSeNMn78xQ/Jiomvf/0XmZwliWL69oC38IMf/EM+uPqYxWxGNwzYyQCS23e3GPNzJts3Pv5LGO8QOtSjumnk8fGacTzy4vmnIWQmEsapJU3XvH33M5Sy5NkqeEFVCqJCoPjp579PVhYgYpqm4XyxBGOo6iMPd19w8Y2vE+mcoTviLKhE0bYdQ7cnjSPibE7XBlqFNwrv4bC7Yz7POb98GgJA0Sk17yTV4cjtw4/49Nu/wuRDe1EaxXTjxPbhkXE4ok/IqK/YngHcP9B1W9abC8bB0I8DXobfO+ECu7Oc5cRRzmQDWUArh1YR/TgwmxdEiSYpc6RVSOew3tF5y3o5p5zltD+7Q0QRzlvGqQVy9rsty0VBMctJsgSrBU3bM18uGdqON29vWa9mp0YzGywjQoQ6WWAcT4OhTjDjeKKIpGQnIL+XnGD9Brx8j9fKspC0n6aeKI6YhvGE5wpKZhQrsjwOn9NTEUCcOrwc8MJTNSPjaMLvV29DR3qaIoQizWd8+p2PTuQFf9qKVNR1Tz8c6FtDP7R0/T4E1s4X7N7ek6cBd3dzc8NyPiPPc477PS9fvkREkOQL5mVoqItigR0VeVJwPFSI2KEzQRY5hE8Dvm1oGKcWoSKkSNA6oWsb6ipcJPJZwjT6sIr2A26wdMeW+XLFaAzD0J+KSvYslzlaaz549iHV/oCdHHYa8ZNn/xCwbqYZOHYG5wxpnhFFGXmWsrx4SnWwRDpjyifaekeapiwXZ1SHHZtyznHfh0HQWFSUhNBmHbBEsc5o29AwttyU1PsD4zAw9DVREqGFYBxbylkcKoy9JY0jvAi/H0mSQDQxjmHA7YdQcKIQpGnB2A+MvkNHBWZywVLlw++ZkjF+mvBCc+wO5HmKECOHfcM4lMRRGpjBjAjvyRNN2+2ZFWcI4SmzmL5ruTx/Tl1NeK+Z58vAXm5rrOnQyYAfztB6AtFTZs9o3TGA12lYrVYMtqGuDY/3Lav1jPky5fFhS11FdEODcyuO9sh+L7i/rrg8e8b3Hh746OVL8kXMsxcF3/+jz/naJ7+MMYZ37+4oiv8/DJRNHartCr8mmyWn1pOE588tWVawe6hYnG148ew59/e3RIlmvU4xLnzwDocDSkmkgKdPzvjgxYrd/ohWmq6bcG4KJn09gFMUWUYSS4xJCLS7mOYomPREVb/j7GyDoGA51ySp5HiIGVvB5HqixCDcE6beMck90zCSLgxZusSMPcJrqsOOrhGUsyDrj+MtZ2cX9L2hlo/c3T0gRUScKOIkoW8Hvv7JN/jsp1/w/IOXHA4V9/e3FGXG6mzFw92OcbwNB5m2LOeXdG2P1hkPxz1Tb8lSRRwXOLukbRu0jylm4XC5uDyjqWqmsUdayeP9PbPskmPbkmpBksKszFlvZsznc/b7A1k6Y78/cn9/z2qRU1WOfnSkxYziTLB73GFHifUtZbnB2Z40Tri92ZJmCUIqps4RyZR5OcNazzh0FGWEnWI+ePaMqnlgGI9cXHyA1ANpkoRe1qTg+Ytz8tIzjZ5jfYszmlivKIqMrj8CEmMhzROObcU8WwTUizGMo8Eah5CSKIkxk+eDZ1fUbQVywDvBbltTlkuQGU1rSWJBNx1x3jIMwYurI4/HUL27xYsjsS54/bbHuIbZPEZLj3c1/diwWCxohoaq6cMw0xvq+sh8MSOWE8fqEWcFcZzSK0Vz17BeL3n4/BXYgBvKsgQhFLePB8oyZxg6rHvNxXnGarHh3btbtNY0dY/1li++fBuGaaVI05xphOPxhiyTzMsldpBM3tNUNSbSTH3MWZfhnQnKiRNEUUoaxzRNxXDys3yFyFJacHt7TVEUHL405InBc8fYWUxSEpURkfGMxgX25RSqwxApZZFzrPZYbxFSh1S9TRlGg3Mjk+tw0tONE1pMTF2GTosQ8vItUaQokzVaLHF2j7eOrPDc398yjoY8K+jbXYBiRw47hJv89e8+nhqmQvNInufkiSPOK7ALkmjNci15uGsRYo1XO5JcI8gRMgbZYt2AmUqUjEmSEWMPIDW31x1/+S/+Dv+ZP/+v8E/+8Q/4w+/9Hv/6v/5vYKfgYUMaZL7k1evPOBwfuby8oh8PTGNoKDlWW169+QF/6pd/i77v2O0PxImiLAs++8mP6Nrj+zNxs/wI8IzjANYxywrevf0ZaZoyK85o2yORngE9Zpyoqmu+/snXSPMF1p3QHTj6fqRr3vDso1+lnToin9MOPYkS1Psds1xw9uSKcYBpNAhlEQLubh9YzGLKZQg5mNbhtGQcBrIpYzQHBjfSjYJcB16pF4pYKz778U/YXJbMFxdMo0BJR1UFGsd+v+XyYs5yuWQ0A0JpnLXEccrj/QP74zuemHOs8oy2RziBtyN2nEBMaB3TdR1RUuKlo+k7FnnMMHZo5bDeEKcJrvfIOMZiAjuziDlUDcY6slwxjj1SC9zkqaoDZeTf+yLHacJLQd+MLBZLvvtPfo8k/oTNZhEsKM6ghQxhSe9J0xOk34vQlnQqiVBKobQEB904nlBbX1UEThjjT6gxTVuHljYGg0AhpaZvJqpjFephBQRyf7jga62JogQtQ2OY1po0kxjTorTEGEfTmZCSP30lccRqUwIbrOX0/QjMBM6P5C7gqoyZmM812/2W7XaLdROHP/4p80XGbrfDOcNisQhEBqlo6iPPnz9n99gzm6eBzTw1jG24HGsrKdIEqT1eSHI1J8uiUxWywbmOptlydnbGixcveP36NVGUgRB0bY9QkrPlE4wxNG3Her0mXZe0bRuCmFHO3f0N49CjSMiSDCkhz3KG0SHHCKyn0HFQl40gTVcgJYXOOYyPRJFGGs9qXgQaxrEJYHMdVNrjMTTTtG1LtauYlwu6rgl4NC9x1pHFJXkRMw49WiuGoT9VvYZSBIdlXiahBKN3LMqSSCdUxyCOje3ELEvAwnyeM009wgcE29BZRtsSaY+xHUUx4+JyxeFwREWBv+ysDhcbaVjNCuxkER7iWCGSGDNNFIWmOg5MvaYol6xmYWsw9QmNeYNpE/IiZn/cBaXcaFQ0Mo5NQOIlgV3d9jX1TUMcFQyDR+uIeVkio4bjzpPEMWY6YIzlxz/+MWU247u/V1MdG4SHNIuZFTOk/GdrU/xPNFBuH5pwe2wG8sGhlEApjxeafd/QVj2JKqi7GuFEIL4LzWI+4/b2nsuzl2y391xeLinKmPu7m5B86h2xjijLkof7ijjSzMo5oBmHGiU1RVGw31VMw0TnWs4vz/FWEOuEzlqwkovNcw6HA1n6MV2/px8fWC7OcD6jjJfoxJHnc+pjgxkLsqwCUsTocYPEDIrHmx06CsnySDiyNEOgmKU5Ty7WFEXB04sPyMqCH/zgBzy5uAQ4+So71mcLmmZguZwzDoI8zxhPq2YfxSgZuJFpUnJzO2BGxZP1nONhot47lmVGlDiaQ0wZ52TJnDSfsdtVnC1XrFYLlvMSj+H5Jy/o+pGXTz7ks+Q1+4dHXOc4XwuiyFMWa2LZsJifc3gA4zuSNGY5XyJ9zLvbd7x8+TL4V5RA2RRrHM7DrFgzKy549vyKu8dX3N3f0laa88syoBvyJcY64ijDuZppGkmymCKfk8Vz1qszPn91ZOgNcVZwqCviLOb24RqtY9I0xUuHV544jjFuwnnB3cMtoxswpgckQmt21ZY41kxWgJiIY4PrHZ6gKAzjkdGOeO8oZzFx7GnqGuXCWkvqAeE7ynmBkwMfvDzjeGjZ7Y6cbS5AJEBgrc1mBeNoGIcBYx06sYxuz9mTImCjvMdMPWW5JClXjIPDE3Aju73l7v6ONM4wxmLdCEAUp+GhTMzx0GEMzOdnTGOHtQ7hJIqEJ5dPTweOxPsYId37lfdkaromJ43DTb7rDggP4zASRQkSFRpQyHGxoe8OzIoPcUPG1CuO+zcokTPYBsQU2HW6oGt6lHZ0Q42cMoRocYQUqpYlxsYYM+Gdpp96BJ6phyJZUuSzcKlMPfcPX3Cx/JDJ9Hi5I0oykmzNODjWTzR5sqAfBOMQlIBITzR1j1KSOFVYB3VnsEeF0gfSxFLVO5I8pqlu0ZGhnK3CA951SCtCWM9YzNiRRgu6dsDYlv/y7/wbfPziN/nii1e8evNjkPCtb32Ltn8kjiOcjZl6wWev/gEvX3ydWbng7jE8rOJyzh9+7/dJU3h29THj0GGnHqsKpIh4+/aHaMr3Z+IHV5+E7mVr37dy3T++4msffptYp+EsZCROYh4e71Bq4OnVp4wKuqYjdSCznNubN7y4WvDs4msc2iOzLGc/hjapm7c/oygdxXyDMxYhYex6ZuWKu5trmv6Op8t/DqEjvKsRXrOcz2gOR+r6gbPzMy4uP8Jpj28NOs1w40A5A51doZKUpjUYa3De0x9bvDOkWbjUG+nohlC9WuiIceiYz8MwhHQMQ4cXEZFUdMNA1z2iop6+y8iIUMIxDoYyDcq9ZkBKTZ4tULJDGJhczlHXLOYZkxHEUSiJEKdq2KYJ5ISyLNBJSjsMQWFMEjATjzdbvva1j4nTiNev3/LixQvMNOEidbKoKJTS7z/fYa0t319uQz4gWEyEUic2cfBEf6Vwei9CgNRrvPRAUEGl1Mxnc6RUgYfqPWYaydKYyU7EUYRMQm+28GDsSFaW9MMABP5qHMdBEXUiWMKswVkZKn6nCedUKJHwGnxGHIV2qKU454PnX8PaCWOCX7TvBM+uIsbRULf3Iacw7kndku3Bcn+7ZbuzpGlA5bVtT5HHSO9Yr8+Yr1dMZiCOUqYpoKuGsacsS2alYug9XV9xdnZGkhT0w8D5ZsWxqVFSYoXETTB25j22DwtpPOPpZYTwluMhIOHqZs/+MSj1sUjYHw/kpaIsc3Sk6DvPbL5CjoqL5ROMm3hyNqNqK/q+RyOYup5IxuweH8nSgjRKcNqEAguvWJXhub3d77DWni4qHXms8E4yX53h3MT+eKSIS6SaQkOrtmAckfZ435EmECeK1eySaQxMXuUnZouMthmZZXOYzajaPVoFpV84RdO1ZFmKp6cftwi3QAqLlB47QKQ0F+eXtP2ORIekdywdL59fMg4W7yaeXC1I05jjrsGrJ/RDwzDBYTeyOZvz+LDneDScXS55eHhAKRUEqI3leGwY7MB8tsKYlmFIcb1Bnz4bx8PEerFEC01WOmbLgkWb8PkXr4nkjDiWFPPiT36gnC8ukEKj1I5ZuWS73YIYaRuHF448Sbm7e2RWLjCjwmpJHKXcvKlYLp6j044PP/yE4/7Amy92KFkikjl5FGH7hvp4JIkkkcgoi4Jq31Mka7b7d9hOoCIQTiFJqPY9kU4xg6Eo5hS5oasl6/kM53uK5Jzj8chqVnLcDzg/8eTsOXc3X2J7R5ZNzMoneGkQasQaQRrlqGgMsnEnebrJedzeMJkGb+DxLmIrghn95uYGhaDIMuqqQynNen6Bl0eyKMdPEmkijvua880L4mxEeEekS8zkODzccbFaI+SCYdyyWV7y7NydBqSR3MN8GdN2jjj1xCoPBwoxZtDc3dxyrx548eEHvPriLYlM+PiDK8ZLSTVeE0Ur4mROml5heoG1tzy5+oTzixRjLOvlFR9+9AE3928BgY6hrzOyVBIlQd5+e/OG28fXVNUOZzLyIuX65o6mNpRzxWZ9RdUcidKGslwyTcGg/zDs2R3uSTOFE5p3797S2wNd5+maliRJqOvjewUgz/NTojnB2oEo1kQypKfHqSEtwo3cSUMkM8p8xeDbwCyl5snVObudYLXqiOMlXTdQ5A6lIqahxbmRPJ1xbHusHcEHXEOSRDRthVSQZJp4GvC+J4kU1kKalhgzUuQ5XdeE1J0Wp+73jmkM/jidwP32mvlqiRcDzTgSyZxh6lE6pj125POErhqQug2r6l2LcJ7lcokxI1VTQ71ARRHWHsiK/JQyDRBvZyMEhiROaJsDm9XydHmakSQ5h7FByhYpLWmcoMUlSk003Vuibk2WZczKOYd9WIGmeoadcmScMImgIGmbkKYpowmp3K6rMcbw5OIpTy+/ye/9wd/h6tmc7e6e8/MV797e8MHzr/HwsMWNFqETcILzi3PGwVI3B7zw1EdohWIae5IkOQXbAiLMWss0jKFuMwp8zTQpcNKQ5nPM5MkyR9NKmlOtZT8cWC5CGKFvap49veDxYQcu5r/9N/4XKPec119+xte/8YL//f/h95jPZ1w9/YDDYYfHE8clb96+4nh4y2//md/BuVBhirCYKQRqXrx4QZrmHPYVOgnDU3Xs2B1f8dHLb78/E8t8Tjs2J8+U4vFhT13d8+1v/zbO2RAGMANZmrJ7fGC1Ltmsn9BOA1GSokdQ6YK2fsPZLKNIV5Bk2KFhmixGKIRsubpaoJMlXbvDq5FZMWfsBkZTIeORJDkjyWa0j+8wMsOjaKqah8drPv7aLzE6iR06Yh0FoP3o+eEP/zEvv3bG5B1dX+EMIPvAMD0eubgMinw3Du/B9sY49o9bzjZFaCZC4pzBWIXQnqqqGKc9MxmRprNwARDm1FzkTxeQiUjFSBXjoz60N4kILSNmZcSh8sznS4yYkAhipRh6c/LGTQg03gmEVNhpDNWqdiRKIzabDU+ePOEf/L2/z2/+5q/j7IQS4L1lGEKxwlfrZaUESRQTz0rcKaE9DAPtOJxW/SGFK05d6l+VBIxjj7ehzlNrjTEjw9CfqkwtAoGSMHQtQqnglT1VnXov0FoFHE6ehyDX0BMpRawU1k9YGdLozmnwp7pVr+naFqVinL8/XTSDl9+PEVJr1ImzK+WAtWOw48xegrRI8QJEGFhffgzW1VT1NpQXTI6hrznsdtRTjtsNYRMy3OP8SJrGlOWcprKnz0pIvGutw1CnNf1QkacRRVlirKDIM+bzkvt7HwbK0/s9jj1FlnAtH/HeUxbnpzaumKpu+fjjj0mSnHdvf4aSnnEYeP3FIzpWpOUM6yQ319foKGO/37I8mzEv5zxud8zSedhU7Q/vPbR911OsSrb3O7xQlPkMM43M0pQkUUgipmng7HLD+WZBdezpu4hpmFgs50Qy4sWLF9zcvkMnGjsOCGPwWcxqtaHqtnTtgaH3GOlCq9ZQEC9yxmGPl44sTRimnkVRkiU5UZLStw0vX5whhaCvPHHs+ORrH/O42zHL57RtjxQa6ybyIuJ4eCBWEq0H0njN2XLFYhkoEX/wB1+wKDPyRLNaxaTJOd///hdszi44HG5ZLp5xf3+P0iBIGMw1RbHA2oGmchRZyf3uNVrk9GNMWWzoO4vWKY8Pez786HngDf9JD5RxrOk7w2KZIWyCnyKiZGA9n9N1NUkUIXPFNBhms5ymclTDA2WxwI8VUmV0zcDUGc6XF8EAPVqSPCHWgjyZB19JmiJtxJOzFY/7A4IYmXSUxQY77smTBUmSEkUaWwx4O5HInHJZMJoDXgi0nlHmBbOFBldRFAsUkjK9ZJZ7FrMYpSFNVmy3Rz65umCRX0Dcsz08MJ/PcL5BLM4wZiItcqpjTV7GdN2AcsEzU+/CMJVlEV2XUNcSpRVmAM3IopTMyp6+FcSZCOZeOpbLkKrGD5wtnnF9+zNcmjLLnyHFxOasJI5jjsdXjO2SxUaRRnO89+yra3ob3u/f/Sc/IMs1SaKI4iWzdc60XTFZhxAVCENvBubrgn3zJa/+iWc+L8lLkCp8iKM0ojr2SA/ToIlsiXUVu22Fo2E+WyHijn6yJGnK+jxUCz7sH0PLTA+je6QoZqgkJlYe1JGHx47Hxx1Cw1B7NosZ8dUTHg97FvOCm1dfcLF8QUONpUQKx9C3YOcMY1gvRLI41VrOmYbQYV7VbejmnpdYE/HwOJDoDJ08paoHrBM0hw6sJcszIhXRtjA2AoMIjToGimJOXR+QEryN8ZPCSgtChq7oLgwSx37CSRVSt2YiyyP6YSJOY6q2Jk9zZkmONgnKZ/RuwOGIIknf13giNEuKssMRBTpCP6AiT109hISkPdA2O3RkyZKStppOq6bAoURMRPGSpt8Rq5K+FpRpQZbnDGPHBx+BGedkeYy3OX0rmc0K7rcSM40gesb+wAfPvkGS5WRJypefveHqas3tdo+1EZvNmi+++AKEI400xob3/fbuju3uwOZJQje2WJuyO96Sziz321uEslxcfkLdPGKtY7dv8E6wXJ5hJs/tzY7Z3HO+uOSwr3DWk2YRx/2esiyJswQtU97c/ex0wXNMo0SJmDyP6fqWONaM44BQPfkso+tAyz2Xlyuu39wSpzP+tX/1f4IST7h9+AnxquB4PHJ9/T3+8r/wG7jYMe0iSGqyzPN49yUff3TGxeYCZzxinJCJp2oth/vPWZWSPN1Qtw2Fi4hXKY/bay4jxfnTD96fib0UeGMRk4JUsL27RSWOojjHWIkTgAlry7auWRRztHhOrCRjoxkyg3I9tv0Ry80FvRWovsX0ljSbaFqFGI+QLZEqYhw9DgU4hG1JZc1eJchizVgfUUkKQiMzzXH3gLITx27OpRYwWgZGsuWK+vNrXLsl17+E8SnG3qM9SAHbQVAkj7y4+iaH3jHZjkxs8LqlG1qk77k8Uxy6kno8oKeY3vRMwNjsuVh7vFFUVjEMexa+QCQx98dHssjwfFPS6Jx22JOKmDGegxnIZYurOnpbEs/WxK2lNgaX5rj7A4vUEscKdMLoO+I4JSPDNBHV9sjF+YLqcOTs+QuefvJt/t+/9/v82i9+h2masKZBqxwrNM4ErmMaSSbV04011niiKAGpmWV5WDEbw2TDqm+yE845xnYk0imIcJk0Y2je8i7gwb7qi4rTiDTPg2qpTjDw0WNdD04hsZgxeMm1UpjJghdhxS0dWVqgMo/3UfCT6wgzlcgoRrjsfeOVcw4zGaQPwRYnAzZJSk0zNFj8ezXWi1BnHOkEIQRZvADhSBJNUa64ePIRxoTXWK4CiNx7e1r7T/RTC8LR9COJjujrUNBw3O958eIF9aEliiy6OHltx9B69bjb4oylKAIM/nBoqJuW1WqD95ZymQY0U5IyjQZ7rEnjGWmkSVTP+Uqzr45EMiYtM+pYUpYl52dzmqYhkRHKeryHPMsY+0dyteL24ZZIF/SVYWwMkpTJKMzkaER4hq/OSx72W7zNMQISFbZly3lGrDUvnj1n39xyvi6IVIw1Ct1aNi+e0AwxdnCcr6/o3J6qGsgTxeVFyTQYPv34I+5395jeMIrQWIMAAQAASURBVGpHNEsw+4hEQDzrGQfF+fyC9dOOZKEZ7I7nLzYMTcVqeYmQiqq5J4486+Un1P090ZAyK2Lu7q9JszXWtXz08py8mPH69Vs+/dY36bodzX3HMq94fPdINl/zm7/0dX7y4y+Zzy6p/UC17/BSkC8juv4N87MVbgCbjNTDnqzIqA8jm8uEL998hhvTP/mB8rDb4xyUs5K+3RPFnijOyNMlOkpIs3CTnKYJKSTlvKfvc5SWTIPH2gD4nKU5eZxS1zWx1ihrKaKEyY7EaYBh18eJoT+Qxp6XHzyjqQfQhsuLK7wbkcqTxBFSBsyFGwVpIknTJzgrg/fL9/R1wotnz9nt71ByZF7ESJEQKYF3hmlwbFZrpHLstw0qtkwjeDcipMXbgKAZx5FUZRwedvSd5+wiYxwUnoa+GuiOIZ09K5YI5UBYJqcQzKib4A+NXUbXV1jjkVIxTYDXrBYznl5+yth2HI6PCOFJVhv2uy0qMpyfRzhfMo4Tw2A4HAZA0zcxm4tAxa+aimFs2e5TzCSYzQseHkOqME4tXa2RseHrn54zjQZnA+Nsc5Gw2z1yrGLyPA3omXHP2XJDvhA8PvbIuGcaJdaPYODxODKbrei6ht2xIk0KohheZIuwHspatvctRXZGuQoHcbnMOTzWVA8GlUT0ZiKbfY3Oh+HBO9jVDcVcME0erCLOAAd2Etzf3hBHYY3Tti0IE1YxfUoSxQxTxePD8VSRqFivnrLf76mOLVEsUcrT9wYpPV4T+IBdh0Az9CPCnvA/qQYU49igkxyHJYkEVjj63pCkGiscwzSgEn1SbSaklIy2oSxLLAZjgn9wNIYsSznW74jlgiiJwg0/NswXCeM4EUfpyWdT4V1AtETpqZcbAE+SpJRzgCXCJcRxQqQD/282m5GmZxzdA1I6+qHCeU03jDg/EEUx42R5uIeuf8c0htumsxOjG7h7qFFxwt3bPzhdHFO6oyNL59zuK6yv0ZGl7xbkc4kQMWn0hGO1o+sfkMoyS5/jraOpH1E6AZdQH78kTj3TNFEdIx5vH08A+IS+74mjgidnH3L/cMswdaznC6T2TM5je8PZpqTrK9bzjGny7I4j1gqEVSyXMVl0zhdffJ/f+NU/yy98+z/L2GX03SuKWYmJ4ObdgZefFPzSL/xpzJiACtxRbyV3d6/I85TN6jlt2yKED4GS6shqoVkuy9D8Mli6zhCtVuzurjkeaj795Z9Dfr9i8BlrEU7TtFuePXtGmhZY12NN6Pcex5662XFxlTGbzagPB3IVY7ynrVqKMiFfzGi8BOuItUJHEY+HLbvtHS++9jFCB06uEKew4X7Hu+vPyWZzVqtNwNsQ4dwEbsn9w1sQE8vynCTW1F2H1AlmMDw8XnN2tiLLQvOR8ISgg5Xs95/zbJ3RDw6ZxiijIQuK1MOD4epqiXQ1kSrx7oB1I1YYmmpgluekUc7tcaTHIYSha45EKqEfWs4TjxUpbR8sLToN1bR1fcRhkElO5mYYISGKmGUZWkVU5pHzQtANPYkcyFWGsooRi0gti6xgniXsrKGqO779nU/ouh1/8Ic/5Dd+/TvYIUU4mOwBGWkyIYmEoxsmpEpxSIZ+RHvPJH9e66l16KiOtUJqTewjJgwgcEKhdIz0hHAVAiFCh/zY9IxdQMztD/csFquTPUiitSJJMqZpel//mKTBFxonMaM1gZcJp3W3AzQq1u+ZmIpTVapSCBn+PSFD9WhZBlSOlBIZ6ffVq19Vsb6vRrX2pKJ7jHNMY88wDIzeM89LFAprPTorESesUhQpJOIUUpqYzV9w9YGjqo4kq+AnZ5J4J2mnAe8CHzRNYw6HA7v9ltWiYBpGjvtH8jTDSB8yEsYgjcdHHVfn60AVQIWZo1nRDyE08/Rpyd2NYFGu6A5vqLY1l+snTH1Cmg9Il+OtIyXjuN2Tq4iEiChyNMd7ojJHdTk+Nrz6yWvOz8+5uLgg8ooffvY9jHEsiw22nfjxH/0QtOXq6opZOadYrlmeSa4fe0gUv/yr30CYhP3+mvTDKKAS7x7ZuYlFCgc/cXaxIYvO6GjwZcRqViLkknpoQO5Znc94vN2zmb2Eoca6Aw/3jo8++g52anFiy9AeiHXG1z+6oG0mnsefYGzLuvw2Q3xkvgm4s8vzD/nZZwN/8S/9Ng+Hz7i6/BDJgq99/QPS+Jzbu1d0g6apHvj4m1dUR4kWMdImFAUIOTFblxx2j2zOz5nNc+LUYKboT36gLIuILC5pj23oOU5yytmSvg+U+nHy7Hcty2WJM56r52vsANfX1xTFmqdPrtjvt6c6JMU4BjZZURREkeZQ1xg7kegElxmUTABJHud07oazxYqhG+m7nsWsQKuYu7sH8IKz5RWrM8XQad6+26LTI0msOD97yjhVxEoS64Q41UgR87i9Jcsl3kW4UVAPNTpJONYVWkuUFPR9YPcdDxVxqRlNhEwi5nlCM90jZAp6pK33lPmGsb1GSUXXg/OKpj3iGamqig+ef8R8VmAmgXFHxl4iIhinPbc3Ds9AHJUBbqqhrm5xcuDDD19Q5At21SNtA0k2Jy0ytvtHsiTH8kiRzSmXGduHCi8lo99TtQ6pLJOvUCbHy5ayjMlLj5lUaBAaW7q+Jk4UH202GJ+y3x0RegApiOKYzdklSkOiL3G6Zxg68t7QNgPL1Ya67kLvshX84R/+AfP5HKknlMvp2oq40Bgzsb0/IPwc52qaW0leDMxnMyZjiIoMiEN5va3RScnQGiJpQg92kSLlQKpWWNGHwJAJ4FipR4yvQXi224ZyljCOmq41RLEgK3IOhwNxEgbXtEgoioxxnHDO4xzM50uUTOm6ms1mzeHYkiQ5T5484eb2FVEmcYNlvVnQti2Puy2zWckwhAR8CODU5GVC3exwVpLnBQgVfJFDAyJBuGCKHjtLnq0x44idBO1gMeNInGq8zVEyITUjQvSAQGmNjiKMsUyTxfuOdhBcXjxjVixpqyOPD0fwBe3xGBKrztAPDc4Z9lVLnudcnn/IZA03dz8iTTXephyqe5z15GKBpiROwkMriWK8DQ+gxXyDdaf2n60nigV3tw/EieJs85Trt69IxBGc5Osf/Sl+9vmPWCwzjvst2s/JVE4UGZSdUOLkETMDz59fYaYHzLQlTRYU8yXbwwNpFJMsk8D9nDwSS55qVpsFTdtTznLy5ClfvPpdfuvX/wKx/0X+z//Hf4+/8V//t2hMG8oN0jVfPn5GPx6YL875/7D3Z8+Wbdl9HvbNufpm983pM082NzNvWz1QKABVBAESBEn5QZZCZIQjFApbCtvh5g/xs+0XhR8sW4JskxIfKNmkQYVJAkRTqPbeul02J/P0Z/d7r36tOacfVuJCfnBYES691XzKiHsy4+bJs+caa4zx+74szWlMhe9HJLsUSxbcP3iM58YoVaKbBt/r8ubFz3BFTRQdk1U1CIHlWAhh2C5u8eIY5F+rFx1LUDcapEVZKN6c/YLBuP18FXUbVLClzWazoqgW1FUPpEBUDVgOtuWy3SQEgUNSO1RKEfsOlaqxlcfi7go/sHDcKULaFOkOO2rQOuDy/AojMvrdp7hOxGq3bhmzEoqiJCtXFEVJrzskSdeta7sW2Ahmizd0enEbUjQ1QhjKuqDrj3Ecm3S5weq/g+todO5QmxbxU6sVtl7huntE7oBlknyF6imLisNRiCUklhWgGzBVBdJC1UW7rtAsucXFGVoo1ZDnO4QT4bkuWbMhVxaWF2FrQe2BNAZPuqzYsljNCToPkcpgZI3tOginzya5xBIbbKvAtk7xQ4/Nastv/9YP+KM/+hf8/JNXfONrj2iqHVEToaVFXhXgwnDUZ7utsGVbUFsYatp9xSytsIxuPfTSplYN0raI3wLjjbDejnwdHFsgMGBan73tOLiOA0YyGk3ehoAUUkLVtOxmtCbLEoC3el/Z7ph6Lk1ZYAcurtc+nlWjkMLQoJHSoaxrfNdtwyZvx/JKqRZ9o2pct93LVEohMai3TFCl67bL6jhYUrYYI6VwLAvL9yGKqFRBVbV3W+x5SNlyeG3HajuytoXv95GO3f6dbAfLdhHCIktzkt3uLeezDau13u4Sx7UYH9wnsOy3qxEFZZbSJAlVmRKGIa5lE4Sd9mel1tSVYrfbYds2o9GIbJcwS9vEteOVfOvXPmK93FCWJbc3a+q6ZH88pahWnBw+a01elodWNsfHQ16/fk1gB1ycvSLqnvL+33zAv/7TH/L5jz/jm9++x6PTR4w6A+6uz3Bth+HQxfY0qszxQp/Iihl29ki3Z+wdDvAcC6TFYLiHH/ooe8uSAcqXDCLB/YOYynKo8grbH3K3uOP+w+8QuxWXr18R9iJ2u2uGdpdIdnAGcP1lgTAxn3/+Kc+efkAQnHD26pxe0EOVhm6/IHAnpNmW3XaO7fiY2mP/YMLr8y9wHAff92ElEMbm8HDCdlXx5NF7hFHNR9EzlsuCs8uXWMrw7tMTXn5xzqN3HrFLb8lTw+m9E7qdAYvlFcO9AEuGv/yC8v7+iGRXYnkx3qCD0YIiV0gKTk/HSO3RjzZk2Zq9/WOO9ve4vrrjZO/grSVhh7TLtoVewN7+ENuRFGWJkRX9nqQqYlzbw+tqpMixpMtuu+R4OmUy6JClG6LDKWVZYruCw2EXKSXr1Zwg6GLJhgcPW6yMFA5luSUKXXy7HWH7bttm78T3sB2fokxJsxV1o6nUmigMmI73yLKCylPcztYYnWELF89raGpBUczo93wO9qfc3t7y7fcf8YuPX/Po2UPiuMv17QVBx+X2OqAbP+CLLz4nS264vb6h1+vT6QlWq4r79+8hbZvtWtEoi7R4w3BwjNIZnhOx2SW8eP6a8fgJP/zxz5jsO3SiKUl+het0kH7W4gqEAVEinBIlU4aTAUrBcBQjrRDHcZjPWtVbC271GAz6vDr7HHAQxuX6+pK0sAiCgNV6xUZk6EZgOzAa9dlt1+zyAssS7HbpWzRDm0wM44iibBgMJjS1IVm5uJ5gl+5g3ZoOMB4FNzjawXE9SiG5LTOkBDs1OJWioE3j+f6WwLfJ0hKtafevlMsmO6ffH5FlNdIW1KpAYLFardnbOyDuSIoiw7EDpCXYJgndbhfHEyTplsgbIx1DXmYILCzbwrUchGzxTz4u88XqK0PG3bxNa69WK2wbtErIsgzXdgjDiJD2oizLEoSkqEp8L0Y4bsuHpGG9WqA1OHaI6xXYlkNdtngQ12tHjMooyrKmrCrqeke/v49ShqZJkEKijWG33ZI6DlLGCByKMuH6+pamzomDmH5nynw+R9qGKOhSVm0oqFGCTjSkrg22I7m6mbG/d4gWmu26wVIKz9Gk2ztOjx7QNBUvX7/k6DikrjMGgxCMg0UAJsPYdjsOlAKDoswLBt09bCmwjSLdLrh3NKXbGXGyf8J6c4st2gdPbnLisA8ovO4AV4IV2QjRw3E6zHcbbNcl7HTYrlJq1RD32uBPtz/Ety2OjlMC/4h/9a/+NT/47r/D6cHv8R/+h/8+/5P/6N8n6tjcXpQEVgdb+Ly5+AxExGBwQFat230zSzO/u+Xy8ktc7wnmnqFUObaxUVaN/qsusxpTlxobTZlXWFlCUa2I+mMG0+Ov7sS6KtFaIX2XYlfQFEuMOnrLOWy7QJaw2ewSkCm+e0qlDbZojUW2F5NvNmhvQ+306EUO6Ib0bWggSe5wKLGsHsvlktCxKIqaKtuSZjt6fZ84nL5l+YLWCiFssnxDUa042H/QMnNR7cuzUJjakBQLtK6IohHz7ap9ENg2daW5vn6Fvy0ZvnOPqs4pmg2irunLAdeXL3DESzx/D+OVrcnFVFRFhS0ks9vnHI4MWRW/tZZYpElFGOSsF3N6kxRl+xRp0n7OLInvQpPXjKYejQSlILBABgG6aqA2uH7N/ftTlDlA1QId2CANtlWjCsVwGCPtHYG0AIlt2SRJyt/5u3+Hf/SH/4hPPzvjvadTBt2I5eoWPE0hGsoyo98fU2QZ2pLgxAxVTaMNQS8mSbZ4odfibXwHRwrKxpCVGQgLN2xHgbZvIS359qdCYjDYrk1TK1zLxbLaYXgQxBjhIOFtoEd+taPZoqlAG6jsCse1MapFKmkapLAIHIHj2l8Vk3+Fc/qrsXZLXmkT9nmeA+2eZ9uxbL/WdW2kbIM/f3XXOZbdJqilZOL3AI1jtzuarmVRVVWr6/UsjPFompoo9EjzkqasyOqUugTp+KS7hDjutsEQO6Sua3qjCWXdJui1qukNezhS0OmGWFIjpSDNdm0hLQAp2vWmdMdgUpKlG9J8Sdh1CYTNfHmFSnsMhxaNVvi+zwcfPaCuFIN+h7wYUteKwLOIoy77k2OWq1sGvfcY+ZLio8fEvkNa1/z2b77HfHXH0ZNT7u7uEMrw/rNf5+LVOePxmMGoy5vLK7SSxKFPkpaMwg7peUZuwcnxHkYX3F6saFwHL7aY7h1i8pzVTYYyDZQ2nWGfKAhJ31wyvNehTi2Wdc7+ybtcvrlGdnxevHqJFEcs849bJujlhAenIzStbU3IF1jikKvrczrhBMddEndrHNvl7HnGyaMhb15dtKpOo4n8PcJQUmUhi8UlJ4ePSDZbpuMOVV1jH0kcW/D00TNM5WIhGA19HGljGgfPCekNA6TwfvkF5fF0hBk7bLeaTXJJb9BjcVcy2u+jKPAtm8lwCOaAKAgpC8O0H9Pp7OHYBmUaLi8bsGK6/R6e57DLUixbEQQBHgG9vs96c0fT2LhWj7xIGA87TMZ7rO4S7p+OiP0utTZk5RKtBHt7B9zeKcLeEWmaEnccykJyc7Wi2wvwPI9XL2/o9vust3MsPFarGZbTMBiNcP0QRYNPiFYORbWiKhNOH+zjBZr++AAtG5LNhr3pIXmRsL+/T7Jt+Pa3vsbt3QXP3tvDCwWCil/77gfM5rccTCfMbzN+67tf4252TlIqTk4H3N0kPH2vT1luyTKF41d4dMHtUTcNYRxwe31Df1JTqYzz2x9y8qDPennHZC9jcjTgZz++oD+acnh4QFX41GpNqc945/H7FJmgEw9ZLTOGo4i4a9HvZ4wHh9zNXuP6Hut1uydqWwGffPIKAN/xubm6fHsZCO4WO3p9lzyvuLm9xnNDgk6v7YI0JdPpkCxLcC1Dkpfku4ay2uHaHRzpU+UCY2yKphXY22WMlAWOX+IYnyZNcJ0RaZORkVM3c0J3QFPllCagqUDpAsu2aeocjMFo0e5PBiFNU6GUwXM7qKZd+LYtH6UUUccFEVEUGQjFeLJPXSlsD6qqoqkNNu3FWilFmmdoVb9dnG85aJUp0Ar8IMSxLXSpmQzGVHWNMG06UmkDxiaK/RZirQVGaTzfoijaQtG23q4hRHssFwlaS3odjzRZU6mGqtRMxsck6YL5as4uqQgTF7Com7odUam3HRNZoDUsFglxZDMed3CtLovbFs2T5QmqbNl3B4djym1NpWq0hlcvLxgMfQbdCctVgsUWNwjIdglPH5+QLHe4rsd3vv5tyiqnqDL296fMZjPq2uDHHcqqwbIE/fGYzWbF/HbOZLSHa0nwchzbaZOpeYrjWASOZDgc8/Llc0a9kzaUU28RWqKKEOlaCGWxf3hC2dSkeUqdVTRlgxt1sTB4EvKkJm1W5NmI9e4z/v7v/s/59kf/Nv/x/+F/g+0q3n329Xb6YXlo45Cla3abKw73HxD4Q4rimqYBR1tst1uELOjF97F9i3rVJqeX6wWv33xBL6rZ6/cxokXhgKRMKhbra6YH91DS/euCsszwAx9lYD67RoqaIJy2APBcY/ltl3c+W+JFkjCcUhuJ0QZpNEYZkuSO5fqW0fGTVnXalEir7VppvcPyHLSxEaZGG4OpBbbr4fmwSwv63SM06q0lxSHwY744+wQtS7rdMUEQsdjO8F0LgUWepLw6/4T33nkfpe23VpgKWwrKsqap7lC9GMcOWOdX4Fp40mOzLJHS5fDeIcoECFFT1ym25RIFXXKx49GjHrChWlkYv8KzbbRrkycZg4HNR998xF/8tMb2Na5jqLWLKQtWiznd4TWd/a9TlC62UJS6wXJc0tWK04dT+n3B2VlJb9Dq4lwsLFujy5rvfuOIbbPly7Mu2pG4pqIoa3ZpQhRp6kLx+mzL6ENJbHKqNCOOQ8IgxBUVgx5cz2/pRR7TcZfbmw1R7NM0cxzHML9LuX//BCMKiq1GWOatUUfR6HblyLQNSjAa129DdVHcMnuLovV6J0mCZTlfFXK2tJDS+mofsi0o271MpcRbzFEbjlKi7UCWpfqqG/lXx3Vdqqr6KrkexzFCGHTz3/7aNlhkRLtO4zititUo/dYK9xburgW27SLa7Qu0I4l7XaLYw/McAml/VQyXZUlj2qR8yyfNOBiPKfKyDS/WFaFn4TmCScen1+sR9KPWWvQWD6SVaMOLWfb2bnZASpqmoatrDG3SuSxTiixFKE3op6RpycXlGYvlHbbs0u1GYBTnl1ekSclg0OPw4ACNpqwvUI3F6ckx8zRFC0lebHn54oIPPnzKyekez8/XvP/0fV6+fkNaFnztW9/m9vaOz758ieMG9LpjbFtycm9Atc1p0or9oxG2CNhsLnh1/SXX1zmhG3N7WTPqjzCmy9G4T5M3lAYmvREjmfD67II4HJJXK15/9gIZGoxZM4xDXD9i2DzC94eYaoglE1x3jmpc7uYXoDK64T0Wd9f0+jZX1z9msu9w//T3yLMEP3AJfQ/kgI4zxaYBW3Dw6ABb9qiyHVmVcnTUyiuEKBFSU5cOTw+/R+javHzxGtt26e0fUtY5i3nyyy8oPTfAshxcxxDEIZ1ewP7+PmHkkBat4qvdTxzRCT2KosF1Q8KgfeOpspz94Zgg7jCfz2mU4fToPtvtlrzK2/FRXTPq7eN6AiFKtO5QVQrXtjh6uM82ueaLy09x3JCs2vLknQ/YVCUy8Hl1/pzT+4/oDzx+8clz/NDFDwXKlIT9EqwK2y9xHIGvGmy3wNhbgmhMfzJA0HBxfsf7Hz3BEoY03fHR/iOEbZEkmk43JoxsuuKANF+j6LJJ13hhxHHXZbk5wxIdhLCIgjGb7ZzTRzGD3pTyLzYcnAbEPcnd7YzpwZD5PGUw7KFFQuDG5I1DXqRYVoMbu+wdDEm3Dnm1oShyXDmg2xUcHx0z6j6kPxzghTl17nJzm/GDH/wAxxpye3tL4EZ04z7rzYybi7plVr3d5cnyHX4gcJwYdMRvfu+7zObXgKIxc6oyo9/vEPgdsnzTjm6PBem6NXD4nmL6+ADX9Vit2ovkYDphvloT+ALLKrFkSBjauH5NVUaUVUPs2hjH5vVZymQw5N5JQLI1FGqBH3tYddQuiTuCQb+DFILBoEddQZalbNYZlqPwA0NZ5kjRQtGj2CVLExzHw7U7SMuQJElLATACISV1JdCmBuyvTBib9Y4obgHBaZbhOW7LYWzawiwIw9ZokFYMBwOiwKExDY4jmS+XdKOYQjVv7RslcRhjTEmabSlLC8+NqQtDZ+jjWILXFy/ohAMQmrvZNVHUYbde4bouq/UM15KEwYCyzrFtD6MNlrRatp22CDyPqiqpipJhr+3Ml4nECkOgINnUDEcjilIR+wHb1Y7+oMfZ6yt8L2Q0PKAuNyxuFigEg34L3he1QdcWnThmt9sRBy6R57LaGKgF3bCFi9u2RMoI2/JQqqbvdxk+GJKlW+q6wnVBlxZRHJNXObPlml6vx+3ljsO9+ww6Y7KsoBtO2O42+F5IliVEfsD8eo7alThYxHGX3niA7zhoVSL9iCzT4EUkScP3f+sfcLL/61xcvub8zTX9QcTh4QG7tHmrMatZL7csV885ffL3qXU7EZGuxhEud7eXhB2f0fiYXZJihIvtKMqk5mZ5getNcAOfpM5RVY7n95nfrJitrnjywfcQ/60HuaZNNnd7fe6uz0Bqer2WpWpZIUqVmLqkqhMsS4MIKIqM0LWRwmAa06oZHRspfPJ0i5QgfIv1fAZyR1YaxqMpd+sLctUQhjFJVpCWM4Tl4Qd9iiJF2BZaNzhCsktWSKeh0xtSVi3ypi4z/GjI9eVrdvkdg/HffrseodG6QVohu3SNJWycuIcyFXWlcKIIVMN2u+D4JGBysM9mEZOVOb4jKZXElCWH+wMePQq5Wwp46WOZVrerBawWCV/77X3CvsUmTRgHkt1ui+P08KVivTznnad9ZvOMwG8gsPA9G9eJyFdzBn2bbV4SRiOUZYEpcB2LyvIokgRHKMpSooyLoETKloixWRR8++vvcHxvwh/+Z/8N84MRf+93/haf/eyHRF5AXiiEL0jrLd//3lOqtOTi7EseT+7z6vVrnuwP2Wy2HD7c5+HpIReXL9gU17z7zlPWixXbZMEg6DIddLClQGmFMhrbNEhdIo0i8lxq2yLuhFSlwaii/b6/ZRdKoRFYhEHQjqKddoytlXkbiGlQiHYSYhRCOLhuu/Ljux5KKaqiLUK1MTRVTZlnKF3j2e3um7GcNqUuHRTtzqKUEqP+ek9TiHbqoPwSq253MeM4JnADwjAkikOEMLhOa40BCOLW5CRES3TxPA8/cN+C39tUfJplaKPYbBZtIrzW7WhfNswXa4SwKPKKsq5IkgQsSV60L/1COFhO23FFtQYkZSlE0MGRLoPOu0SDp0g8hFBsdtc4TUYke9Sm4eJ2QRDapMkCx4754sVzws4Y38u/ern7N3/2Q5Zrxf5ohCUHHIwPubu95JPbT4jjmEf3TtlsE9bzW548fsRP/vINoSeZTPv87MXn7NZtGEiZiPeeTihWGZ5jY0jxIgtkTm1B5YTMCsXegeGDw28htaRMrthtDN7AZj4vOJgMyFTCcntAP4ohtAnsPXpH+2x3M4b3fo27xYL9yRGO71AWEKXf5hcf/5g4/JJvfOMHTMYzbi5vmYyOWc6uWS4Ve9P73Fwusew5tmPIFxV+1+XB6UMur3/Mdlly/+QpTW4zGNyj19tiu4Ks2iGxcez/HkbedihbiXyxZbjXpdYN3Z7dkuK1IQx9tBIICrSlkH7FcrsmydfEUQ/LralrTbZZEXU7OI5H0+Q8eLDHZrPBCTrc3NyQZmuGnSFJonDdLlW1RjgltZBklcKLQvrDEfUdpFXDze05RbYl8C2u7l6w3AQEHYcodri+ucF1XaKeAbnh4Pgdkt2avcMD4mhEXec4XtMaBZyAXzu6j+1pNqstSrjkZc1uucKWfdI0wTQWTZ1xdDxG0uH86gWD/j62FOxPnnE3n/Pq4jP29vd5dPiQ7SpjvU158sEDkqJhtVrx9a9/nazYANAb+EgZAoZJcIqwa9bba7yg02opiy4fvv+EL1/8hKenhyhdUpWKMDDkWWsxCDybe0f30IXHutgSuhMavSboOOx5R7x5c04QRNRlQl23IZRa19hWQBRGlGXB/dNDlvOc7333gCyvmc2vyPwtQbyP50ZoscSSFRIHy4Ug9thtczr9DhN/wuwuodsXhGHYBqh0yYPTEcpsEaZLXUleXJ7xwf0+/6v/0fv85Y9+TiamfHq2YL/f497BkJtVzmbRdmf6/RFNVXN4uM9qtWE49vHCijAacTuT3M1nRFGIJGrZYoFAa4GQGt8PmC3uiGNwPQm63e0JI4v1ctOCsF2P8TBmsbxttVw17LYrel2FYxs8L2zTzl7MJi/YLncs6ho/CnB9j7KqSGlxMdv1lr2DfTA2u/Uazw/I0oLdcsN0OkXohkaXb1OOhl63R1Nq8sQiDvdI0iWWgJwS1w/xQx8/FW0B8nYZvt/tcusUdKMpczXn8LCPUXB1dUVZeFSl5uBgDyFrVFO21hbVsFtvGXXGJFlNupvhWC5x5GGkYbfb0uv1cI3Pdp4Q+C6PHzxml2wZj4f43rB9iDUwGY7Ja43jpghjoZG4QQeta0y9putPCCOHqqrxA4fQ7b5Vpd2yP51itEVVbcFY7NYFruvQiXwEremkyA2h67LLUvJdRa8XI9E0taYRDYPemHU15/f/zr/NtP8dzi8/Z9Ad8OXLT7h//z6B32e+ukIpxaDncnnxGqV2TEePyfMdILCFpCgqFps39HpdjLQpGoV42wFa384ZTlwG+/to0WDqBoygMZr15pZ7Dw6IgiGqLv8/7kXn7fjxbv6a6f6UKO4BNUoXCBRpUpDlM/aGQ6TjYVSF7YSIRrPZrFnvrjg5edCaNkxFpWo8bbOY3bHYXLM/fkaWZdiWRfU2Mb7ZrLi5e87BdEIU9lgXM6QVYNuaIs3Ybq/wPIfB+IBaNTi2j6rSNvxgKn7/7/0NhvEx23RLWZZIS7d7b5S8/6GHZEpWpUjj4GIjHQuUxWK2JG/WdMMHuLZEKbAsQ6UU477L8f0x/+rPnyMYIzHURuOFHmHikZaf8cOfOJTFfdJkhzENwim5uJjz/d9+l+9+9x7/yf/tFZ1YUqsKUWmE9BDCtOEE2cOyI6SwcByfot6S65z9kYvLkOUuI+hW5GUBVmvSur2Zc9zbcO9owg/+5g/4p//FH/Hetx7y3g8+4sXHP2YwHXGxvCIcRgSjPov15/SnEdFA86x/RCfuc3fnEHo+0k55+vSUySDg/skhX36Z4B9MyfOSbiXwHIFt+yhteHwy4NGH97m8vmG7q/GCGMv1qFxBWSikbBE/vt8GPqHt8kHNdlN+1a0UWNSqDQEp5FsIu2oNL17rg7YdCymdr7qMFgLXdRHC0KiaqihbvnC7tklZV6i6xEiJ7XkMei3jsLX3GARtaK8sS5qiIq8FeVqwXe8YDHtUboHWNZ5vvdWAGrabgiJTFEVNWf7VOL998fJ9/y1Tt+3quiJC/hXD+m1R7TgOo16XvdEeWb5tqQhpQdMYHNtjqWuqRpFnO7TtUlRbwiCmako836ZpSqRw6MTHBJbT0hXQINqXpU58vyVrIPB9SVNuub1ZoasU25HsH3WIQ8jKK3Szj+84LNZbPM9jOBzj2CG9aEy+LXj0YJ/56oa8qnG9iMFEY7kO+8fvMB0H1EpTNxk6N2ySiji2CYKG2rJ4+s5DxErx2YsvcWMfPBCBy/H9fRyZYokGOy3Ze3CP6/kdeb7EMRFGhRwOD8iyjOP9iNi36PWecXt3w4P3j3n6zjN+/vlP+dM//a8Io4Dp5ABThzhWQuArdpsZSJdH7xzx2adfkmUFbmR4ffYl19cXRM49Qm9AVt1xfvMZruuy2aYMJkOubs/pDWL+ux75//tLfnV+dX51fnV+dX51fnV+dX51fnX+v59fFZS/Or86vzq/Or86vzq/Or86vzr/f53/ziNv7bYpVp1VzNcJfuRzdXfBbHbL5GCMR6tP9MOKu9s7lC4RykfaJat0hnQMdVEzGIzZZBviuMN2vSYrdu3ScbEFO8PxSpJ828LEdwuE29Dp9zm/mqFKG9drsTDT0SMWiyuqasH+4RQpTcv/8wZgFFWdMp3uY4TGtsdoLbGk5vGjJ9zeXWI7NfP5Ds/tEkYOti25uLxpIeCrAtcxZHrGYOoxv71C+j5u7CEbzZvbGZo32K7H9fKKKApIVyucUOF4NkoH3M0KlK6oig3kiroyhJGL0g1VnXJwOKbINNoUDIcDVJ2x2y7ZOxqQ7rpU5QV7exb5bsvR+Cm2l7Nd9yiLW7pjSZGndKOHZPmKJCnQZk0cHHB5e0bU1TSNS12VOI6HwMJ1AvIqp24q6trjycMPeX3+Gav1NZ1sTODutbutTclylXLyoEsUnPL5Z89xA0le1DRViuva6CRjudowngy4vL5CmJjesIPngyGgUQVJMacXT946bzX39ib0BzWNXfEP/4O/wx/+56/xJByfHmNUDZQo7SCtkuubM3ynz8ef/AwhNN2oR5pUQMluWxL6I+pKUVVzfD/CtmLScg2OoqwaxpM+jUqpirrd54xsLGFj4WBLD9+J8X2fhVpglKQTdSnLEs/xyfIVOq0Z74+4u72j1+3QFIpc1dgyZLW8I/A9dKPRTcV4MGS73pEmObtdyqOHT9gVC6oqZ73c4LoS25H0B8dkSUFV5QjZctwkkjDyCAOPMrdwXJesSFFNy7Mz2uDYLlWZohwXx/fZ24uoqxt8Z8rx4ZSi3HF4eIDWEj8I6HUN19fXTCat6ccYl6h0iAY9HKG4uVzT6w3o7Ek2u4zR4JDe/Ygyy1nOlhwcHCGNxpWCoOMT+wFGWxhVkO40oWvRqBmu7WOaHr7TQxtDlqZEwRCUwbEltWrwvbe7uFlCXuwQODQ6o+tN2K0rthuN0C0OKYoDXEeTmho/iNCmdYmHbkRaZfzGr/1DusF9zs6/YDw65tXLT8nSkr/xg9+nqjdYUoMNZQp5Nuf+0WMCP265pjJHMGSxmHG3eMHJ0a8jbRdtWmWnaiJWdzdUaotwHYwAoWwaoylUzmJ7QRRDrzeiyP96QV1aLgpDsdtSqxwZdBCW00KmdYErHdIkJy+WuPYBvu+jdU2a7ogcjyxJSMsVTfOIvGywUUhpoRpDkSXkxZbJ9ASjG1RVY9ntHttut0GTUpVDhHRQKNDQNBpTNizW53Q6DbbToWxyhDBI4SGk4uz1c0znjOm7fwONoq5LHLfFyuySJd/7wbd4/bOQ87TARbCZzZBxTJ7dMTxYU5YCxwnItnmbZrZtjNFs0zWff3mDUhOUqBBNjRQueVEiheG9D/v8+Z+CalqntlEVtVxTVA2ul/Pxz36BaYYkuy39bguorJrW4jOdDPji0sLBwbcbtCWRVoBIHRw1pxIhjTdCFhaOrTBKoVSJlDnvvzfl6s0tw95v8MG3XvK//d/9E/6n/7NfZ3oyJHJ9TNBhsneILg17hwMWtynKzpiM9uhGU+Juj2HPZTGfsV0n7O8d0jQNg14H2zI4jsJd1kgLqqrF/xyNfdKwYuNlqMrgBgrLAxB0wgFZ3q4rKVUQxy51XdIfdlBKsX8wfgsub9PVWVqgTKuFrJWirpuv0uFA+2vdqgCbpmnZzha4rgM4iDiiaTQCCUh007InlfkrFqVmu16xMe2IXDcO0mrH1hiJUhrzFntkWRLH8vDDANe16XTDr8bbtuPRcQUdWdLU7Yi9LCqULiiyog0wNQ3GSmlUhdBtGBfderiLuh25W3jYjqTXG2DbkrrJGe8fk2Q78irHpJpGOeS5Q84GrRwsMaLRG7A0SoJtVSDdtwi0ANuKGY1kK06JLYTdwxdNa3tSEk9WpFVGsl5yPS9B+Oydvo/vu9xskjYQKgxlnXE8ntAZHKEbQ2SPmM9LnE4bhvr052dM9o4QkWZ9e8Ogf8De4T0+/eQznrz7lNkyhyyjNylJkhrfC3G8PndXG754cUEQD7g/HtLUJUHgIWRNELXfIztMCCwPYwSb7QVQ4NgalQ0wFIw6J4RPDtmlKXvTPrtlgdBDHj0KuLq6QmDzi49fEgY247FPGIeUhebXv/1bXF9fM5u/odE76trFsQMuLlfYgeTk/hE///nHv/yCssoVRbYgjByMNqzXt+wdHNKJT0mTAuEYtrs5q11NrzukLDyKvKQ2ukV/hCFx1GezS8jLJaOj93CbCD+WCGzuFldIKRnvPeD6aoYdgt+L2G0zbuYLilrj9QSuPaLX7XMzuyOOBnR6No4Lnj0iywp22ZoHD084O9tycLDH2dkZ8aCizkP6g4Af/eJfMh4dMr9bc//BI16+uCSpHRzjEfY16+ScnBLp9MHkLNcFftRD4dFYFavtmjjqI6VDluQoXZGkNbWRdMMBDgVlc8N6ViGEheM1xNEAy271c7q0GHVdXHuI3fXZpvP2AhSSo8P3SIoFTrBkdTsj8LtUjWEw9NFFl+nApxtOWC5nGOFSJhndYMAm/xwRuyzXnxOHA1bbBf2xZH6XU+Yp6yhGN4owchkMxvz8k5+9tR5FFDk0ZcnKesn11ZwwjlnME7abkjDMWsboLMcRPp24JnCmJOmGYd+jSFK6/pDldoYlBmyXCsuu8eyQ5e0NFB0alQOgRMF6dco//i8z/uP/4z/H8jS94QFJolkuN+hGY0sXU1lUVOg6oRMP2GULdtsdQriUeYPRkiTfUVYOOLBJZmjt0w0NSockSUVdV3SiHnm+QIqEwBtT1ZBn4AU12qxZbCvcwGd51/DOk32ePv4Onz//hNvZhsHEZbXdgmiom5S6FFh2Q2A5SPeAuknwPYd+36GpLWLfh/0+y9ucIl/RG7hkqeTBwyOC0GWzS0ibBsc3rGcLpoN9tNbczl7T7e0zHPZZLySDASyXBlYNgrq1WwiFIxpc22G3u6LbDWmaACUbkvyaXq9DZ9hnN0/QTc1idcXD068RRAqsNdulQxPkBLZPnuQ8eHyCg4vvhNjWSyLpMNwTiLqDIEQgsGWf9W6BdCU2Np4bsbt9yaAXUpUKVTtUVYjjaFzhYkyNb08wUmG5CegIo2IaXbOe3yKMJOjYaJO3gHvRw3Esjo6npHmBJqVWFmHkEob7OA5UOehGMEtnfOOj3+fewVM+fXnJuNejUSnnb77ge9/b4+ThCbsMSmOwmppGOWTFDH+wTxQEFFWN0S7GM6wWS6bjCX48Ji0Lai1xbU2DYJPMOD58Qte5j24MtVbYnsBoi+u717w7eoAfSKryr1PeyBpbuix3CZWc4bhPMUgaDUJ4KCVbPWlYYVQHZUTr+JUGEYXMVh+TVwnS9bGNbDWwpqHJBjSyYHw4xkgfJR1qJXAtg3Bc1rszbM9levAejWmwlINlC6TlcLFdIv2CMD4EQnS5bsNeukHlcLU5Y2gFeE5Arkr80GBUgNaGSt1x/qrkNr2HhUetGxQGCoEM4J1HU1arAYtbjbFspJugcwvXCCbTDkHPxwtucQu7DZI0ORDgeSt0PSFLG4zY0agIoxU6sTBNxt7RmOs3gk2R4kc2eV7id2OqdMXh3oTBqMN4F7FblygbMBLXdtlsFrz7O++Q5QnyqkH6LlWhaKRBZyWBlHT3JMuiz3ox5/u/9jucv/zP+Rf/5Yz/8f/6HQLHxV0vuPz8DZ1Bh8PpgNnLS0YH79A0NbfzC7JihZYHvLk+52jvMWE3YrNZ0d3vtUVM0sPyE0Dg+SHG1Fy9eo2OFKfTJzS9M66Xa4JujcozHu8f4cdjXAt+8YsLPN+nM/C5eDNjdPwR57efUBd9vI7ANzZ+N8YpHaSdY1kOVW2hTENRt0D+rKwwJkaZLZbVQekKaXeJAh9hFNpycB2JqRtSXWNqjWokoraxLEUQSRynDZNaIgRZohpDnlUY0+oupWg5nXne/j8oXVNWGfUyAy1aUYHro7RGUWBZDk2jsYRESt7udIrWRiQUQkYEQUBatPvISikWm10Ld7dyHMdhvUvwPAfHcfCCgH63w77v43kCRI1WAkxDUWToWrPZipYXWkrqWpHsCuraoagayuKW+aamLHPyymp1kIBltX+2Nq15yA86aLsl0CyTApEZHCfGdUSr8AxifvzpiiD0iGKfn5y/Igg93NwmSRKOj4+ZnsZ89tkF/qBHaiqevzjDjyI+/+wTbCmIOlBlDqtlypQYIefcLa9wLci3V2w7U3yvgy5ixn2PbboG08eyB/RGUG52PLzf7oZvdjV/+aM/4+j+KY4Xc7d6xSbJUa8tHp4+4PzNZ5y9EqRFiRtULLd3XMwKPvrgXVzhcr28I9+4ONqmzEsmkxPudm94ffGc/nBCmVfMLgSDYPzLLyjDsMN6vSYKh0gJUbhHr99p0QJ6ha40w/6Y1WZLmXn4gUOSrHHcgOFwyjot8UVKVrtgh1zPZmQ7mxja5XsjyNI52A7d0YTVdke37xB1YurK0OgVg67LZpmy2q4o6iv29x7QVD5h4HF9c0UQBAynIVmR43pdPv7FJ0SdDmUZYJRDsnWRok+SrSjKjDfnZ7ieS1Gk1NzgNccUaUivH5CsOnQHPaSl2q4VBSprjRi29JjP7+hGPfYPjimqOflOs5mvmEyH1Mpl1J+gyHFcwWx5h2Ubht0pZZESiAlNA4vVgv7IRgNd/4DNZkfTpNiO5qB7j3LbUFc76myNH8TcZQVh4NHrh0i3YbOdo3UfZVwsWbPLDK63ocoV1+cX6Cak0+/xxYvnxH6XIDhidpsRefusNlckSQxCkVQVUmqM0mi9ZToeslmnxJHAtlx8P6QsYDjqMZvdMh73qFWDxYDlcs6zx+9Q1hk0NUEYE3chdO9TlhWT6Qgp4eLyNbvtkigYUlQdgsjBFhG75IxuOGK5SBFkYEIwPo3OqTJBN+6x3ZRYTk2aLSkKiePZYKU0lYttReySBVj7FGVB4IVok7CcrbFFxHAyQJcu00nM1nNpdI4QFh2/i++7dNyS7WpGXmywpOL0/ojdLiXfGo4P76NVTaYT4mDIYn6BpMPefp8kXbDduoyHpzTOksCPGT88ZJdc0u8esJglDDsRYdCBpkZVSxrT8M6jZ4jGbx/gpebrH77P8+fP2ZZLuuIebuDQG0nEa4UtBFIKvCAgCCTj8TGGgiJP0Dpnf/8UV7osZtdEkcPx0R6PH3dJkw0Yn9AfM7wPquljGWiqkERJqizlw496/PTHTxl0FZ7ls0q29HsR6VqzSmf4keTm8grfDQmDim7cwxINnu3je3YrFBhPSZItVbNDqQTH6iI54HbxgijqoK2UYe+4DQiYO5bLitFkyt50ynI5p6hyXNfBdl3W24SyXNPtjWkai7xY4wUuRyffIV0PuLpZ0Bt41HWF0RZ1XXJ8cojvB281lTW2bIHq6+0l7zz6iFoZlK4wtC7q69szOv0A1+lgORpVSaQFm82GpLgmaNqFfqEEjgsIQ5JuMdaCuPMtGmWom78O5biWAAR3s0sWyxmu4yFoH06mMTRCs95eUlYlYdyhKAqMqLFlTFVVJNkdw9E+vjtpO1BN2SKr9NtQj3HpxH12yQbf89Ba0VQNSreGpOOjh9RNges5KK2xbZft5o5OFNLr7mNZNsb2KaoK2xKUVcrt9Qv68ddBB1RqR91IPMunrDK2K8nPfnpBf3BKQ4PUFZ7vsN0lnJzcQ1VL0q2gqnOCsEOR+y0UW/rkaUZVeDSVg+N7VPkS34soqpL+ICAMesxnZ4ThlKZR2JZDnuVMJkO++c2v8b//l/9P6rrX4rgcjXAD8iznu7+1zy65Zjk7pN/bo5EFTdPaqBqdcHUzY7MG1zmkERrH8qh1hao1Rm65WhYsVlMs12Oxzfj+3/5d/vA/+c8I/lOP3/x+xNCyOT7qkWYNy5nm2bNvUTYFlpI0dUnXicjuLhg6Ar9cktcGQ0WtIC9KwnBEPBojpMBgEFIQTB02XUnhOzRNh8O9fSJfMewXpKImu3vNwO3yzrGH8TLyHIJgiyeuuD86Jstu6Uy73G5rZJOiOz203WW7WWGJiiIH1wswxiLutC+bZaVQDaAFWXlDuvExtaGWFZEbYEmD48eUTU03CnGkQAgHVTfkCRTVAlhQZjUGG8txsD2rnTQajWksfKdDlrfyBI3CtjUCg3Q0NRm272Jpty0c37rQm6aiqtpk+26XYvkWWmuKosS2XTASKe2vsEUY662WucZxHBzX4hefvMDzPIwUeJYhCvv0+iGO3U7DgtDGc44RUmF7NcJWuEriAp5WaNGjKEuKomAkxFdSiqouSJP0K2xTmbs4vkOaKqRopSuqBo1DowyW5SEjl3We82a+YLcTDGXM5nZDVRleXL7iz34UozR4TobrCIwqmI477LZ3+J6FupT0eyG2W3G1OEdKiyzVdLoh2/UtxSuPx++4ZNWGoNyn25mQZHconfLxT2bcP3nI3WyD41rM50sGAw9dr/F9n34UELmSKofr888QpmDUe8D+nuJm/opR54DQXTK/2NLvTDkYn7DbrhAiwuiG67vXCGx64Smxa1OlWwQe/c7ol19QViqhN4jaEZGWLJZztskO17URtoUiJ81Eq0syNWWpicNDjo9OWa/XWNayZdj1Yyw7xNBQ1wYjKmaLFf1xhGVPSYqKbfa6ZUIuFf1uRKM7eJGgLFuQaX84YD4vWa8zxpMBQnhsdilKBygaXDchyUr6o0Ns20JaDbtqjTA2nX4PpRST+B7XN2dEMUhh0+0dst3UhCHE4YAyKcnTivHUI01CPNelMQ1GtwBWSYPvSQK/w9Xll1jaY9TvUqY5Wb5mMDhBSZuiar2llrRBO8S9ijIXdGIPihlGn1DVa+62Z0hp05QVND77o2Nuby9JixTfd6h1TtQRlEXG9q4gVxnj4QnScekNQm4XcwbDmNibEjmvKbMTlvUKITWBZ3P/6AnKWmB0ipCCp+88ISsSNpst+wc9su2O6fiA8Z7LarPk4aMR3U6fPBWEsWQxWzGe9IgDn7rJCPyHeG7AaHhAd9Cw27nYBy6Xr5eE1j77j/rUekbVlFSF4P7hO9RqQbJco5qCuoxxw4RY+JRZTa/Tss5uLxL6g4huv4dvj5nPFxRCsV2XLLY3HOw9YbtJCaM+UgbMZjNCO8BihyMUjlUQ+yOcTkh/GFDkNTJ0cUNNVRuqSuIHFrblURUZw4FhODhluZ2RbCWNUmyLnF7HZ9iVqKLL+RcXyKnF43vv0dQay2mYXdcMDvYQ2rA3PUSYgqvzOXGnS+D57E8kZbEmrRRlWjLunqDdHYu7NdO9Cb1Bl+7ghDCEwPU4ORwR2AF7x122f/JDbKuLEALLEuxPDwjuB2glWSxn7I+PsEX7MK+rnIfHMcf3T3n5/BVRGOLSEAVOi4QpYbde8+zxU+pmS3LxipNDH98MOTnc0QktbudrHFyS7QqjPKKgJi9Set2YLF3guArP7WKMTdw9RrgFr15/jJflGFw2q4Ju32OXzAmDAb1RgKX38fwtednasXZJRqMtgqjHbDkjCDy0stjtdgShJO4GdJ0BUg/Jsmts1+Fg/E3+8P/6T/l3/oePiDoh8+0NkT0grxOWyQUPT/eQdKhNhtIloddlmy55c/WKb3z0O20hq2ss6VLkFbezF4zGhtAfUZY5QgQIIUiSHUl1y8g8aH8u6hxpt4zT2fyaMLLoRGOqUmPbf40NqqoK1wmYLS4YD0cMe3s0qqSqClAS3w/Yphc4oU8U9kC2KKgWG2NxfvWiNYc97LWqRmNhC4esTLm8fs7+wQijnRbJpP+KQ1iz3swJ4hqtHBqVIyzQusWulcWKJEk4mgzQWlGrNuk6mkz5+c9+SmQb7h89Iy8apA1N0RBIl11ygXZfE/Ue4wYeqihQGESt0bri+dknHBw/w3FMO75UCdrkVLUEa8t3fvOEN2czDDGO2z7QpXRJV3M+fHdE6A8w6qbtzmoBb+Hu944PmS1vWG8sirKmrjRStlOtpi7pjSvWa5DEIBpQNq5jENpgOQXYGU3Vx3YcalXgSBfhSZLlBtffst4VNOIRwhgKDcN4yO/9rd/in/3zH3N09F3GX9fcLnKW2UtC5yE9AbezNwz6UzzPptc9pqlXNHmNsST9TpfL9Y4syzi5d8hmU1M3JUJohGybI/vTfYwPkbsmnAyotaAQDcHgAf0iIrddLM/Fdh3SzQ0mTbi3f59VuiGwFd1Rj9tXL+jGNovZkkX+kjiOKE0H23Vay1YZtiNu35DUDrd3F4ynRy0034R4rqFGIbBYJzmi9tDuBZbxuXpzgzE5lghpmhSBQ+i7OLZHEHpIaXB8MKbBsRwCL8RxLYzR9PUY1/eQjkVVFV8xMPOyeDuWb6hrBQi223Xb4ZSSqq7oDmK2u5o8LwjCHnXdKj2NgrJqwe6KvC34hESJiqJpx++bxRrPdzBliOGc2pQYY3DsAEt6eL6DtAWOafADFyk1nt+6uYWwqBqDagyWMFhCUGbt5Cz2Q4wxYCSe5+MEkCQJw2EHW9jkVYlqFIEfkWQpVVWhtSb0g7dQeU0UxfR6reZSNC171g08bNvG83zquqYfHtONQ7brHbXJcV2LbhSR5znN9pZaCU7uPeLFi1fc3BjyJOXsiyt8L6I/DJjNzujGe7zhCsdpMVM2bouTKyrssMODg8fYXqsync9SnrzzlHjg8sf/+s8J5CGjk5BB5xm7VU6aVBwdjBFHYz794hW9vkutBZtVSb8TUTdb8kQxGQ6ZDnu//IKy03VJ0wLVtDsVtqPwYxswSGkolKBuchwXNAW+1yf0prw6O8eIFOyG1TzHcUqMkRR5w3SvT1mW+E7IapnghwFaOaRpgevnqAbmywZpbbEFvHmzwAs0trdGaYmFZr56iSViHE9TlBuEJdlsM7IiZ9CfUqZQNQvQXdAV3U6AVi3oNfI7RL5LYRRaTajUx8T2HmUicCxNrXLmNz62M0KKgqaumU72cKwBtgjZJTeslgnCMjQ1NKqgyCVhGIJMUUWHXbZiPH7E4m5J1N2wWzRYbkq+MliW2xo4wh6pymjMDVo4JKlCyZ+R65qH736du8UZea6IezFpskabhk4Y4lkBvh3gSoHuuRhb4QLTaIB2Y/Yn91im5+z3R/T6AUVtIQk4eObRlAHSu8KxDymrlOPjY+7de5/F+hUGgWv3wHh0Y8mb2/+GwJ9C84B7JwPSbMVulxKEHpNJn7NX10wmHV6/fsPjR/uoygXxirpwCLwILwavO2S+0SSrJe8+ecr5zQIaSbYcEg8MQdhHmTXDUYTERjQ9Hr3fw7YMnThAmT12uwnrhcKRDePBiDDwMWaHrrt8+MGU8/Nzqsxmb9Jndrdmf/iQn3z8z5nu9VB1B9VsOT68h5ANr16ccXh4zOMnR1y8mZEsahzX5snjRxyMhqRpRscLWaYZ9+6P6UU+D46nFHnFer3meP+QbtdDm5S6MIDg9OGU29s7HK9pESJpD8eueHZ4yt31Ctv36N/bR3oOdZVxsD9GCs3x/h4HR1Pmtxs8RxC/8wzvao6hxf8MwpjG6RMPXZoyZdQbEXgRtm3jWDa2lJR1yr3jexTpks5wyocfPOXV2Rf0+4fMljOs2uGDDz/iW9/4GmVZstsk+Mdd9veOeSeLefHmjNv551S6QaF4+vh9dDPm8vpTarWmyFPW6xVm4vLBw6fc3Ebti2OzYTjeIy92OIEhzROirkWS3yKFjbRcimaDMhFRzyYp2s+1sTRlldDp98izCiUyPNGnZs5yt+Tf+lv/S/7T//M/5m5+xaOn99jsdnhugJAO6XLDcn3Ge94zbBmRFFc0taC0ct6cP+fDD75Gr79HkiQYIdCiotxWaLklzX08P6Ymw6Dx3A5XV59guxZxdB/XizBWDsbDdSLWmxmhP8Jz+nheSFHuvroTHcehLGt2yR1h0MWSPmXZauekLUjThE065+nR15FWSJZv8L0Iz5NkecLl5RVP3/mITidmubvEtgKkcNglS7bpinv2I+JowGJ1hee4OI7Dap0wu73gYe+UbmdCWqzwrbBFoDkhV5dnaCmYjh9SqQIjWl5mnpf4vsvBeB/XtmhESl5VSAtqlVAUK377b36T5K7L3d2uNUhZGlv6GJXxg9/9Nm5QtapIf4CWKY7jUeWSMBLk9WvWKwN0QOZEUUSWKYRx2T8MefH8As/vYHsSjIXttODtbr/EdQbM5iVOPybJUpRUCClxKJjNL5jftUicutniuEOkrVjebTm5P2A6VSTzEcrIdkWlqMC1qYqCZ+/2KJDYYghIsCHJcp688x5/+eMf8Uf/j4959uT75PJnlCpgMnXJdtf4bsns9gtcJyRLCuoywXNt4iiknp23BW0oqbKcZLNh7HlYVsu4tS1BkVWguzgevLmc4YeK2XpOGI4Zd0LuH+5zu1pRaIegv88me43BMOr32SULfvF5zagbEvUUl9clrhDki88YTT5kmVxQ5Asmw2ckZYmtfIxyOD6JSLJzsqzhw4++SZopOr0hm22CbTfoJif0H1DVO5IkI+41FKmL67Yg9N06p64M2yLHsT1U3aJ+PEeQFhusysVxY8LAYbHbYaQBJHqTUpbtS1tjNJbQOLaHlJLppNNC/uuaoqjASOxuie7GX6GRal2/fXFugenaBGjdYpTMW/akbdtEcYBt2/gmx3EsHDvE8dqXj6Jo3iokuzRKvt0d12R5gm2/LejiABBIu+2E1nWJEC22qH4rq6jrjGxjcCxJvtnRNA3aGFzXZde0kwfHKOpGYURDz3NojEYIiCKfIPDpxx2U0URRQFE12LaH0rBerymKgk7PxpIjpLERxmMyPuD03jOyLMXxJXt7XwflYYmKszefohvDaDTCD6dt8Z6WZJs1rm7QpcJ2PfK04nz2hij2SZuc8WiK58Jnn35Ksm34zneecnN7jqy6mMIwHsaMuxLL8ihrxbNHz+j29rEcm7vVa+oiJ4w8VCNRylBW619+QVkW0In7zOdzqqrCtm10rRGWg5Q+tguNzvHCgO1GEZiIm9kt0tJYjuDyYkkUO3iuxmgfW8bcXO3wfAg7HfJ1zouXP2TQn9AJT3CES5avyIsZvqcxZoCWJVXtUpc+UFBWJZETgmxNBX5U0TQCo3zqcsd6NceRA8J4QJIk+F2HxWLGg3uPOHt1zrh/j363y8/P/w2H93068Zg8NTx5eMqnn/0Cy3bpDgXL9QWj6IDttmGxWNLUBVHHRhuB6wUoo3A6AVmtieIJUegSdODmHA4PHrDbVJRFSthx8OwJiC3aNJjGYb1bYFSB7w64uYUwUHR6Do43IqtveHP3CWlioc2Kfu3he3u4dslwHPLi+Rv6vQ3D4YCyKdCNwAkquv0x61lB1IH5znD/3jE4OUbtYUVzdqucoFOw3/81Xm6e861v/jrL2ZYXb36IK0+IIp/rmwti7x6KO4Qao82ITGXt3mTYxzIeTx68R5ZVvKyek62HOJbLcGBjNff57MUZ3/zo+3z2/OfE8Zjp8Jj7D+/zIvoxtjpCHdjQuAw9OLy/z717H/Bf/df/mIePDsjzGelWM7vZEPtdHCk4PN1nvaz5kz/+M3739z/k9maFbbt8//HXWcwTpoMRo2GNaXpIoejFNo6VcXqvh1YOqhK8//gBN9cbHj8+5dn9YyTtW6anVxxMRvQHMcP+hNiLUcowGI557T3n5GSMKGo8K0dheHD4kKS8ZP9ghO/1+OFf/Ih3P3zKdrtl0p8SuT7a1IS+jy1denGHwltzdHSCsTW7pKIBNss5Lj6T/hBVVsSBJNmtmPo+gnZvCaNxLEld5GSmaotJ30UIjWVZ5GWOlDVxHPPhu9+gKm4Y9HuUicuH732H9XaG7x4jdMXt9ZbjUx/PhyDoMLuyWC40nt9+/3qdRwz6+2yyDUaGSEZ0Ro+4uPiULEl57/h9ilzx8ed/QRBEjCYdbmdviLshPeFye7OkVBmR5TAad7m5u0U3Na44opZzhGrdwQZDlqV0uh43qys68YgqF2TlHYo13/nmv4vvHvHzT/8Nj58+w3W77LINylQEHc3d7RzHhbh7COLtqM9YCBy2yR1Hx11U0+owXdfBcz3mN7fUasPDB79OpRqMsWh0inH77NI7Op2YTjR56yEWGN12/dabGaPpANfvUhY1jaq+uhMFFmW5w7IUJ8eP0VqCcJGyNQotkg1FURC4I6TwcF2J1q2p5O5ijsBif/+QJF1jjGptJ8Jltbmm24sZD0/QGqTkq6DG9c0b9vZHnN57H23aQivPc1zXZ7dLqcuMoNvBFiFG5Lx9bqOV5OrijN/57T8grTrUdQ7GpqlBiIQ8T5hdTbBVB6MMTZNhh6C1oakUga/50z//ETbP0DpHCoMlQ1RT4UftA3O76eIHDqUpSHYVvhchtIMX1lxflHheF9vWGEugVIHrSLqdjL/44afUxkJXOYG0MQKyrORgmIOA9dInjis0bYAlcG1s6TBfvWR6MkLrLnmRg1sjXYERgjxbARkOI7LGw3M1lmORC4+kTnGQ5Pmaf/R/+YT/6H/xNa71l+hc0IkGVKXH3tDj+npDmW/pdDqs1zfYlma92tComuODQyzfZrI/ZP3jc4a6VeVo02BsiRP2mW+XdMcjAjfDkw1SeNidmi9efMrxyTukaoulXE7vvUchc0K/g5v7nD7skxY56AX204cIETB6/H3OXlzwG/1DynrDdr2lKgV7Bz2SrCJJYDwdMz6+T5Y1rNMdi+0ZdhQy6N97a8WBRvYQQjCeRlxc3hLHHZrKo2leM4xgXx2QpEuqOsW2Hea3a+Kgi1GaKl9wPt+gtcZy7JZ32ygc6ZDX+i1TkzakY9v4vscuaZnLvtcGeBzLAWlhlMJyHFzR+rAbpamqBqVLXMfHtR2w28/KX4WEVFXS2T/GtiXdvo1rewhj4biSpjBU9Q6l3nrLG00UnFLXrfe9rmtsW7LZtSsrrmu3BWPTKiHLsuV/2lZbgFZVxXa7xfcDNrsc2VhoAONi2W/H+UbjOq3nfbsrmc23vFFzLBtC3yb0PXrdDpYw3Bv36MV7iECSpiWBH6NUjdKtFKLbG1OWhtLkWCbEGMX7X/s+RkuyLGMwfYLRNroWuJ6gqtpVI2NqgrEmL1PqJsfapLx5dUmvPyLPSyyr5C/+5Cd0oz793o67VOLYcHt7i9aa4ShiPNnj9dlf4jgWB9NHXF2eYVtu+1LtSaTz13feL62gTJOKNCnRyuA5LsYoiiynqQscVzHdP2I2v2qXcZ2Gszcf47gdJqNjqlLhhg2NKambQ4SEveOAN2/m+J0x2lgMuxMi/9dQunUeB96QphAcHN/n5mqGH0TIoMayDKvVirCj2ay3lPmY0wf3SDZ3bBcpo+Ex6zzl5OAeYRhze1XSjSz60SHnl19ydHhMkRtc22Ob3LJJzhnvueRJii37dPs1n335Q/wIdtsCp0wp85rr4ozA7+L7Pqtsg9IB3c6Y68sdfuBgvArX6VHqjO1ig9js0PWAoOmhVMnJyQl5KtilG4zxkXaBLay2iK3WBLZHr9O6zOvaoRFgi5jz69fEfRtpQnbbnEAG5OWKxWKH5zkU5Y7txsUKLe4fH/Hm+S13ZkFe17x6eUUY9Il6Q6TVEPiS5dLHdWvyXcPVLsHzoSw3fPn8DC+usZuQoK6Jw308a8Q2v0AIQSR9fCzyQmPHkp9/mpDnX9Id2uyfTrg7v6MfHpJtDbfXP2Hcv8dytuOdB0+5ur1jNvuSgbjPpHcCTU043MeVI4RaU4mMzz/713zvO79BWWX40wlltWN+d0fQLXAslzqpiEPFH/zdDzF4NEVrlKirHa57x2Swj+Uc0DQOl5dvOLm3j2g6DPvfpakzkvmc3fKWB5M9nh0fUlQbZndLHCvkD37w23z6/Cf4vsuo2yOK97BsQVULHD1kND5E1xo/kHz66efsTSKeTp7h+S6z2x3HBw9xKk1gFE6oWC5XPHz4EFXV7UWcXjLueUS+pFI1y9kc1/MJbZvbqwWh9MhMhi1tjKqp8vptwSKwXQfb8phOHM5eztqPrG7YP9pnMS+p6pKwV1CkA169ekUcxvzi458wmezhuwF7Bx3Obmd4jsur8ze8Oc/pj/cJvD5Z8Ro7GKPXMWl2SdxxePnqFsu3kU5GUV4Qxh5+cEhtvsQPIwyGwiw4Pjzi4uoWhAei4WaWsCuvcUOXPPeZz+ZkWU5erxgNLfJyxXprEcWtYnW1WlOrLransd0S25PM5ku+89E/YNR5nxdvfsh6s+M3Dt8H0cLhPeFS1Yr56hXH9w9w3AHb7ZaqKvHckPV6S63m7BILx+lR1RuSYk0QHzNfXCIcQ934SAlFobBdi6Y2pPkdXiSwHUFVZxjjYNmaqsrJ8g37zgGu5aFN0+6HvT2u4/Hm/Iw032LbfaRl0eQVja4JbZvNZsXp6SM8t0tTa5RWeK5ACJv5fM7eQbeVAeQ5XuCghUGpViwgjSTw+hRF1vqeBdRVQ6N3+JGH0DFC0KakLQcpYTa7ZTDwODw6xbZtSlUjLQ/f9Um2Ca9ff0roNjx4/Gs0dY0xDbYD6UYx2R/y/nvH/PmfLLGlixA2QgryTOH7Gsta04l8dqui9YOrEmErLCxsf410arSSaFMh0IRu9HaVyGM86vDD2zV+6ILIka5NVghGk5gPPzjm//5PfwG2hePYoCArFDor+I1v9+h0LRrjkBYJsb2HHagWkL/bMOzsePPKoSwPsV0LbJt0l2C0TX9o0xtE/PwnGVFHtJ2w2mDbsL6p+B/8/a+R7hr+T3/4E/6Lf+Lwwbua2/VzOsN9VFmz78UcPXyfy5tXzHYFUdTjbp7y9P0P+fLLz/j01Ut6vRFRYDOwW0uOEAJLeuR1wXpxQxi06wDLZMt0fI9aaurljjgccT27QrrQ8Qacv/4CJ/Lw7/Up72omhz52N6TMLDqThny95Kd/8gWT/pTrHZSloK77GJPy5ZsFr56/4d7xU/q9fdLbHV98+gUHp6dE9YD7Dyfk9S1ZCXm6YTSeMuwfcXeXczQUvHzxCYf7X+PDxyN26TXJVuJ0Ooz238dyXLYnW/LdjlGng6010tGUVUMQBGRZhutY1GWFRCNsgRcGNI3GcVqjD0bheU6rZkwzKt2QZyVV1bDZ5dSNIskqdkmOZ0m0tt/uaAqEEJR5imVZ7NYr4jDiRfoKjIXrSCwBUSjpdWOqUtOUDZlqR+patSKDVmP5dh/TsRj2+yjV+r9938W2bcIgIPQjyreg9jZIZNPrTdpRvmyLxyzL0FJQFQVKGZQ2KGWo63bnstfrYeya0WiAampGoxGhH+BadrvD3DSYlcFxPCwtEcbCttr94yAWdLoBfglx5LfTFVNQmQbXaSjLmiDoUQQVq127aiPdEbbTro84xsP3B3TslEflkmSXobVAq4w8W6Iqw+V8Sdw33N3uWC1rTu8f0OlFXF3tSLIK6eTMbq9QpeZg7zFGJKznW3x//5dfUApjiOOYuinZrldEcYAwmm7cJrE/+8Vf4LkBttOGJo7375NXNXm+ICtW2FaA5ZQ4btbuvQUPiP0hi9ktvYHDdlXQ7fkE3hikRVa9pNvv4siQvb0RZaloqhA/NMQR9LtD4iClyBXpTtCYim5/DykcpGUoqg1CalxfYmRCmQX0ukOKMqeqNsQ9j12ypihKktRiMrFQ5pr1MiLyTlBmzS7dMBw9pttd4zoVqtHUuaHbGTAcdHhzfs7TZ49YbS7Y7Vas05KDgxM22S2BZ2EHBeutQ5Hv2rSzcej2Qnq9Dp98+iOGgymz1SviqM/V7Wv6g4CsTqkKw7R/xN1ixnR0gB9YnJ1fE4w6JHWLExFijyDoss7O8IRANwG3l9dEccirl3cgNriM2C7mDKL75MUG2/PxnD5JCp6ryKsl42GH7XZLb9BjPDrg7PxnnL9e0Iv69HtLymLD9377Hj/8V2uiIOSjd3+DVy/uePc9gx8KKpXRj485+PAxn/78M9ZFwWg0ohsNsJyaIOwiGg8/LKmKGQ/29qFOeX615vSpT7XrkOiQg0mH0XCPvNgwnyVkyTkH+z6He8fcXWumU4fNbkdeGDART+71GE9cLt7MKAufyPGxvAK8HsP3v0Fv0OXNq0tOjvZIkozR1x6y22ZEYQ8pLV69uuN73/sIrWyU0Xz3a19nNI7Z7lbtG2slKTA8e3iK5UjW6ZbtJufwuMt3vvOI29kbbq7mGKF58HBIuZ7jdlzCaMiTxyFhbLX2CwV1HeE6JUY31LXPg8N9zi/v+M5vfIc/uv1nHO712dSQbHZYtmF/OkDrO2y7xVHk6RrhjOj1Yw6mR2zS1/huyd64x/lVhWpgtbvAi3rczmz2Dnts8ldoa8qPf/Y5AEUqwc/x7TGXi5d0eg6UAavlc/b3eqgAalvTOLDa5fSGHptqTbYLeP3mDb1+iLK3xKMO/mDMLlsSdT1Gewfc3J4xPXEZpl8jTRosJ8NyFZ2RxWYdstws6fU9VNOQZ1v8sCEIXYwpiSKHqk6pG8NvfPvfJbTepW4y1us1VQH3T/dRymBEhsWEslSk+Ws8mnY83Rhspx2N3dw9R7Fg0P+wHWMJQRQHVGXDbHGGJR2ieEClSlxPYhqLbb4iK+9wg36rcLPBaKtNei6WVPUWz3kPz/PI8xSl1Fd3orbAmJqmqbDoYIxBy1anJy2PRlWUhXq7x20QxqKqM6QIWKzPGE8jfLfXrgUUW/ygQ51nDPsRWdEjjoYUZYI2Gt91UEpwc/clfrfm6PgRWbVpXzsE1HWFVhUP35lgmqDdf3M96kZTV4p0t+UP/u53MfSRQkItKbKEft/hLk8Z7+fk9WdoPcazK0wDVdlq8R6dRlT1FSgfVbu4ToABtMpZrta8/0RSlYJGhUSBT1YW+KHPblMxHDUo1aCqCDsCywopVIZjh3jOkrpK2a0llmMhpMFUBsu2qeotjnT4+GfXZPm7DIK20MjyLUJPMBSUzTV52qUbeNS6RlUa2/HZLjOCqCIpduhmQtmk2I5ANprIi7jJLxg+dIg6a/ojn7/8y0s++Ojr5PaKYSR48+YNdZWi9iF/qzEscgdL+hij6fU7LNYVP//Fl0zGIQ/XHY6qGikEfuCz2Mywjnu4XkSDRtU2rhWR5nNqZbPLVpT5LX58wMJc0+uDLguuX78gUVtE1uHhwTFvXn2GG0eUZs77Tx7Sne5zef0ZSbLivae/Q6NS/sUf/THf/M3f4/rqC+6SG25mCV5kkxU5tcr4+OevGQyO2DueYvKSKplxuy0YD98lq2/5zgdP2e1WZOuCJk2IRzG1UvhBijY1WqX0hiFZvcWQMwoegyjJsahtieX5eKHEcSWXV+fYRYrnBaiswnEMYdAmtiWautZUqsELQrJiSxzH5FmFMC6+E6B124mvK0VRVyhlCL1OqzeNBwgDjbDo9Rx63SGe29rZPCdonzV+2wU1qnn70qOwbfk2xKXf/jeNbbc4L8tyWsOfaQCDH4TYVoDnO3ieTaNrPN8hSRJU3f7+NMmxHclqtcJzA5Rq150CJ0IIi9Z2qXF8jyTNkdJGC01WF7jSwjIWTdW8nUY42E6AIz3KpCTbrnGRZIsFCBshNbYDvvQJwl7bFS227IUxWkmENGw3M4w0GG2Tz29YOha+FxPHR2hTYIjpj0+pa8XBQ9VippSN0RZClLw8+4Sk0vRGB0z3Q+piy9mL18w2W/b3e4y7AS9fnf/yC8osy8iyHEtIXDeizhVCgC0k/W4HQcVgLHACzasXd6i6Ior71DWMOvcp1TW6HDEe99E5ZOsdnV6H8eB+iwwSNkoX5FnD8cEz3lyvKXWJNFtWm2scx2c8PGW2+oIgDEjTHENFFHuty9mVuG7rPu4OBtQVBIFHVVZs1zWht6PXmVKUGcbkLFdz4l5EURdUqkYaSVM79HoOZVHw+vJzjg8fsdndEYcD6lzi+SBdm6KomN1etf8gLz9GWhZaF1jS4vNPz9HWnIGZMp2GSB2Tl3O0aHc7ml2FMQrfmZDmd6y2G7ZJQX8kWZddirwiSQqUFVBWNW7gUmQWvd6wdV9nCUJmlM2cJLnj9J09rq40jsix5BJDQRxPyVObm9dzIs/hrPiCs4u2a3Z4ss+gP+Hq5hrHb1jNOtx/ZPP85ees1gu22zVP3/mAMofF6jmWUfyzf3JG9/AJ/njIL15+SVOuOOz2sMgZeh3eOfo6P3nzp0ymQwIvIIoESimEJdib9pjseXzx8eccHN1nEEK6rDmYDtBVTbULefTuA16e/RxpDuh3IwbdHqOVzcHkmO3a0Ht8g6o2eGIP3enR607xA4vF4g1H/X2CaIrApzec8OLNl5yePmndp0Ofcf8UVX5OfxDiyB5uYGO05Hd/7++x2l2yTRIa1bDKMtI7hdIVs9mOw8NDVtWMMOxi+dC1Qx4+fIjnebx+/QbbNaw3NU+ePCHJr3ly7yOy+pY8D5gtL+lOplA4aCfFdQSBGRJEsJglPH4y4J1nE4zJ+Qf/3u+yXRWsn3/B44djDsbvYH56RRjYb/VlgveeTZkfu/iuYDIMCVYjbBpOHo5oyBCOofcoJM9r7j0cEHctfvyjLYf7D+gPOpTFnLTIiYcPWc+3rBQcnR6iCo1eCZyOj+fVNHUOaL72/rucnT/Hj+HBySndKCQexKTlOXVuE0UDvGDDZnfHcl1ydLxHthvixg1+Z8tscU0UjdgmG6J4jONvqDLF++8+odY1i8XibUenoswTbNnlvcd/QOi+x2r3gnsHT/mzP/uS+0/6PHrnmN3mLYLLhavrW4Zjj/v3PkRRIhE0Tcvj2+yu6A99Br1DtGlHWNK4NKpBmYyDwyM8z2sLfash8ofM5xfkxZrj8AnCxDRqDcYgRcx6vcZxJYfTD8jzHCHaMfhXl6d0+OlP/pJSFwwHBxR1hWWD0i2/7/Xr1/zmD96l12m7H9JysOy2C+n4OUfHUyzRpaxKLEcjhUea3XFycsSTpz/g+k2DEBrH8ZCWpswaLKchjFyMlki3wfM8dF0TBh6r9YzJvsCmj9IlKIHWBtu1qaqMJL8h6nUoM4UwFVHokaY1ZbNi/1iSrAVl7uBGYNuCWjhU9QrHs/BtC1NMkZaPJmn5ssJlMKo5fXDEctagKp+wG1DUWxxboBrJo4c9mkZR5jZ+T1DkFWHgcLtW3H/WxbVs6kriRx6lrnBdl7IusJyG6XTK/MtVO7GpQdpbLGNaz7NU9Ec22aKLZQm0EDS1BdLG1Bnz7DVf650gRUBZJzRegDQOoqopjcX57JKDaY9nz0754z/+Y/7lfz3iD/7hM+Y31+zt7XE4OmI2u6YxUOqU1VLRj0bcnc/oDwds1hlPn3xEVc7JLtdYlsBoTVnkeE5LTTB1gxvHrLOS/OVzOtMRnX6EDlIWrzyaXU5/0qdKcqZ7Q27Kgnx2y+e/2CKajPBgSL3OGe4dEXljVssvaFLNu48/oFJ3xD2X7/3NX8cPBMH4MUVR4NsjyKEz7CGtPWQJm2RJRUZn0EMZwNhU1jWahrvlhsbcke5iJpN98tzQDRzOf/Tn9Hodjof7fP7iBb3JAZfzNbfXP+XdDz8iLxSuF5FkBXVWEkUB8WQPXWqUhjCMqcuKQmh2ZUZdFjiuRZrkBJWm0YYi2yEN2BKkrZHSpqgVltT4odeOoF2f8ThG66blZ5YNqtHk6wXbWmFZAtc2DAZjUnuD50b4gYvvW1iuQAgDnt+ujgiXsimpa00w7qB1G6wTtBpLz3eQ2qZuapKmoK5rdtsKtEQ1EtdyCfwAyxLcP5lg2WAhMAaaqvWzl2UK0kbVDdPhqB3zOxZxJ0CrkpqWL+q6LqvViqpa47sBKIUjXYpKt6xM47WJd5PgBBbprkCX0DQODTWhF1BXFb7XoWkqgrjVEId+QKMqmnyLahpqEkoqtHbxQ8O2stAmQVUtu7Q/PmGy79PUAiUEMjrm9L13UdUGo7bskiss/7+HlHfoGtJUsdhd8/jRU6rSoqpTlpslVV0gPI9CGF6/viXb+QxGCmlv8EQfS1RIv4cOU9JyBfhsk1c4VkPkONRbgx8XdMMxm82cu/WPqKoccoPxF/iWz3TaIUlmaF2xTTKkNUfImO38gsl4n2brk+d3aNMu1JvG4vpyh+dHdDsDEDvyHDy3S06NEJomA0/1OTwMMSogrzI29RajUx7ff8Rmabh37yGb3ZpGOly+nOGOJKETcLcqcaRGRg3FtmaXbDnY+5DxpKaoXCbD+6xWb5AmJwjaf2zjpdzdbsnVjLLOeefRRyQfvyDbvaD7+Hf54vNPsKjJdwZHrvEDG21cjCUIrVPmmwWL+Yz333tK3xEETx7w8voCj4qd2jIKHLxgRNxxqMUJ46+PCbwR2zdv+N5/8O9x9abip3/8j/jt3/kWZRBSIqlsw+2NoDf6iMMAnm8CPBz8sGTUeZdiUzA4Utwtd5jXOa4vOXoyZX19Q5WkfPSwz83lz9jvHnKZ/YLDocso7HC7miGiLp9/8Ypvfvu79Po7TvaH3KyuuffwEeL8EmlZFCIh293w8PGUcpmx3SR4AxfbjUi2OR+9+4BPP5vTlIKT+1OubnYoT+PGE4aWDVqwXJ9jgF6vx7Mn3+T19Ze4do9Hp0dcvDrDi31EMEFvNpxfPOfw3nv86U9/QRxl2LZDHB9zN/+C0UFAnrt0BoowUtxezTl+sEdZOXz58lOiYEhT5KT1kmKR8NGzZ/Tudbj8f51xMIJCWRidEI76CJ1zPf+s7cR3HPIyoS5A2RmWG+D4Npt0Q9PE1LHg+3/7B+gipixr6N8i7BZJA4LKSRhN38cNV9gIAqvH3kGfzWbJ4Wmf9W7NaH+f7XbNOrlitmj4xtd/hy9eXNIZBdwkCyJXMul0OV/+nM4oJuwe8MXsL/CjisvrNbbdZzzqcv/oARevr3h871t8+erPWCzv6I66pGmK5xzSHYRc39wwGLl04gOK/DXrZcNy82MCbwSixnPGXFxcMJnGNGVI5Efc7L7k9mZLVW9w/ZpOMKKsE/qTfTrBYwLvIev1G/zOkPkqod+f8bf/rb+F7Yek2wJpG2oVUOVLvvGtd1DZmDrb0dghRvkkqxTT7PjgwyeUK0GjLYRQ2AJ26Q6/W9KJHuA5fdAGozJcK2A+f0MU2owHDzBWg9Y2TaPwlSJJZliWhetEtNGWGsv+6ytztd5yeLKHsjKqqkKpEiVKXLvHbpvzjV8f8cEHx7z42MPxW/i168SsFmsePhrQ7QxJljnC0e0ISzSousH4n3J3Z4E4xHFdoELgsd3u+Oi9Y+5W5/huwC7PsN2axhh2G0PVvObw+Ovcvh5RuwrbsqGpELbH7fKOd6ISkwukF7HdbbGlaffBasl+t2F5M0YJibIUlrYRwsIgOZi66KwgKxucULXdDSpMYyPJkXVIYP2/afuvH+nWLD8Te7a34SMyM9Ln57/jT1V1dRm2YU2LHIKiOBhgRAiSMBcCBOhCEiBIkKD5C3Q1N8IIIjgy9xLFGY6GQ9fdbFtVXVXHn/PZ9Jnh3fb2fXURp4sSBEE9QM8G4ioDmUBmxrvXXmv9nmcXS18iqJCqgek2UfU1g75gNi1RHB/DsNhC0lQUNeed53u8efOKWahiuCpe6RGXOVkScNRvMTgUVC918sRB9rb8RFvrs4lm1OIaT2sT0aBAoS4VVBWkrEmyFY+fuBQbmziW6IqCnioUSoim+WhVwGDYYTVv8O7DQ+5u5vzyxZe89+IIy6+oK4lu2Ji+oO+1GY8z7CZ0bItMTZlMCippUEQLdNdD1zI03USUJXUt6bYGpE6I7+sUubYNvSQ3DMQhVlMwvUlp7bdRVRXTtWkN9oiKGFHNycscf3DG/dULBjttbLePUfjM89cEERx0PTbrOUJtYdk+qnrN5fkEv9lF1zv4Vp+MkDKWNFsaRncXq90lWH3DdCZ49sF3URWFq7e/4vb1NcPhkHkQsF4sgAabzQq/cUBYZhSbjJvZmNpsMbs4p9euaRgmL372X5LVTbx2G9+TFJlBQz0jyG/xDIM4MqmKEs2UyDjBaDq0nB3qOKJKS/aOTllMR/QPzjA0DUOVGJ6DraRYtomstqHfIhbIGrIsQdUcaplRFQp+o0EQxwhRYps6ZaWjGSqSglopiTcleQG6oaOoKmWtUOcZugzQzQa6XqKqEg2oRYGm+lhqgyKNkXqyfWDBxjRBUcB2JZpqU5UCQ3WpBNQioywFSbkN4dXVt6N6AYahoukGaV7+ungM45RalFvJiKWz2YQYhkGj4ZAmW45nWdbYhoksa0xLUpYpqlQRhYJKhW4oWIZFkiQU5bab67ountegrsttGFjqlBUINUe3wbO7gMD4dq/VLUp0bQvRL8uasjCpZYVUSjRFQ0iVvMq33wuXVu8Qv7356y8odctFKTc8ePyMKK6YzScImdNp90mzgt2mR7kx6DknHPdUijJGZBa+5xGld5iqjaCJqirs7ik8fvZDJvcJ16N73LaNYTTQFZuGtx2ZWJaFaznbRLlmksUGaTZHVbZprTBY4zZK2s0htzcjygJaXUmaVGiqTqujkiY1aTEiuF+gGJJuOyevBUUpcJxjinKE5XmUdUG+maKqGVJ1CcMAx/BRVcHV9TVxHuB3Bgij5vx8u6d2sHPIahXQa/Sx2g7vfPg7TG/ucUzJojQQSCoazCZvefbwAbOVwu34NX/v7/73+Nkvf58PPv5NLs5foak2hnfA3SdvOOi3aTR73Ml7eg2bu6sFh+8+piwyAhFweLTLcfuMJec8PnlCtN5aE5yGS11nODsNNjOVdsPk6KzP7WQFsuLv/Af/fZLCYH33J3z048doioniqWixy05bJYnueOS2QO/wTmvATrtNahmE44Cj4zbtno28XvDe4TFxVZKmOY8ePt+inzKJ1DIqZczxwz4tWnSsAarlEYVrHvWb3Lz9JTJacPcm5PDsOeQVel1zdfcVzz96Tm/4jHCSMg9/DnlJkXc5PBnwzZtPeWtlPHj8jE9//uekmw0N08C3PMogQ0gbxTBoDvaI5kve3ixJ8g2WpaJpGjfTEYt0w5PdR4RRiuEbeMU+imxgOVOW4QpT2wU1RlFVmu6QdH2BpVrc3214/Py7rDcJWXrD3vCERusUQ83pxAV6r8DWDdRSZf+4xdXFWxRXw5QOaXGHbbi4xhBFcUDEWA0VWbXQUQjSNa7WY7Fa09mRdDo7vD7f0GrGrDb3NIKMqpZAhWFoSNXnfjLGcxus15Jaf8tsuUMQgtMQJEXJeDEiz1NcR8Gzm2iWQiFuuL2P2Nt5l9VsTpIUDLpPccx9iizBVh4xHv+Uhw8fkqcqm3DKavUpnfY+by7e4PldylKyXq9pNBqkacZqGdJsa8RJSF2buM6QVqOL25TksYfneYzGVzx9Zxfb7HB3O0eqBr/9O/9Dfu9v/A9YbtZIbc7LF7/AskrK0uLTX8WIco7vtKjLgtF0xA9+6zmnZ/tcnFc4jgbo5MWWGbdc3+ErQ1StQVlVKGpJlkhOznY4GvY4X9tkUYnXytH1FovFCI0UXXaoS4Fp16RRTlGVBMkdXksjiXO6PYOyyrEsAwWD65u37Ox3MQxjmwitt/aXv7w2myXP3t3Ba+wTTGs0w/rW0lKxWE740U8eslyH1Lh4lkYttwmZIBrx49/9mKuLklUhaXoNirIEoyRPKp5/cECZ6JR1ByFCyrJGEZCXG569c0Q/2aUsSyzLohRbEPRqteSHP/wAv2FwXWVYmNRljW5YpEmE5a44PmtyO1a2gRxhIWVFlkXs7XtIAqJEw7YkVZ1RFSWm61NWEY7vcvF6iaYcI8oCkDQ7bTbzkMFeA9vXef3NAtv2UVWFhtPAMlQgo9cZ8NmvrrBNDw2JarSoRYqiCpptg/GNjSh1nJaPqFIc0yFKUjptQV1INHqUdUCZW4haYKgJUkpabRNTd1Cwt10mWVHXOULqQM3R8R4Xb+Zo1iNQJVWZo1oKy2WGqJe0GxbBdIu8++DjZ9z+l5f86R99xf/of/JbzO5fkFYptrGHzCosTcVwbKRRkm8yHEtHlxVCLdAdHa+vf8s7FSga3N+8xTv+kE24oeOmRFGE6zeJo0uWoUJUxaipydmT9zB0iVWbmK6HrRQMOt/Hbevcvkox/TY1CV999hXNfh9ZGbzMCgzDQJVTCmVJmVQIAS9e3NDeWZF1N4Sza1rNLlrjIdNXP8XyHHqDIfr6mtH9C+ZBxcHJMe/0j6mSBOk4dPsWvZ5LWNj0dof0ex2izZg0FlSKx3tPdpHKkmJVYtuwnGW4Yk48WYPoIB0NX0tp6C7L5JJO45BkPKc023S1CJGuyIqURneXJN5g223yvCCrYuJQUCs+nqmgGEs0ehTVik7HpsxrOl2fTRxQ1waGteHy9UuarTM6PZc6izCUBmG0wVK3xa1t2eR5TlkWaNp2yqOoJqrdhFoixXaqIYSgqlV0o2Kd3qKqApHVeG4DVdkWpEiLMs+oFYUsq8nyMYqioyo6Qmyh7bXI0QwwbR1VNdG0v2RdqjiGjaJoqKpBXWuISiPNMjRN30oiqi1AfrMJMXQLqKlFSVluuZmqqiOlRNdVNE1B1UoaTRdV2ZJqNuuY2twm6der7WpHXdfUdY3XaLBZZUi5nSSrao3tCmohKfISTdPw3W063/j295EVKbomWC3GVBU0/CYt/78BbNB8NUbXLOpyG+o4e7BPFJbbpKxvE8cRRZ4x6PZI04AyB121WK83DPcfbNEgoqCqc3yvz6tv7mm2Gwz2jqjqkiotSGNJVlQ0230URSGOY1TVJM1gs1zx6J095rOQ5SrAdvooakiSZxiWz86+zWJsUpcTjk6PMUyFJLpnHc/w3T2K0uH2fk1RzNjbPSYtV9xc39D0mux2h0gM7kYjOh2bVmebEo3CJZ2+R7e/y8XFEtcyORs+ZboIsPUO3X6D88trNBTedXq0Gn3WqxveefYuX3z5kidPnrDfsSlyBduBgfcBq3FJv98n2mhUUcVR/4hl7KA31pyd/AZZmRHlgqPjh5zs5TRtk/H9DWnW4bvfeUbHU/jjF3Nm8yVVHGLUGaaZ8/5hG1G1aGh3OIrPUlSUaYKTLhjPP+H+fguf1tSMTXbP80Ofu7HCQFgMnuxwcXtJw3DQfJs4mmMoPR6fnGGaKqObWyyRMJ3fsXdySLLcUKLg9ZoUaQHZhuFOh2xTkHJPpac0vROQgjQZ8/2z52iWwXI1oUpnXM8mlLXLwd67kNpk1yP8niAKVrQch4u3P8Vv/QaOO6BWGvzJn/6SRw+GNBsOo/sl8aak29plvlyiWAZuC8IswLJgE13z8OAp09Gapu+z+2iIVlvE1YyitjHtmjD8mmH3mCxL0Mw1m3VOt9Pi01/9a3qdDrblUtZwNxqhKzp16XD2/IDr6wvKcoPIatJKISbBDwqWmzWnJ6cskhF1qdNQW5i2x3CvwTIKMAwbxx0yXwTkhUqtQJ2nuK0uiqpyN7olzGIqRRJEEbasqKoa17OpKkGcJmQ5LNcL+oMWmuwwXyW0/BPqKscyPdKsJE8tbLtBKWveXp3jeD2Msk0al2RVTJYrGEafoki5uv2Sh6ffIyseoIkGWTbn+OiMunS3LM56jVQ1Fqs5ZZVRlHN2944QMqesElqNXYoyIk5q1tEtVVXx6Mkz3rx5wbsf7hNtwDDh5OwB09mEvZ3v8er1HcO9Iwxnj5/89t9gr9fkH/7Df8ho9CXvfrhHLUpsvYmqzLC9grJ0QH6rmMMiKxJKseDwcJ/xWxfTlJS1gkJKlAQM+hF57iJKH8/XUISDrBTKcsPJ/hm+s7tVPRY1lq0SRwWe28Xrvk/L7CGr7Y6RlPW2U6xFvPvuD7a7V/JbcLWi/vpMTLIVvjFBV58hVQVF+VaJJzRQE1QzIQpdpFApqhRNdVCkTlGm3Nx9w2TSwGs+oahK6romCiu6HZfjg32uzy2CpAK1RtUgywp0o+Ls4T63N5LppEKxQFV1TN0GkeO4OZcXczRruL3h1SVS1Qg3EY8eeATJhKp6n6KOqEWFqkjSSGJ3ShTLYrpKqUofRYAqBEGwodHSsRsVea5R1gLPsUGrKHJBWSsEyTX3Yw1FeYht+Wh6DVuUPM2GxtHhkJ//8RLPs0Hb6kS3mBUFqWzYbAo0zUHWOSgVpuZBCU8e71HXNRdvVritNnma4bgeaIJwseZ3/94zzt/eI3ID1QLDUskqQRkXKGaBbkhUmsR5galr3zY0II5TjvdtkmBCnh+iapLDoyFnp4958eVLvv7yPd57p8frz6/ZPy0IS4VKeviWg42kvePT6XR49dUrjruPKQxJHb6gKkp0Q0cKSKsKaxPiSpPraUxjx2F++w0dc5dG06WUFqsg4M31JZuo4Le+8yFX0xsOmn1KNeXq/BZd89HMXfLsnvaODyZoVY3tLIljg2a3xzJakRcB96M5qCYXLz7hO89/j0dPn/Hikwtsd8kiHVGEsAjvcK02YbggLwW17HN7P2J/74DTww8x0AhXNxyfNdHqiqubOcKI8fsNSDRUUfOrT7/m5Nn71GXBk3c+Io5mVIuSN+e3aC1tO3ZebVCLhOXlDVGcsn9kkocJlmVxfXFNY5BycrrPze05be+YveEhq8aILIjxmgNyaaBpkEcGo3lAr7HL9WVIms3RDJVG/5B1cInuFTSVLoZqU0sVw6/INwVllVJXW+pFWagoSHRVINSCMM5xNQdBimXp1JVKr9mnEiGG5aBpbXSloq63KCAhM/KkBikIkzmu66Fp5vaMEFv5gaZLfLtDLQVFLlAUgRA1YbihLEv6/R1WmzV+w6Usc1p+g1ani6ZvV+C2TFqJ61nfdjpVyhJA4DgOlmUBW+NQXW8fIuM4JYo22LaL69mYpklZKrieiaJlCAFpWtBuGUwmayzLQUFFUSTr5TbVrioKlmWRyAoht51OVdY0TIO6rvEHTUDdJt7T+K+/oBSVhdfWyYoNq2WEpmlYuotupPhul1VSoVkZSVVQlzqGYRLnOVkesnyZI0XJoL9HFBeU9RJT76AImNwvqSmxTBfTNHC9JnGUkFcJs/mYXq+DoigcPugymtwTxAlC0Wi6O2yimCLPaLg7NP09QvuSdr/P7f2Yzk6GZhkY6QnzxZhmx6OIDeKwZG1U1DLA8zxQc0qZMV7fsnt0TBqqqGqLJBAc7p3hNSS262F/fMSbV19g6xqnT3e5/voVx09O8XQT2+lwsrvDzWhDUdnMgjt29lRUZUG0dFmGV+wd9nh0/AFvXr6hP9whmq/44Qc/YHyZc3g0oIgiepbKm/uUHbfFQdcjjx3KLKHfOaW9p6FkNdLReWfvBzik5PWUeLqilAV+q0NwHTLoneAPbJabCb3DI4oyoRyF7HsWcSHQPRMdgVoZ6NUFcdWFqU7LPSYOEvb6e8ikIEljpvUFe1aDTr+BGURUqsnLNy/o2U2q0iavNBQT7NzExSYtIibZhtO9AaNXv+D44JDaafP2ZgGAY5sszt+gWBrdTo3vSSoFLi9+jnNt0Hc9PL/Pj35wTFwl7A56qLLCrRfI0mO5EOzsHpCnCW7DwM1q5otbylTndPiUMLzB7jcooxTXlmxmc7yBznQ1otX2UO2C6V1OFdck0ef0Wn0UBqhOyP3oGs0q0VwDNEG+2bDeROwfn2E4PT795AtkLdCsjJZlMZkKhFaRmys8f48k2+4Det0W0/uSkgolvSNKK1xzhzeXV6BUWJ5A0QSl0AijADUKSSINw1rSa72LIWL2ugamOaMqJaIW+E4TY6fLem1TZhma3KcsbijqOZv0BsfqIDE5ODhmsVgSxStMw8U2WthWzWz5inbXZTx7RaMl8N0hhpWxWY9otQRxHGMbHqI2KeuUOBtjOR10zaLZVnHMIYvVNXmsYFgVWSqwjAFptkHIBYPBEFGbrNdzen2TIgXP2WW+WiAIsbQn3I0k/YYkvbyg0gTtlke6GfKf/p/+E37v7/8d4jjCcTWK3MXSDZptjyTR0I1v96DQEGVFUV/i2t9BVg0UO6OqVGzDYrXIsTpruu1jJCVSqRFVi7isEUpGs2WQhQqGXyFRt2PyZEJ/P+DRo2fcvNJIkpRS1Piuxc31Ff/ev/8THLvNeqpi2Cqaon27hL+9NCPj8fMurz7JyXNz6yQucjTh4DUKdBNk3cd2t7zUGgjDkN0dl7K+R1FaSAqErLEdizDIsNoBddUkT7YWEF3VqOoQUZg4nsJiPWG96lOjYKrbYrsuJGm2wrBqTP2QsNBp+DaVFAhVQRHg2ILxpMKxmmhsbSfNlsv4PsJtSXJZUssWhmZge22CzQqEQpqvWaxzbHePcpkDJpoKUipUVUGrL7FsgyQBRVMp6wzT1AGVhm9wefOGKFKwPXXrSFcL6kri2gqVCEkyE93UqMst3UPTVSxDxzQlYbIhyVRUu8ayJXmZo2YqKjVC2RCFOoZhIxHkZY6mG2TJ9qE5SSviqEP9bZFfliW26xLHU/pPXAadYybjLkmVoxQa7733IRdvXvAH/+oTDobf5+jY5/b8kmlU8eDMY3kz43vv/5hgdY5WqQhNkGg1nmHidzrYpoOUgrwqcDSDSpf4TZfGbEY4Tnj27DeYXl2TxhY7fY+mdBFI9vcMXnz9B7i9PWaxQKLTdDxqmTBfvcHVPFx/wMX0hkenD5mvUsbLBYssotcbUiQOz59+D8MMUORj1pHkejLH7miUogDDpt1yMVUHzTJ58foLuq0+d3dvsfUtO/qrly+AlCKMOTs6YTp/S6bHNAZDUqGw/7BDq+mwszkgr3IcS2e8GHN4PERvqxy/8x5lFpKsIjK7ZD1a8vplwI/+zk+4vv2En//pHT/6G88RqsNicsPV+Vt2d4dUqxd89eUrHj17wusvfsZ7P/xd/HrFxcWKVr+Nba65u79nf/8B63iJKG26JDx5+NvkNVxevcLTPdYride30JUCt+HiaAqGoZOKCEUDS9WRBfT8FrW6RBXN7Q5xs6KuEjRpUxQpQg3Jaw3TcECR29Gx2H7ufb8BqLRbPmkWU1cSSYWUgtVqhZQKolYQSoaqqgi5De9JKem226R5jqrqBEFCEERbjKFt/toQ5DhbhF2Wxei6jqKCqW+7k5rGr73r63WEoij4fvNbvmdCUWQIUdFoNDCNFqZl0fRrhKw5ODgiz3PqeksOqSp9S41QVfRvmZxZUiBEgW3brJfb4tG29W2Y0BAo8t8GEf/aCkqsjE3ooSkqcbmi5bYQ5CzDNa1uBxEbaEZNEMW02j5FlZDLgrQOydI1+/0DFsuAokypZY6pQ5XkGJZO02+xDu7JU5WbSbxtScuKbq+FbpnUdc1yvSCMakbTGa7TxLSWlLlOFAXYVsmb12/R7ZDXr2d0WgekmwFVnaFqAU3vlI5zRN+T+A8HOKbF1dUFp4/anL95Szir+cFHv00UluRmjqErNHYqos0C2/SZLVVopjw8PGazWBFtQj589BG5nvCT3/1NskylqAtsQ+HByS47+wcIXfDVi39Gb2dI2z0gDUPsfkG3Y/Fo54ypuCCNNhwftxFZgebvk8cR7z/u0WoeMl+PcKwQ32qzXCQ8aDQZja7IA4WD/VMW8w2Z2SXVMoIix5Qa3aM94iIjKnNO9npcTUOKGogjVKPk6MFDLs4/p9HwsewDOn7GKkywixrdKNnrtrl++xLP9jEUlVmwYueozXw6w7NbOLpBb7+DrmTMRyFe3aBUYvymA+0BOiXDWBBeLdk9eMSr0R2Pzx4wnt8yvfoco9em3esjdBdVE1TpGqHDo6cf4Ogu8zigVhSczoB2w+Gr15+gFAE7wyZZWKKoNotyimGq/PQvvuHs4T7zaEpL2UeaJrk0cPwhcZWTqyVxXdJWVaStkeZL4g24zQaFIlF0jSCcEqw3nD7wODwecnW9ZDbfYNsm6AZe02U8mSDte6zK5XTosIhMylohyy4xmy1WgcpByydJlyBcagF1nbNcRbz/+BmTcUZRCTbhGKEsebL7Q4KopKhTqlKhym00XdJpnlEWCgifNN0ghQLUoIDjmsS5imfuMVu+RVXvafsH2KaGoqjf6jFVpvd3GI6g13NYLRQymeFbNY7dpC592v2UIvYYred4zhMWi3sabYV26wBUg8XyFrSKZnuHMAyxjTYHu99BRcEzhyhGQVELWt6Ar7/+c7rdLieHH7MKRuRJhmkp3Fxv6HZ1pHJFr79HVXrM7js4vZRNkKMqBqVac3Z6yP/2P/qPuL1ZIIS7LeJzHaHEbDZ3FHVBuEnR2SMtQnzXY7mcs3fgEm307UajkKAUyNIhr+8ZDFVublZYzglFDYYZs5mXPHm2j2YJ4o2CUAWi0vD9JudvX/LofYM6K6hFiaoYVKWKqBUkGZ2+xeg6AbFN26uaSpb+WyZbWkywnR3yVEdVFap6C0GOFwXtXZhMpiTxEZoeo6HjWBppktPsl6yDGp397Y1CCtKkwLRK8nrE1WVAXXwXVYvJc4GilURBxaN3NW7urpHV0bfdjK2POE0KWh2DssyRordFkaQpiBJF0SmLkEG/habvM5kUGIYDUiWKElxbw+tkvHhzhW49QZQr8qLGdnSSoKTd9UBxWC91XNdF17e6Sk1zqMQ1WTHj5soB+Rip1uiGiaLm5JGCbglM28EwLTTTwrBBQ+dmccN7z/fxnIir6xWee0BeQ12CooGqFrQ7CmEMSanQcTSSNMW3t3KNQdMgT1fY1j5JJVE1gY6OkNuQRcOrSKOaMNKQyrbLYppb7Jpl6BwedLm6ekNeSAxHJ4lTdgZ7nD485u03r/mXf3TIf/gPfN58HtFwfWyR0243uN8sePbuE17+4nO+8+EPwbLIxlOk0cD2fKo8o6xy9g8PuNNqlsEtzaZHHWbc39/SG+yjGiGr9T215VFmGg0sGr0ByWKG4vmElUS3u1R1zmDgc/nmimm8JpQa/+JPf8nRWZPr2SW9ToMgGnGw+yFvbv6cVsdHFc9IyxmWkiLJuX59wYPDQ4IwA7PJjq1xeNRgvhTk6yVdTyeZqCR5QmfHpSgMXk1eIPKYvYMdXr65pT/YZzwec3cHw8MHrMbnJGmIrMa8enONajU42D9jvQrRlZz51Q3FKuTv/oPHXL1d8eHHP8Eo/hBF1BDGpKVGncRUjQ6GtWRyfUG/J/ne9wcU1Q15fMVmcc39jc/RwwGL2TWLmwmdXp/uIGR5/ytmG40HZ7toRcHB8XMarXuQGetIJUlXFMIntxw2hY3r2mRxjgmoRoWiuEhNQ1Mc0rxGChVJRlqsqCtlm+oOlqjooJQ4lgZaiW5Y5FlFWQdYlkWYZ9iGBxI8W1KUGe1+m4qauqowDIOyqIGtJcwwNEDD8La7jJ1Oh7ouv+Vjfgt1FwLPd78tHiuErBF1hZQGWbZd9VBUEFJubTymTlFUmIZLnuesVwlVJbdhJlVBIJDU3z7gCeq6xnI9hBDbexzf7mH624T6eh1hWVsNZpQEZNm2yPyvc/2VC0qp2/itJnGU4nVaoGvkcoXZLfnm+gvKNMe0FPqdXSbja5pNF1lLFDRsW0XXNITc4DgeVSlZbL5BqAqziU67XzDoHFMLie/2MAyDTbBAEQppFJNlCZbTQNctet090qQkTtaI3GNyF/Dw+DFu/xlJPqJp97ENF8vokhRzLq+W7O/v0vJMwvWK4bFkOrnl9MjH1xscDg6I7QA7rZEVPD15xNXVJUk2p+H6dHomrm9wtv+E1y9ekpSC090uRqqj6pIkjHh7seR7zw6pzYLd3pD1dMb99Asa2oBgumIwaNE2u9y9uWA4POLu5hXdXpMwtJAtjdFFiu2tGA4PEaXG/f05ju1hyCbL6BzTdgl02P/OKaNX91zd3TC6XSAbCkkUEecT6iLF0nt0fJOr8YzgQEFzFVp1g6gKWJQBm9UNQVoi+xpRNMHxDtgzclytIvIlo5uMvaOHfH31GafNHVpNn9oF5gFN74j55A3pxmQp1piaijbVwJCM8gQ3r9A0l2bzgMbQYzJdY3g6sQipaoOD5x+BoSHKEt2sKBOFNFYw2yGbRKC1VHqdfX7+iz9msL9GKi5pmGJbgsWmwG63WC5HWFZKOc85Oznj/O0VzWafUhhM1+fMgpR0GtJsVQhR4PTabKqE0WiE41S4zQaLaEmSzml6DdB8FGfG9WRNf+eMrAzo2DZVVZGVAtv3yLM5HdtFKSTUexwcwItXP8cyfWzLJssKlsslhhsxmmUMbB/bbVCnKUESUVWSXGxwLANFOyQLLbIkxLQr8iRlMBgQRxlxplGUKX7LQF/Z/2/jVUFSbNO9SVoQhgnPnp9we/cS09tBUyxkDav5iiwpabX7VLLG0GseP3nE+cVLVLWFlJIq97HtBkVeEyUjCpETRiphdEGeS6RUabYsbs8zOj2f6eyWutig6wVNx2O2inj4dIf72wuO9g/QcPj680957+P3+fL2Ja5nMBjsoBk56+iebB4hs99C102ydE1ZKagY7B3s82/+8I/5f/xn/4SD4wOmsxs6nQGWYYJaUchLksKmrp6hqCUoKlVVEkVTHnWblImOaQnqyiIvpiiage8IHGOLqvEdA83QUZSMNFvR7uskqYOiGSiqjqAmTjb4DZt+74AiblPkJX7LpawFcZRjuzV1XVAWTXzLolYSpFQIw387/ukPDGRpUhYqhqVQlTm6paDrOkW+oOXsM69tNEdDkVsrRxjEeN2UXu8htysVyzUQdYXt2CznC975wSFhHBBFGbolKQVUpSTPapLijpbfYr2osTyoRYnjNEiDFYWYUktBFOfUdQwYCFVDqxU0LWSdzFkuTMq6QkVByBJVN7bhgjrC8w5IQo1MCLK8xjFUsizjyfs+SZxQ1R00T6BoW91cVWSkyWSrn9WPCSIL09vepEzTZBFX9Hcsbq5mWNYhrudt1YiGhaYrDHbh/m6MrvVQFA1NUdEdiMKcdsvAdBJ+9vMJfuMRUs0oEgWSFFPVWGaX3N8VpMUJaFssklAqJNti4NlDhzJJqaSNrmlUVUalW6RZSdO1iPIFGD3SvMSxFTRDZRNtODx6yJeffc0XX73i9vXvcXrwALe7T8M2KPI7Eq3GyVR6xwPKpeTsnSH/8T//z/lt5ylVnlPVFbZloxgavY5Nthb4nQco7orJ3RV3o3u6TYeD9hnfjF7R8W2KOKPSSs7293nxek7rsEFW3BNsEqKNxXB4zP2bMfP5mEWyIX8Fu4NTDM3k9ds3hPmStJ7z+sZFETcMek0enBzT9AeI+paanOubEYbp45lPefHqBZtYcnr6jJMn3+Xzzz/ddtRChavXX1GV8Bsf/jvcXE0Z7g6RdUrLaTAbb3j52Z8Rpgq2ATuDJlWdUYQpi7WOsODy9opBS8fuP+JqnJFbG1ZFj/d++D2++tXPcPoqMqlxfZ9aj9gZPuC/++A5r67fkhV7LMdvEIqJ3eugtyrCeMzzx2d8+auXaGVOuVZ5+WrMybMfUsZTLl98RbxO6DVdokWIsD2Ge23OL9/wzne/jzNoUyYxvtWgrGGZh8g4RSFnvd76r3XdxLJ1nMaA6+srVm/vGA6HHB8+xnY0kiTDtAySuCArEwypUuY1iioo6gJFkZiWgmN6xFmERANZk+c5lmWxXgWY5pZ3maYxZVWTpimWZeE4Dqa5NWCVZYmlbzWWqqr+OqRTVyVZlqEoGq5rbf3nbPcdNc1A17df1w0N07JRFEmWbTMo20CQC4jtmB5BFG+DQnm2LWQVRaHMtoWtpums12vSNOXw8BDHcTAM4/8DlfbXVlDWlcZqtcCwSiqRU+c+RWkRJzHNjkRKsV1YVzN2h0PiqKKsEjy3y9HxQxbzO6Sus3u4z6e//JRMjvHdU64uxjx6Z0i0CJFCwXU9TMOk3Wyyu7vL7dUNWu3T7zlMpwu6zRY0BVmWkeU6Hz39Ho7moxIwaO0hpYVt+Giq4M15yMPDE7ymxFErGt1dLEXBqFSePDsmXFmojQJTZri6id3ISKIR7Y6CFjosNymblY2iG4wWb/E6Cmpjh7qMUEwFkYUIz+bJwz7nr6/BgMX6jiqvOOh2KBIDxWyTJSbDk2PSUmc2D7BcjUW4oSx80vs1CSvc2iGYLUHTSdIc27WIYo2bK5PBUHI+nZJ8M0JJC548PeNmdcvAb9FuNrFVAylsBHfMxxm+5VDGa+6qGlVPaSg1WWBiRAtK1Saa5yi+YLUaM9x/yHiesIpCJpuAxw0Yj+94dHpKQ3H49E8+5fTxETf3b7HMmqwMKU0Dv+Gzmq1AN3h8ckIidVa31+z4h4yvr8grjUqp+Or1S6JFyHcPfsDkao7f9IjjMegGSsPH1zRkkPGHrz/n8YMDDMcmjCsyppiuSoVHpsbUIsRsS4qiosDg7fUFju2RZyV1nTCvdZIiwfP2yMsZpulye3eFqHN6TZesVHCsLvNJwMGDh8gSRjczTE9QFH0WywC3CbZnEQYRcZyg6gYH+zsspyFeu+Ryfsnm9RivXdNo7+A7DVRVo+GbbLIxR6cdbiZjfEdF5BW1ApvNhKMnx8jaRCoJuio4Otzh7eUnOG6bRsND0xRsv00crVitUw68JkJIFFWgKCqO0+ST15+jaRrDg+e8fD2ht9Pg9n6NaZpUdY5p5HQHB7x6ecnuXgdUyWx+j+/7rMIbxrclB/vHxPUYz9UQkc/h/iH3owWHpyavXl2RJwqWq3J49JRe54g/+bN/QbcfoyhbW0RNypdfLmg1PaoypSxrdvsH/OoXn9DvH6GpBm/Pv+DswQl1aWOr71OLXZo9g6rS0SyTKA4o8pj/w//+P6bZa4Bms5wJ2i2Vmppok9MfgJAmitpEVVNUzd6y6YoVrmOzCcztIat6OKZBsAxoOA47zWe8keGWB0lKXWmkWUwax6SZhqo435p6agQeQglIiglvXkzw/O+SpCGGYZPENf29JppWISoP6QrKMsP12uTp8tdn4kc/OuHLz96g8B66oVLk247Eaj3mqG9SFw0EJqBQiRpVKEglo7/XYHJfoumSqi4QUiDqbefQ8jJmSw3DNJGqiqaVyNpE1UJ6vTaLqcB2WuiaoJJQFTlxssTtJehGi6xQaLsmuSiopYFl6MTpLeP5DFk+wLIcZLnCciDPLNJ8ie943N35IEs67R6rLMU2FEw9IUzPSQuJYT4AQ0VBQdNtgnVGoykZ9PuQnBGgo2qCIlcwDECR+M2CxczHcVwMTaIoW2ySZ9tE8QhFU7DcFqou0QsToUrYVOztNajzjCproOoaeZ5iGVsOX5HnfPB8B1OPyUsLY5vBoRAZUlSoWsmwZ7LcWNtxtyzRVQVUlSotcXcqukOXt7c1tSLJkny7gyZyFHR+67ee8Qd/+Jqf/fkd/4v/5Q+4uHnFcPCA9coijZe8nS4Y9tqs44Cf/upP+J2nj7Cu0+04X1GohSAJIu7nGW23i5KX9LxT1NMdsvINXtWgJqe3N2AdLAizhGZrl0LtsHNo43bgq89fsLv3iGSxpFRiFssxq+sJiifptJ6jVwPyuObZg7/BFy/+GUpt07ZbHB5YRKXFqxcxj099kkCjFCGaXlDFNav1kmgl+Oij97k4v+Of/eN/TizWoKs8PnyX3Z7L9VXGcnOP6WpMx3cE65B0bwfH1GgYGkU5Ics0Ws0HjKcXrJYzfK/FdL7Bbx5zcqDwF29vCNYV/d0G08sr5E6bdmuX2lDZHUrCuaDRdGi2OqBsMX665vHocY9PP79hsOti221GNyGz2ZxnH55gGV2COOL0Ucno7g3ugzY//MG7fP3qa8p8QJFm5Jkgcbv4es3rz/41asPDUVXCWsFsNJCeg1YI3KZB72GP2TSl0fKpqowkqmk3D/nOOz+gliF39yuaokGaCpJMImqdQkjm0w29ziG6naKgkhcJ09k9FC57+x1qYZKnyZaZmsQoikpVQV4m2LaNqlqoylZNKWudJKqpqi3iDClBrQGx3cUGpNSQEjRNJc9L8mybPM/zHEWV2LaNENtA2tamZSClxLJU2u0uWZZ9WxSW5NW24+q7/tY7X1XkeY7rG5imQ1VVtDlA0zSErEiSiM1mRaPR+OsvKMtsQ5JEOK7B29dj+v0+D588ps5Uvvn0c44e7GCaA8bTOcN9n+kiRtdVJvMZpXRQTBB6k7c339Db3aXIHXTF5N/9vSN8SxAzoddtk6YJYbhkuLuHSkKv57Ez2GcejHnv2fdYrRbsHbgotc3oboxrefT6He7uQyb3bxEi4ezoEd2ugW+0cG2PpnEIdURWxUSLBNd0iRYR08lbDg6O6Lg9Xr/6gr2jrWh+NBpj6g1OTvaZLuYopMynYCgl0u2z2QQYuortOoyChIenQ/TdjPlkxMBpcrJzzPXlDe1dFa32qLQlP3/5JxwOniPkmGWR45pHqOqMbr9LXJXM6wZNS7De3DIcHjDLUwa9HRqJyWqd0+pbMEnYe/SM2eIStyPp9E6pRufMqwDftdkZnPF2+jWKNDk+ekLx9mfEmzu+Wi747vvv8OT4N4k2gsnkitPmI255SRkrLIo1/YYDeUIcj3l+9pzp7S2l6XJydILTskjqgo0w6Qw8qrBELQz8PZ+NqLhcjDDCmMps8/LuiuFwlzwyaaQZGgm9hyesN5dYpkB3DbSyj+YY/OqrL9jz92i2DTS94uvPvmJv94CwCjEbFllckWQppiMI5zOcpovvHfF2tKGsbtl3W1SFwHUt8tpDMyesNgLD84hDnWRToykKuSdwnS6r2Zw0WzOfSoosYbfXJox1dHc7bui0HpMkCUidhmHSMh3iVYZmCUbjJeNVzMenv4Hth4RZhJZnzBcbpNxn7/gDfvXLP0QxBE5TYOsKQRxxeHjEYlFgWRpREmF1SpIwY7f7EULYOD7cjZZ49RLLgNeXl7TrPZ6aBrXcqhfv76/Id0aUaQc3HWP4ETejkNnimu+99x+QZDOCMKMqU6Qecb+4pt89YLmS1DXoRhPDuSEIZxwcewSbkqrQePX6ika7JE+OsIwBifKWIhtyczXl1YsvaTcPCVYL0ghcSzA4MVmvJMvlHLXp0O8cIOmw13cpxIq70UuePvwem/Q1tnZCER3R23Ho9lpATRhkHJ4c85/+H/8THjx4wCefh9ieznI1YRC6NDWL9eaek2d9TGfAbJagWwJFsdisE7odh4dPdvjTtzVlYSDUDaYOcRzzt/87CypxQ5l2sfwEIXTyKsWxXM4eGHz2SxVZZ1R1hab4FEWBVHOE8Oh2nrLelOimgqSgqgSqqrCYByD2KavtWRbH6XaU/JdnYp3g2gNiRft2wR5EpuB6Bk+ePOGTX1zgOY8pqhxDh6qqabRgGbwkz063gUY9piq+3WO1U/JiTRzayGqLrzEtgzKPsb2CXveAYBWQRwWKCooiUBFsgiXvfn9AHMcgPVQJSBXTdYnXa1xf4Z33P+BXf6ShKxGGWlPXFlmi0u3q+A0VVQ7QNRupRLRaDZS6xG+YtDuSzarGMhqg6xRljKVZ1FXG6ek+wx2dF7+SWI6DVDNMw8Y2JKqyJC9nZOEBjUYLQYQqdFRFoioWT58+4+uvXqBgYlkqRW1T1BJEgutoBIuYKldRtRpZu1sLUV5QpCG+76AoPnFc47klCJWqLkjCiGbL5aP3H/Bf/cs3lHWFoYMiBUWVEwUxjccQZAmThYOm1MhKpRAKqm6Q5TH/4d//bYJ5wn/15/+G93/+mIOB5Oc//QKvVxFkGSGwyl2yOCBPbc46LbT6DlFLVF1DqRV02yKZzdk93kFr2Hz66lcMenv0ez3AR5E28c1XNBsWlCo9t0GphGg2IG1UvYtUmrz30RlfvXrBoPWEp9/5Hqcf9Hh7/oLVdINZeQxdlU3jAb/33/rb5Jng8nzG7mGT65svOL96xcHBCWl4R5212B++z93NL3lyfMb1q5Aiq3j38UM0ZUiYrzg9GhKEJpqxYnjY4+58ytnuDnnf56effEJvt0mVVjQVHc2wkWYXf5CximPWq5idgc9ofM/noYHtaOh7XfRSoisF9xd3aC2LrlnRsPfJh3PULGQyiqh0k93dExajt1TdXVzfAdlivV7S7Gio0uXR2TtMJhseP+lxc1vT2TFI4pwo8/n4g9/h/O0rTp8fk1UxqiaYj27p9/aI44jOoEux2WDXktX9kqOBTbws2cxWVMLgxfVrWu19dgZDamZ88sUtirFGVDaV6KOoOnle4do9ev0Wru+hSg/HNShzDYFEUFLKnJvpK+6vc/7dv/23qKoCQ9VQVZ04jpGqQlnmlFlJnqfbrqWlohsGSrGVHmjalgRh2zZ5ngIqeVai6RpSym/94xVV9a273LAIggDP3RZ8jm1subPfQt1Xyw1FmSFlDUqNomuolYmilJR1hevamMa2uDU0A8dyKKqMPE+p6+2o3HH6pGn+Vy4o1f//b9lert0H6eBYXXZ2z5DYTKYLaqVm72SIpzfwnArX8FhPIw72OhzuHtBwPSbTV8xGU2bTS6oSWs0ee3tPaPfa6HqGrepoaoXnauz0+jw9/hi1VImmG7JFTbqYs2O7sA459HrUCx0zsTkwB0TXl/i1QM1Ven2Lk5MzxrcjotmSs2GDZLag4QVkZUYlAtJ4haFvCIMpyJiLty9BWTA8bdFoDjHMJp3eDs1eC92y2T3YwdQa5IUgKUzyIONgb5c6k3S7J3Rtg5effoUtBM8e9Gk4DbJiQ3ffQKISJWPCoMBSFLJ8he2o1KnCfP4Kt2tQ1E1MY4DdayFUhf7BgNIE6bh8c/GGnaMBtFSSbEamRsTJGqWG4e6ArIyIOz5HR2dopkqyicA0Udomb17+ijpa0HH3eLD3AW3rOdejKzq7Bo57yHiyZKezx2T8AkcV2I0ux61TGv0hJhmFqSKFzkLERJnE6zaoCkgWAk2pqCmolT5e5uHpgk53l6JYbTVcooUgJDEFu+98hNfd5SqqUI46jOOISAgmkxm2olGpEZPViobt4/pNUhYkVYGqWNRFiFJIkiglV3ymk5RlNCGRn+MaBUWa09zfZ1lsWEcvmU5SkiglWr8gmN/T7XkcHJ3g2E0urs5Js4iua9FUFSwJo/mUUqhodJgEM65Gb1ksZt8iRnLCdI5tOhRRhtNoctQ7xuvo3IxvEUqLSZ6j+x0UO+Ozr/8paZGi6jGauovTbpMnOZNFhCKm5FVOp/OAII5YRLc0Bx5BEjObBBzu75IlkiyzGR4dkFcpVV1SV4K6FtQiwtWblKHAlDq77TamlDw5+DHj0RVhckUYpNxPrlgta2z9Y2zrDEXTGN0nOOYAW+9R5Bpv3064vFhwfHrG/kEP2ysQlcpseo/n+ERRRBSr5LXNzt4AWUG/pyCtClUcbbVklsqb8ym39xt2envIPKTTMtCUGEWLGPTOKOIdzs7OGO73abWaOJbHs8enXL1+yddfvOLp42c8eHBKJSSbKCLJJEUSoysF739wSHBvo1Q2cSWIsgVJGXPY7ZLNYbYO0OwCUxVUhYHqFPzND/49ymVNbVkIq0JISVlaeG6CoZ5QFx6lVNCEDkpNEATs7Frk5ARxgSIzauFRyYwi2dBqxCBsKnTyMqXWFKK4xvGWv371Wj5JaFJLBVFZaJpKltZ02hWNtkRVe9RFilJLBDUGJlE4YbVaUScNqBPqSkFoNSg2m+UbNK3ANg8RGFsTlSgwDQ+1VEmjObK2UcgRFdSKRlEJdD0h3MyYT0ws06BAR1MEHhAFCZtkxf3tCgUHIXOkaCIqSZLes3fQpOXtU+SCXIG8FtSyQlW2oZFWd4itPwFsanI0TUcq26K6u2sRRAp54aBqoMotYL6sVKQd8uD4BCnaYBfoWAhNgNYgz855/eKfcbcM0S2VOC/QFQPFEFSy4HsPjxl0hqxzd2sdEiVRGaDpKovphNHsnNEkQdTlFk5d50hZk2YVg57J55/9gsnGxLV8pFJT1du/eS0r+kOLxc2cKrMIK4VaqallQiU0NCkwuhm//Tc/Jg0j/vN/+mecnT1nMPTo9B/z/OFD9noOqlDp2A2eHTvcJnPKb38OUgVFoYgDHpw84z645c34knbHIM+WjBcKi+mCZZTx6PkPcfw+uweHjNch4+WaF5/fMp0uGA52CIJXLDYhjw7e48NHJxw+2yeVBxz0vsN7Hzzl3/nt32UwGPC7P/4u9/MlSTbHsy2SZM5u+0OeHr2DVVcEk5SW3qbhL5E1vHk9I0wT3j15SLdj8c3VLZVo8Or8AnQbLYfl1Q1pPeLNy88IgksOd3ewpYFKi97Dpzhmyt0iIsolukxxOymFqtBqdWn0dW6vIpqWSaEusdotmv097Fqi1QY3o3ugYpVuiPMlWl1SGQaWb6E7Cg2nxrRSTvYf0HEL4s2YX37+FZkimEwzGs02480cu+GglwWb+znvffA+Tt/E0H2ypMHe2R5m0+fxk+fs7O9y8N571G5B04p4/fKeqtpgygVeI2TvoUKy+Zy3f/FPGV28oj8sWV3eUM6vqCf3hJMpD4ctvFZKmugkUcz05oJVVPD2zT1JVHMyfIijPKGhHvOjH/8Gq3XOapUzC0ou5hNKUaOUIetgg+PmdDoeDV0jiQNuJzeEmwChbR9wqmrJdLFGKVRU28agxjAEqqNjehau4bC3O+D46IC9ocOg66EKMO2aTAYs1iuCZEUQR1RkuJ6Bphm02026zQ6tjkuz7THYaeN6FqqmgCIpypJNEFBVWyd7kW/H4Hmeo/6Vq8T/WmDzLrJc0fQ73FwsGe4fE6Vj/GaTjrm3dV82dTJb56N3f0RZpNxejenaFcNul01ac3S4T5ZsWM0XDAYDep0es/GMLCzY9buIJMVvmohiTcP2iEWNrhg02hppUXBysMfVRYDuFCjUGIZKZ+gzDwIGfZMoc8mCglbTIUsqGk6TdqdJGM8pKkGn7VAXKoYlaDX3GE9WVOqMyXpKs3vI9WRBw/NRLY9amExWMZ2Bjt12abW7mEaD0fia6VLQ3LVYBxqbqMRttpiuVFRXYx1taDeGRElAJSJ0x8CxazAcyiKhrBQqraTdaZKWkpvpL9F0wX6vRzBXEYGKNCVZeouhqHz2+Q2mZ2EZKpKcIH/FoH1KmkpWi3sc16PGJwsEyzrkcPiMm/kIQ0voDnaoU4f3nh0yO1+ws9NmM6vR5YxFMKHrHNMaWFA3CTYbOnYLvZBAh4ELbm2SOQ6y0WUz/hLTMohWIfs7AxTZYDxf4RsrVhsFddDA0F1s1YaqIEpShKowHSVkZcFAEbhJiyhNCPIFrVaLJ09O2awTVCx0a6vsTOsZopqj6w3iTKJoJoanbD285rZFP9x9jFaWFFXC7d1riqLgeK+7XXIuNGbTAtQI2+2hOms8Z5/nrQdcvXlL/3CAoUvGwQoMg93jIfPVkr3BKXkSoziCnWGD68ucpvuYsipx7JpSMbCsLsgA29EIohm2uc86/gJZ71EmFbYJ0dxAySMMN8A0GkyX59hVm5ZfUMuI0WjEkyff4c2rc9Ks4Lh9xMXFa7qDHT797I85PXtCu7WLEBfUdYWqQLDO2PngO8jqDbf3IY/dMyoZsApKhBoQbypajSZPzn6Hzz7/FbbeR1agqhnD/T5X9z+l1z6g3erjeD2yLOdu9JKinnG4+yFJNqfZ6NPrZag0WK9K2o33+ebL1/hexaDvU88q1rMNlSh49vA9dtyIp88ecPH2Kz5457t89eac52d/n51jyWzcoNseMthpYrnOdrHbl+wNB/zP/6f/M/b395nN5xiGRZGvaDR1NuEIke9wtC9BzdHUBoXIMaWLrivcr29oniXMZwqK3EeUNqLaICmw1CY//foPmCxVDNVEpjm6YpGkEZ5dcH75F8TREaatURYFhi4osxxDz3DM3lbfaDqUhMjKRtdL4nS03eHUQ0yjt+UvKlto9l9e9/cFCvtIKQGBopao6MTxijj2ELUBugA0VFUliXK6Ax1ptUhW/navSoRYlk2eqjRbNmkIeWKiqNV2N8p0iDcxihFTCcl63qXleyhaiaarZJGk2dmmQn3rCcuowHEMorjAsRWSbMkqmxIHRxiGjlAFUbjAa/pkac10/hbregeVB0iRo6tNpJKRxDGFGDEaR9Tp+6giRVNNqqpEN1zi9A7HafP2zQ2GfrLdw0NFNbepc0VErFc1tdxClDXFwjRNkiCl0dDQsNFlF8W0EAXoikmRrdFNiWInFGVJWUukKtDEdl0iSyQNr+Lpk0e8eJlSC0izgrpOUFSXNA0QUqLoDmFUgS4xNJ2sqCgTiSZLRJHT8J5xdzdh76FDmmb4fpM4jfAbNevFHYp2xMHBkJdff80//r81+ft/90Mub+/YLHMs38Rttum3G0TLjLMTD/WTVygoJGmEYVoIreb65lOu5xn9nQ6nxyZZuKHONDrDmunqHNt/lzKUoFagphiuz9m7D7FMhYZn4ndcbGt77s/WFcv5jN7Qx2vD1TiktF6jNQxmNxs2WU7v9Bm9PRjdr0mjOcN9C7JjTFFyuPeIMB7TQOXp8TEZJV5dY1UVjw8bwIwoMvGqE9p7bVajMQ3dRRt0sZ0e/UHGp19eUlpb7rCpPsa0XvKLT97ww7MPWQRL4mDG4cEBX765JUo3jKYT1ouEtDMBlmRZwZnxm7Q6K+YbyaBzjK4paMLFbrsEi1uSMEBpdVmu4Xbxiq6nsH/2Dlm2tfesRhtcv0XLVlnNF6x1A78pCG7u0WghZZv+mUZ+66J6EG9GhOMMZVCTz1Li6ZqnH38PQ1GpKhOpWvhqjrsLr1dXHLdtlPGKvXYTTXVIg7cEpmQ0VjB1AXHMcpTR6vvoaUZL03j4+BG3939EZ+8Ix29S5TplliMVGw0TSzFYrWIcq42uWNzMYlw/pop0Wu09duwVdZUhsyWa3UQpPdIiwTNyoqWOb5oUaUQRm2hWRWFJRL4VIcRrjUbLwfRTNKWDIKJ3ZJJmGxRFI00kdVnhewYIlSSN0Q0LVYWyFJTlFlemKBqatlVIJnGJohj4vgOIbRBIkf/fBeH/j+uvXHvu7XR5+ug5lmrxt37yE1p2i3fOPuKds/cxyiZ96wBfa3G6c8b8bkWn4dNo6QyGPo4v+c5HD9GlwunBHqfHDpaxRhE5lqbT6VgYqsTWmtiGR6+zD3qB23J48HwfRXeopM1yk6HbKoou2D04olQKNHOXTaYQBCWj+xVFCRgKmtVlvIrYPWlxN87JcnhzcU+UlszWAS8vX3FxP6JQFMJC4fXdOcvkjvv1JaGMCeuIVTbjm+vPiOWc0WbK67s31KZGa8+iMXCZRtdYTRNhJRiDFW9nXzAOIhI1RHgxuusyXy2phc50lbIuzlls7tGdEtu2mY5DugN16wZdRui6wHFVPA8sO0PTI1rtBp6zg+P4KGpFf2+Xi5s5jtvF1SyqJGUyuUStbNp7bfJyzVH/ANew2d99wMPjBwTLObke8er6l8xHv2Cd5wwGA9b1lEb7FFkG2FVEJWK0MkN32gxOnhEsF6h1zuLmG4b9ZzQaLQzLpagLsnyD7Sj4TQPTtInLmGa/iWkbRMkMKTRMXSVNYgzDIDV0IpmziBb0+g1sQ6CWCl27Q52sqIMlhuygqy12e/us1hP6Ow6+aVIHC0oxYme3TRiGBPGKQiZ0el00TaPVdjE1E0TN3k6fw70zBsMGmmGCXnN5e4ulG+zsdrm9u9iCjmuLvf5DqlygiJiqSIjTitvRNaP5Gml65OqC5p5BZ7dFb8eiN4zJipJ+f58gGrEKLmm3urw5/wzF6nJy9j7f/8GHHO0P2fHex5H7aFKl3/TIUotgBWWh8umvfkmeLfA9h4u390TFPW/efsHDJwNmi0vSbIWoJZpqoKgqdRVx/XYCgOOprNYpnV4b003RVIt3n/0QRercjc45fdjE8jaMJze8/PoG3+7TbQ9QFQspNKpSodvtUpYlR8OndNoad5czXAsGnX00adP0JYIZJ0c77O/ukycl6CXHj5r0+x1W8xoUBU3ZQdLiZv6SWkZ09iTjkcbNhU2j1cQ0TXpdB9/u8O47z/m//p//LyxWSxzPYbFeMRpPv0W6FARrnShMcDzJ5K5kNs9RVUGWFagqiGqD18ow9CFZmaOoFYqiUVc589k3fHM5J8paqFoCVY0soSw29HoKatmgqrYrBJVQKfIcKQqOj5qsxyWK1KlFiZA5mmoRp0tarT2KvEaRLkWWo6k6sirp73i/fjmOR7TZ7jGV1VYtlycZliNA6lSlQy0rimqLDFktVvT7Nu++8zF5XlJTU1Wgaw7rVcT7Hx2xs7PHJkwwbQlyW4jmmWB42NjCmbUmuqmRlvE2PJZIdC3fBq2CEtswKaoY07TIihrVzPjBDz+mzjuoqrodZbkGtVAwVI3+wELXPOpaQdVAAJpeo6oakjWPzj7Gs07RjJJaFBiWSRqDYadkeYhvHyMVBU3XETUIBEmY0O9AuKqwbBeJQoVEVaHIYj549xEPTp9D3aSqBBoaWZUipYKiKLgtePX2FiF1VE2hqAvquiTcxDw6GhBHc+5vc9I0JS9Ssnzrbc+SGLSIKIU43ZqShGCLOapVRLWm6QquLiJQFfJkBUJDKDaLxYoff/8j3n/+lNu7Je9//F10peQPfv8Cw+7S6EpsyyXOI7K0RigmV9OX1CJFR6OqSgzTRoqaVVWRNQ958uQh3313l9VsgtGQ7O5BOC4wOCXMDG4Xt5imybA7pC5zpAlBFLLOVlzfT7m/H/PN20/JFQ3DccnCBTKUdI1d1pOUaD4HNUetIn720z/hi5+ek8kWGU1KZZdG9wFHx+8wHl1SJQbvnL2D24Re9xDNcUjzNsPDE3TjjGfvfo+byVum6zlhrtC1D2g7HqksiIWG29V52LfYTG8orRxN7vA3Hn8IzX18bYBeJtwv7lDTkgeHp7x++YrFesbF5TUGfaqwIli85OLFHfeXazRV5frmLaPJJZ/86s+RhWQ9mdFSHMLJPY5ioeLw8tUrHLfF64tPsX0HT5c4ik8WhBTpAl1zsI0dVsEGzSq5vwqobJ2byxG3yzUrLaPKTKJEp//ej+jvPUT3m2ySO0SxxtcVDKPk4KCBawVoZsgmuEY0TI4//Ak/+Pg7ROMX5EGFbkr0copR37K6eYknN8ST13ToEt4ssIqSbnONb4W8evlzos01yeoSZI3QSwxzjG/YPDx9zjoM2WQbOl2H9lEPr99hWiX4nS77e0PQfIQiUGqB53l0hioNp4uuN5lOZszGI2y3ZjKfsFitmUwviWYZq9WCupaUpYbjunR6TUynTVFr+J0miq5Ro1CW28/GX4LTi2K76lNVBbq+ZfEKIVAUjTz/bwIbREpdZfQHTXp9ncUiodXwsS3J44dN7s4ndBpdVJmRJBsur2cEUYRl++x1j7Y7MHLJLz/9BtdqUJY13W5OKaESNrpnkKYhi/spR6c1mdg6kSvVJwgXHJ/sM5/McU0Pv9Hh8y/u2NvdJjej7AbqjFSJyNIlXWvIOjynN+hws8wwdxTmN+dEUYKwAsqypt3t0NltsAxGWxuDNieM5jh2i87wXdarhN5ej/Es5uJqRsM3CKKIYqHQ7ucUqY+mKQz7p/zFX3yB7YPrGTQHHq/vfkm/38Gkie7V5GJNEIyoCwXX1lhOYixp4TsKydrFtA0m93c4uqQ78DHsgvlixOHgOd32kBfn37DT76BqgrK0GZ72WK4v2e0cggmz+AarHJDWCd2uzvJ2wfHpEzbhnDw+p9Mc0u84iHVFt9XlYhVhaz6b6RLVjVGFRrttkbsaUbqg23xIVhb4vQbreEWnbXA9+owoqBn2O4wmCTv9HnfTc3K7T7+rsQxDCrVEqSt81cV3O8yX10gFhg8ek05LimLF4/1jVpMb+t0mReFg2tDu+KS1gmZLyqVAqWIcYePoAxQ/x28dc3lzyVW8orO7g1DWmLq2RduYFWm6JtXaRJHK5eWXPH/nEQ2vw/1dimPtYLs5s/AtlaixWx6rJMHwNO4XF6zmC9rtNmk+J45Djk+eE4YLiqrG94bcTr6gYZ2i6jrT8QTPbmC4JUfDjwnjKXXm8+G7v8eb62ucRw66aJKEF6ThEk3pMGjvk8YxZdWh3R6wqHL2dl32jiz+4pdf0HCPqBKXVruNbfi0fA0tzLAsk7qqEELjwcn3eGGsyCOLd579iJKAz7/8Gd/56Lf57NMveHD0HMc0ubu/wPParNdLdnZcnj17xu3tLZqhULLidvYJTx78mCga0fT6bBYJq+U9J8ePSfJ7krBks5lgWRpJccfh7nvoio2u5ridQyY3K2wn2u5IaRaz+R1xfc9sFOK6LtPVms3iPfzmPoquIBUbS3dpNxpcvHnNP/pH/4iDw2OCONpaYxQVz3MRZUUppoRBE01vsFwuKTKfihzTTokjD1UpKYqKyzcrLKtJVqxQhEZdwcGxxfHpE352WaOrNpBRqRlFUWPoErX0KCoN29SoygopVEyjpNf2uBAmopBoloqoFKIkYLgz4PCwx8WrBMsXVEWFKTQ2wYLv/ubxr0/E+fQahT3yIsawVZI4Q1NUDg5aXJzfo6vvYxg6sA2TmabBcnlHnBugPtr+P5oNsjQFmZKkc4pSxzR3KCuJqlkIWRKEa4L4Ema97ToEOqpibQ/7co3TjtisMxRNQi0QNVS1xNIrimqJYTZQZRtUEGWFItVv+XcZv/ndj/jiy4ii2qY8NU1QVYI0jtGMhPk4Ii900GqEkNToRFGM49QE4Yy63KWqNRR16wY2bIewitjf8xndW6AaaKa2TbKbJkWRoaEwm8XIeoBpWGR5Sq1ux/+6skW/hIEKcnt7ErJEaiplVTDo2FBXiHqARJJXOXmeImWNQsLhQQ9NONhOE0kOiopuqkTzjNMHuxwNm/ybP7qiFDZ5amw94GaCqkIU3PL2bUSte3R6PXZ3etzezvmn/8Wn/N5/u8fr67cUlUaZXtO0bR6dfshqdUeartlFbndXNdCNisntPfsffsgymyC1Al3pkeaSxeKG1nCf0WSL2ytETLQJScuK5eY1j88eUMkSw9KpqXG8Jl57gCgDRKTQ6rbQAhen0WY2/oJY7vLx9//HHJ+2ef3FT/n6iz+h3dkhWN4RyQWe7WG3u+wMDtDMkOnVmNcvJ/ytv/W3mKzuSeOAOAwxeM77z99htbgnNGKkrdJsNJmOXpIv9znY/YiBZaJrtziNJrO7nI8/fp9PP/9XJHGL4fGQVaJw+k6DL7++ZX84ZGe/z2qWcna6R9crWI5WPHv3xyzKOSoOj86ecHc9ouUfkkV3tDot7u4v6XV05vMbmvYTPNcljO5RFYekWnP5xS0dv42FR5FkmNJhvUx4cPKIn3/2SwaWycngCUq3S5DoBMkaQ7d5/OiEi/ANn351ieYbaLJgfn1Fa6+Hawt2Bi1my4AknbP/4JDp+A5n7wDTHvDd3/wheRERyh4fPz4mupsybD2E/I4wGqEoPRxvyZ/+/i8Rpc9kNuXDH7zD/PKK3V2Xr795w+PnByTzOWU9o0hWPDoZoloF8/MrMq+BXWrMlxGhn7AqYoykZNDs4z1oEEcBIm0QKWDkDpouuJ+84JOv7wkih+9+929yfKhTBBV+YwddV8nyDWGoMF+usb0SRXQIgjWGYm7B5ipo2nZvU1V1qmrrSndsnyzLqKuavEjJ8/TXvMq/yvVXf6dW4zY0yrKgrAu8BlR1ynhyS5pt6B/2eXM9ZhrMqNUlrqfguxaijLB1gzdXV1i+y/GDhziNDg8efczZ4w+RqkOQFVS6hmI1sLsuUS6ZRyNUyyKpAry2Sym2gOIkzyjqAMurWQQLknqKbteM1yM01+Ts0VN2Dh6immfczRPawya3o5S4rNE8KM0Yv7OL6bRYhBOW6wWr+C29fpM0qcnyhE14T1VlTGbXZNmS9TLnbnxDKVI6AxMhbJbrCWUB9/djDobP0LCpkhZ5HrKzc0gSWlvIqQGrTc1e/4hHD8442D/m6aOnVKUKSmO7wJ2m+G0Xw9bo9/dYLkqOj97HNFvMlws6fQtRS46PHjObLoiyKVE1wW27rDYlhjXAbytc3F9Raxp7R3tcje64XozA04gqg9fnG+Io4/x2Scdpcb+6p+PoZNWS/d5jikTjbrZhodaMF/f86R/+KybBglZ7QJybrAOFtMrwO7v4HQepJ3R6bdbJnE1R0h0eIQ2DQpG8vZ4TVwmKZqMaNm8vzsnyBNNosQlTNNNAqjbzcE5Sx8SFidB1VnnGfXbNqAh4u7xnFFxQ1hlFKvD8IXvHO1xcf4KoBJNRyuX5WzQEddJiGU6QqoHnd7i6HlEVTRTpsJwUxNElwbJElpDGMZtliKxKkCE7Oy6yzGj5RxwOnzC9vUQVKb5pcn+1YDmFq5tr0jwmTsH2PdbBgqLIMBgQbO75sz/7l3TslD/7V/+a3/8X/xmb5YjTk8eYro3QOqzjPsvViFqoFPk2gXx7M8dQ9mg1+7z/3vdw9B1efHVPp3XCq68XZGmBYZhIqbNYbPA8l6PDU8J4SlkKdnvvEoUF77/3gC+++FPmq3MkBbNxwcHeeywWC9JkRbvlYBoKmgqPTt/j5uqWqtS4urpGUSRFalKkAg0Hx+qx239MkTTZ6z2lKCqazSZCuMgo47h3wAcP3yNblnhal3Az5fS4gWcOSLKcOj9B4wjLkximjWUZxHGMphr8r/83/yscx0EIQZKlrDZr6rredqh0hTy2KauEd54/xzWekaQltbKhrEvqQsMxTHTLphL2NulYm0BFGORUeYVR7BEGOSqQFimlHhLnAXsHPZLQRFN1qkogZU2RVGT5jCItqQubot7uDVm6TxonNFoGlul/6wsW6KqkLAuqcjsu/8uX7+xT5BJVk9uxruqi6TWHRwOom2iGS1ZubRimblBUIbt7GqtFja55aJpBJVJQSnS9YG+nS7z2sS0PsNE0DSkqqjLi+Gxr3JJSkhU5tVAREoRYcvbQZ71MKTMboYCiWqi6RpwEPHiwR9NrEkcVeb1d9DdtmyCI8Jyam6tLog3omrVtG+QZVakRBlO++70n5JkkSQNU1UTUBmWVURUFnqfQavoEmwxdcyhFsQU+lyrrzRRVVqSJjVChLmp0XaWWAtM0+eijZ5juDlIzqGRCUSRIHeI44XvvP2Jvt02R2aioyLpC1w2KvEKl4OSoievuEgQZRSkINhGSgixVMLUC18gpc51gk6AqkkoKJFCWNZZTM5/PEdLBtFzyuKYoM0BQVxHDQ4Xb+83296fCw0dPKYj4/d//hs3SJsymyHrBwcEhq82at9/ccDJ8iNv0qaWkqnMUqdN299nZ1UgzCAMN121A3SJIS6om+J7k/ZMuThWiKjGGq1GUsNsd8ubVS87PP2OxmNFqDPF8h+lsxGI5J4gmvBn/nLf3F0yCBd2DU3ZPj9h72OVyNqK33+HB432evPOcojLx2sf0jvY5fecpoTLhZjHHsPf423/ve1wvvmG0mKHIFk5TY7V+i6YLwiCmYQ1QdIVSd+j03sXRJdPJiF+8fME6ClnPYq5vvuAP/uhnPHv4IUUxZxmm3N+8YhYIGrspUkQopcujh21+9fmfU9PG7Hi8ePMC0/IZjV7y+psrBt0DTLuk1TsCy6a5s8N0Pf4W4r3EsRvEyZK4LLha3DBNDQ4OjjDsgNLKuZnFDAcD/uBf/BMeP/6Qgyen3Ly5pdDbaCzplDqxmRGKOUYwZb4s6XSO8Fr77Dze4WpzyyIqUByHxu6AorfLeSh5//d+wqvZS15+9TXzWPBnX1/yz/+ff8Fnn64J5iZX4y9ZTzWS0KSgIlraPP3gkP09jfc+foRlpNjamm++/oLj/QOMAvrdHYrZOaTXjM//nPvXf8bllz+noQqy1TXzq69wlQ0/+eA5v/2bZ7S9GVoekC02iGpFEcxwvIqT40d88Pzv8IPf+Pf58e/+Nr3dHYLQwG8NgYrZbEZVu+hGTa/fwpBdTE3DVBUqanRDo9FofMuV1TEMDdPYnjWL1R2WI9H0mk6nwe7uLp3O4K+/oFQNH6uh09vd5X4c4jX2tvsvikGjdcTd+AbdsNgEAatoydXdPesoQbVcZquM6WrEYpWT1zbNfp9VvOH2PqAzOKbVb2JaLfxmi7ps47gNqlLj5HTIZpGjKzaTaYHt7CNUQVxElCxYhheMpndcXt+gG5Kd/iNmsxmvX/+Sy9tfsA7umIymlOUVlRrRaA9w/R5hnnA7PidMbvjBD36ErBpMxktcf9vxSOKSmgVv3n7N7s4p7bbN2dE7WJZFVdaIyqI38ImSCFFDs2VTZpJarMhTMJRt2GG+mNJwn2NZXaRikVdblEVcgjShuSeoNUkudPxGh91hnyCJyQq4n92RVCvshsVyU+C4Xe7uKs4ePSYOBaq2yzReEVcxi3XMpig4PnnAaJLw1dsXeG2bMC8odQe1CXo7o9Y0qjplmU5J9Q1ip8ndtOQ+e0UoFZymyyJcEWdrnjx5gtLuMQ9DSpmzd9ake2BydXdPUuTcjZZsohqzJQjSkvvVhFW0BKnjNTvM4zGlYVJJkyC8w9RbVJXBukzJnTbnmxil02YWCu6nIxYpzNchBoc0zQP6jQPqXGW6vCbIYP94gKo3kULndvQFq+gcy3G5v7mj069pdHS8pkelFuSlxWefXZEmJcFmymY1Rpcp68UblDpEJBKjHKAWPQzZIA0qBDpJqDDYdek3jokWkoZnESwlqqoym80wLYUwvUdkxyznGcvlaxrOHqDQ9h/z49/8Bzx6doLddBlPJJPVDde3L6irDFcfsl7fo9obLi7ecnzwPkVRULMgDUuyrOL7P3xCUd6wt9vFsrVt58tQEayRTsoqCZhHF4TJnMG+xTz6KZvsnEKETOevcT0LRYu5uvmCbt8hTlY0/D5SmMRhRZmb7A1bSKHT7kmy6o5KrpmvxliuznqzAGlQiDFJWlDVNZe3L7Edhzx30KycNxc3xHlCISdohkKRd6lKDdXwSaMTFKVFmqaUBYg6pNvq8i/+5T/m6uaGbm9AlKa/hvhulWIqoirJiimWWVHwiuvbL9iECYp0EZXNehVi2AmbcE2cZqiaJMvSLdxbCp48fcynv/wzLM9CKCV1bRCHPk3PZrL5hHkYYuratujTFbIs48MPHyMqi/vxCtM2EFVNmZcoQkM3V3z5+SWOa6KgUZQJSZTQ7rgMDxq/fk1HGa7jf6tqzCkLhaxYcXyyTxZvcR4Auq5vMVRKjO/rKNUuApBahaLUlKnC/l6fZsMkiRwqmYFSoqgVcVjRbGyh5cFKRdddiryiFCl5JlGExeHwjH5nHylUhKioKoGhqRRZgmsJylTFNBykIlHEtimQZwUfffSQcLMmDEBIDakU6EYNUqfXb2DpFnli4PkmUoCqb0fSVRnRaqm0vT1sq4FQQFJj6BaqVNHNhDjNqEQDRVfh24eGshDYjs588Zrr2w1C6kijRFElCIu6zCjyO9brNVXlUn0busmzGgWTNAnJyinTWUiQFORV/q2hxGA2XdBpt2k3Ha6v1jieu03IagYAZZbieIKyqshLFSkV6qogy0OCIEJHR1Ez0spHxUPRCo7PnnL24JDXr8/54z+65cPvfB/TkURJTS4USmVGsLxHqUx0TUdRJHVVoZsdzp49R8gFw0Efzx0yW0/IiwDXdLg+XzAZxejGHg3/EWWmUOYbJpM37O7s0HBbZGlEEMYEwYam67BersiqAtfqYjZKluFrfvnpX7C6P+e/+L//73j12T/hzcuf8Woy4vVogta0SfRzzu++ICt1+t0neJ6HaJqEVRul9Qh3p4vSLKhMk6DKWRcFh8/3kbbGeDljs4nZbFaM1xtsO8EzK8pKMl9Meffjx5ydNghynVZb52pyjut0WC/OySNwrDZX1xdkhcLR4UOieMN8vCJP7ijDDEuNqNM5o8sX1OUt8/E91xdjbkcrdnfP2Ns5o+HbhMuQptPGNVU2m5DDfpeL66+YzFPyyqI3lIxWL3h4/Ih0tsBUHaJ8xWR+w87wOb1Dj6+/vOSX33xJVj/h9FmTv/jZH3J+dcmmikhjWK4t8BrUekI0vma8vOb1my8xHZvBoc1yfoEZqbz34SkfPtyl1RDo1YaXX/4xL95cgN6h0TcRdpfW3gmlVpGXFlIzabQegLYmiu+4Hr/GclsEoUGclpSpyvMPvsv17S3TpcKPfvghO02V1Zd/ys3FHbbnkoxf03VBN+aceAF9L2J69XNkfkXLntPWY2QwxqiXFOGYyWqCahmk1Qqp2qC0qPWaIMtJCwvTcikEJGVJlGasw4ggTsiLChSNTncPVXMoa4VKQpjEJHn0Vy4o/+ocSgxMp8EiDPF7bWaLKevlBE0via7WNJs7dLs9Lq4v8RsNbsf3VNUUv9FACIV2uwtqjm7ZTObbjsskeEmwTOl1uri+xXKzRNNhtlJQLY3VSmKYDSazOVIzMXSP+WaJJ3yW8xUPzoYslwmNdkmnOWC+vGQ8vsdtCHRTYW+4S1K8Yqc/oOGegDEnSFNKUaFoCoPee+zv/C7oL9GMHEt26A47XF2/wtT2ePLw+5Rlyc7OKavVHZZhoigeO8Mms0XG4ycHpElFFK947/3vcHt7j6IILDfm/8Xan8TItm/3mdi3+71j74jY0UdknydPf85t372v5yMpUkWKlIpVBRdsyB7bsGGg4IkBa+aJYY8Mw54UjJILqoJstUW5JFVRpPTE7vW3eff0J/NkH5nRN7vvtwdx9TzlgAmseWRkxj/W/q/1+743vxxz7/57KEZENI8QBANFheV6TpgsqVo2r1546IbM48cPeP7sGUWzhyRqGFWFKPEQVJGlG6KZNa4mt9h2j+6gz2waEkYe19MpigpiWbB0FQxLoN3b4yZ7i+PP6G03GU6vKNO3mJqOUrOYh1MsuYMtdjifLpmvbxEqHYTSQ3dEGoaJlOT4vosnQJKscf0JYqli13uQLHGdgLZtEcVjTLUBUsBqmVOt6QiSjGKI2O0dbq5vqeoVOp0GWq7RrJlkFYvhZEQulQiqRZYJDKdnGNEYSdERtZzjq4yd3n0CZ4ZhlBTlkhevLzi5eMvTR0d0Wu/zxbNn5Omc/f4hFavHT3/+Of0uIBcolZhcKNja7VNeluzv3+N8PEIttvDWCa1Gh+nihoq9MTslssR8MUIUErypT7NZ43y4YFvoIikRd3e+xcXtV+ztbbGcJ9zMf8ag36JWOWC5XPHkve+xjlNMaUKu1MmLlPHyjFKIefrkEd4agniFIm5uGbXKESkujU6G594SORm2XUUsJVS5SUUTybNzJEkEcnb3dvgqjAmiCUZFJMpeMxrbyOUWZRbR7RiI0iFFppAVE7Z3W+SZyGLuMpq+QdcMDKVJHI9oNHeYTlwODu6xWN4S+rC11ce0DF48+xEHhyU5Pnazh0iV09Nf0G5BJEdczOb4rkdaRkxcgTTxCCZv6fS28fwDsqyGbrlIRYUs3SjvLi/GfP7sX7J3sMvZxQgBiVLICf0AARmhBEEqycMaai8mDNa8ej6jKLt4YYCqypSUaOYSRe3guCmy4QIxZVEniEZc3lxgKQ2iVEcRA2QxJowENGJ6vQEXb0yKIqPMC0RZw/OXuK6OXdsDInKxQC4FsiJBEUREycdzUxTxIXE6Q6ZCEkaE0YjpYvarM3E+A0kJQJGQJZUizbBMkdFohFA0KEmgLClycJYuD+732epGfOYXCEpGlISoskaeQRit0LQqYRCjWCJJnqKIMmUmIUhLqkabImmQCSGaWqLJOoEbIQoxvpOjCA2cLAJpg+YpUaAosW2V2TgmCkvkWoGUCciqSlm4dLoCWVxBEq3NWKssNruFoY9dUahWa5SZQZrl5EVMKiYYikXgO+TFguF5gyK3kbWCMEowVJPAcajUItZORpAIJGWALVmIZYEoV1jOv8JdFyjKNgWbnU5NUfHSEhnQqwVvTq65ua2it7vEvosoGfh+SKOhUbUtvvjJBWmxSxD4lGmGZVXJ0pBWs0USpyxWOW7qYVeEDaJFkimKiKdPHkA0JQw3+Lso9yiKjNDxaFowX3i4iU2ZiWRlwiIu2Nk+5Obihs8+v+T3f/89DNOjVoXpZMn+3n0EKUAocgQEFEUnSRPm43Mqe4foisgqneO6OVolREgsVtOYwwcDVn5MtFaYOG9J/SmGVKfWs7i+PqNqHNDqlKC4uG6KEQXs7vVI8hrT2Sk3qxFWZZdqtU5FqVDttvH9W3TdoCnWWA2P2dl9iDMXSZOEW/ELikJF0TJq5hbL1TE/e37FnYM9Cq/O/YfbfPnln7N+N0EqbMLSodbdol5rkGaXSHYF119Rq6qgy1hSDUWw8BSJ0ctnFEYFexBSzhMOtna5uLlg7Y1ZOjmzaR9FgK1uk05DYrWYsVi+oWW1afUCxrcrOsoeuXxOHposE5/CrbDV0pkMPXYG28TuBCXUeDzY5fnr5xzsDdANgyBccvJihVmx0cSMNLhmcVGi6DIEIMs1CkHh179T8q/+x1+QKq8QJlXGoxW6ViVaZgxaOlkZcDE8RSxS9naeYs/G9LUqPzl9zttA4Pu/8VtUduaMJnOuV6esVyGd/hGDI5uV4vGTX/6Qpt0iWaQc9rawW00u3l3T7Ar4qyXBMEMVawTJGrNW42Y0RxXWbHUaPD89YRbEqFKT2Shm5S148eU5Rx+9z27Uwmr1WPtLgvmanacf4y5G9Jo2RR4hFD5qnGFIOZ4/Z+7pGB2TIi6o1TvkgoeqyqycnE5fJYkF/GDDypVUA1WvoGj51/zUbENpkAo8z8UwTPJs85AoSdJff0N5cvYW27aRJY2KZaAZNSrVgrx0MZQGt/NTFukNkmESpQKSrGFVayiSzmK+Zqtfh1KiLMDzApASDLNAk+u0qwNunWtKqSCIM+JiTlasEdI1glZB0SJkaZcCh0IMCLKMQkhpNHa4Hv6CmARHWYMSMNjZRVZEEGLCsGQ0WrM1aJKlK/rtjzm/+AsyYUi7foDj3/LP/of/IwcHD4nDCmpdJopT6nYTw9CwLIHlekSanaCqNrbdRFYqjCeXRFFGsHbQNBNdt3l3fouqGrQ7EqPbJc12B0FZEEUy/d42njejLEu6PZs87RP6BTtbIp1Wh+PXz7GbVVbeFF3pgNLBrneIE5+VO8FuN4nSgNKc8id/9gpDqtJuNzm+nGBUUnbaLZz8CjlXmc42C8DOOqRwIoK4oG/vkxQx6+mKVuuQ1XhBok8QMpVBew+hkIjiW9RKl/HcBTlGE10EwUQSYX/7IeP5KUa1ZDrNsbtV8mKJqtQZjSbY/ZJO9QGL0CEOfe7fvUfoGWThJYYtsXQztKaD7xdMFy56TcAPYgxNR1Ij9re+zyJLaDf6fP7Lz7FbEit/QRKFFHJJItm4fkitbqOIbebThFqjjbcqePVmxPOTMw63e6iqznji0R4YpJnIl798xcfvf8BPf/IX9Hbu0Gv3EbJrTCtk53Cbn332I1rtA9p7R8ShQF7EDMevKZSY7nYLQdbZ3umSxhkfPf4up5cvWTpzVFnbqC+tAEGQ0GsWRexwObxAknVKSaTQRMyKSJBmePkcVXMo8xaiZCKJFRbzFe1uldk4pqKBIBb4bgHkPHyyjSACwubGKU1TciGg09G4Or9itVjz8fu7mBWLOCpxJyVb+9v4Xo4t1wjcNYtZTLffIEiuWS8zyixHF9usvSFpJpDEPdLQRpAm+OmQ0XlKo2NzfnXKw/vfQVUs4iTBrDZwvQhJkYAKtl0jz0LatQG341M+efopzryOE3cxrTpRuEYgJi+v0ZQD/v4/+L9w78Eef/jf/0sksUKWl5BuQhSiIG8MDbJMmIzp9g6QaJDGC0olwg9zFE1mscz45Jt3qNZ0ylymTFNUSSOOCgox5PGTI4YnJkHmYmklZAZpsKLXAU3ts/Jyanq+AQJHOapa0m21efP6dBNWEkridE3V7LIIrmi2FS5PDApcylyjLAoU2SCVUhxv/KszUdcekCOS5gWSJOEsHQY9CdM08fwQuSKQJSViuQnuZKnD6ck1Io8pJJEyVsnIKEiRlITRjY+mdYgyGUmRKYG1M2J7p6AsBPJYJpH8zUg2AN93uXs3plqVyeIKZaEiySWIBUWRIhQCe7s9FreTTWo1nFKRdSg1gsDBDUM0pUYSFxRmQJHmqHKNslzj+UuSVKQotI12UpRRFIHYT7HtGt/6Zo8XnxsIpY4gJoCALKsk6RKzlmM325zfZlh2hcxJNtD6tKRixaxXC9arJkWpIKBTJBllkUGWg1xQqw7woxAt2ZhIJBHS2Ke9rbK1s43vTolSHznTkfOSOHEIw5BWU8UPfBw/R62r5HkIkkCR5+gyeOsJ/lLY7KkWxdfJ9JIgmmFbJd3+ET9/4WzS2hqIqsrdoyecHT/n9Zs3/LN/0uLekwpqXNBSNYZnY3b3+lhVixIQhY33effoHjO9Sq1QuL5+g2A2ybMIf7igrFaZz6AUZbJshaQKVIwGflRyMx1SrXQoi3KzH5sXdAcGi2GE71U4m5zQlmvsDw4QYxutpnF19Yb+0X169R4ECropo1cbGNUAQdZBLEgTkTJr0O7oXL0bM2hr1O2UdtvCVi2ef3GOJEsIYgu9LlAOwbka45QmkvJN3OQVmtFhHaQ83B8gKykXx1MkvaTzoE7sZNS6v03ceM3anTFZRRwe3UdbjQniEYEfY9QONnt6YpO3V68YdEQO73VZTSbMrpbsdg6paOCsrlgEM5JIxfcV9PQWXbb4td/5G/zpj/+IWm+bJM442N/nT3/xI44evA9liWJkLB2XUp1QM2wa23W8ZAWawNSp0N/aIxFLvHlO/2CX7a0qxSzCzyViJSILCgaDbRbTS7RBjdfDF7Rtm0KJ+fLLn5AHBYPtHiMxxGrWmbo31AcN/FWEUYi4yYKmrDBc3CIGPpWqiaDlyOKS+cRhe2tAU5NZTqc0TIvd9mPKYoJHhcP+Hmav5OJnIw7ubPM7f6dDUkrk82te3szo1fdwFz5/cv053S2Zmt1ivhhiiA3qlkbMGFG0kSoe0WjMdOqztb9Le9Ai9T203EXOBAy1ybuLFMuyyNmY2vIiQ5HkDXuySJE3bkeyikCcRGRJ+itN41/l56/OoazoLFc3GFbK5dU7ktRFr+gs5xGqAbKokKcimmJAKVGr1ZDkAk3XEQSDd+cvOLt6yevjZ9RbIquVQ9XaohRVzqfvNmlFoYpd7bG/06Gi2mShhKZaUJik8QyEzSEgCwn1usX51S0PHn3KoHOHqt4l8urcu/s+WZbirNZoUpea9ghNFTFqOueXz1GkgI7dIwjn9AZb3Dv6NqJgIokWi+Wakoy6eZfFMsSPriCvAjKCsPEm/4dYfb1ex7Jq6JUcxxtj12UajZLLizFCITPo9Cgjk/lswnD0glZrm9F4ymIRgiCh6RaO63E1nFKr9Wja7xHGYJgVdBUuL65ZOmtkNYAooyJViN0lmiwQJzmqVrK/Y9KwNaJcQi4lAl8mCWeIiYwXrLCsNmmW4BVLUFJkUcV35uimRF3dpmZtMfeW2HYVSRK5Gn5Oo16hYbVo2Hcw9CpxknN2NkbC5PbSp7vfRdXr3I40LLOFJCmsbiyCPCHyI6J8yR/98IdcjE/ItJLP3rwm16bcnFwxGY8oZZfT8zkXZ3NevPyCl++e8+4q4vpiwsnbMTJtxFTh7uFTlIrN9WSCqra4s/MRreoRvq+yDhYEK5E8FKnaGvVKH1k1mE7m6JrJ4eBjomBEvVFgGk0qlk4ZFFwNLxktzxALh+XoHFk1iJIQopymDUF4yd6BSewaCKHC7/7gBxC3aGxXeXX1llhw6Q803n/vMZpqEYYxR4d30EuNJMxx1yvqlT0UQUQV6oxHc5zgBsddEcQScZIRxQWOO2fhvGa9WNK07iCUOVXVxKyWiJWS06u3CIIMuUBZlDjrGZZooRRtGnqHhztP8dcRt9fXCImKpKQsJ2MMNWO2esditabbHZDlCSImaRZRMeqs/RnX1xO2dtsE0YJGrUGnrVLRVe5sfUC3WeNwsEfDVrm8PuP4/OdUaxphdEOz2UQTTALfISoyUiXj8Xuf4KZNHGGPrf0BNSunohtEZcrW1hZ//j/8MR++X2U4vdisaYgieRFvLECCSJ6nG8ZZKSMmEmYTGq0GYaEzX68hLMhjlSxISAmZjBPIIooywy0i0rhEy0TqbZub2ylNRSNIEnKxRCoL/GzEYhYiESJoJWUEZSmQxRmSdMFotoRSQ0g8JCqEWYIop8hiSJzYpEVOFudIioTjuNiNmHBp/qri3CAkpWATdAlih919ifPTMWkIeVFSpBmKaeC6LrYpU6/ssQwnmzOl8JEkiSyR6O2GjJdTvCxDFlOkXCDKfLI44v7uAarcJiCHooRyTZbH+C6k0ZCVc8t8XSJpESIQBzlhVGJUEjrVEt9XSKQYRVQoIwiyAFWT2OnUyHKToEwo8xwZiyzz8AKBu9slWQpubKAIMikZUiIQxQHdPQPBF7lZLUnkTXreUCTiPCYKBWwt52acUzFM8jihEGQKWSMJHe4+aBJmDcJMR5Uh8xMywSWloGJW+eTpPlfnLoVSEIcZEjKZLBAVBf2mipmsGc8DSkWgSHKCJMUPJcSi5OhggFiqeKlGmblkZURJhu+X1GsW9w8rLP2YQgwRiDaNslIh9CIOBjXKEJaLHMNU0bQmulJg1CW6W0+QhQovXl1SN0SeH7+mauuslye8evM5x6cXZEVCnKQIosjFxWvWsxGCpuFkGl2zQTRJ6Dy6Q6dho1QSau2MVlckzaDU63R2utStDlkmY7VSzLpMHmVEgYCThIzmI472tmi2TKrKLpVajkDO0b2nZEHEyvH54U9/wtwXQKgyuozwkoiQGEmXWC1uyALQdFiFOYeD+4ROwNnNCbIZcvfRdyn0ECmGar1GobnM3DHj8xfEaw9NVfGCFW9eH/Pjv3zJynORpBq3t0sWbsnN1VfYzR3mQw+728HuGBRrkNOAVThkHcXMbhfEmord3GQkTi9n6LUuCCmVqsxoMqSqdtjrPEaVLJptEz8qmKYpn706p1ap4uVThkXKIir47m/8gPee7rDwHN68mhGT0exv8e56xovjK86Xtzw//inDmzeMVufE84CyEtK2q1T0NpGUEcVrJKGCWq1zennFMsk5O5lydO8xj+49pdnbodM0kLQI09LYa3XwvSWCnvH68meMJte0DlTu7G4j1wWqlkhzxyaKpsyXK7RulW98/0P0vsFKSdh6ss1O32Q8u+DVzRi5qjEPJ4SRw+6TCuN8zRfnr3n+/B1fLieskhbjLOBysSKTHKbrKb/8/JTbmzHP311zvR7hehWSWMBzc0LZ4tPvfI/Qz3hx6XA+uWR8tiRJ2sxWa/qmRulMUNIz5GyJniVU0oCjXouDoy3uHO3T78hoZcR79w44PNIppelff0Pph1P63fucn94ShC5JpKIINQ4ODpiOQ5bzmHZzm/UqIM0dFC1BkS3mizG7RwVb3feoV7vs7Q8IfdC1CkWmQalh6j2ajQHtzgAvWHF5fUW/36coM0ohoCgK0qQgyRY0Gx3KQqRet6E0OD4+JvBK7h19SBKX/OLzP6ViSHzw/qe4wRWataBWbTO8dJnNb+h2tom8Cmkk4HkOnr9kuZpQ5BWkssd6vUYxVvQ7A9bOkiSSqen7hGHJcLik26ujCruEPkTpiCwPMXQTVW6yciKiJCEvIYpz4jQDVKzqgPVCYHt7GyhIiynzxZhu3yLJ1jRaGvPpSwa7IqP1GYPtO1hWSZlkDAa7dLbvISouiu6jiD3KzGJ04xKHCr63wvNWSEIFz/NRjIyqtYtVM5nOLwiiW5bOC06u3pArPutoiReFxNnGSdyy6yzWr9Esm/39v8l05hNHUGQll+cvqSgSll6g6y38yMEwWqSJSiklvHn3nCxT0Ssab89/gt0RaTT6NFsma/cGWU1wHIf5JCdKM+I4IYhCSsGn1ekRhzY1q4OX3NBuGazdG4yay9JZ8uLVzyiFgK3eQ8aT14TFDFVXKMUAqzqg3uhz9GifVr+BWFkzWd1g9yrUWipuvESzDFbuiHfXLzi6+z5aNQd8ut0Ka09i4VY4OviYO9tP8MMLzs7O0NMKSqygaCl7D9q8eP4ZphpxeXyKRISzWm9umhKVD9//AQ8ePKIsJdqtbar1HRTVYrI8xq73KIsEzfBw1wusiomqpPjBGkEIefXqM5JAIIlzZDmiZra5uX6Lv5jhTyf4qxuyNEYQQFVUDKECxCSBx4Ojp7TbbXZ3jtjduYuil5SFSKUqc3HzGllW6TTuEwQB19fXkNvohsF0+ZI7B3sMOl1Sr2CvdYf19ArF36YhPGR4fY4iqsSJQpYoeNElvfY+RWZiVC2W60v6O03MaoXd3QPiOOHd+QkyD2m1e6imSMXso8o5e1s9hhdLnp38d3z63R1GwwhZVikREYX//1BElDZVljmCYvDRjkZP9nBXAoVUw41jVv4Uoy7y8P4ON6MxgqiQpxlCoeL6a+4+6hHGc7I8glKkLCXyvCCNcu7efUASWeRphSRNidOUtAiomBLNVg9RtEGCvBCR5K+tE2aBIluslv4mTKPJxFlKnmcE8Q26rv2qZEFHKIQNozIXMJU6nj9jMl0iSBJxliOKAmEQIJQlippxfLxG05uEsUNWKAilTp65HO0/RC67GzVkIW1CLJmIocs8evSA89MheSYhSQplViNJfRzvlA/e/5ThpUCaKsiCTZTEqJpB6BcoSspi5VGUEnmekaciGTlJlCILKWKpcn6xQNF0QAAhI8sFhDKj063huTGKKpGTUxQ5BQVyLjNxXrPwZYpEwtDWBK5Mlm+MG054jaZLpIlBHOcbjp2YURYJWRJTZjFNu0WRi8RZTE6KIpvEvodURtzOLlmsE5ByJEkgzQLy3CWLcj567wk3I4/VCgQxI89TojAjCB0sy6TeKfG9At8LkJAoEhUyiSAI0CoieqXPzXWKJrdIIoE8E5BEkapZo1qXeXv8Ek2vUpQSxdffO3laoWJJyLrD6cktv3zl0Lnb5Nqb0rt3FwydqqVAIUCRkaUxeZgRxyX1WocH+w0WyzM6Wwds12wU0yArRGShj6busHPQIy+vkSSHwFtjGAKd5i7zm4DRzTmXF2dsdQ6pVyxss0ospQxnI4JE52r0lpPTUybza85Or2i1Gkjpki+/OOFmfMmrk5+jhBKLkUOzYfD8+Be4yzVJHCCKMu8ujzkbHhPFPl/84qc8ufMdytka173h0eAOFU1GbcdMZzeMZm/I04KmfcDB9kO+8f6vk4SgqTZpGZBR8ur4HdW9LpVGg5I7PP3+J/QOnvDtj/6AyM+IpILp7BrN3qVb6ZNlkGYegqfiZBn93h6SVTJzbtjZeshO+zF3to+w6wW/fPtvWacyd3cGPLn7mO39LQJf4M//8jVhckvVDNiq3eXVs1MKIcLzh5u/p1bFWaWUUp0gmzK+zrm6PObP/uyHuK6HpMisnCWXo1NuRksMy8QwbY7fXvOjv/yM8e0Kxw0RkHn1/Jparc90MeLzzy4w9D5pWTJeeLx6d8LVbIwTJJwPZ+g729iHFvagz7pIsK06jw8PEKUUJ/bIlYhqX6PdgrV3wWKx4mq8RJMSTKtGvW/RaW/T6ZWMZj57D7s0Wx3SREDSUkSpglWVmc0SXr0b8vZywvn1FEXQ+flXP+F6dsHq0iVZe+TCnOVkRBmsCIMJW42MLUuiJl6zY8/I5ZdMV8+JLn/C+MsfU1FuEYU5uTckup2j/dWpQX/1hlIQSsJogWXVNlDr2xmLxYzXr45ptwbcv/eQ5WpKXq5ptCXarS18N8OyLGK/hqwI1KwttjrfwDR6ZMXmCbLRsml2LHQrJ4wcRKmkajVZLjwMvcp0OoaiQq3aIUvZPM1ULNbOBFnzcP0ZcTblxesf0+01CP2AslBx1yW6ZhLFLm/evKDVVdjZeoAi1Wh3TUTBQMy2CPycNE1Zeq9ptCVqVp/Z1Ofd6XPqlQd0WzukiYhtt8hTnfUqRNYCgjhC0UN63QFJkpEXEXHs8+DhXXRT5fTiDdP5OXZT4/b2Fj85pigz8iIlCGS0iookKSRxzpu378gSleWkgqErPH/5MwShRK9UuLpecDM9wXVCvEV7owaUVkRhzqB3B0N6QrtxFxGb7b0tZNXkavwcq96k135Ix97jcPc+FUMjDEeoKqRpShivOD27YjwJsGoaoRMxvb1C1X2SKCIvIuqNFm+PT1ErCY43YrlY86O//Cmz9SlWQyDNIzqDPkkZ0entcXoxJPQqPH74TSyz8TWrUiLwBGStYL66Zrlc8PTJdzg8uM/RgzphMmV//xABhcFgm0b1iKM79zYA884dup0ONb1LEsUoYoV6rc34dkKe+2QpRFFKRe+wv9snT+Wvb3wiBu0jOu0+CGum0ymSXKXXtuhYD0kzDbMhcX58ShlG+MuYmITOVg13KrLdPkBVBIaTEUatQa1ZJfKXaKJB5IGzDhjevqXIJdLQ4nb6DlXXaDXv0m71GQ6PUbWUnf4D/HVE4KTUzUN0VSPJFjx5b59mvUfN7HB5/YoomdNsDahoFYRYxtR6yLJCXqTkeQp5Tqtdw6xJLNcurr8gyG8YL85RKyboMUVRYW/rGzTsPjPnc6LYpVl9gu/F6EqNg/4H3Nm5T81o0G8N8Bwf27Z5c/YSUc/od6rMVmOMKrx693Me3f0u9brCaP6COKxSt45YrUMWKxeEmCiJ2Or8Lls7u+iWRN2ogephNRvIuchP/uIf8cE3qzx/+5bT1zGqblCWwsZhC5uAiCRQCiV5maFKLv/539lDK4eMhzeU+XiDAluX5MGSs9O3xLFEUuZk+QbPkwsxTjzk1etnXzdNJWUhURQ5kpJQsTJuhjN0QwZENr1sTpw4bO3s4roKpVAgSBppniOgUDETvDXkmYim6AShC6LAaj1jb7/Dap39qqI4RCgDREFFKHXKLKQQ1iyWDpIibzhuysbJK0uQFwuUSpNCEPGjkDiOEEWRyHMpE4WK3PuaBVeQ5hG21aEkQa8IVMwB4tdwcKSEMEgoC5XFakiaF2RlSkZAGksgFATBip0DnTjSWS5AVESSPABFJA5i3nu0iyIaFHmVOP0Pe1QJlAoUEUt3xHyeUhbC10GfBFlWiMOUInd5c/UcRekR+wKisCYvPZJUQzNWaKZIXnztKy5k0jQnS6FIYho1iSBMyDMZSdxghcpCRRZLGvUC1ciQlTaiaIFQkBcJQiGjiSq9nso6CYnijKLIiKOcsgR3vUBWCjrdHc7PVxSFSBzH5Nnm4aUsM2Q94Id//mPSVEaQoSgKFFnf8CNl6PdraFqXKNx8H4iSgCxrrFceilryX/zv/pf8wR/8J1S1PUzjiKvrgK9enBCGObqqoYgiYgllUVIIMqYu8Od/8q9ZrySupiK5UmeFyGR2haQteXnyC372k2eMblbIqsFk6lFmVdSaxHIOagWa7T7373+bdTBGUEW+fPaWNLMxGgnj2RCJBlHgYwp9pEzGFCrU9QHf/t4T6qbF06PfwJ8UnF9e8PLFCQfbj7C7DUShThisMYSSYB7izFeEQcJXb35MWNHwCgtFsxjeHiOUFod3HhCGMr2tNooEx8dvuLkaU9E1KEp6AxtJ0TEtG0OyiJMVX735Y27GLzk9O2ayuGDuXdPo1qjVYD6p8uib3yURJXQTuvsFl9cj1vkETcgpigw/c7hdXuB6Iabeo93tUalurGZ5qrNar6m1VJSKRrd7B1HUefvmnJWzxo9yTE3FrqqIeYZVhmjlikFVRVd0GtUazZbM1vYumVCgmCmyLNPZMnDWPu1Wl6JIKIqEwM0IHAFnHSFKGX/6w3+FTEnbbtLrdMnigouLC2S9xK63uB6/Znh9wcXZhMsvfcQwww+njEfPmc+nGKqEoErkZcDeYYvl/Jpet4YuqHz4jQ/JghWeF7Dwp2zv9tF0ifc+PWTmeKRyilXV6fZbtLpVuk0df7VAU0t6vSrf/977rOcz4qTGg8eP+OSJzYeP7hPlOmmQY6gG+ficy2uX09sluwcfscoFtgePmL87RowEqHgEM5+rl++4OL8mTEbgBn/9DaVpGWSFx2CwjSwZHB0NqJgae7tHqFqJgIpdb2OZLXTZxltlxEmAJGdkcY0iV7hzr827s69YrkZU9NbmVkHIOb34nCC7QVREZEnDNLqbvY9cwzBMHCfg6mqKZuiEweaJIohXOG7AnXs7qAYErkLgBRweHjCZXfDizZ8iyxL3Dr9N3TpEEWwEKUISdQx1gGXV6PYNYGMHcZYiy3lInm12u7rtQ8R0lzTNabd6qJKJIDr8/Gefg+iiKRtn5tXlOXatwXQ8R5MsFhOX2M+4s/eUTz/+LQzd5mB3j5KQPE+oaH0kGayqyu2lxN7+ALMqEBY3LBY3yDmoskspR0ydc8gLMn+OInYQRZgvL1ksZ3S6DZJsRZZl3IxfUrUrjIcwvB6hGjHLKdwMJ5DrvPhswuT4Bn8UI8YJoTNjPJyiqiqe53NzJXJ1skQWMtxVwOuXl7jrnIq6j2l18X2D2+ENv/4b3+Xx44fUzTs0qx+wd7DPzegNW1vbWMpDPnj0m0SRx3B4RbezhSK1+Pijb2HVJB48uMv+/iEfvf9tvPAtq+Ar6nYFQSgp85jQT5lPrylSGUqfR0/2CdyIm+ErKCOKKMCSddLAY3+7zVZXZzUdUdUtes1tpLTFTq9Gx9b46ou/YDl7Rez6rEYukjiiLNf4Kw93vsTWLMzS5KC/RbdlYeoSsTMhSUIOdraJhpec/uQ5W12DtXeFKkeUmYyipIwnl9hNCVkNCMI5XnjD0h0xvh0hCiUVuY6YGyTJmMTTefrgd+h2u8RRQavTgFLDcVxUI8SqqnTaA6IoAkHHT110W0PRNEpKEDbKvqkzZDFPWK5SVv6YtKxSaBGJmHIzXVNrDCBTCLw1tUqPqtZCFQusSooorOi1+1Tb+3zx+gVGQ+XyZkiYrCkLld/7/e8RJTMKSmy7T3/nIZWqhYBKv73Pwe5jslxEQmM2v+XozgPyPEPK92nUH1MKEjVrh4pUoZQUBrsH/Hf/+P/N7/72Pq4/4mLyc/xwgqYpm9Sz+LV1oRQQECkLyLKMIjf51z8Z8Q/+6Jq4NMgT8OMYx/dotHUOdw5wHJm0iH41Og+ikFJyaTf3iRORJA4RhJI4Tjb/M8xxvJgkiSiyAlGGJMowDZn5YkIciUiSRBwn5GIGuUi9plDRewiCQBzHCIKA47gMBnWaDYPIq/yqytxAFEUEUSHOYjJGVC0BUapukuw55HlOGMTUTIndgxbzZYRuSOhKBUEsmS/GGKaIVS95d3aJLMuIskCWZbhrB8tUePn2S26GLhWjSlFujBaiYNCwZVrdkOnMAaFGGPpfKwZLZEGmKBa8fnWNoGhfQ9t1srIkSzL89SmXFzcIUhVZ3jRDglCSpAK6ktHt1/FcBcSNr1iSFLwgQjZL/u5/8r+max/ghQmSpiPlCkWssHYXWBUJ34lZLzc6S8rNe5znJVG0Ge3NFhmiLEAeU+YJCOLGzZyMmE1ilssMSdQRBAFEhSgsSZMVjjNh4UqUkoosVciyDXw5L2JEdcnJ2SmTFSRZ+TWbMt2EbgKHWjPDD9fkgopigGJsXlMc+hSpR0UXWC1F0lxAMxWKXELTaixXI77x8bf47rd+l//8f/af8Tf/5h9Q03+Lb3/6v+W3fvPv8Tf+xv+eq+saaQ6iIlNmBYNmByWXeHR3n/lyztagx/TmJf7awV+7OIuChtkgy2b0+l2kskMaS6QsGY9dvPCaxdzBMLap2CpLd0IUJaR5hpwl2NoWmpRRr29ep2pq7N65T6Yl+PmI87O3NGp1SnfN+ein3MxPGeze5+bmOVdnN/j+nPliSVoo7O8/RpZ0ZGnN5Ood43yNbZq8uHiFrOgUsU+UGWztdYiikDfHp+gVgevxZ5yef4mIwNXpjDgMGM++Yu1MaTUOuHO0RxRmDLYMknRCr11HzFQKtaDRdDg+e8dOT8AULIq4xdGgQuxcEAYjBEFiMhsjmwkFIaObGbpcB1HbNFuzM2JXwJuuqSgly1sHTVfQKrCOF6Rqyfl5QpCWLJw1tXqHo4f3mY0Ver0WtZqNqe2zcmIWqzVZoKIrFaazMUmyYDoeU7PqWJaM644xDIFmw6Tfb2LIBd16k1/79kecvnnFejrng4f3qAgqq5s1TzpP+eTpI+4eWOzvNXDdEfNJTtXuEfjnRIFPRdHYHmzhjjwCR6Ol7yGmVby59/UZJlCUOS+OX5DmIo53jRskpFJBe79OquWM1ufkRUq7VVLkOe/OfsZP/uzf8tWX79CNmMszhyTK+bd/+m+JhQw/m/DqWKTobDPYrtFvVri4/oI0cHCXPh9997fp39mmnmtY1R47Tzq8ev2Os+cBuvFX7if/6g2lZcqIuY3vpJSlgyCWlIVMFPsoWsHF1Ruq1SruSmI22Yw6i1zg5mZEnN3iepf84hd/xnwxoShjFC3Bqm7MHY3aPZxFTOgU+F7CyckJiqwT+OXmA6MJSBq4fsDxuysUzaDZHqAqW1xfxYCNpqnMFyM8N2a7/5hatU2lUqFm17j7cAtZ0Wi3+lxeveNm9JZqtUIaq9SqAxRF4OjOXSTVJYxW2PUBjXqHhfOGwWCbleMynZ6hawL3jh5xeTYijpcI0RahK7JeDzk6eAi5jlCCouRoisFiklKmGkUeU2YNwnjBcj1ktRixXK6pVAM8f0mZ1ZnMZ2ztSbx8cQZpg15jm5raYXxzhbuYEsUTfM+hZiskScB0vNnpQFoxGAw4v/lLguQdXjDn3WuP6XROGCRf37S06BgfcND5lHhp4M9yDLnk7h2DvW2D2e0Vel1B0FQazS7bu032dnpock6tEiBnJR8+/Rb+WsJ3MuJ0xNo9Rig06laP6eyENJ5SxAK6kSKKCSvvBMuyGN861OsdwjBkPHnNchIgFRv8z/nFG+4+aJMkCetVQCG4TBav8L01Qp4Q+gFP7n2EoiyIQpciz1DEmHAFvcY9jg4OWS9cWg2bes0gDiMq8g5/67f/Lg/u3afII+pWg9Ap8aMhaqVKvQlZeEvPaqIq8MvnX1Czt5GLLW4nKZeLa0y7zdG9O8iyyHRxwxfPv0CVFB7d/Zh+Z48oXKPRhKyOF0zR9QZH+zt0ay3SeEXqC+wP7iGJOZ7j02o1OB++5er6HTfDOfValzRNubqcIAl1dM1isbzF8RNyoYYbRJRlgSKriIKKVb2DrkOtqnx9uFncnDvYtSoUOYEjUcQuaRjgzuYYskXVFCBbUtVqpMkI5/aKQa3NdmuPwE85u35HrsbcrlessgRRL3B9mefPr2k1dvGja8hFylSmahlcnL6iUk05HX5J5O7RbX4XUQ0RJR3SnDgv+PjRR/y3/+X/ncMHtwT5nGZznz377xJFMoomUpCziXWXm2aBzU5jngv0t0Qad3tMohZpklGSU1CyuA04uFNFEjWcZUmSB5S5SpZlWEaVp0/vsJhmIKgoikBJThwL6JpCrWZR5CZCbm6A30VBEhR88v4jksgjKyCKIlRJJhdC4jilYdu8eztG1iDNYkRBIc9KYi+i1awRRfyqgsgnjmPC3GfurWm327juGsfJSIscURRRFAURaNYVLi4uSMM6eZqRpSFFLrBeubQ7Eo43wgtyJEkijD0kWURTJZz1hCBaoMhNSqkAIUEQVQShRGDB5GZNkXZx/c2DcZwmmwlEMGerX6VaG+B4AVmebM5sSSV01zTbAgXgxxmiVJAlm/RnGqUMmgbvPf0AVe6RxBmCICBJErJqMFuO+PlP/hE3wzkyBlGxppAKdEMgCwJswyALDSTRQpIFkjQgE2OKssRuSry7PGc6F0HKkCkQhRyElCJXePB4n+3uE9ZrH1mNN7eQSUGahdTqJbP5kBdvRuSCRJr5pGWArOrEoYwo5aydKafDJSgSSZ4SpR5JkhAFKf2uTbMxICsMVM2kLCTiNCRJEnRF5epqyPBrQH+aRoiChihKGJbCb/z630QoagD4SUSlqm2arcwADP6r/+qfE4QpSZ4gKCJp4eCWKRdxyuBol61+m25zm2JR8uTJp4hFh/k04fBgh9H1Gdfnp9hVFVMVqCgKqeewbXchybh894YyDImXAUe7O1zcvmUynVKzLZIkodEpqVoyfvgWUxMpBQtDL5h7HtVqTq1b4zufvI/SOMPzPFrtGnmusLN9h3bXRDNiSnLyVKbb3qWtqviLc5z5kiSR0bUOYXrJV599gTf1KKQQVWtQq+7y5OEPyIo1ojjBXw8Zna25uLhi9O4SI6+jV3ocXy0ZTQKccM0yX1Gvb2HXdqipKpPLJYpsodYVssxiv/OEFIWLq9f4boBRqBRuyMC2UbSU2TLk7u4d8sghTxxmtxdIJBhiBVO1GHRV/id/8z/l4dYWn37YYKvd4XD3CQ27SZ53ePzxFm9PnrGcuYiCSaPZJYgCKATEXOHJg/c52L1DFvssl7eUaUarUSVLXCxTQiSjam2TlQmf/fwr7h0+5MnDI8okw1Isntx9RLWhEqQ+yWoTtpSEDMuIcLOQeqfLbLUmjRMazTampFLf6pKXHstgyHp9QSHoCBSkUUIcC6y8KZcnNyiSjxNEjNYJct3g+7/+v2D3YZNFIPFb/+k36RzsIDUM9L7MYiGBJPPV8RdkikVCxtpfcutdICkyo8kKZ52SuwHRdMj8+h0np6e8vpyhdpo4XomzDDk43Ke9b/DHf/byr7+hDH2RTnuXxXxFnpfMpg43NyPa7SZXl7fUGyppFiDKIapWkmU5ltnArg2o1StomkFFb3B4OMAwTKpWm5OXDqcnZyTJnEZ1hyTaOLqTZEUcRqiShSo2satNyjxgPvZ4cO+QbrtN7Jd4XkCjto/vJbjBFQeHR4iSTpIkGHqNxTzg7OycFy+eEYQerusRJQta7SqioHF2foUsywioPH30G6jlU+YzhzAZEsY+Dx58wBfP/y3T+TmWVYNSxvcnPH30MfcPvo9lWezubRE6Er4/hDLBD1ZY9QwnOGMyP0YUE5bLNYZRYMjbJKlDEASsVuck+YI8NQhDn+9/8juocp+9oyP8ICdyfYJpSK9tY1QgjlPSNCcJZLb791iur/nq2eeEUcrl9Qhdq6BXJJ7e/z26rXsM+lX2DpvIskR/u8nu/SpONKPVadNuDeg0t7i9uCLzY4TARJUsltMUb53TbHSRJR3PS7Bq+5QyXF4/I4znCMqcIJhRMTTa9Q47/R4qNe7f7QAODVtnZ/AYQ60xnw2J4jl1q8p8Oudw9yEt20RXasQeaAaEvsJ0PMO2q2x1D7AtDTIJZxFT1SvcXK6wtC0atT0ULUFVZWRJR8RGVSx6AxvX9QnCGC/w+Ysf/3tms4A8ukOZDSjFkkZzizLX0Q2LpRNRqXRZzGakUczDp99k4rvsNxr0TBFV9pk6CfWOgTd3acoKg6oGjBnevGA1nZMGDqqUIWQFd+/s4oczbq/fMRmN2en3+fDpIWkAnYZNXp4TBgH37+5Rt3oMOgOWU5c01gm8HLveRFRKRDFA1UpUTaFetRERiKNokzBWJEyhTbt6h3vbP0AXa6jUeHzwLQ56OwxPX1KrVLi7/wBDlWiYJjV1h/cffI+O3aWi1BFQKHK4OY843N3DNuqkbkC2jqlLClmWYFmgG2Pmi3PioEIYJ0ymQ2LfoVLJiNI1hnGIKr2PUlEpRBk/WOO4twwO9/iv/6//L5rqax4+uIte7aOpNm+fD7GsCooqIysbleB/aFA2QTeJLMuwDYnwJkFYiaiCT5moZEmOnIaoosDV9Zg42tgcklSEUiF0HZzliMATyHKJLI8JQ580TcmyBVmc4Lo+CCl5liJIKkIpULVK5rM1aVZsmqyyRJE1ssRHVWUqlTZu5CIpEmVWEoYhW30DQ6myXKa/KskoybIMQVRQ1SpimWJXuyBUEQQBWYSiFMmSlOurZyynIaUgIxQKeRZSZDGmYRBHDtv9xyjK4GvdmUApCASuR69vEEYZaaYjSgWilCOKBb7rcXgwgMLEDzNEzSfLEihFwjDB0EPKPGM6STEqFoqgIgiQxgmaJlCtySSlTBAmlHmCJIgoqk6chDy42+Xdm0sct0SRFLJi4/SNkxS7mlG3RZK8ikQKZUQmKqSlgJiE6GZAFBcbTaQso2oSiOB4KaZu8O1vfUyns0+SJMShiKZsGjVFkUCcM59eIwgGkgxpFG+cxV7Idr9OdwAnl+eIokGaJciKQF5uAlEff/gRdr3NzWhNTkEUb7BMQVRStWx8L2R8C5puISBTqZh0ejXSWOToziN+//f/58S5QZxGG6C8WDCZTPjwg0/pDbZJshRZsbCqHUxLRBJztvoWP/vRv2M6vcJzIlSliqLqTGchiTsnW/hMLzxGFzdUFAmzKeJEMbKp8uj9h9y99wgRhe2ejTcfY5QWiiGhqTUQIhbrMa3WNrImo1cE2u0OHUtDF9YU/orp5QX3+x/QrmYsLzIahokiF8zGBZVOydvjU3qtp5iliVzsYDd2eXP+GsVIeXf6BbOhR5GYG3i8biJWFLJ4hStWefiot+HkigqdWpvHTx7Q392m3dlBkAXOhm+4Gi7Z230Pu7ZN6GbsD47oNbeQxZzYnzM6O2FQ28euNUg8eLhdZfg//oJBrBJnHrraYDw+xwlnuOsJiyhBTGzee3Kfva1dtra2KDKBxcgj8W4Zn/2Sd+9eIJYtDu6/R7XZp9rrotegKHXEssFwNOJ2NCcM+0S5RCmVjFcLLq5e02wcsbOzTb/TJfDHrL05g60OSTqn3Wwhlzq3lwEVU+b25pQoLDArDT7++GMWiwUnJye8u7nCsvfRmhmT5ZQgzhBUh6IUcYIxY3dFmEjIZcxqMmK70eag1cLSE67P1himycyZsopD3GBNUSsZrU9w8xmGbTL3PIpcomqYdNsd8jQl9EJmk1tm02tCZ8LpizP+5f/3H/LTn39BrWVzMTxlHUQM7h6wEiJ69xVC4S1y28QetKjXLPK8z8ef3iG5fc7p5IQX4zOcRGcZaeRKEz+c4TkuuaIgWTH3Hn7Md37zUwxd4P27/b/+hjLyDZLU4d6DHRq1HtVKncdP7iHJAlXLpt+5z3Ti0O/ZHB7u8+GHH9PrNTAtjSiKELGQZR3LrDLo3GE1FTjcv0O9LqFrEs7SxzQlKoZBs9lkvpgRRiu2+odQmNw7fMzuYJ8yFbm5vmI5X3Bn/wBJ8lFlCUkuCbySWrWDJCssFgENe5fJxKHZGhCFGfPZmvt3P8F1ZIJkQX9HwXGXeP6Sf/aH/yWv3/2Qb3/nG+Rxk+HNOa+O/5KPvr2NrClIak6vv48oqQSxx2LpIog5pmmgmxXCyKXdaZIkCYvFAkGCooxYruc07C6jYch6FSKUInfvfMTe7j0CV+Tk7HP0SsnoZsz5ccrkZrm54Ugkqnaddv0xl+cJveYBYinT7x4hYrC3e4eDg33s7oqSFGdqML+Bv/zLf0mWXaKySxk3UMQuS2eMZMbMVymTlYNsyZxfLChQCLMZg609unYblYy9nQFN2+bHP/0FvZ0+1foOWtWk03nEdv8RpQCmZSMpCXG6YDo/w9LrJIG82bkbX3F1dYUidlAEizJe48xOUSWV26tLTk9/RqvWQhV6GEqDi+MZmpYiSAG+HzIZLcmTGCEvsSyRipnTtPeQJYE4nuOsAvzghs8/+3dMJufYtZLAWyKLKkk6J0mnvHj5I569+iGC6CMIArVaEy3ZJV6lVAwBpaKyWt/izmbISZOFs0aQfFbeGqveRzZFhvMElC5pWdIyB/RbW1QrEgoqctJiPZ6Rh1NOXp3Qbe1RliW6kbKcerjOnOX4itlwxqDbIw0CFrM1u/27dNt9WnaHdr1Bq6UhSgWqWMPzdEyzSh5HjG9mCKKMIMokSUxeTLgcf8nF1WvG8+cMh7f0+ipvX0wJ1gFPHzxhtbplcjMmdMF3QoK1i1w0GF4uMHWdipBz+uoZ3mqNIgsMBtts9R8hy1VGowlZ6NAyKxy07jCo7WAZBmlxS7PexraqtDoqUdSiUfsdrFYNQZFwoojFOqLfucv/7f/0f8aqv+XXf/M+gQeaVuK6b7i4PKbWqCFLCpqqo0gyIgKSKCCKmzFrmsaYVZvddoeL0yGybhJkK8Iywcsy2oMey1VK4HmURUFSRl9rN2PuHuyTxjKlAHkaI8syCDmVSsZqGZGX0tfBvoQozinFFNsuWK0zskJAkASCOCJNZCQ5ZDYfcjteUK3XyLJkc3NXCMTZBa+enaPJvV9VWQiIuYmEQRw4FNHF5v33SyRFJEkSZEFjvVzxve+9x8H2YxbejKIoNko8XSYMffa2D3j7/IwsFShLgVKUUFSBwI/Z27Op6J2vgzoCaVyiKAZhkDLoy1SrEquFjyio+OGMrPCJwoIiy9jt3SWMC5AjimLjkReBLPeo1mvMFjGlpJLGGYokkZYFkpijCD55piIKGlmeIkqg6zphGLLbbzO7KpguFWTdoMhMskhEKAyiLESuRGSljKht7EFJkpCnGWJeMuhVkAu4vZkhq8ZmhxGJNAO5yPnk6QMur8dkQP4rPSSkUcrdwx26dh9Z2SGJSwRRJ4oKCgpEOSTJzlkv1siiSRR4xFFBTokfRGiVlDhfU+R1cnKSNESWZfISZFll5Z7ylz/+Ea6nIpQFSZIhiiJ+4PD+++8zHo83t4+CQCqmyBULVA3VsPmv/5t/tNkTjyLWa58kC8l0EaNm8PTRAXpVY516TP0JZ/MbhsNjXGfJauUQRRGGqWNZTZqtAWGy5vJ2jmRkeEFBb6fHMppz7/7HLNwFXhTy6YO7mLmILdvUdJ3Td285OQ7p7e2AZjEar/n4g2/jL1OSXGSxWLGYT8kjh6Jw2Gk9YDaZkpXQv2cRywsk3aZeb5CFMY32Dg+e3CHKNe48qLJYXxFHBY12jZk/4tXxFwTpikbPwIlvOTkfoukdDo7eIylWiAqsohyzYrNVqyBEY6qmxn79Duc/fEU1veTVL/6QLApQMoFgesN6dMbk6gVJuGC702Ove484KFnM17jBFMMoaesdKnFBkqQMem1evfwZL784J1hZHOzewfEWRHHIePKWXtMijBcYSkZVM3lwdMR+o8mP/ujfs791iLuYcXM5YrVe4joRW4ND8nyFrOh0B01UVWVv/wGWWaXTs/j5Fz+lEHQktUKaR3zx8oeMRiuev/6cWsPGMLaJiluWa4/57ITlbUB3+5DLqcdw5uI4MqW7pm9LhCsfQc25GN4wdT0++/kr/EjCWRXMFjqKVWHv6C6ukxC7Q1Iv5uBwl06jz+OjJuv1G1J/yFa/pNWuYNU0hldjLNPGDXI++uTXODi6w4cffZ+tOx2G0zP8cIzcnHN19Zq0LtLsQJTICJZFKIucRNe8nDrIgoWirvG9FGfxlh/90R+DUODLf/WZ91+5odR1HddzSGKfZsuiKDPC0Ge5XKDpCrVqi2q1iufmnJ0s+eqrr8jznOU8IgjXuP4QVa5C1qViasTpijC5xTIb9DsP6ff79HoDauYWVaNHvWpx516bJHPxfR8SmXZ1H6koade22OodcXtzzsXpFbtb+7TqD5ElE1k0iQMDTWlydv6OdrtFHG04cYGfsnJvcL0ZkS8jljXiOMWsinz86X3arQHT6RzEgGpNYTg+5dXLCWGyYjQ7xwluCeOI1y8WzFZnPHvxS6aLIUgrprOA5y9/SZStuLi84eJ0yc3Q4927U8azK0ShIMsD9nbvkcUyWWwRJxHb/UdkUZ3L4S2tbky9pnCwu0MhKXjFinXg8ODhHpJYcLj/Pp99/mPm8zXj0Yzba5/FxGZr6z6htxnx3T96j3sHnyJJAVbVwAtXbG/dYT1X2T+4t0mmZQb9Oy2qvRq5atA61JCsmPZujdPLN7jhhMOjPhcXPyZNzwjWSzJhxGR5TKVSIY4yzi4/Iwgi6maf2eyck/OXvDk+5eG9b5ILM94e/wW7gy5bnX0Ot46oWQ1EyeXjDz7h7PQVcTQjzWL293ukYZ2twSGu69Lv96mZDQ52tqEMsat1Li5P0EQfJatCYNCyLGoGlOkKKZPZ7jZYTEY0zA4Pju5Ss3JaDZlmXaduKQhJxKAVs9MUMelyc3vB/XtH9Mwu05uf8Y0Pv0ccFewedpDKGuvJCaqQYzRUTKuDXgqkUc7J8zmGUGe7vc1Or0O7UaFRa7OauURpxHg85tmzn5NEAYPuHrqqopQmd/eeUlXbFFnI/p6FoZbMpzNqpk3giLTtfToHTax6nyxWvx63hiAUyIpESYasNkmA46uX+NkESemScMnCH1KpVtDZJwiXePFn3I5PiaIEJzin02lxfT1EMnPuPvqIsFyiVxo02lX8YMnocs5Ov01N20ZISgzNRUYiz33enn1FXFyCcs1sKmLyH6NV6oRpRBhLeFFIu9vmn/7Tf4zG57z/ySFefI+H9+8xOjnn/aPfJ4o1ZFVDllU0RUeWN+YSQRBAKCnLnDSNeXCvj7f0cb0YSbJqHMSsAAEAAElEQVSQ0oI0TLGqCk3LZDQON7umFJSCR5JFaHpC7PukiUiaBV+rCjep3sdPHhG6OqGro8p1CiGlQCLNPBbrcwJPIM4gL0sQBdJYRFEzPvzgIau1j+tHJF9zELMkot+5j7uGUhR+VYgxilIiFB7ees6n731C3ewQBglZlgEFSZxjqBrdbo31MkFR6mRlQpmrGHodSZB5/PgOs8ULBHw0TUMUVOI4pMhEju7ukCcaSZYhiCWqXCUvIkph8/uuxgZZbJPmGzJDFkPgxzSaCtPJmvUyoSggSwWyLCFNY+rVCqPpjOlis7KgKdqGESgUBKFDkfsEbrLRFWoKWbYJhyV5Rrtps9XuIosmiRiSlSGGFJElcwrNwY8EvEAmF6Fgo4rc7FC6hMFrTr68pma0SNOYUo1AjUizmFrFZK99SB5vk5MiyaBqCoJYIIoJhipSuPs4CxlZ80hzB1GWEKhQ5BKDrQrOIsVbJ+iqQhoXxEmGH3kE6ZB7j/qEaYHvx6iqTBIXlIVKKaV85zcGfPXy3xHEJYJYkqYp88mMR48eIMkFi8UCSVKIMh+zNCn8jO2uzbNfPucv/vTf0G33EClJkhhJkKHImfoi58sFolqi6HX6nR7NOOPpwy002WVyM8RZx2iGTiZlXE+XSFaDne0260lGv9Un8ifkaYhcGpiKzeXFG858n7HgMs9KlNo+t6sTcmlFmq+Yjs84etzh5OwXGKnKt77zbSRR48GTj1mMLoiWMp0tm2rdptfep4yaOLOIOJ0jG7D7sIkw0bj6yRecvBkRhW1k0UWzNG6uc0y9wcE9jThNMPQ+ipWSiGOenfw5BQmdnRqmYPHpwx7J6paG3USslyhNm8gKCAyb8r3/DPvXPiGOAoL4horZo27c4+R2zZ//8R/z0y9/zvXljHpFxChkiAPi6AZ/LvDB02/y9P4d9DJir9Xg0/s9hq9+hjec0LNtWjUVo+xBkqFXchRF4uLqmvPrG9rtLX7wO48Z3RxjKDKHB23azdqGiBA5VBs6v3z1p6zXS/KyoCDH7lR48eZzTk5uSBIVzSzZ3RmwvzfArltU9Q6nb4dMb1coYhdFNsjl/oYRfXXF7sE+TpixiB1k3SIxdAb1HWqWAUWKoEDPbiEWGTUz5Wz4OYapsw6XVPQWihCwv9NBEit0e3XyYEGQWTQHTYIY1onH7fqY9QoyTnj+8qeQz/jl519wc3WColpUzR5GtYZQqdMd1Gj1+simDvKMVXhLaiSousLuvo7SU5jEAe8ujjk/89k+uE9eiFRl/a+/oex0m2QxiIVKsA5JA5F2U0UWmqyXCcFqBYWIhEBFv2Z7EKGaE3TFpm3uUamZ3Azf4cdToiJDkwyW84TZwkMqNjDgy/MxVqXzdeoP5qMZ6/WQAh8UnYlzTJCsSGKB0fCaJHEYbPW5uVkRJwF122S2PGftXVEwpdVqoGgWhtVga/cuklRnuVjQatdotg6Zrx3W4QVi3mB4EaEZBbKgEwYeqiLz4O4DonWBUKj4jsTbV+ecn17ghacosohQmoyuU4qyRJRUylynWmmwmnsslwtkScO2m4hFjdXKpWe/x9nJmiROefHlJVWjTRrqrFZLoGC+mtLo1EEQMLUtirCPqiuMJzBduYynHnX7LqW2IMFDUGWKXMP1FnRaFXS1xHV8XN9DMVJmswmSWDC+nSBIIlfD11RtA7tTAyHE9yLKVCGPfK7PrtCVgooucn35AjkvMKUKpZfRNRUSNyUvRIQwxz0/5k7tG2xX75CvXQovoKYafPeTH3B7PafMCra7O4Rezmzu8XZ4RpKM2ep8ROI30CSZ+/uHtLUei9GEra5CGC8pEYmI8cqEn7/8JasVjK9n7HYMVgufrHARJQfkgkxUcBYhs+GK0XjITruCVihogoksWazcFZqlMFvMuZx8hRMJXM2nePEcoRCYrkDptVmFOe7oHXZXZr3wuLh9Rau6jxedg58yWU84cU4QjByjWkepR7jCDUKtynip0a7fo6lr7LR3afS6fPf7v0Pf/oBu+ylO7DMaLVj7E/aPutzc3DC8XTNcLFErFRSxTrROCZ1TsljGHS9QKwHbWy0oNoBZSZAIwwp2z0YtDJIyxTQ1tEiiKFMUpUOyzFn5U1Qlxja3Odh+gijrvD79CsdzsCyBs5sz8sRna69D7CeIQhVRhoePtqjkDaSspKoZWMIholblePiMxzvf4F7nMeH6AeeT95nFAsPrCe46YrEaYmkW//qf/xE37/6Qv/0//RaXFyO27lSJEwelCuPYJ/QkdFlCEwQqVX0zchQEBAqyokRQoCxUBk0oxRzPV5CUkDQPyKKYhiUhlR5lYpFlGX7okacWcQS2beN5HvNJjG5IxBSUKEhlRlVaIcQFkpHi4VNIBXJcIrJmHResVgmCISFEGYqmECVLOlWVyE+JJA2ZEiEqCYSEPCvZ2TYZ7O/hR/mvqiwk4kxAlBXkUiVPA/R8B9VokysR5AU5KbkSESxvWPsxQhaR5hGiXhClCd1Wl/2eiZg2SDSZNMvQcpm0MGjWIE1O8ec6sqqT5wkIPrlURSwD3NUbYi0hFSV0SvK8pBAS8jRClSJuRxc4abEJQyUpgpayWnq0Ww0GtkqRGZswUZaRignBOqHXqqMVPp+9OqfUq6SBQy7JFLkCRUqtUhLJEOYFFBvTTiqIRIXGdkVCX4xIigoiOVkSI4sCeSaSJQGdrSqiJjGLM8pUJSVCjUQoJSRzgr+6ZO4tUbU6UgqSVlDmBaYicWfb5PX5TwkTD0lqIEUGqlDihxN22xX8mwuGpyNUpSSTNiN6MVfxgoCtTo/CjZkOYzI0siSlJCUtRHRJ46j7LQ4Hv4EXLilKCQEZz/MY9HZw1jGCIJEkAbJoUKgpkirQtrv8/f/n/4O8EIizklIQaDRbSIKEHyq8d3CfPHQhSkmSETfegqJqcnGzIhVl9JpMWo75/IsXjFcrjJpFXjp4QcjdB3tEmAyvFyxnDm+/+pJev0m7W+XPfvYMigaQYDc0tvr30ASTreYDKmaDMs1pDVq0D9qMJhP2D6qcXy2odFqcOyd88fIZkloniUtm0xH393dYJDd4i5ziRCMY+dSrT9nfr+NEN7S6LWIvIk2G+OuMjvEph3v3qNVNNDUjjmN0rYJoZFwPfWI5o9L+lLtPvoegK9Tr9yhCjU71Lu998k2auwZiKXN9u6K1/x6emFKKc466H/Gtb/wmdS1DyRMy3+dqtkZtP+W3f+fvUZolb5aXvDkeMQ9cpos1ZqfJ0QObk/GQakejKDyO7thEksZiFRCuXVbRhIopcXN1yrtxSY7EZB2yvdPHm09wl6c0WxaT6RK72qRi6ii6gO+tmV/fIvhVBvYRnXqDndYeQZLw+plLLsrEZY5o5RxPjnnx8gTFUHDWMYqioVPgulPW/i1nk1cM3QxLr3ObTBjdeLQ0BVOvU5YhWejQ7u2gZAXOLCKY3hKj0KoOcG991EIgDAqQOjzceZ+ytLC3LVS5Tl3tI6s3vH2zQNVlzi6ecz455mS64rNnr1DrGWdXt8hpwXSsUVDFC6Bp76JafSYLB60s8ZycF8fP8KceB/v7KBUF0SwQNZv77z/8628oiyyFUuH87JZms8bBQZfZOKZaVQGRn33+jFanzvsfN9ArDlaljqU9JFciJu5kswgra4iCwssv36KoNiUaaZoyXg7JsoLdgwFJUTAcX6CaKggGINLv7TObT0jiDLthUODTbvdp2Q3yYkGnJ7NeLri4fI1t21hmgyyLGY+GOO4C1x/z8uVLnOiMMJBYLkWGN2PMSpudwUPW/g3tTp1Ou0eSZAhFnTLe5dmXQ1brBcPhkNkoYL0QmNyUdLttXj9fYlbaDLbaxEEVbykyX55wdT6la3+IKvQ53P4ON2cFqV+n23zIcrkJkKxXcwxDYjKZsLt9H1PrUjdEnuz9ACPvcnH2DF0WsHSF1WxORTcpIpkijbDNOnKh07EtJpdL0tCljENmiyGT6RWK4eL4l5yeviNML5nNxlSqCZ6/RNQKSnGNXe2Txi1CX6ZjH7G99ZBe+x7eQqfMZA62PiTw1yhiYwN9lbfYNvvYhYicuHzy3kccbO3x/Nkzmo19Dg4O0DSbL7/8jMvhCVm2MaEMb06xGjmlOMfxRJzQp9BmaJbC0h8TFyG1powTREyvFhwNtpC9kHIx4UF/l/XsnGorZrwMMc0GAhK6qVAUKV40odk8BDEnSgMiscHUD4hFF9lIuLN3H7mwUAUVQ2jQqZnUjEPywmB7q0sZZjQrJrbZ5GY0QrMazKcRaSxS77exmk+ZOxGNZptG6w6JB51+k4q9x3hxTZFA4N9Qihm9/SPipMGd3UMcxyEo1rw9fk5FrfP+Rx8yvJ4R3EbsbPUpI4E9c4dvf/gNBDWiPlB59vaSuqqS+gKKJ7FarjEqJrpmIkgKiD6CLyArK947+jX29h6z9CdUzS53Hu2jV1M6+zVSOcJxNVb+GqkS0dxSiLICWdmit3OHyfwaZzmmJRt4ZwFtraRI4PXtJYmcs5gueXf7Etc759t3P2Wrs8O7UYOfntQQRZVlvGQ+n/Pu5pi8tPjnf/8fMx7/c/7Wf7zDP/tv/nu2d6ooUcrqcsX9Vocv/91nhCIklRxVVZEkCVVVEcWNqlCTFSQENFWh2kz58vkLgiT/+uZJIosEFCmj08lZLMckqUKa5oTJksjPqNUs9EoDXauSJhtuY55v4Orf/8FvkWcWeQZiKSNTZ+2u6HQNDnpbuOsNdFxWBIosJw5Sut0mUZQQhykiGx6jWAoIZFhVCd8LyfP8VyWIEogCfuTT6hg8em+Xi+GINAqRBBlZlQmjBIkSVZEoS40o0TB0nTjJQSxZzC949eYzgsCGUiMvQkopY7lYY1kWe91DBNkkzRNECopcIEk8ZKHggycfQ14nTdOvE9opZSHjeQ5H93uoWo3AzzdjfzEniUviIKe/E6NrVfKkiq5rwGY8XRQFobuiVquh6A3KPAVZoiw3TMg8CXGdd9yOHCTR2KwskIOUEQQu/tqFVEUsLLJCAFmkVCSSLKZRMzja7bG/f4jnrsiLiChKKARIk5J2u4UgGyRhgSpDWSRQSmRZTqPVZLDX5uJmQl5WEaVkc0NcaAgllFnJxx9+D1kxCDLIsxJRUEhLD7FUsK0qVaNPKebkeUCWCghSTpZEWGbI1fBHTCYOBQJpGjOZjmi0W9Rtm5ubG9I8QVYV4jRBKKHTavPF55/zJ3/yb+h0OiRJhCSJiEJBWpRk8YRfnr5GlQ2a2zbL1Zy0DIlqJYvFBEMtmC9uEBWLarvGbOJTFiKy2kQIfZ4/O2Yxv8GuGNj1DocPj3j+5oRXby/Y6TfY3u9Tb+2SZAKPnrzH/tERzW6FfrPBch5hVRokfo6uFbx+c8Gz58/xQ4Gt7QMaYpvcy9E1i53DLSaLJe8Pvksl9zg5PWbwawOUAZhGj8KRyTwDXWkgpBVsM8cP5+RpyO1lQBoaPH54n8cPv43nuDRbNVq9Kq/Of8Qf/tH/h9PzGau1j6x5nF4cIxkSs8UlUbLkG5+8j6IlOMEtoqbx+JMaW/c6HNy9R+3RDk5txWj8gjevn/Hy5pTunXssrx1SMyBRPGbRikDKiOhQ69WY+w7zUOJ6OSZZS9w/6lPkLvJCIl4UtHs7LKfXCIWGJCmcXt0ycZbUuzVevbxidFmgCy3OLk6ZTiOQLTJFJJYdckWg1CVSEeQyp24VrOY31GsWTbuKrOQ8eLLNfD2hoamkZYyf+Jy8OUdTbLbbh4i5wCoNCYnZ7m5TiBHhzMHWderyXdbLFEWu0DZVgoVHxUwRdB2houDnAZlkoll1zi9fUSYS4TIkXOZE6YjLyxG9LYVmtUu31eX64hqJmLZt4swiLEOELGUyfkkmRhx0arT6TcaTN+w2GuT5isRZ0awozGKXZRygtRJ+9IufUe3l/PLdv/rrbygvzxdUq+XXQHIDL5hQr9vY1SpR4HP33jbD4Rnzecj4usvwSuPLz6/IcpdqXUSRNPSKwenpS3YGTTr9Jv1+n8P9Q25mM96dnTKf++RCwN7BHZr2FogCrhMxmb/GrleIU4c0lhCpgpAiSiUiBqPRDfWaiUDObDJmtVhTq1Zx/RHPn/2INHW5mZygVURyIWK6PEdSYoJwCUWNwKkS+iLrRYoidpDFKvPlgr3dI8ht7h18wNP3HmJZOr/9W7+LbR7SblkE/ookSllNU1aLjMQ5ZDXPWC9HfOPDDzg/uUIua1A6QIEgRpimiV3v4To+dw73ePfuGUXpIFdaHF8fU2g5vd0Ww+kFspGj6ha34xNkNUWQ1xTSLZrcY7WMefx4G8tSaTVttgY7GIaBiEkSgySadFv3+eC9X8NU7+KtcpAKyEwujy8QZf9r7MSML1/8jLycYdcMdnpb2FaLx4/v0O1YuMENdttElKHIFjSaVbKiznKRsL07wEkD/DhFUmVuxlc0WiqL1QVR5iErFeaLgGrNBnkzIn118pqFs+B6NMSLAtwoJUhSMr0kknwMq7mBss/GWGaDs/NbkjJBkqEQCvJyiednJIlKhkO11mK1ihi57yj0BYZVJwplzi7PuB2N0Cs1jKrFu/VX/MN/8S/4b//JH3N8fUZYrvjpL/6crMixajKnN2/Q9ZjQu2bpHiNIG7Vd4meQGoSJibMOGZ6fIIsaxxdfYtXr3MzOmYw8htMTzt5dMrldgJhh1U2EwuDqbMLam5OWIr4XQ1FjGWh89XaKqdpoqcvdowHrWEVUahzt3SPxYuIkJityZEliq9qhsyfQsvvEUYo/G6EKBXW7ymQ4wUtDanqfyXWEXRNp16rIqYkYV/iPfvB3uDw9o1i6tKsW9brCZH1Gpq9ICwVdN9jf20KTRDIppWf30cQc13X5oz+65cXQoDRMREFB9HyiwqNlD/gX//Qf07BP+F/9wTcQ3CZ/92//HrooM1+esffoHnb3CFGsIskV1FhCkDeBHFmWoSgpioKyFEjjHE2CVreDILbI8k1TiCgShxG72zVq1ZiLy1tKoaDIRcpSIE4CdrYbJKFNEIVU9AaSnFGIIVm24qsXf7ZJMAsGWRYhklLkIp1mn269imJUKPKUMNjsxoV+hKYWxElJmYmblL2ukEQp1YqGLBlkmU2clL8qTdMQJHCdCKMi0Wx1WPkxmqIiZJvASJZlqOUGMXY7cdGsAn+doigaaWTRbxkcHOwwnkeU2QaplBcxiiChGRAvI8azzTQiT0s01UQsUvodmyhIuT7z0FRzM2KXIE1D6qbNnf09pjMXz3PIsowwDBHQKLMISRqzWEQEvkAUBQiiglWvkaUF3WaFg4M7IDTJsgQ/3QDJPc+jpqk8eWRTlhpZkVIWCWVZEiQZznrNb/z6d9ja3SUvBESxJKekLEV8d8MN9dcBX3z+FsMwkaUSQ69SUBImKd2OxdnwlizRqFoasiKSpjmiIFOUGZe3b/BjkThPiPOAogwQ5ZTIDfn4w8doFYFXb89QTZO8SMlygTjVyIuUX/u1R1yeD5nPCgR8ojgginTSNKZbu8fv/fb/gdncJyMnyxKSJKHX6+K4Gyyb7/us12tkWSaNYmq1Kv/wH/5DHGeFovwHNaVInicIZUnP1mlYArWWyb//0Y9oVHoshkuEIoRcZHFbcO9gh7PTKwy1galF1A2dmqkiCiVmu0F/u40q5eRJyfPnz5Hykr69jSL4DK/miLINQspXXz0niAtGsyuyOOS9x/e4vjxm4c6ZhWOcSOC7v/49/PmKrmiCHCOLLu58Rb+6hyJXaEg6kV+g9FSm0zlfPn/G6fASseITBCtkKWLvcJdUFHHcGZ4/x3On1Cp1zo7fcHN+Q7VapdkycWcZpeDy4acf0eyauAsXb5GRRBGrdUSzMcBQB9SsLlEU8ckn/xHVRpOoKAizkrVU4q090kBEsE3MXZ/Tkz/m4tVrqv0KoqQQeQUVXeD6do4Xj4GIxSQmjCLmN0sqskySyMQ5uK5L4AUUaYBVSJzfnhEVCaKh0GzvE/k2YZQiSB7D6ym21WR0c8ndo/cwTJ1m16S3rzOazSnFknprl8OHe6hWDc3okOYVHjx8jFVVePTgI7Z3W4wuTpAEjYeP3qNmSNh2hcvxGy7frdkdtJmOrpm7Bf1Bk1JQqJgSRmnyYO8BR9t7CJTE/prVasHMGZHnOe56ysXZFTVzl43uSqaq2fR729x7+A22ex9jKiaSIPLwwRFZpKMroAiwt9Xl5uYF1YqMN3dYeiG3F+9QopJG3UJDY9dqsNPdJs+qLOYe89kS0zSZr0IWTvFXbij/yi7v/qDB0VGHd2/H3E6u2N7qk+cil5dXVPU+3//WI16+viSMZsRCQLMlcFivcXE2odPbRkwWBElMp1FHIkFWclarBd12h253D3cdcDV+hbOasbd9xF73KWkYsTXosVxulp3jKMVTlpSlgCyZLFZzOt0a529vMI2M7a2PWcxdkthjMo5pVHchX/PsixGqAceLEYMdlYH9Ho475vRmwf7OPo4z4voqwa53Ob0eEecrLEtHUxr85g++yen5V5wcv4JSo2IoSFIbUZgyuo4xZJNuT2Qxv+Tw4B5x2qJaV5nNUw4P7lFKE3rdHbISBDHh+PiYv/W7fxsv8ElLl0pNwvNDtgcDlkuJOF6zdkYooolmgK6btBsWuh0QRSlWrUEULmg0c7xkTpKmaMoOmAHN9sd4TkGjnhNnU2TJxPfmHL+9Zu+ww+3IoXe/xU4LFr5Ps6FBVqJXO8yGHvP5BZYlYugKi5nCcumyf/gek9mYXA7J44JCqhImK3a6TXSxymQ5wVIVbpdDSimhVquxWKww1C4yUIYLnFVGXviEgYcuNLBbbZJoQpRGTMY+9+5tsRz7VHY6HF+9omDO0cGnjMbXbOkD+o2nLMPXKKhkcZs0D/CnAVq/jyJZVK0u7uoGTTbxipI0W2JaKobR4HpxiqykiH6fj9+PaNX7iGqJ3d7nyTd+D7t+F92o8ur1jzg7/jc8PnpKHAusV0sqlQhDUahIsJZC0jAnCi/Q1B6SlFAgI0seFxdfYDf7TBdDLMVgPi8RZQ2yCGfWZGvwCGc5JLOgUnUwhAUvLq+5s/M+B41DYneOjsd3vvENLq+O2RnsIhTXqLpKEqfkQoPFeE6R9jFaOmJqkic5k8k57jhhu3WXQlXZ2/6AIH1FGFVQ5M3O2NXkFEGTCHUZS7GRsoA8jujesTh+9QaZGs1Bg8hXUEORZmdNmu7wF+8y1vkAEw3D8ElcMKoVfEPn3/yTf8Cn2yH/xd/733B+ek1x+4ZAMtBkiTKqoBQKs8WQ4aQkKRNqooqXBvz/WPuTIN3y8z4Te858zjfPQ87jnW/NVQAKAAmApESqZYnqsCmGW2p74fbG4Yh29MqOcHhjhxXhdocjZMlyt9WSWmqFZEkURavZoEiQIIAqVKGme2/VnfLmnPnN85nn40UioC0WyoizyE3mIjP+5/3e/+/3PKIgATdeakEQkAUJkClXVAxVZ7WIQIhuLDERhEFKXtOZj8H3ErJMuXHO4iMKOusbRa76c8JUQo7npOnNFmmt3kLX2syW45ufpyiQGvjxgHKlhDlbMJpG5Cp5AtdGyFJUWeHwTpvzM/NG3yOkJIJMkgQE1oLNjQ0+e2KiyPEvzsQwCBBEGVXUMTQR07RZufFNgcwNkHRIU5GDrQ5+sGRlZWSlCI08YbzENGUePiySZCleKqLIMjehoYw0cBGEFdViGzuwyLQMVTAI4wTXsnmwUWU5H5HE1ZvtmAiClOI5ApWCj2KYjCY+4s95jZFvk1d14simkq9jzgRiIhRRJcnAdV3CMGF3q8Tp+SVXgwDDyCOqGpnvE2QxRQUiT2Y4CtAMjTD0yBIFSc8hSWNG449QtTJumFDQBWI/JgwzpFQgl1cwg4AwqWBoJcJ0SRDeAKjjcE67XsAw8kiqhJ/EqKqKIcPSXtKqVeh063z+eR8/NOk2iqThjaAhjiKKZYVYyBjOQ1SjhZAFSKpHnOZR5YzYdxmOpjhhGSMTiSMRRU1vaCWjIz788L/HdG0EWcG2LCRJpFQqsZivqJSLeN7NNabv+7QqFa6uLvmn//Sf0mo2SSOBMIxRZQVVlXBXNn6QUJNSbGeJFIQU6wbF6m0MP6Vx2MByxmSiTM5QyBsaVqwhUqBeKjLqjYmyOZO+RDGXp9wtIgUNiqLC4a3bCOLrzOxLzs8f0222qTVCXr36jO2NQ4bTJRPfY/Owy9VZj9BKKet5Mm9OTpTBV3nwzh1EL8Jdhrx88gWWkKNRCghlkUCKWSt32LvtkoYZRVGnbhgslwuWfkJ9q8N8McbQRGo1lWJeRxJrZEi8enXK4Z3b6PI1CHUkQcfQFLY3mkTJAjXTifwVqAbmak4UvKJcgzB0kNWEJDGI4hg9cFl5IfVWFTExUA2dTNLQ8wn9oxeI0iZiYjKbKOSaCaVCjeXEJHLg4M49BGEDczlHEUu0CnvsvCGh5WROe6cIukyz1iFLREq5AqbpkKY29VoZQQJJ86m0atR2ytjJmOFqys4tODn9CkPfxgny3Nnd5uTl5xT1Is7cJAEebO+T2At+/P3v0719h1anSCmvU6xr/OkP/px8uUa3cZuLkyccH7XIlzTsRMH0l5iuQLWUEVoJjqXgmgK5fAk1Z7AazjBnA6aTJaV8kZxRZe9ejhdPr5gvXHL5FWfXNloxw3M3KRQMVkuLNFDY3V3DD0YUDI3VfEjg+qBnnF+c8uC1u4yvnyFIKuZ8QmDauJLP6tRB1GVKqoqeacSKwHK2xHN+6b3jLz9Q2o5JnGwgSAbNloKqa5yfn7NaxNTqLi9e9BkN5zx8bx1N0xj3pgTugPVmlcXIpbVeoTeaUK90kDOFYX+ARMR0coVkGESpxcb6LkGpBVHCajKhWa3x+Pln5CsakqiTChFhcoNRsVyXxcJC1UVyRon97TtYy4wsCbl9eIs/+v6/olQrsL6+yxf/9hFbhyn2Sib2JNa+LvNP/z9/RrOTUTO2ePqoz9vvGah1nWZbZmdvC0mscfTijMloyOHefXw3pFxRqVcaPPnyC9bbD2k2pjTqdeI45Fe+u0bCjN3db3J6PMC2ZpRqHn4YcHr1kq2tO4zGM9a6t1kuQxRZo92uMhmvECSNJMwzXz1HVhTiNKFcyRN4AlFUxCjKeO45oqQwHJ8gkFAs3GPcH1MuF/HDKaZpo3p5umsbTGaXFIoSQhrQXWtQyKmAz717d0iDJb3LIbFg4EoqQpIiKQGZMiFXSelfjzg4kBiOFwTJAvNiQT5XIk0TWq06vueh5ELmywFhJJAvV7AmCY3GJvVmk5U9ptPaxjAMkmxEQVawljIiCdWCRmCDLrTY36kxWh2jaQZ+4FIsqCSxjqCWqDbLrMQUsaawcBZ4q5doEcgqLFcz8uUqnW4bIwez2YpWc5vgyqXTrhLYCuvN+ySiiVGE4dJDSrvoLYGatkGttM9i7hCkNe4//B625yDKAd/69t+gUqnw43/3zyjoPs26QECXRqXLbDgjsEw2t+4Tp1WWU5OCUEYUNKJEIl8VSMnIySJ6TkdUcxCAIotMzBlFqUhtq8rSz1jMTdrlCu/d6zCfuLSqdfRiQBbm+PzLz0l0mXYQIYkScRBCmpEIPmkmk1RiOkaFWTTGDeesV+qYdQF/NECng6qAJHZZOA7drU1QY8azY5qdMu4qImFOMV+g0NxlZRrE8SuE2MS2UnJpjC1qhOzxo5cWK6FGEYEwCwh9lXZtjfP5Y1783o/53/y1Bg+/8R7T6zVqtQrLbZmSLoLTJJN7mIMJ5spnsloiyzmsxIFUR9QUEu9G+ybIAggqiR9TKWnc2XvAn/xgctMMjjxUJU+a+VSbCkg5LAuknIDnO5T1Groq4/tDHFcgCCPyBZlCUWF8bvPON9+h0rDpT2ZUarsk8YpI1MjwiYKQ6aKBG4EY+QiiiCDJRLGD608YzW6a5wgCmaDjejO+9d7bmMsrer0Bql75xZmYxSCpAlEQ0e3mGI0viTOVOAmQZRBFEdu2KOdzVBptZA1SSSLNPDTVQBBduq1NXhwd4wQSutpHFdooeQURAVUN+eSz5yDUCaMbyLokq0RBytZGnU5H5Ic/W6LoLcLQQZRFksyCNCUK1hmOXbzIRnJr5JQClmNTr5bYbG3xZLTAjyMK+QKIPmmc3XAzBYvLoY0sb6IoIo7vYyARBgnd9TIvX3xFyjayLBKGMYooE6cCOVWiXEk5uYyJIgXfDW4QSJJIGHpIqkA+b2CHCW7mIQgxgiASRgk5TcKQEoYjhyBT0XIavu0AHkIWU1AFHt65w+/9y+eImX7jB49VVK2EKjl02kXOr5Z4aYlKKUcWpQgkxImHpgTYzgxRaeLi4PsBoiBjOhaBk/LOm/D89PeZzRtIosKof8l3v/tdirk8URQBIvl8Hk25wVjVqxX+1v/t77KylrRqDZAUwjCjUMshiwKZBHlJwrv0ucSnubFF5rtYMw9lfYeL8w+Yeybt5g6LeR93KbJ20GTpThl8cEEsLegat0iZsLCGiKsmWxvrXJ4POb68JpdX2FpvUNUqjIcuhVKDWgEW0wVuNmO/cwdd3EDJQrpdmeNnS2I7x8H+Dr6b4U1NZtMhlrugUQtYmDZWuIXtpTTbBWbnn1AtVHESl8lihbSxRXlrnWw+5Pmnj3jj7fdQlYw08Ylii1plg8FoSKlYv1F7xjGyYTAcrBASl3arhjlzqFbLxIKKntd58WxIq95mPoGUFYaeRxAjkjSmtN4mNiPSKEd/OufNN7ssBqeISYHt+jeI8iaLkcObX/8aL6+esPI0YkWktVsBQ0ZOFZAyBrMLFOp01puMFpdEQp5aSybOZJq1JicnJxhykdfeeB1VyTMc9gjTkCgY8+rFJYrU5db+LmZvihyZRO4Sy3G51FOSTCKJJda3a4S+z9XlFHNwRrtdICfpFDbuUsDkkx895c7r38BQAjQvIVnfRdQyfElFFXxq1TaqtEQttBCLI7KiSM0oYUYeQmYixjJbW1soao7RyEbPacyGDnsHt5gvr7BWKWvVEsVCg4veitUqYGdnhzjwaTRyDEcygiYwHaXs79zBNB0e3nmHq8tjslhlb2+T2bVLljlcXI55+7W3GM8/ZzLXUAWZONMx6lAoCv/hB8rReIxRkBAoUK13Wa0siiWVUn6DclXCC0VqDYPx9ZKVdU7ipxTkOppUYBKOmL26ZGNrn/7omkLOoDe8RBQFhEzAHSXYnk1kH6BKIVkU0qoWGY/PmQ3jnzPfesRRBmmTcqdJPhfSrG1yfrwkisr82R8/YX19jY3tDX74g8+olNbwLJfQMbCXAosBPLz/NW7db/DRR3/K9laJNx6+w+5OjsnwkNnQQpVHNJsNwjC9IfbLMwajKY3mW6hCjVIRzi9e4jgr3ntrj8df9UnEPp2NvZuC0SikVJwzm4/Y2qlQKGgMBwor+4qVG+OGDpoBS+vnHL4sR6lgIIslRpMTskijoLcwckUyM+Hrb3bQJIfHny7IrR1w/KpHpdAmXxCZ9vvUyil7O7e4OJuwc7hHho8ghoiKSZYUCMOY5XyKZc6olHWOnj2lXi0hqbCzfo8otBhPn+GPI8JYppBrc3CwzaB3yXxpsr5Tx3UtIkIMrcxiHoHo0eyWCPyE0Ew4Oh2yvrFGs1ViOl/hWikHOx1Goz5uMOf27TsMUpOC3kIWXYy2TpokFMsZV8OQ7loTSfHJiTmSyKdbb5IyQdcs7BBsW6bcMcAI8fyIt9/8i7y8+BTFyOhf6VRLFr3xp9RbG8SZR298RqlYQVEhp+9SlHdJsgX1SKVcWOerx59w++7XQbD4W3/rb1KsV7n3+rtocpEkddk82GI5ucb3UpwMJNkkEWF/f5/zkwG7exvU9yqMR0sKRZ0wKaN5Buvbd4hml5gu2H5E06hwfP6S0lqJtapMsHQ5rB+ysHsIUY4oStnsdpitLNJCAd/30apVpr1zNK2Aqt6wIQUxw4g9ZKfA3D1iHIATimRymbkHhpEhF/PISZm8PkNRbnA3L189od3poOtrpFmAqDvIaLiWhOl8ilbaYrNwgO9PaGzvMDrvEes7/KMPJFKhQlmKiLIUWS6TKxf4+Ge/R31xwT/4P/82xbUcT5/1mX/5j6l3NSb2gKXp4iGjlTfYrOeo6gr9vsXGZpNEyEgFEQHppjktiTeZvBR8P6RcKtEfveD07ApZA02VCcObHO6bb75BfzgljXMIsoecKHgWEE/J6V0GfRtRkW+MNI5ELqcRxC+ZziQkXSZKbMg0rGxBMWdQy6tcjZwbjmaaIKMSJxmymCDJ0BuuEGWFNE1v8oOpgCh4NBubON4JgvrvxbZpetNijoIQRYwoFdvEcYCkiiRZiJLqSCKsdXKcX4/w/AhFySGJEXGckiYhI/MTipUyqlZFFjTizCdJRBazKdub91hNhzhuRKFawPcWyEKRJEpRtBhBlgiSG61iJqZkmUgaa9TrLqom4AUisprHtl0yI8ENI2r1PEKWMF+FBKGAGviouRBFLBKFY7obBU7PPeL4JpIgpBmyruHN5uzurTO6TokRSKMARZJIs4zQjZGCkLzRxHUTRBGCyCcVwFANlpZNPl+AWMGLUwQlJkszICXLJBQpo1kqMejFBHGIJsmkiYCqS6iizNZmhYvzcyYTj2qtgCEr+GlIkt1sg9PUZjha4QYC5WJIlkWIQo40drhzexct7/CTj54givcJowhJDDEKNUy/R63SZH19Dcu+IJFcKuUyb77xBkEQUMgbVEplwjDE9D12d3eZzWb8i3/1e1RqtV/8DWVZRRTBc11UWUVMZfqxQKeUZxi6ZFqbUkVEdIdsVG8TJdfETkpD6VIqK8TOkiySuXNQI8lvEIchkppDjHSy2EESRN597Vf40dMvEIwq9kSlVk1YmJ8g0CUvFens7vDFsUU5KXJ8+lM8P2A0rNFaVynkNFqtbSwnQEo09t69zdn5MU9/esY733iDs/4lmqbx2Sef8+1vfB0/EhAkiTu32gz654hBjEKeRqVMmnmcXw7Z3riF6w358vEJX//ar1CpFhFEkcG0iWhEqPk89eImYepRrK+TJA6mGROm7s31eFPDXSWU6i2WyyWG2KXTLSBnApawQhSgUki5uDpGNUpYfsTF8YcUGmXef+u3CBhhWwmysaRYzpEvtJnMzihpGfPJlPWDfVJJYxHaFJpN+tYAVa5S1W9sO821Jq4NR2dDZCEmTRzMWcYq7HH//m8ShCZGSaM/dnA8g2rZQKHC+XWPTqNOKqXMAwtJVlglNtv3DhEEgdnVFCHbota5TXffR9M0kmWZciNi5i9RxSrN2036x8cIcos0cZAVhWK7QiJtcH7xGXN3gRxk5NUKgiay8CPEch693iTxRXRDx+sV+dZ7f5X57IyffPCYtd0qR2cnXF7AWrfIoy+esre3RpDZNKpr1OtFssSh3z9G00Bt5AitOV4YoOdEdrplzHCBKOcpNbtMxj2m02taSucmKvdLfv3y2KCCw8qaMZ6f8OTp5/i+RKXaJQht7KVGdy2HkXfoX/cY9ExyeYVSSeby+gpBSwlTleFowsnFKWf9s5sgOSr5fJOHt94mc/NoqoDjprw6nvPlsx6j8ZTtzSKz3hxzHiHGbXxb4atHp0x6MOsnnLy85I/+4Ic8fO0QUYHx9IxGN8Io3DDpzs+/4sGbee7f2ef+6y08N6bb2eJ3/uff5fbdTeJI5OFrd3j44DarxZBiocb15RzXNVmZlxgFj17/EZl6Qq6Q0Wo3WNuosvR/CuqY8+uvGE2PsIMTwrTHZHbF7bvrzGYzpmOLSl3kwcPbkBap11oIgsPCfI6iusymI+LYplwyWHg22/caxGqMXqyjFhNGy2dUN8/5j//TBvYywNBKTPqwmiVsbGxQym0TRw6tTgGFIo7jEQY2SQSX5yMkMcI0h7SaBUrFBmW9TaWygax0ubo+IwU2t95HBDQtQVYkwtCnXG1w98FdSDXKhW3K2n2ioIRRrCJkeVy7SCIWWDgjdvYbtLaqXFyeosh5WrUOcWyxu9umWupizlNIZCx3znwZs7JMJNVhMLxgZ3OPg51b5LQckR5TXJN/jh7J6F/6TMc+SD65fI35MsCLUl5efcLcWhARgbJAMfLEUQ5zZWMtBdY3uwiKx2Te5+nLj4hCi3ff+jaBrPLiukdxzWBsXnA5esrB7i1Cy+flZ18QWn2uz66Yz2NSXWO2ihFQ8J2QYqWAqDRpdzdpNzu4Tkx7ff3mZZ5BSWwxPl8xWS2ZTvvUCyp6LqPRMijpIpPBmPX2LRYrE7VcII4sCFacXz6lUAkYvPiSyHNRs5R23kBJIEpitFwOUVYoFtr4soszTVgGLt6iz3Z3jUBSWPZd9FqNgXOKXNY46vV48N43EPNwNT3B9Cakgs/L46fcffgmb733G+ze+RaNrW1KVYOHd7bwJkV+9LTMnz9JUYWUouYRJaCXitjumI/++d/jV9YV/vf/xa8jlSVOr0YYWsDtnRrrhsaDtQc0imtsHbyDoci062VOTh0CHyzHxgtuNIRxnN6wGWUVUZJQFIUsg/WNJm+/96t4gYooSESeQJqIqGpCu2MQBApRFCIKClmU4boum+tdDnZvMV9ZRCGoikEmgOtCGsNskhJEBdJQRhBjFLlGbN+w4JaeRhwpSIhEgU8UhhiGgEwOxw4p5gtkiUCcBIhphpBZPH58RJToZHL2iwdRICImThM836TXX+C56c31mSIRBxlJ7LOyevTGS6IkI00sEhJ8L0HKdEq1PCenS7IsIk6lG9tPJoAQsrHRYBnfHNORHxOTIWsCnufR7lQo5Ds4fvZzH3mIKqmE2YrbB4dkYZXxdEEU+TcqN0Q8N2S2uMBcLrm8uMGjCEJyYwLyQhQxYDYfcjqY/6J4pPy8sBNFAfPlJblSB8+VyRIIs/hmkLLnbHQ0JFlnZvrE4k3mmTQiCAI0WaW7WWVhJiSZjCqLKJKCLApYjoeRh+VsxuX1CkUxiMMAQ9PRZOVmsFRdXNfGdW885mkCqegSJibFfIF7t3cQs5+/yhIZ5AxRT8nihNgO2d3dpVrbIEkjokQmiyEIAsLIpVpvsrBNEsHBni/59je/Tr1aQpUFfNdhNBogZBHdbov1tTbf//73eXV0gp4zQLhhg+q6jqIKNxKoVCJIJIrrReZRjyj1mE6WSIjoSp4kCFFihaJUYqNdp16q0K51eXDvENvxuRgMmJhTlGIOo1imXa0jqxofffQzdjprlLUik+kxXz59yUb7Lq8/fI1Gu0ZOV2nm64TxFZ1uk3KpSrNeplVvMBz0Obp8wcXikk9ffMWP/6zH8y+OsZ0rTl78KZqWsr5W5fV7B/jzOebMJhYFnl18gRdnBJlFolqUO0Xm1hVJkuD7IpXyFvV2joU14PT0iIuzK2qtIrlcDj+cgpBHMXSG80sENSOMLbIo5PWH91BEAaOoIks6SCGlao1Wp8bZ6TGeG9IbnRElfZ4+OsI2BSrVPPdu3+Ph7X1Go5f0L0eQLqgWVaylQOLlqRbKpLKMUioyGk4ZXV2wmK+wTI9cUSRERUyLCKgMBzOiNMD2rnn+8gVZVkRQRfZ2twmCBQVDxvcGtBtt3nn7PuPpFbY3JLB9ltMxUWSiSDAe9ikWFRxrxeB6QK3TIS8vkJR9fvtv/q+5vX+LakskkFVu7e2iYyNYKYlXxs8mlIoqtUKFmSUQLGfUSxUQPRSxSLVZpzebsrIDgszhuj+kUJM5P+tR0loc7u3z6PMTYpbMFidst2+xtdHC0HQs02Q2WuK7PpYzYNAzqTfKhHFCOA8RhYzz8yF6KqAK+ZuMe5Sxv32XVIjJl/O0N4sookG9Uv2lB8pfekN5995rnJ2MWMyXaJ0Ci9UVgtDAj21EEa5PPMZDH9uKCFFAMQjijEK+zswZoebKTMdjdg/uk6FgkKfb3ORnX3zGatSjUWyQeAaT2QVqScJLEspig8GFTRrYvHb71/mTf/cZ07HL2kaFyD+j37vgzYfv8f7X3mI4ntFaL+KFPfJSiWbpbZJuDLLFRvcetj3l/OqYbneXb//KN1nOenhOSuDLbG7XOT+fsr29jay5yOoSz1fJUhnXtrGCgER2+OJxj1J+G1nSmU18bP+Etc4d+j0HSZkTJwlBPGYwcvFcn2Zti/PjazrdJutrdQRBYL6YYugySVBhf+8urmtyev6Sw+4tgsUl1XIJRTMpVnRW84zhSubpS4flKqW70aWzBY6V0u7mCfyI5dKiXK0ThiEFtYOTnKJL23TbMcW8iLdMWGve56c//Yy333ibhWvSn55Q0nNYVhU5lyIpHppawvFXlEt1ppMVUZSRLzSYTEaU8hA4Nq1SjmU/5vrsiJ0HLdrFFg+3X+N86KCpLq1am0+Oz2i36mSpjyLn2Nne59Hnj9l5uM7k2qZWq6Bp4Pg1otild+2zvf2QlW1xcfqcTqdDHImY5pzX3/oNEqHPy1fPCBhCajAfODiOyLd+9dfwrUvkpEJDzpGFLpqmEcYJ6903EMVTavUiSWhwcn6G5ckEyRzVyyPrJbSCQK5SYVtKeLj3LXq9U+LVHGshsvv2Fs2ywNI6I1x5VHd3OT/rI2kRY/MGFzE9GdJs5giiJbX2GgXb48wU2FpfQ48ibHwqTQN/HFDt5Pno+Qvu7e2ymPVpr+VwrBhV6nJ5fYWbpsSrCzY3XuN66VMIV6iyTBwnxEGElBNotu9QbhywMD1ee+M+XuozHg641Vxn0Y9QDYmLy2vWOnd4+uSKzY1dSvkGmdijfz7nO7/yWwTTjNOrP6Ba0Vh/7a9wGR/xg8chf/K0h15pYCgJWRAhKkVSXeLzHz5H877gP//Pf51aU+R8saJlJsSSR6WzyfnjIzrtPIUopryxxgAf254iqgVeXA2JUoFEdJADDXR+XsS54RKmxKRAlmU06jV6A4vrwRRFVxEFgyhMKRVUcrrG+ZGNJKlIaoIU6NiWx/Zug2KuymqeIYo3kRwjV0LVRMoV8LyUOBHICMmICAKRvFJCinQkzSDNpiiyjqIqTFYz7h9sksUqYRAj6zFEAoohEPkpBwdd5otTvDBDRP3FmZilOoqo3BTGWGG5GaCQiRlCliKkAoooIisZK8cDVCQ0MkIkWSFhwcb6W0wnY06vXXJ6gu+HJJJIsaSTJBZng2sUZR8x00gyGT8KUHUFx11xeWkiiAYJCRIRSXIzuHuuzfErBwQBRS7huiaC6BDGcHinSa1WZ7kaI5RjotBHSDUizyUTXBRFxyhUEObZDb/S90kLEoIIxbLI068uyYTXQIjICAgjjSyBTrvIyxcviOUt4lhA9EI0WcZzfQxdIcqWXI97yNktHMslp+dJUhNJEkAIUNQ8omqgihpp4kEqIUkKmmywsVXm1ckxji+g6iWEBFRZI3AVFCXmtQev85MPvyRLE0TVJwkNMlEn8E3eefuAfn9Irz9BNg5I8MhI8W39hnFaL/LpFxdYS4Fmpcxf/I1fw3Ec1rttAPr9HoahkzM0osjj7//9/5Zmp4Pv+yiSQhC55AslRClDFiRSICpJXE/PWAUSd7ZfJwzP+fToQ957+zew3QmVepNiwSBJQzJRRzFSvjyakKUWlWITP5whCjqet0JxRaRCjljLkCQDMbDQcwKa1mJ785DTkyMUNQ/WDGu2wB1MiaUWcq2I0pQZXC8oVwu0NnS+eHJBsbOJPfqMb3z9DpG2zcmLR6iVAs26wl7a4vPTVwhpQGjZaGJGvbnGaHkMvkanvsf4KqZYklmYxyhuCVWukEY53nnzW7w6ecKXn12wc3+T9WaZwB1ycTpgsRRplANiNyWTM2zTxLFjJMXgxclLWo1NhtNjBKlDvbPFVrdJFr1NoV6h/TfWsFen/PCH/4xy4TauMyYTmnS38iRnDuP+DOQ2mbIiCX2mPRellifwhuSNFkZ9jen4JYntYakmYnUPUYFup8O4t+LO3hsY8isK1QC1lCeNZGazEZNZStkwcP0Z/cWYQrVBnGRs7O0SWAle7GM6No7nQd+naZRRiDk+66PLElezf8D17AF5SSUzdBDGLOcBm5uHvDx7irWMyBDJ+TpyqpMYCWlgU9LytOrrYBosTZdGc53VYoZnOpQ1HS+22dvrUDWq/PH3/zVvvHubjb13+fijP8GgSndnjen8Ga1aB1URyWIFYoXI9xiMTigoeaSixHI4ZLPTwrVUBM9Dk1TyuoE5WOCYY2qNbdpGh8XUolj5pefJX35D+fL0BaQCRSMjtF1Ojq549uKY8fIVF/1znr3o8firlyxXI9ypzOcfXDOcLLHtEClQGZxMUNIiR5eXSDI8fnbMP/7H/4SlbXFyfQmlGZ989iOkuEheLJCTba7Or4kFjzfe/SaDyYR3v73N3t0q3Y0cO1ub/OrX/xrFYpHpfMnb7zxEFhqICgiyjxubSPmEpTfgxx//j/TmX9JqNegPn/Czz/6YuXmN65kI8oqLk1Nq5QqFvM5ifk0hp2AYM9r1OhvdPQ4P7lDSa9w+2ERVfV57/S5JZtJpbrFazmi3ClSqdyiWajj+NQvriFypQCIUUHMSRlGj3iqhagVyBYM4k0hFidPrZ8ztOStnyGh+QaPVRUgVFpOIwPZplXe5+Hybrz4U2Nheo5xv8v7bv4aKzmqeIEgqklyiVKyzms0o5wuIUZF8TkQSMtbW9sgVWwxHEe996wFqRWBl9aiW6zTXGgwXT+n3TsnXGswsF0Wus1rarG2s40UCXuxQ6+RZ2zmg3b2FoKjY8ZA33t0jr5Qp1g75o48+4OnJh3RbbVbmnFqzQLPb5P691zH0AuViibffeYC5jMhXYpx0zJdHp8h5nfFqTLkt4iZzvGiMJKpMBiZJaEO64snjf8eHH35IriCRZWUUsYlRLLN30GDcmzO4Cjl6dY4ihAiSTG84RZAN+sMxqaCTiDd6LCee0Gw30LQ21eYOqi7Tqu4gZiGKWuRi9pyhbZLKAmLOxB3YFHMKOaVErbHDfDJke6dLXqsTmCvymYAYjHDGF6R2xqunL7GCKW21hG1ZTLwp40GfWW9JImUIYZ5aWefVxVMCMcT0E+RiA7WSpz/poec0OnsHXA0vqVc0qq0iaSYiChmSLDLtX+MtLrm1+zrFms4nzz5H+rlZqLjfQVECgrnI7YM3qGwKuMkEJS9yvTjBs0pUizkUAz569gPcaMnR9RX/zT/8H/h//qMBP3wxQi/nQUrIdAOpbPD4y895+if/HV+/f8r/47/8X9BsgDe5op1vE2YSjUKeLHZQKjkyZcmn5nMyvUxdyNh/8CbDMGF44YGU4vsiru/h+Q5BGOElGakUkWYJiqiTpnlyTYvJ5Asca0lBNYjkiCQzKZZyJILN1fUlruCTw0DUBSQlIp9kHL16zsqNkMWbwTRKgMSns96gPzIRYh9NUchQMVcr9g82UIw6X3zyinyhQRykhKJHEqmI8pLh7ByBCqIskMjJjUFGFdAMlcVcvLkGT6RfPDd4mgwxDgmdgGfHKwRJIbFSxFDBTzzyssrtBx2CZQEQidIAQchYWRbdusHF2ScMF1NEVSVLLWRk3ESgmtM4uzwjSfIgKGRKQEGVCR2RSiHD8qccn1yRoaJkCbJkICIhKyIrc4qer4AkI4kuspAiSwauvWR/p8zQWeAmEVmiEjgr0jjDDl26jRLt/V0mk4gocgg8/xcmG8u5RiuoqMVNothDSFLE0CBTYogSZvNrMvXGPS+EIYgCYRaRZiJRssRZWSRxiyiIETUJPzCR4xvVYaOaY+bbOMuEIHOQBJlELpIkCaoe0qwbpOImQZaiShkiMUkMUipiaCn98TkvjycoqoqYaQhJiJJECJlIImdcXc5JE5FEjBFi8KKUBI/E15lbC8LAYDq55q/85d9ifX2dQiFPq9VENxQEUmrVMt3uOv/23/whX528pJYzkFKZTJEQM51quYIgSjhhBAhki/Dn212f0ewJo/mSnZ0NrMWEdnObjb0OulZEVFVWyZLr3gmhO8VJLXQ9j5SVcJwyV+fXvLzqcXl9SaVeoH89YByrpKpDKiz57POfsnJNrievmK3mhKKEKfpUWgV8Z8irp6/QCzpDx+LJqzEYIUt3yPZem6Pzc+Y9m/W1A4bDx1yPUyaBgKSEKGqBwzuHtFu38RKLNCkTBSrnl88YmhdcTca4QCpX8GODif2cTz/vsbX7Ondff8hoYDIYOFwN+whJARmH/pVNpdRFKao4iYcdgmRk3D54mywu093ocnbV5/4bv8Gd19+ms7FHLFpcjn7IdPEMJVsnDl3K1S26mxUcT+LW/Td58OAb3LtVx1xes3A9qi2DMPZwXIHYSYgWVzRydRShhp6VmUz7ZIGHb3nIasRosGLncA87SBmPljiLgFatysZ6BzfMCNIIRQXiImtrdygVczhBjG4ITOYzFtMMa2GytK+4uHjF7OoVYhGi1YR+7yVJtGA5PCYwBQajKY+Ov0Azttm7s4GhNdg4vIeey5geDVmsXGLBI5rmqbQS5tM+sq/SbXaRZZliu4UqZKRTl2lvgWDIaJmPO8xY377Fy8vnDOdTGrkm660qiuSCKSDGKePVBc7E5My85HT6JQVdZdwbIeQDfDnF9j0kOeXTTx6Dl7u57UxU7r+2zeDa+6UHyl96Q7lcgq4uECSDavU2V/M/5dGnP0OOq3zve1WSVOXgcBfTmpLJsDSXTOYuQlag2eyiqi56SWL+wuPjyQdU9C7vvP0GpVqVN9+4xY8//jO++b0SoS9QKlVZziOqusNsBFOnhx9YXF0q7B00kdUZWZDj1v5dJC2l3mjx8uiSes1geuWhGyJL5zleWEGRNR6+vkV/eIzpXKLpEpraRBAFVt4FSSzRbb0GokOzXGE8i5hbYyoVFb2gk2Qmg2mPr731DbzAZzZ9yUcf/whN3mXlPSZNM5LUJfQCZHmJHh+gKnl0WSGnWfiOTf+yT7nQ4PTkmmIlJYoiBr1janUFWaggiyLVTpO8usfF7AtEySQy79LrZYiKyRtfK1Iu7aBKOR598QF7hy1MZ4Zj2yhijjQOaHd0+r0RipEjnzcYjR1WlkeuLOGlX5KmbxN6HoICOTWH42SsdXdv3OWuiiCImPaIXF4iiBbIqotlRpTLVUzznOU8QlQTNg/vkMgxcipQ1gxu7bZpbWxgz33SNEY3JCarExBV6t0qL65/SOJ1yTCYLpesra3jJx/y6PkzDne+Sxw2mc5fcvvWa3z1xSmGkeKHDn4UUq4INIw2y8UcXclRrWyxNBVM7yWRb1PQ2mgFjZkzhFCgViuSxgsUWb1RYI4zfM9m/+AAy/YQjQgrnJCFJSRXY2FeIQoqhbiErEhIUpNWq8tgcIR5foW1lGg2tiiVS1yc22g5D0kOmS5DcpUqopQwmSyIRZeFq1IuLggCEyOrstdd5/j6BK3a4Kura9a6bXIljZXlkKGjKAsqtQKaLrK+vk0upzMIhywmc8yX52x4HrqukdPziGlGsV5laV/z5u1vcKFs0e20+N77e4QLm273EDf4jMGoh+0skPEpYuBHGdUayMk6Lx9fslgYHNsqp2OIdIVmS0UXCqSqhj8PODr7Ke70Kb/1rfv8zv/hP2Mwesm/+OD7bK91qMVlCC08LOyxQVQO8ROflVdmrVbh+OQFzVoRxR0xn84YTYYIahnHnqFqBQhDsjgjiULiSEJRcsSJQ7kIchpzcnzM0oqptYuICqyciGa1y2a3he8JuG6AqifEkUYYJDQ7GoKoYFo2+XydTAhIAhFDA7LwRo6gCHhBgqqrIER8/a23eP877/Pff/+/YMmUWMyjZTGZFBJ4LqXiDlFokqU37fM4jpEUHze8ZjgxkZU6afbvW96yLOG5CbqWcOd+h6++TPBln4pSwBFD7DBkTXM5Pp3gkuKnMXKi/zybmZFlEa3mGh9+OkVIJAQEZFkk8gNmiwnPn8/wrC007ebMEMWM0AnY3MgjSQppXAYRFEXBsW2SxCOnZLz3fo3PPn2BF4D2c05wGkWoQoYqZjz6/DlBoKKRIMk6ggRJlpIvKPR6Q1YLkOQcSeKTiRGqaBB6No+/+BG2extVVUnTmCQRSNKMFI+1zRKWaeO5N7nIOIJ8vsh0OuZgO0etHnB67IJcIwpDZERETSBceRiahqbmcaMIRU0QUYnDkDiWUGUd25kxnTiIUoIsy2RJjCHl8eyQrYMyS/cZpjsil9siTVOkn7vpq1UZUVxieSYZOlmWEWcimipj2ibtcoFbdzb5/r/9irVOh7/8P/mLWI5DrVYjCALiMGJrawtBEAjDkH/yT/4JBaNEkMQ3iwshwXYtNjfvI54MbopKcUKsRFSrVWJ/wcHuPld9i2KliCInNGrr9CeX5KUauqrgJ0sEoUukWeTyOr3+kHpD5nrykl/93l/gcvAp1xcDZK8Gok7gn9FulbkYDqjk15FVASGp0Z/3iIQ53c1vkK95JPkygQ960UA24erqko31Q6oCSIKCJYhoah1RGLBcTMmXLhmYCXJSo7VVY+65jMw5shCznA4pF1UUrcCd9jqvjp/QvzCJwufc3nmf7sYBP/zhD4miPYqVNt3OBvlySO/LPu1ODtvVqdc6rOwTSuI+putTaQakcZFipcF1/0OOjm1Wqxn//J//n9is51AjgdXSJVUXrGwRqWKgNRpsuDkur0Zs39Z5+uyMglFBln0q5Rqj8YL6epfF6QlvPfwm08U5hCIFXSOrgdFokZNixETHiUYsl0vCMEGK1qkUY7QsAFnACVeMhw5xnLKxscH11QXWykXXJKYThUazxMVZj9u7r+O1Ftza/HXajXVyBQFRkOlfXtFYKxGENsePjjgf2pQMBzm3jSqGXI6O0JYa7777NnmxhTMbUS0GNNs1FmYPQc8RJDHdehNzPqJWOeT29gPa7SrO2MUPYlJRorupMh0NiWSJUkHn3UaD5dgm3olQcxm+V8aMJ9SbeYoiyFoHbTomE1QCJQeVFDsJKTVLDF6e0dC3eOc336R3PUbNGyRKyuXVkun08pceKH/pDeW0N2J0OeX5l8dc9Z5iqF1u3crTqdZYXtnkciVePD+i1z+l1x8wm3m4XkKunMdJLkkTEy8es9nVWWvssLvZ4Yd/8pRXj86Z9DKMbI+iUSKK5sRxzHQyZ3t3hzfe20JWNQ727/O97/wWewdd5uMScVTADnv0+n1SMaCz3mA4e0WaZQSBSJZlZCSsr+1gmwmV4gZRHNJZbyBrcH4+J0lbaHodrWTiJjO+fPk5qiFg2itSYmYrm+lqQrPTZHDlcnF+QuD7CGLKePk5Ob3B9vpbRJGCIRWoG/d4/eFdcjkdGY1iWWaz8z7FQoPnz59TrRXo90ckUcbe7jaKKGCZQ+7evsPOxtt4jkiz2uberbdJ05gwvlERXp1HCGnEdDxCUVPSzCdwVVqNXRRZQMx8POvGn9zpqjQbaxQKBcJ0yXD6EjcIUfU8T756SpKFNzaNzGU4PQFlxeX1UxzXZLma4voLTq8fkS/oFAsNFpMlhtJkd7dJXswTrmyuL/rM/SFfXn+AE8Q8+vgLpqsek2UfxZBwXJ+jkydEqYSmHuILS8aLE8LEY2kPKBY2uH3wqwwnZ1z0vyBXVPnjP/1XmMEZXjZm6fXQCjqaUaRQzFPKbSKJGqIUszKHlAv3UKQ1pJxLqlkkokYop0ztAQtvzNIxyVc1vHSKUJjy6OkzLvonnF1d8uXzZ6z8a6b2CREJK9PDyBVZmAsSaYVRqGJU8gwWlxQqOdRCQLGc5/nRZ1jRNS8vnnDa6+NlGmYEWU6l2Mnx6uVnxImGmVWJ6+scuQs69/eZuSsqlQqZkFAo1dnY3EFUHXqjF5jWnE5rl/nCxLQFtg43McOA9b3byJqCG7lYgcs02OfTRxVa7e/y4pnN42djfv+PfsbHH33Kv/mXv8fL/hM2d1tIik65LVJuVpksfBZuyh/+eM6//MGKf/GzlD87afJ8UkDRaggBHL865cvPH/Px/+8PWDz/A+53J/xv/3ff4mvf2+Or/oKXp8c8yEt0DXAEhWenPaJyipZTkL2MfGSiyTpZEpLXDSaTIdfDM/LlOqalATdbyNCXcf0I3w1IAhdZVAjjjExQyJKEW1s7WAsVJB1RAbIYCYNKLocmSKxcizSQSFPQdIGcIdJolhgOUjzfBSFFkA0iL0GOfczlmMnYIxUiRCkmS0OEGM6unvPoi3+N4xTwnBQimyAVSfyMr3/zIYPRnIwYSc5u1JCZQuDZ2P6KKFWQRIUslX/xiKJIGCQcHuxSr1UYjW0UNOIsRidP7MrcuVXhG3c1xFVALn8DdyeTsVcOO1tddnZ2CH0NSRaI45Q4i0jjgHq5wPvfeJfY04hCD1Lh58MUCKLDg/tvcHXlIogJtusSZRm6ISNkJpa1Yr4IULQGXuihGi5JHJJXY9bbFbLEwPVj4jTGdH1SISUKEiTB5vL8Ck0tkQn8PJ4gM59Oee3OLTQZxqOINI0RpexGSxkJWOaMJFtSKFbxvQSyG5aoHy0RBIFaqYGUxSSx9/PcpkRKRizEhJ5PrSTgOhFWKKJrOTIkUhyENCVnJEwX50SBTD5XIc1iBEFAVVVIIyTZ5/zyS6JUIhUUZFlGEHUk0SCOQ4pliWKhgipXUSWJNE0J4xvLkR9NefbiI14cHfG7v/vXMQwDWZbRVBWyBEEQSJKEarXKBx98wGePvqBRK+PHCUkqQWYgyjIb6y0ERNJEIBMEMkGCtMTBzmv4logiQSYIuL7Oo+c/YTI1CROP2WrMfOJyfvkMQRI4P/Wp1LZwwwjUBZ9+8TErS6C1to9kGEiqQZrIXF+beK6CG1kEqcjurQNKlS5afoO5d8HzV8eYVoSiS3z17IjpwKVUKNCo5RkOh1imQ3ezxVenP8V2i9y/9x1Ggx6+aWKt5gyvTLI0RELBtm06zVts7uxSzHXQ0nXuH/4av/Vb3+Xw4C6muSAO8nzta1/jwYO3uHtvjzv395jNV6h6dvM/pNrEscJa9zbz1QXrayWIRcxlyrMXn5IRsZh5bHRfA1llGWUsZAVpe4/267+GUN+lvn4bXe5wdPQCXStyeTLCsX1AxNBvCqj2SmA6Sfj2O79B//oppu2QCTKxCLmaQrWssloE9Prn2I5FEATYTo/p8BrXipmORxCJfPX4K0I/pdkoY5sWsqSzublOoaggqBZkGof7GxR0Dc+M0dUya3s6pcotJguT//a//q/44pMZSRLx0Ue/T6FSodqqIegh9VqXr717hzdee50g0ji+fI4TWUiKztngCENJUfOwcBSEUo72nQpZUSJXLnL26pwX/QGhlHCxeM75aIWiFLGXzxkNJ1wHMrqRsVxmqLKAvRLZfW0HpaDhLBSSwCcIBBqdNXwlI1El8nKRllrltf3bqFHAyfnJTWyDFVfDEwaTORvbt37pgfKX3lAWlRpbG4cYd2Ry+RY/+emP6Gx26X+5pK4c8NFPP+Hg1jovXyz5q3/1V7A9n+FgSmuthO8XKWopekvg6PExmiGRZha/8usb7LV3UaUclWLEYlZCjuvImcCD+7u8fHLG9m0VUQhZLBacrz7iwcPb3Lv7OnPrJZpuEGQTPv7if8AwGuRLKktXwPHGNKvbuLZAv3/F9m4Ld1Gi7xwhpApr7S2W01OKpZsXx2RqEUQ+sqyQJAnd9sZN3sDYZDyekYQ51HiMZugEQUauovDNh1/DXsnIokI+l2Ozs8Fnj/6MmhQhqgFeJIKkkKuGPPnoFXvbtzg5f4wsqMwXI2I/Y727T62yIg0lRvNrWo1NGvV9gmDF7k6Hi/MepfIarZaMquUolgxWToZh1FGMEZf9l0hZDteG0FsRhDGtjT2+fHyEltcY9m10vYC18rgevGJjcxsvnDAYXpDLKwiSh5Gr43rge2MqTY3Li2tyeZ3r8JjQE9EkcKwq7XwZQ4rQi0UiQWC+mHFweMjl1TF6RWU0WlHr2Fz0ztlav0cQZTw7+jG1RpnIb1Eo2URxwHg6pFjReHX+kkIxx9wekvZ3abXXWKyuWFoyoqxgOw6yMiNOfJqVA07OXjGan6LnDOLEo91ps7SWhPESMVUwqhmrkU2rWsAJ+xxfDChVqhydXlIql1E8mXK1QxgUMS2fTBBxXRdNLnJ0coSQxfhBxtHxjzi8vYaoFDD9OWY/xLRC9h9UOb94xe7+LSZ9n1BaYtoT7BU05BqVWovR7AKUjNOrc/b2X2c2m5NmAWqujJ2uiAKfLJUZjQc0mx1+8MmfsrO9R6VSp3feR5ZSCtUOk+MhYRSQAVEY8v/6f/8DPhEy/i//1/8KQ5XJVUrsHXTBdagWanzwt39ATiiQSjJxqOEFLoIgkaQGiVokkWz0vEjqymhRQOKbSHLG4Z02Ww91dre6tNo1MHym0zlPegPuVjfZrBZZrBJqqUJJtti5vU0oZcwnPUS1wp39Nzm+uqa/6NNp1KmUy5hiwjBOcd0YLSfihz5SlpIKEmEQokgyYegjqBJxJFPKFbh92OH8eEwkBMSZgpAKhL7DW6/dxhx7nF0P0bQmy2lIc62CmIkUcgYX1xaBJ+DYIXIGBDGv3W5TKMpM5jMUXSfJIhRRRkolDu41cf0xbgD1bgV/HpCoGTIZjVqZTz5w0PN1gtAmS0WSRKfdqZMvlFksrhCVJppS+veHYuaRRAG2mSGxDukUQVTJhBAhTcmUjEr0jK/+zU9xnN8kkGuI+GTZDbOwWhN59uIpspxDklIyISNDwFyu+E9/9zsUjGvM1ZDNeoMoShAEhTRZQWZx1b9ClEtkxMQxqJrKcm6xs36PfD7HaPSUTLAQBBmEArY9o1qqIRAwm4ekgogbuCjIeL4PUcZ6u0Cl0uDpU5tEsFFknSwTCMMVulbmcP81njwPSeXsZoMXJ8hajVopz95uk88/N0kyFV0JETIFSZHw/QlpumA1n+NZZcLYR0YmQyTJMhRRwlpcEwcNUOukUUoYRui6zGrhsr9XQdMter0ZWVpCUlJiP0IVCuhKxp1bh9QrHubqOYohkKUmoqyiyBqSmKHoPtbIwgsictUMIZMQRJk4CNl7rYOgBDSbDX7j17+D6wSosoIkiViWRRyEdDod6vU6f+fv/B2KxSKBn4IMqiQgJRKyYrCxuY6QfYmmaZBlCLFJ6K9hOhoL54ze+JpN9S6e5SIpeXZ29hhc9mg3c7heh3v372GFz6g2ayCtMKcyppWys6fz9Ktr3n//W9jmguFoiKZplCo5ag2d49MvsXwXSU+ZzCbkcmvMxid0mwe43jWjfsDe1iFZ3aFUuM101qfcbCIqKqv5jLu3mnz6Z5/z3nu/wuHh2wx7xxSrCYlnU5Fq2IlLvnQXN16SK7S5OH/GWnubWr3F1eUQXa9y75sHDEaXCFINqQDH518yHWbUmwWKpSZu4GDkK8xWI2R1D1mOubqYout5NF3m9t4tPvjJZxRLKtOxS2d9n0H/mIPD+zx99ox2Z53Nbofr01OqpTaIPqenX3H//tskwQDPjTFNkzBMKZUKFAsKxycvueq/oFDoUsmX+fz5Be994z2mlzMWlk2zrjOcmLzz1re4OjvBDfpMBxHf/fb3OD96wl/57ncplje5nj5n5ljkjTz1epUgMsmnXYqlGCmTsRYW1apGf/ozPvr7V3zrm7/Dj3/8z7h1b43PP/4Jf+Ev/d/5m/9Lmx98+AGTWYaBwIPX2pxezpgsxjiBSLWWMjFFjNYuhZxISfRQBIinAmf95yRhgK6nNCpldDFPoxYiBwm3ahv0e+dUt+s8fT5Fbdoga6T2irXqGvOVj14RyMIJhAb7hxtc9y9pd2t01po43gylUGWv0cCcj5kJEYaU5+7tN/jqq2eYQYKWF1l6lwzm4X/4gXIwvca2Anwn5up6zJ3X1kjSAp3dEt//wU/4i//RWzSa6wRRhhM4CJJAqaZydnFEuaojCRKhKXDv7jYCbd69/z5nlxek7ozp4oj7D+9xeRqw8nNIik8YqGzuV9jc3GEyuKZca1AwZBRZwxFOee31u8zH4AZLRCWmUtUIo4BCvs5175Jbe1Wy1CKOJJrltzkafsHdw3ssFjGevcQyp5TrCp4jkkQqtifxxuvvc3z0ioU5oFSp8+rlEY7j4FkyrYaEHVTY2buF5c6YjzVK5Qwpk3A9m9H4klq9gO9KCEKVOLtmPI1QxCr37t4miR2qFYNKcYdirkDv6powSChWisSuQpQOsJ0yi9mYWsPAdVckBIiygGtrDPoKlbrIbDFnvHBQdRlBVFEUjcV0QqWmk0NjOFmhl0KSxMO252SCTU6vMp706LT2CDwol26+LxYrXF84qJpOEPU4eZVysP+AIIy4vu4jyh75Wp7JtI87cciVijjBHEXXwFE4Pn1OpZpn1I/QDJvhMOT1h9/j1YsjCkWNNPaZT5ckwQqtkGHZHpqmIaYF6g2QxByq0EAvmLhWRqN+CECzvkNv8AI97zIehIzCa0p1ENM6gugioDKdWDhOjOulqPoQy72xapyfXHFr7yGuIoJsstXZZjKb01nfYTkPWeu2uew9xvEyKsUdFKlAlNhUa01Wzgl6UeTVyStE2WOyMrm99z6TvkO+GdFs7XF9OWNr85DJdAgZ1OsGs/kx6wdfx7YmJOns5tPd1RmKUqVUKzN3j0mWArMgolDJ2N24zXTZp9vaIMsyxpMrKmURQ6mRxBnT2QhNU/D9EFmROLxT5/nllHKtQJC4ROGKRz+9olXf4tf/s7f46wfvs1wIXM+fkXoxg1FKt3Sf1fgrBP0mGjBcjKl3oNmos7++T7OUY3fnDZ68OmK2vCZNJCIrolqsIgseUbQAJaGytoWFzNXoCCdWKRcN0CUWssWH11PuNt8mEFOc5QpdNphGI1ZejJ/aqNkmouCSiTFZFkMWEscyiqSiqBG+F6DnMpLU53o4QDNkZDFPQoSsZDRbMtOJxXKlIuaW+IEGNNA1iVx5xtXnp8RZQoZLSkIUuGSpAmILP1RQciJRlCKkP9fxJS5mWCcInxN5EpEgQhBQb2oALJfiTQYwixEUmcByyJcFvNAHKQdCShr/+8NVUQAyPG/IfJ4niETknEyCSxJahHGEsbbPbvNrmJ9fUpZF4uzGhiMKGaE/JdQyBEFGJCOIY3J5A1WUWC0uKVcKyOrNlbUoJgR+xHjW5z/53Xd5dX7M3PTQmi1i3yPyQYxVEumMTNskEwQkUUSXb4YcPV/CT/s4dpGlGROkAloUIYoiQZAgp6ALAe4KfDckV9GJo5AgTvG8gFa7gpYTsJ1zclqBMA5ueMCrJTv1EmkiMRhZQJ00jW+ykpmCJIgQ+0iZAUkJI3djH1JkncCNEdOU7a0up2cRWZYgiAK6ISBJOkIaUCxo1OrrCOICxIQ4iDEMDUEQCHyPcknDcSKiSKbYECHMkYkSnhOhCgq3Dg54+WyAphcRM5k48hBUkSSKCf0VJydL/qe/85/Q7bZZzpagCPj+TTN+e3ubWq3G7/3e7/Ho8WPW1tYxTRNJUm5a7nFKq2swXzzFdW1UTUdIM4TUQMkl2JFFsdZAt0SCwAN5gqp1GPZOCYMAsnUEyaSU38JOTimWQ4SsxOZ2juHQ4uJ8yN7+Ol89+5hGrcnGzjqriYsoS8xXQ3Stwa39BwyG1+TzAp9+/GMOdg6ZTc+J45hyvk4YevhxiJFLmC6WBE6EW/FR82XkwMDQHvPVpz/k7tcPsbIZ4WSfvd1NHGfMWneLXm9Jp1Pm6PEJzcoWk4HF2kYV+8pHkDMG4xNcJ0XUfC6ffkaaKBTKW/i+yu1bD3jx8hnmIqOzVkTWFwjLAggxpjWjU9zE81ckkcL+3bvMl9eouTrlusvFZR9DK3LdO2NlukjFIufTMc2aiO97JHHAYjFhY2OLKFKRJQEt5xIFC5arJbXyQ7RCyKPPPyFE4+Wzl9QNhbn3imzZQFJKvDh5hUEOLReyXirxo5/8hL32NpOhy49/9FMO77Yg01Bkg7xR5vq6T6WwRhoHXFzM0fQURaih6WMMVeWP//AfYTkCh7c2qZUVXpz9PusVg9fv7SAUWqjekunkc2arOaK6zp2DIuNxn1yziqSFzAYxabGGlFkIyRUHeya3tt7k5dOvKJQkJvOYbnmLainj0aMnJFmOsSXQbNUoNH2seUDZaONlKUfDGfd39tBTjVa9wqvRC3puyjt7bZLURVdivMDl0bMxSOCoKbrswKJIpa4zdyNc3yEOfTJR+w8/UFYrZVqlEnldZX/fYLGU0dhgZz+l3ta5tXfAeOjwznt3mM8DZpMVdjBAU4vMFkd0u10mkwStpnB7t8uzo8csViatRgTZnJcXCaWGSF2QmIxDjnsvaa57eJlGqZ7DT86olLe4Gh7TbKl88OEPKZZkAl+nUGywWqTkjBpFtUCrtqQ/uOZw702alQNcf0a+oDK4XtHdWCcIl7z2xm2KhRq96wGiHLBTvsuoPyD1ZWrlEr4v02w2+O63fptqucBPP/6UWFhw1btma3OXfv+S696Cva3b+EGA4DfxowGx7yAIAoaRcfTlhNt38pQMibPeCFVVWS1HLGcLtnc7qKLGoD9BFmNqrXVmsz4RLv2egGic0Gk+5PzqgkSO8YUpJ5cxRkHBtFx0oYvtjNm6u8ve9us8P3pCoSQyHbtoGkSBRpKZqLKOJBZQlJgXL16wt7+N4/kYRp7AE4ljmYV5Qa2R48HmWwjyioJYoVzaZDY/RRZlgkBGrcHYnSCmOmpcQdYWOMEUIWlT1kwCScU2fWzbxnUVktgnl6/i2BFhMkYJdxDlOXFYwrWqeEHKdDzhrbcfcnZ5Tbe1TqmsMRxP6PUu0NQizlKi3WoznZ5SqnWwVx6VqoFlXZLP16nWughSmdAVmDsjdHmbNKpyej5la3sDJJlitoasFJCyEhsbKf3+FYa6xtbmHvP5FD+aIiCysAYIWYNG3SDLBFbmmDj0UOUKC+cZaA26nRa2OSdLLVQZFouYg90dFAnmvQu8cIEg5ek0vkGr0WS5GBAvbWpyG1s2aTfrxHFMmowo6iVK6+D7OWIGpJlIrdGhd2Vxa/8+afYhmpYDMWXzsIb3bEApL5AlRVRNJMvFNNfbiMUZSVyiUdGodkrc3vgmprjio599ym/+9u8wWI7JhBH5eJuJvyCTZJaLMXlF4yeffwFyRF0tUTJKXPUcTq2XGGqVVXWdDUMlng05XVpsdtfQ8kWs5QxDEKnIAsfXlxwPI2IRomRBvi6zmWvz6vEYx88wBI8k8BGMEnlZZmpdU9DrJElKFAfYdsiDB7dY7/4Kp1d/iEQRMZNAzsgy2D9sM5xcYfkOrVqDyJOw7ZicrpElMpNZiqoUCQMw1JQwiNnZXuf8sk8Ug5LESKqEkIGhZniRz2Wvh1ao3NhSRPBTDdu+YmX1SVFAFIgjgZxRIApd8obEgwfv8od/8CNyhkaSOL84E9NUxHU9fvWv3mOxOiVINJQ0JM0kVF3EmEVcL0q8nCyJ5DxREIIkIMoSYpKxvl7n4qpPGKfImoYoZgiZgCRAsSjw04++olCp43kBWRah6HlkJcOyl/hBghO66IJAlsYoWh4vDBClKd//wR+xmP8lVD1HnLiIGNjeNfWKROD5zOc+opgncAOUnMxyYbFXr/Dm63v8y9/7KbK2QZqJpLGJrBkoosYbrx/y4sWfsrBt9EqdNLlBBjmuS+feDoETEcUqYSyixtINjigJSdKQ2XRBuWAQE5MmEbKo4AceulFAUxKq9QqLRyNk6caOlAgBciqSRD7lUoXx2MQNxBsXvJIjSV3SMKZULFOuwPHpCbKSkWUeoiAjq+DaLs0mfPHkR7w6uUbW7qPIIhEKgpAQxTb12iat+j7vvP0dFosFmZiiKQaaqqBpyo1YAPi7f/fvoRoGjheQCdrN75FTkgwO9g64//AQ6fdfIckSYeiC1GJt7ZD5YkIcZGSpQ7FYon+uUWprmDOTZqvEYHBNGkr0rl8SCgaGLhGrA85evSKJI7719d9hPLlCyjrIqonjLjD0HIaqsXQ0SlUdRZMpl4ssrRmvv3mLKBAplppYJqC6LLwljqkQZn3knEq9UcLzLPL5IlJWpNy4TeilnJ/Z3Ln7LdLQRyklkFW57Dk0Whm9ozlrzTzXvS/QjFvMp2PiyKVT75CmMb2rPu9945Crsz7Fqk6QDBldKXztGw9pNBo06zrL5RI/tIkyC8PIM7OuYOKRskm10kBVVcIgQ8tcyjmdRqGO780I/IDUi1AVnbdv3+aif8x8MuVcvKDVbuAEV8zmGpVyAcsMuTo+RZZjirUNDGOdjbWM9XYFX4gYmz2iNEEQZZD8G1OOqqMIGuWKxM5hlWcvPscLQmrddUqdKqPTU6p6nVcnj2nVmiwW1wShQLXeRtdVAjckSTLSzCWXD7j93ntUy33sscuw/wnnn6YoZZVo9Zjrk2t2t7ssI2hUypBlTAYuqW6hqDmKUpNYyCjny3gLgXr5Fn52wRvvF/nzP5ojVAqcTE9gITOjwNfff5fh6Rn1ShVFFxDlGNF3mZke5bKC54/Rci1MVgh5GSEVsLIVvfNLzJVEo1rB8kdkakpZKrOcmiTRJd3mAYJ9Regm3L59h9F4/EsPlL90hrLbqVAqlbh18Aa397/NX/ntX0ORVOIowLFtBtMjjJKPaUaMJ1fMlucYSoU0Ubg8Vsnn7mHPfUSxwGnvp8Tyis2dIp+/eEmpUmSyfEXMjF5vQhDIPHztFv0rlz/8g/+RRitG08FLriiWVIaDCQIFlPQeurLG5cWA41fPue4/QVQXrK3XuXvra0wnDo+ffUh/+gVR5lIol8iATncL309/nssDaxXjBXNKpRKbW03azXUq5RZvvvkWgZ8xGo3Y2NqkUm6ys32IY7lkqUexVGNm93H8CCc7QS/kcaMVg9GIxSJALy2ZTS+ZTIc4psT1eR9JNdELDr3Bc07PzxjPl4xX1/T7A1SlyM7OHf7sJ98nVJ4yd59juXNsd4Wk6UhSkdH0iqX9kvHsiBSXpy8/4vmrT9BzMVEUgxBQra7z3te+TaVaZzDqkyslaJpGJvh4vkWWSixXI6p1nXanRrmmYORrDGdf4YYTFuYI2z+nVJMplIqoeRXX81EKDcb+jPK6S9koslY+ZDa9JFZkfA92du+wMAeUGy4xHpmgsb61iSi0STOFjc4brOwRTnBOvhRQbkaY1pzdrbcJogWW5RDYKlkcYdnXRPGEOJtSKFYplKSbTOdihSTk8PwIy1liWRGZJNJsHTC3Lqm0Fdpru5h2iuPmCH2BcnGNQqHEdLKk1Vyn02ljWnPu3b9DBiiajmkuEQWF8WjJdDLEUBq0m2ucXv+Aja0SxVKMyIrt9TWcpYMgpOwfbHB1PUJVW5SbTTZ37qPrJUaTV7w6/Zj5fEwiKdiRg1TQ6M9GBBn8+Z+fECUh1eouk9kJilSgWb/Fo0dHKCpcnpyTxUAqQpJyuFFClAy81CVIbMJohZzT+erFK/Jai8HwnJXtorHBx3/6I9yhiBbl6D0/I54nzF9OuPjyFY2kTbyMyYwKy1Rhal2RhTMEV2UZznAQ+Wt/6W9QkqoYzpjl5YJLJ6WRLyGJPi+Of8z1YokkFBhczNlqPERspNhKilHYwYrmJGqAWk544903iKIASVBJ44DVwkSUFRIRkGI0SUeXSiSyxdR2CMIGqqiQZhEJAgISc3PJ6dUIpBsPdxRmeKFPq61grSLmC1A1iBOfyE+IYyiUyiCWSUTI0oggdEmJMHIJubLBbOoSpgmKJiOnOZwgZHdji+nymqVlkwoRcFOCkSWB/b09ZuOYOBLwPJcsSX7xSKJKlqVMZucoSgkvSm8QOJmKl0jkCyl7TWhULSLBI81XSH7u60ZI2d3uUCl3cT0LARFJUQj8CF0VWd9o4foZMQKCBLqeIwg9dFlAk1OiQEE3ygRBhCCKjGZzLq5O2Gk+5Ld+9X+FImuEkYuAfFMqsXO8cfcuUqoQhRJxGN1s7OIQzwsQlZDlakwYG4RJSpylyLJM6PtkWUavd0y/N0XTDfzQIwgC0kQiyxKWywFhAH4okAkpZBFZkiKmOoW8wn/0l7/OvXtvYDkykqhBJpDPa3iOS64AkZCRZEVE0hs8lKghyRkIMStzjKFXsCwBJJE0vYkpibKKpMQkTFktQmTJQBYVQCTNRGQhR7GkIwkiitRF129a9qViHkkSUGWRSrHC/s53EGUD13NIs4zFYsF8sWA8HlMul/l7/81/zedfPKJRbxHGCYKUIiKSJBKibLC7uY4TTIiThDBKECUZL13Smx5xcfEcTcuo1wpcXSwZza7ZaX2Xje4eUZjDTwJKxXW2dne46PWJkpj5JKNW3qPT2uP09BjLGZEKc/xoxPW5S6PUIY1kxKyGUSjy4w8+xrQtHFsmV2iDKOEFCvWuyMKcoqg5yg0Rz8/I5Vr4qUihUIB0wXD+lFy9wsNv75Ovx4wGTwhii6veOU+fn5EKNvOlTbXe4vDWezx44x7f/I0mF70jRKHM+sYmkljhna9vc3TU4/6d97Ftl83NLsVqxAc/+Qg/nLJYDUnSkNcffpu9g7tMlwN8L6NaaTEYX1CpaYiCTKGQY+XNOD0b4FgpjVIFJdbYbGyxVl9nMbYIfHj37a+jq3kKhQJB4JJkNuVajsDVyOUL1FtNXOcVo4tratU1VK2KhoprhRjKDqpWI0pCjJzCzt4mfhTz+NEzxuMxe3dbvPHObZSiyVfPnt4sgVYOQiIzGi5BSMgEmdVqgeNatDdzOEFEJsWouSap4PP4MwfbV3j8xYiz6SseHZ/QuzQxitt4WZ6l4/Hs1RNenlyiG0VyukIWKDS7JZaTCYGfYDQdlv6KF8c5vnpWZWpn1Bp7KMmS89PnbNZbmJdHuF6Po+k5zwdHnF2ecWUOMFcz6uUmTmzz2cunDO2U8+GIopzDMwOa7RayJmLaS9a2d/jer/8mq8WSRrPM1955l2alwVuvHfLGwzsYUpHDvQf/4QfKVncDo6YRSRm5eoWTS5O3v/06al5Ez0PgK1RrLQJPRlWKdNt7tJsHFAs6Dx7e4umLZ7QbZcYzFzKNL1/8iC9ffoArLHj+8oze9TlffPoUx7b46stH9K4vWGvsE63WOX0+p9Xa4slnE2aTCM8q0O1W8X2X7nqB3d0t7j/Y4N7t1yCRCXwLSZLY2V1H1QNG4wFhYlOoCkRMOT27YH//Npe9U8azC+r1JqoqQqbhegvqlW3yBZmnT58yW1xSrso02jKt5hqyoBOnK4pllbXNHJOJw2Q55dXVl1yOX2KU6tjJlKXlU64c4EUxo8k1KT3WNxs4poq5uNGyXQ1OKddSxvMzCoWYyWTEydkpd15b4/f/+RWfPP5z9MqQ3f02oXjGwj2i01lHFjUycYLrzUkCjdG4T+C7PP78GetbAtPZiC+e/BDXdXnr7a8hKxqeP2drpw5phqblUFSJMLIAKOYb9PtzUAKmYwdzGeA4Hv3xBRPrU7xoglpUWM6m7G9uMhnMiASHmTMnkyqE6YR2t4S9yBN6OrMJqEqTKFvhRR4bW4dMVycYeofD/e9SrTfpj68oFou4voNtuezv3cNe+fjhgHJJp5Rr4rgmgecyHF3T750TxnMkOSOMUhAFFuYM25uDWKJ/OkIRU3KqzLg/IK+LNEstssQlDWwkxcH1btAVIjlMc8XZ5UtkTaBSbXKw/yaCbEEqUsm3CMMpUiZDUkfTRfJ6Ece9QDNWaHpIkjgMxiMQFVAEpqNLAiuhVWvSbZWQMpHNtUMMtchkZCIFIgfdTYpxlwf7GxiJwODpCbfqu4QDh7xQpaSVKaiwsVYiy0IkMSNLUzbX9+i2yyS+hJppaIKGKCiEts3yPOXW7ibl5hr1VpmdrSar1QDRcKhu5jHlJ6zCGDWfQgFCeUVszUkmAQ92fp3N3b+Ar4MaG5TlHF89/YLiWhNVSMgXDO6017kajXhyvEJRuvhGRLm7R6zHrOw5ReMue402oTpjOC7x4eM6p7N98uUqr919i7fefsCDO5tUK3ks08NzA4LwhkMXBAkeL/k//pf/My7HAwrlG6d2GMLaVhc9l+PLr4YIaelmO6UnXJwP2d3dpd4oMJ+ZFItFAGQhIcsimp0as6WP74cokoqhF3C8lHK1DFHC8ZlLLlcidGPEwhISl1KpwsH+fYJAQpIERCkjzTzi1MIwVL74/BmKrGHoKrKg/OLxvQRNl2i11rg89ZFkDQmNSAhREp1VHDPMWYxHBjV01GiBLAl4nkXOkJnP+8wWFqKSgHBTIswyiTDwcV0XWVJIMhAkiKMMx/PY2dhEUxTMZYSqlW6ynmRIio6gxOysK0yuAiwzJF80EEQZSQkR0oy3XtvjcOcWy5mDpulEfkKaxfhhiKplJFnEbOESij6ZEBGFGWQKhq6RxjYyRXwvRhRvSo9ekCIIApWywmplY1sBgpAQhB6IEUFoEiceUSjQu7KIiEgTCT+MSLgZ3F1/xrOTV4SZhpBxw7cUIAoTFNHg3r2HlKstAj9F00FIUxRFJY5j8gWZIFrimCqyUCcNM2Q1QhAgS0Vyukir1SKJdKIkRNUSssRHlEASZKqVFgf77+H7LpqmIHDzAQGg2+0yn8/523/777C5uYnrB0iySprJaIpKmkUUyjrdbhUvmJFmN4ipOI4p5KqksUZ7vUa+aHDdG7DefZPf/et/k7PRh3jxgv58SCqWeHH9Gf/ff/sP2To4YOw8J0imZLLH2fk1ZxdPiROYTUJkqUK7W0EmJY5tHL/HcDhEyYEgG1TbZaarPnGU4AfXPH3ylN2dO6RJhCKp5DSZ0A2ZjG3CRMZLbPKFCoJkc3k2QBBLpLqBUTNYzR1Cc4BnTXGCiIvFKf/wX/93PLs85c9//JJXx0uK9SbPT48Yr66x/JgoSxksjjBNk/nMQtNFFN1DkQtY1pI48Xnx6qeMRiM212/z2uvv0F1fQxbz6IbC+VmfXC5HEIwQxRL5coVMKqBWBGaOxSockGYC62ubpGnK3n6Z4fCMYr6LWvCZTMc02wXq6x30QoX12gGHmwJaHh69OmVl+oRjl+2dmyvrMLJZWX1OX03x3RUHt9bptDdR4wbXr+Z0qi3yagVFkjEMhc5aiSidkYka8+U15kJkc2MPcxkSpg6j5Qqj2mDaH5PJEkfHI8RiiK8U6ezVSeWI06sXBGmK54TEkUWYuKRSQhwUWN9u0ZuMWHjXfPr8A64HfY6fHuMvxgSWQLWsomewuf4Wv/bef8y7b90hVXLoSoGDwibyNCEM5ljxAsuyeHT8IdfWHFHImC2GpBgInoPluXz4/BkLy2UZBDiE/OQnP6Ky3mTm2zzuzeknV1wtgZLCKpzhsfgPP1BeDj9l5fV4cvKI88WfM/E+5vj4MQW9SbtV5dadbU5OXyLqS0RZIkuLBKFDLqcDLhv1XVKhwYPb75HFXbrtXYJA4K17b6PmZL7xze9x5/7raDmRRrNDrpBn52CHv/Tb3+H1d25hWxFvvHWbLBVZ2+xQr69TrMcgpLz+2mtI1JHFCqKUsLm2QxKuCD3Y6O6ShEUse0GlXKCgbXJ4uI5lj2m3mxQLNfx4TquwgeNcUV+vMfPGRFlEoVgmV5PwBDg5OUHPxQwmIyRFZrV0mU9NRETy8h63d75D4nc4Pj5jNbOpVnR6vacUSwbtzgMarQNUrYjtLFhr30ER1sgZEpPlEwp6myjS0QoiY/MJt+9u8Zvf+20axl0GV0v+/Cd/xOnoYxKjx2Dc5+J0QWCLqKrLyhrihmNGY48Hr9/m8rKPkY+5HDwhzQQca4oY26RxnjQLsL0ppxefUqu2ycI1/MAiyVy84JpW/RaVeoNSrY5qVIhSh1RUyCQZc7Wg3lIQBYPhaMLKOcPzx4TZBYLscPegyf7hCbWiiSFkuP4VaSYwX51zevGE7fVN5rNLTo6eEIchgRswn9jIyEznL/n0k5/gOQ639+9yftbD913W2vfIkPHjJarUYjqyWE5WxPGIpXlMGM/Q8zHDwZcUaylB4HLdO2d35w6ZuOLFq0cIusXF6BWLxc1L8+Xxx+g5AVUPmEyvuXv4LV68+Ap72UcTWxhGnVCIiFSH8XJOIk6xVworx+b4csnnT2ZUWi3COEJVXWQxo3d9jCKJ5HNz5MSnpNUIfBXTW5EKIfl8jKbLeFGNP/zpH9Lc6bBaqNy5u45lebQ281z2vqLT0rB6JrlUh0wgiELSFFQh4/U3NrCWS2Q5T5gFEKsUSwZ/8McfYYkJrnnJB3/ymJkfordsXNNk3PNoa+/w/ltfo1JdY3JyTc3YIkpa7G93EIIZn371BY6aYksRtnjFYPqI5ficUmMNKz5hFSzZfvA2G5tr6CUZ2Rc56r8kYIJSUBBVDTsu8tGzNh8f1bi8EpARUdQUrVAmkvNIRpvWZpODvR2atTy6JiKLEbE94r39X+c33/8NPHtF5As4yU3pgsRGkW0ktUiaeAiZQaKIyImHkGS4FIgyCcMw0CSNMMkQQhkneYaY6aiSTqKIJJGOGCo0m3kWix5ZmpKlIUg+tq8gkqff/4wnn75AVCs3IPQMojijXixx99YdJOXGwS0gk8rhL55ETmkadUQSjmYrjKJKGN04saPUo1PSaKgx/eGKQJVIuFGupolCo1bB0CTOTpYoeokUizQTEMUUIQo4ufwZg2WALCrIaUoieKRRiiwK1CsFLs4uiZAQM//m6loSUWUZOzJZeR6uEBAkGfgRcSSBEhHOb7KqkaIQZQkKIWmUEmUJpjfBc8F0DNIwIPECEkEhUVNC+5KyHDIerdDUHKE7J0UGESLTo1Iqk0gaaVrh/0/bf8Xa3uf3edjz73X1vnYvp5/3vL1wZjjDGYpFFCUqghzAthDLiREguYgFBEGQAAZiIIqgiyAKkiBBbMdKgkRCnALbtExS5HA0hTPz9nPe0/bZva7e/73mYjPULS+YDaz7tTawfuvz//6+n+fxhAQNBd9PyWKVNHT55vwZIz9CVnJSfEQhIUtUPM/jg0d3+ejxY2I/I5NFSGXEJMMNUwoNg7ubGsdPj1E0FSnJEBQNNdcJ4wBdV2mWuyToYIQICoSJhCbYf27piTk7HLFYpUiWRepLCLJNUSwS+yu+/2v/JlniIysCmSghiAmSoqOaKoZe5J//P/5TdDMmyDLSPEERAiQlJk8SdMXGkFWW+ZRQKKAoGnmeEicpK3dOlE559uIZw3FMo1mjP3rOm1dz+sMZuaogqh694QFascKv/frv4Dh9WrUHjMZzEMqYFYPH7z9h6SY0uyWuen2iSOTCveB6PmY297i+vqZSNRmvvuHiYszW9n0arTVWjszduz/g1ZtfUjLbaAosl0t6sy+x7ATXH3N2fIkYKVSLLcLomtWyj6FsMrx2cBMPtVxAMFesvAmlqsGjtxt4yxVLf8Gjtx6zXJ3y/JvPGYwOOTo6QtESTk7P2NvtkucScRqQSgLX/UMkZAwjRLckji4PSfOQ+bzHz3/5hyiajiBLCPqA/uCa7e4nvP1onenomiTVcP0AP5hTqhfp+VcMBwOm8YxZMKNoFwlXOftb91g5Y54fHCIlKTvtLeprJoFo4a+WWIqHaZXZ2t/l4vQE110QL4somUSWxty9s8foTMRQSxyfviT0XKSogMiCIIAgUnCCgHq9jq0bSEmTZl3h+dOnfPn1V1z35rS7m1z1bri66TOd9SjUNCRBpllv0Cq0iYICj57cZbZa0Sxt8PbDfURAlC2Mcs5w7OAEY2RRwLJNKto27z1+gqwrhLHO3t49gmyMlJepdSocnb8gDSIqtXUOLvrcjD3ssk6YxKyWQ5b9Q8LFAtFeslhccXpwwCRxWSwvyDyHUnmB419wdf41shQzc+YYmkkQjAkTCSdcMZ5OWIQeB69v/uoD5XwaI0oGrVYTRZEII4fL/ueMZs+IQoE4EPjg/Xc4fnOKaaaUqzHVagVVU2g0GpTKRbqdGkF0RqOtsd39mDt7T/CXChV7m+GNi63XKVs7GJrOWncXXelyeTGkWq1DrhGEC1zXgVSkUi2x1n5EFCaMRiMUuY5dvmWiaUaBKE0YTW6IM49vf+s77G5/yGqV4gUj4jhnNPBIYh9VkXAWPk4wJIxFotglTwTSQETKdTa6LZYDA1WuMejPWbnXaGqRJFvguisUoYOozBiNRuxs7bDWfkCt2kVWUtI0ZzBY0R9coogWzipENVTc5AZN6yCINqEv0my0GY7PWbhHGFqRs6Nr9u92UBWL82OXne2HdIsfEC5yfHdGtZ7SamxTKq5RrW5SLjWRtRX7d7ssFzmXl9fIskilot2CmsV10njCfBCQJwrt+j6iEhGIZ6zt60SJQa1R5fJyyIunNxy9OSHwlwhJjYZ9D0O2UESJ2WzC51/+iI8/+h7edAMBlXLFQlfa/PSnx3iBRhhWqDRM1ro7tMofYGl1DGuFIFjYhSKaGeA6E0ytSJ6nRKnDcHYNUkwqOTx7/VOQUgb9FIEq9ep9htdw+GZIs9mgVt5hNHTwHZ1m9SHuIiMWzlg5MxrVO2xt7jFfXjCfQLvdJfYlZLGIJNw+USqywWw2Jc1XCGLEs2++4s6jNW4mV6iaSKNYQUk0Ml8hJ6FRfcx0fsFkOqJUaPGdb/8685EGmYGuVlhrP8AwmmSphpjVEdCxTRtDkendHBL5c/a2H/Lg4VuIosj2boPFLMJLV7y5PCJRPE4vJuT5EjHW6K5t0pveIEoyoigRxiGT+ZAPvr2NIEm3BRcgF310Xad/MyZwMxIx4NG7W2RSwsmbFVvdR4jCjEyYkyQaV4tLonKPnAVJfMw47/H68nOC6SlKrjEdHRPNBe5UPsSb+VydDCkUd5klK4zVClXJuDp/TcWKScMZ25tvcT0q8M/+4Ir/+w9XTMcyogaqlRJHIr6rcHl2yGp+g5JBo7PG2x99wFsffsy3vv/Xef97v8qv/52/zsGNwB/9qEmn84B6o0S5aKApMpIk0Wis0e8PIRaIvDGxHzAejCmWb6f7gqhTLBbQTAkRC6sg0BsecHx0gWEYSCJoWogf9ems72DaawSxgCjFaKqKLspknsPf+P73EbIOYRiDJCJLKnlqEoQRl6MvWMw9hExAEAKkvPAXL+Kc6eQKGGLKJfIoQVNvHeECKpk0RdYS8sRCkTWyVELJ9dsDVcmIJY1c0xEUBUUqY8oqfpCyd2eXzsYuy1VImsYEoQBoxHFMuaIgyRGV0gZZdmuz0VSLIHBoNIt8/N53ceYGSSiSpwmClEAuoqoqsTBBNLqIGOiyQpinICuEiyX7Wy2ccEHCbXjKMjBMldlsye7+HbRCyGJ5i93RtdurdtfxKJcrrG80+Pzzz4kTsHKZLEuQDZlV4rPebXNns8vF6QkyBRBVcgEEWSLPc8yCzvXNAESZlJw4DtE0hTjx0OWQ6c05qiCRZgm5KnPr/87/fEoa0mg08Nz4FlWU5CiJAOmUBJd/8+98jzxYkpEipy6mKGDoGjc3N/z9f/e/RbvTJs9zZFkmCEJkWWE+G2BqOv/nf/rPGcx+QqlUQJIkkiQDbifCsqxhGSaCFNHupgwnJyRJhOcHSJLA3DlgPL1ka/Muk8kIRTPwvZR/8S//GUm2QJYsRClGt3yCcMFiOWU8OyHPREQ1Z7a6QVJUDk6/ATUhEl1E2SASQqxCl1Zz97ZwuLGGLNh4i4xmo8L1+Rm55ONHC1zfoVbdRrMU5vMFab7Asgr0+32SqMbaeoeLm2NcTyYKiwThgpvx12imjGWraJLJvJeQhnNG/T5HByM6axuIKIwGEyQRBClgvfuQWr1MEARsr31IpbCGrIvUqruYuQZBwMo/4/jskJuTM6L4isvLQ1aLFF216PcueX34Gb2bAUW7TZQueXnwDMScy+s3qHIBCZ0kiFFEH6UYk0QO03GCoCmEQo9wpdCtP2Zzp8Roecbx2Smv3pyyWmaU2xKzlYOTnjFcnTKfz1A0iyQfI1JAwGAxcVDkgOubZxiqxoMHW9zcHJGEKaqaIIsCctyipLe4GVwTJgN6/QlpFlIulri7+4DFOCAOBhjmrRnLtE0ajSaWbfDZ1z9hY7ONrjQpFi103SaPTWJPY3TTJw8VEs8nmPmkUUroSKzvt5n6M3LRwrR1Atch90IGkxc8+/oAby4SxzFvDl/R6tjUWyKzkUQ9s8iQyY06gpQwnbpcTBaUOjbOeMVqIlGtG7w5OMH1lkS5w7NXv2S+GpORo6kWoqRwc9MnCAJcd0Wz9f8H9aIkF5kve4yXR6hSnTzT8D2HYmWHJJvy8tXX7MZ1PvmVj3j14gJFv2YhjknDKpVqBdu2aTXXGQxO8V0HvaLQLnVY6ONbDZMo0BscI2LT7bYJXYiECYKYslr6t6Bcs8jmuw/44z/6JQ8e3qVZX2O1WnJydM329i6yVKBeecjp2Uu2dzscvHlFnAvIYoE41nD8MYv5GF3XEcWA8fSGwJUpl0wuroZo5u0OTeBAFMWUiwXevDoiR6NcajJdLJAVgdm8T6PZxV/lrG0UGc9uiOI59ea92yZYqUkYjmjWdgk8Gb2QcXV1xZMnH3F0+imnV19SrVyytbvDeJKzWDks5hmoLghLLLPILz/9Kaoq8/DxHpEnk0Ua79z5AacXXxPrErZd4PIyoVwu8uLwBVvbRf70R39AuVYlDGOGA49ePiCOZ9SbPVprVV68eIFhVAgdmZpZRVVCjl7fkCYG1cIjXr04o1GvUynWyPMlO+u7RF6B2fQVIk3CZEC91uHybEmhYFGpPeT8YoCmx0i6wOGRR0GtY9gZsSsh2HPc1QqBMm50w/WxT5Y5GHqZOM7odJpM5j7t5vvM52d/3izcw1Q6LPIZRiHi+PiMOw83qZc7yHmFMI1o1n3W1u7y+uAFQeiwufYRptHCd0WujkfYFZdGs47rusyXPT7+6FcZD1KuL+ZE2Qwk6fZz3JyScMH5pUeQGwRBQOiGpPkK26hTLOrMF+cgJMTZmK32t/jqy+cIwoJSsYK/KODaS+r1Bo3qQ549+xfo2X1EOabVMhFG0S1Tdegxnh9gGmXu3dsgiUTMqoVqmEynfcrVAgW1wOx6hL5h059OSdIYUZTQVA1DV7CKCbVaFS8IECSNnARNtphPF5y8cXlwN2bQP2buzEnkEl4QUC8ZXA+vyUshjx6/x/PXn1PRoagVGc0SrOoahdoNrnfCYDXnb338m7jLgHrhAjfL6LlLVM1hFOvkqczm7l1WIbx56TD5cUyYtBAMHbuYIUoCnpOxcmMC38cyQEXHHwd4N5esvBVpmqJbCsgSZslGVFIKdoUwNamsbRL6LkmWUWukyKnI//Q/+GPm0zXWtgyyZEqxYLPz7fep1Te5uTpGklPmiwF5mOC4K6pFgbW1x/z8hzfkaYwimiiSRqXUoVbVWLkTBHQsyyIOb6+Uc0mmu1dkxhiEGDAQJMgSn0rFoFjX8VwdyTDIlJA8+dfWCEEVMesBfh6D3CaME2TFR9MUVo5Lo1wkilW8eEGaGchCTijdIpMc7xRVaSJkEkKeECUhmZCRpil2JWc4nZBkRfSCSBZ6JJkEuUKWB1wOxkzGCZIUk+cBkmKTu1AqFHGDawb9CaokQ6KRZwpRJiIIOaP5CV8dvEGSJGQREk0hiFL0XGSj1WDlDxmNlxQ6HXzHQ1NtFElGlmHhLZgsYtBiVk6CqKYkaUqceyimw969bQ7+aEK50iTKXYTURBcyvNkh6fwOhcIWxwMXLdJQZJEshTiOUTUBP8iZOwFSSUAQc6JMRhFMEtcndiXG1z66VkTUbq0fYiYT+hH777QxdQvXz5AkGVETIJfIyUCY4U4/ZbNaBkEhzmIUVSVcLtjcXOfv/t2/y2QyxQsDQCDLMs4uLyiZFaajOT/52T/nu79Z5f/2T19TatVJUgWRHEUwiQMHUVaRESmXTASlQZaNkWQZVdFRZRtdrVAq1kD0OTj8knr5AT/4je+jqUuu+0eoRo7odtnafMjJ+ZdsdLY5PfsGxBhJVbi5uaHd7qKpNtNZD60A1/0+fjLGcTw21+8gCyZHbw7pNO5QqxT5+qtXmMUyeilgMHtFya7w9fN/RehHtNZK+F54KwEQQwQKWJbEcPISUEkJmM8dJK2MoWQovsMnDz9kkn3N9VBk984D4jnUizpe3GM6cyiXWpiWwqvXlzgLD8244NLZ4d52idn5BffW7vH+r/wN/sUf/xGdvQIHlxP2N99FU22CpUylUeX583PIIza26rc6R01mOoup1lSGkyNEYZv1DZvT0zdYZpEgXJH4Sza2PmaRHOGHOqP5FZ4rsnWng5tN0E0P1SlSKujM+wl7WxtE0QJVtEnzORcXl7RaCqbaYDYbcn45wbZ1kkQn8jN6NzcIgoKQKxy9usS2l5hdg9UkQo18ynqbXBszmQdUm01mqykLb4aqaeiGCrnBYj4mzSIqpQ7NxhpB4uHPV+TYWKbIbDZjd2+dm8EN01mIVQwp0CbIVtRKZUZDD0HRscwUVZHwoyViqGKYMv7KYTqNkPU5G50HVEo27rLH3t17GImCWHzBcLbESE2SKGfzfpvPPv+ScHHER598G930iLwpD++8RZomlE0HvVClYJmcn11xfn6OIIgkkU65aCErwl86UP6lJ5Rr6x2iZImh1ZiMZ3QbjyFYY7E6pD88JpOueP78JVke0mxbBEFAoaTiRwPcoE8uT/nFp39CnIhMpgvidM54MqRcLvP40dtMxj3ErMB0smK1WlEuV4lil0Z1jZveGY7TZ7UIaLZq/IN/8A8YjSYsVj3SBO7s3UWSQsJ4hijBYjnj/OopnW6dOzvf4vSkz3xxTqu+hiTqt1cvaYrr+qxttbELTXTToFS2EJAIQodWp4QTXTFd3iCrIikronhFliooes5y4SIIIuc3P0eUI2RR5ezsjPc/2kVWMoSkQ5aYIC2Yzi/JBIdXL494cO+7pFGLIMy5uH5DmAyQtAmVVkqjWUa3Anw/RNdNqg2VxXyFagRk6ZLhqI+7lNnc2uf84opcOaVUi9jbfI+i2WF74wnjxRVvv/0u3c4Gy9WYqXPGcHHBwekAQSuxvvkYxx+jSSWK6jol+QmJvySOZzx+dJduu4VAwuZmiyybc91/hePMqdYkbKOCbkjYBZm7D7rUKo948ug76Eqb0FMxbIup/5xS1WC+WJALHpVqA2eZsZx5FO0CgSfghh5rW2XCbEKxmhGEM/LEYtCfYhc0Ft4ZVjlhODlmOByShAGTyQBFy9CLLrpu0B8c0mqXubO3jyIVsQybWrV4+7/KcsgMFqtr1tbrfPqLl5TLJco1kWoTwiDFW6m06neQpAqm2sXQRcaza7z8hCBdUas2WcymrK83ePzoO2jiHVbuEEVfUatvompVGhswXnzDl1//mPPrz+k0H+FFI+arIc7SpNFusQiuaa/tMpyNWfo95pM5cRDieC4X1zeocgFZEpDyjL3NDue9zyhUJGRRJs9AEAQyz8CQQ+4/rBAGSyRZIM9zMm69vl/88oS9/XsIcZmyYlOTMx53tnj2sxfIRoUXlxPiqczbnR28SR8hE5n0DykWi8xXJs5EYKO9zk+/+RmfffFj7GqZQqnMhmqTzZcsnRWWXCMPnzC4foysf4TeXKPUBVV1GF/PGB1d882nP+fNiy8YT6/pTxaIZonS+jq1Oxvc+5VHPPnu+2w93KGzuYaY6kyul5wcvODpl3/KL370Q158+RmHT99wcnDG5bDPOIqp73f45Dc+4t/+9/4d/nv//r/Pf/d/8N9BYJ2K9S7/zb/9e/zmd77H9z74Do8f3WNtbY3+lYEk6khyxmISsZwGzCZjXh/9PoNeHwmJ0HcQMgNZlCiVFOZLl+M3HpVCB0UCMSuiyQVExSMlIAwgS12yQETK5b94RW5MtaRTMHTm4wlFo4KYS8RphpDriNqINyeviGMbXVGQJAVBFfC9iEol5uDVM5bzFF1TkNQcWVMQKVBr+9iVGESRLAFyGUWTiCKH3b0OUZajaVWyLAUgSSPEXEGRM5IkYTSZIRor/GBOELpoak6Sznj0eJ3pykfVtdvglYEkSYhiAEKIpq/jBjlBlCIqGVGYEQch5apBnBfw05QsFclSCQSZPFNx3CmT2RWeKyMpKlHsg6ggCzppnPDwwSZnJxccnt0gGzKapiGIOUkaQZ7Q6hRZLBPyTEBRboHnAGGwots10QsKMz9F0VTSOAMyFE0nSxLaNYvDNycsFjGCLCEiEuQRYaogiTqKrnBy5eH7ArKmI+oaceTx7/z9v4cf+XhhgCTJKIpy+74EgXajzX/0H/2ndDZzljOII5UMgTSNSYUVcRKgKLfg9lajgbOckwQqIJBnEMcRsiIi4HF5dU7gWGyvv4dllhj0pxwfXXF+fokzrdNqbnLdO+DBg3s4yxRFUaga2yh5gZ2NdbJAxnNm1EpdJBRKZolcXpJJC07OT1muhsRJwGAQ8Pr1DZqd0xueYem7xIlIoVwiyyVEWWE8mjIdgaL7ZNmSOE7pdJrM3ZfMF1Ma5bvs7dxnNTqmu2UQK1Vm+VdkQUAwG9IfHODmS7z4GD9wSLP49jfG6xGGPt/+3jss3SsKWYzkqVTaDdxCwIXn8zt/699DCDMqkkgYunjOjCCacXHV58H9d6iW9lnOEyZTlzgo8faHXabzCzS9SpgNuR6cYphVkAT02GKttAv5hNnUo1Kp4adTUuOKr776iiy0WExk9jYfIwgDZqsDrs/nRK7AajWjWrZZa21haZ1b+ksxplpbo9VtsXJ91jZqZJmC43rMF0Oa7QzDgGJJYr76mt50SKIOsAs1NF1hPJjiLUYUjZDUWTAbj3DnGQWjjrNy8YIp08k115dHyIrE0ZuXTEZjuutl5s6QIPXx0xFvjk9YBHNk3STyBDx/gO86zAYhYRATIyNbBYIwobVW4tGTt2jXH7P0r/FCh0Ztm/l8ys1ogoTBRvM+Tz54TBaDHojUFJnf+91/g067gJJX+NbH71CtGJQKFpZexlmFvDk4QtVydrfXubu/w+BqSBoJXJ1f/aUD5V96Qikg02quE3gS7baAoIzZ2q2wdMY0qh+BEJKUfE5PLm+vq+prpJFELvTpDc+J2aHSqjJZnLN/9wG9qzGiMmcyWdFuh9QaZQb9KY8evsv5xTGDQe92P9ARkdWYTqfN+emA169fE2zmCMhMZzcEYcLKzXAWCf3ehEwIaNZrjMZ9SpUxiGuEvodpakymRzTrLaIgYrK4xrIN/Njh5voYXVNYXMYg+qQ5vDj4M8qlLTob91k60L/4BlW3MNU6NzdvyNKI3e09kCucXnyFqVpYBZuXz8/QbYE4HbEY65hmmTS/JMlWbHTf5eXrL4h4Taf968zncxBClguHZsdEpk21sIcf9Th+M6JgbrKzJ/Leo9/hF7/8CUnusVzIpHEZPz5ha71MHhsUrQKrlYivXNOu7PDVL88RlJj1rTYnJ2PyrMFo+eJ20Xzew/cyBoMxohTiOhmddQPfhc6aT6f6IX/2Zz8ly1XIWmzvNCjVUpbzOds7d/nqq8+Q1JQgTDk9esrWvkaU3VCvrhFEA+Is5ek3z9HMkPGsShqbWMUaQn6DIC7orrc4PHuGYreYzxKEzEeVbJqdMor6BAERs5CgiCKO47K5WUcUZObLBRdXJ+TClE7zAfPZiCTKsHQDL+sxnq1Q1IxiRef6akbKa+q1DrJksb5+u6xv2SqJoOMmArohoxqQZAJrG1tMeiuuB68JhAJr61uMlz0C3ycKi4g4rHce4fhvePLoQ/rDUwI/4vr6hnLFwjA1klgALSWXfBq1NZLMw4tjrm8mlMvHFEoyw+ERzdo6RWOT6fwVxYJEEoRcXYx56+0HXFwtsMod1MWELHNI0xQEkfOLU8Kqyd/9vd/gh3/4f6Be03Hj2+tvRRV49bLP1eWQStmibG2QJgbffPGc9Xabra1djk9/RO/wFfe6DzGTFgP/S37w0feZLCRm45xP3tkhWnm46RWtuzV6YczKkXiw8ZCWeB+jvE+cKYSzCc2Wixd5CEOb45MT3MC/nTzqNbbv3SFOIxaLgMViwXw2xFvOEEhBMlEUBdM0UZUCllam3llHMQSSSCFNHLLshjySERKdlRsQhktGqynehcLghYEolanWTeq1NRRDRVBFFMNEayh8tPc+hnw7yf+N33wH1cyIkxxVNXGdAIEpw9Eh7TULP10Qu+BNHMTYwzLmnJ+f4zmbFBUBCBCyCFmNef36BTL3sS0FSYoQE/UvzkRRjCGzAJM88RHwyNIcWdMJ0wRF9JGVImkiIqkpaZZRUUrM0intVpn+ZEUsrqPlMsQZuqzg5SHj+ZtbPVrQwTIF4kwgimJUQWK6PKYki7dTQlEhIUOWRdIkQZSmnN28ZLpMEIUGsmggpC7L2ZK9tbsEC4vQURBElTiP0TUNzw2ot4ps3q3wwz/5BiSd1dKnWpYRJCDPMQyNONNw/ZiCoZDEIYmXIssmdknC9RYcHkaoeg3SlCzVkXWJmbMgljt88N5H/OT1zymbO3iBgyCkZGlMydZIEwdymyRLicMYTdaI0wxVUkgzh4PeCq3axTuNsASFVEiI0ghFEjG0hIvLMbJWRJAlojjEUlX8xEURci7nOuOwchuWJY3eyQl/83d/h3fffZuz8xOKtk2aZkynU0RRxDAMnr/8Of+n/+Sf8h/+47/O+fEIRdEQ0FC0BClTyeUATdaYzUN29loUCgGut0TTtFuNnyyTxRarRcTO9jZCrjFf3TCdL1BkG9+L6G5qZPGUw+NTGtUPeefxr/Psy/813e46Yq7Qn7whjVW21t/mZvQS11vRabVZCceImYHviLRbBSaLS568d5/RwMH1FQQtYjrts25UaXfqDCcX6EYN27ZJYofIntBt7rFaRowmryB9RKVwlzAMWa5GJLlAp97l+sLF8QSEiYIhb4B8wMlRj/v3qhyeHWPpHTa220zm1ziOw8bmLsPBgrL9GNs06E2HbK7fZTY84Pn1Cz7VfkIYT5DSAuWqhbtK0XWbB3tvMxheICspg+tzLLPGTf81pvkEw6gxnL7EGY+plR5i2iF+FFDMC/RHY8LFKVleYeis2Nh6G0VTeTP6MyICRC3ifHxE7EkIWgPNhiC/YOovaVbfYrE6QVQ0lguPUnGDJEk4v7mgWGmRCxmCKlKu1xCUKY4LghBwfnFFKohkgsj65h6HBzc4nkOntUutVuPNm1+ShAXkXCAjJM9USBSEVMayNG6uegiZQbNtczM8YjC6wLAFRKmArIhUbJNmpUBnY4uLV2+YjkNa7U129zPc0MVZaEhqQhS7HB25/LXvP+HTz/8U09qhu1njmy9eM/cS1jeLpJ6IrsP5yYJP3rlHTfyUtWSF4T/lUu1SseuEwYDTo0vuv7XHxBcoWArrzTrT+Q1RGDLpTzHVEiQalmX91QdKU9OZ+QsKxRbLRUwULFmuIrZ3N7g8ndO7mVJtxkiCS7O+x2R2RqPRQpRlQsdjuhiCKNHptBnNz1nbWufo5AJEjy9ffIWU25TLJYJwSZ6n+IHDw4eP+fLLr6gUt1mtJkhKyNHxN3jBCCHXUVWVIPI5vx5TMGsUyjmrhcO9O++hKDnPD37C5cUN29u7+MGKQtnipv8G33ex7SZHp6+583Adq6wxGV0Sxg7dzgamWSXOPHrDAakm8OroiE41oWRZeMsFnfUCx4dvGI4s8rSGphisrdfxvRBFTRiOblAVgygd0yxaWOZHLBYThuNj1tf2EBE4PnxDuVrDW+nEeZ/hOGaztcfTrw54/NYD7t6tkOc5hrzFl1+8QFYlpqMlzXaVfv+GTruGnNe4uhxQaaQohsJiYlCyizx4Z8bJ+Zjp1Gdjcw1JMCmIH9KsyByfPKPRquH5U+xinbUdndliiYLH868WhPs2etHh4urWErEKr7Hj95iOeiC8RjcF/OQMvWjTbJucn0wpWR2ajSJzJyHGZT4WkDBJYokgnKJZIoIv091YZziM6LTustGt0zv9hrv7d5hPHcplFRn7djdREfG8AM9N6HQlyETEVY6qydRqD5iNlwi5yaA3JHQtik2YLU7Q4jZvXp9Ra0ImxCThJhPnnGLJ5Ph8gEyXNNVoNTcQUFg6CxTZZHIjsrv1hKXXw7Te4rx3QK1UQJZlbq4uMQr5LfJoZXN6ccR4dAOZAcoUW39ApSTQu+kzVC6xrBKZHGFbKtNZyqNHH7K+tsX15XNKhTK2UWQ8GGOqMg+2n3B9fsn6/RonN0fIUoVauU1pKoHQQxRFREmhuV7mRshYW9PY2miwdMYoikWWZSi6zGI+ZnCuce9RES8IwZQwtgQadh1ihyc7XSJBZGqHqDWbbfMRk7TIH/zsx3z43ccssy6QYBttfNnEkCvYps6ll6BLFsHKIcRHkFWEUEXOQyxNpNspMB4LTKYrev3X5JmFIOmoZky5opHnJvXard1CCBJyQWK1WhHHE1b+DEnMSTOPUqlMt9Wm2foYMVcRRdCLOpksIuUqaq4ydUZ4boC7cnh59CXuyiGJFVarGYamUW9u01pTaFbu0WhZ1GsN6vUOmi6ysWETRymbW++Q5iqpOMZdQRREeKsVJ5dDdnbfwlkprJwpYZAQOAvaax5zp8/NdYPUVNHVAE2x/+JMzKWU+gZc9xaYxSaiIqAJECYZmhphKBZZVLttjgsZuZjhhwmyGrPWKHPwcoGu6yhyjizYJLmD591gmxbuogTo5JlPkqQIio6YyqT5giisgyChKhKrpU+5WsJzZrzz/h10U8ILRwTCAilysXQDd5VSe5DiZV9BVkDUBFRZQMlEEtEgEzz0oshsusJzBcoFC9/PQF8RBzFvvbXH6cWfkeW3Leg48dA1nTBwsfQFtdoehhbiL3wsy0DMbsHhtqriBTO04gMkuYznr4Bb17brubSKKoWizvnFNareRiQkyxIk2YBUIU5c+tOYwaKIblnImYciKgiCiEhGvWYRBSKud4Wh+li6RZIsSXyTZtljrVPgi88mVAsFVtNL7u0+5u/87d/jpneFosg4nouIiCzI+J5LuVzif/SP/hdUKhU2d8r84e//gjy1EMSUNJHIEgvEHMOqMl0MsIsC3jLHNouQ31ps8jzB1Es82L/LfDEhS5eUiiaypCFSZCbNCDyJ7c1NCuUBaeJyfvmS7//ab3F+dsLTl1+zsavjrzRSKcQumQRJn5krISkVZqMxzdoWs6lDGAnc9AKEPCdLBZR4lzwKODt9TanYQDcqVOsmYTwjXBjoRpXJKKJZ28XxhiSZy2LusbGxQa/XY3N9m/7YgzhEk0dEbpuVtMBTU7b2tnCGY9a775Dj4TohnudgWzXmswnOMmVraxvTELm4uGA8PMcP5xSqEkbBxFlq1MtbkM+Io4Tr3htEKWE2O0U3alQrZV6+PGa90+Xy4oyVM2dn5wH1xg3nZydoxj1yIaJUL1IUtxjPr0miEEGZ8eUX/4p72x/QKmmsllOcpYOcltnf6XA1nPLu23f44hdDaqVt/PCS2XSJINXpbjT5/Ksfo2k6a2v7LJ1b+kSWhxT0feJEQGaNUinn+eUL3nr4q6x3HYKZTb1colo0WCwdojCnUGyxde8dVt4laSKyCkakWYLjeMhyga31OoqikYm3D6RhfMn+1nv4yyKyMmZ0DXYKp89f02o2GI2P6GyUuL6+RpVsbCOmoBkkSZlmU+Xw5AtMrUWpbHB8OMALEgplleV8xsHxKfX6Jp2OzeZbDbLVuzTKGsVCleHhgHqjzNGLKfXSezSqdd48fcrddzuocs58UkJXM6p7VdZaG4xGI0Yz768+UGZZRs18l1X4FFlSublZUiyovH41xNAL3L23xXIR0h+cYRhDNN1GEFXu7D3h5FThvHeMSJFH9za4uDjj4vIAZ5nTaJW5f2ebb158egtLVyc47oJf+daHfP7pC7JUJIodls6AKHbwwgF+oDJdDGnUu2RZRnNDxzYS5tMYRZM4Ov9X9G5GbG03iWIBPxtDbqPbBoq2YukkbGy2ccMB45sIScmxKhqL6z43/TFVs0geW7TWV5xd/YzxMOZ7n/zgtr0XhiRpgK6VsPQOVzcXZDksFwLlooEoZQzORty984iCpbBazpGVBbLusJxnLJfXWJbE5eWcaqVNhsPKTdDyHo5/QWdD5+rynP07D5gvBszmN+yuP+Tkske7uc5Xnx+z/6CKrd9DtWZY5Yw8tyiWLPpXT9ncO8AoiPjzNvv76/QHr5j1Ze7ffYQfjrHMOpqakWYeqlzk5Pw5kuLTtDcolky+fPmH7Ozv4E0z+uNzgmhCvZgS+nBxMkJWZTImrBYRs0nCxsYGl1fPiOIamllkPp/S7u5jqU1uei8I4gFrm3cwpHWur64Qkdhq3SVZedzdvwe5RKXaxFlErPxzLLvEYrLCNG22t97l5uYN9UaZYqHMcHKCZRaQ5BxZirl/5z4XlxOGownFosZs0qdQEmjU95ivLpCtBZat4XspK3/M5lqTJC4z7C/QLAfP85DUlCzK6E8CAk8gyy4RRA9FbpEyZjpyudfYYDpdIEguV+cRlXqBMBpSLrcIoxWakVAUTEyzxnI1pVyrcnjwgk5rl5Ld4Otffs16t4Cia5yfnNNZW2c08OhfLzBUmyRbUDS6JLLJyh+hTkeIyAiygO+7TNxrjI23cYIxG9tFPvtiRLNl4XkeSZ5jGAb/+J/8PttbEjICZrVOu9ZGEXK23xJZTVVUJWFeiRhOX3PTHyNmCueHPrPFEYvpCxK1jGKISFGEJJq4sxkICYlk0LFMbiY9EHNE2SKXMwRRRjEVghByWcCs1MjThCROCf2AYHlbsJBFBVEI0UQTw1Kp15sUCrdhXZcNBAW8aIzrCrw5HyKJPoqgICkKkqEBOaasYtt11jdamPoehvYuSRQT+ClpFrFaeiyXDkt3zMXVGV993SeN4tvmrSggiQrlcoVi2aZeb1Jrlag0ihSLVYxinSxv873ND8iT9NZoIokETkouLpjMj9E/LBLnU5yZh7P41zuUs6WDKrtcHrnMRjmGWiKXfArlBu4qpFRMmK88FKmFriREWUYmClhFneVshCzokOWEgYdUMJEVBbKcarXM4FonIYDcJs1umZBpvmC92+TkMsANMjRRwjAl4jhFFBLIA8Y9mSzTkTQgjshECUnViLOUYmkLx+shKhoJGRkgCBJFU+fk8Bklq0qajhHymDgGRZYRSHHCY6aTFfqfh2lZMojCDNdd8dEHe0xGPr6vkecZSZqjEhHGObossdW1+OWnzxHSOoIOWZwhSgZJMkM3iiRJwnwegiQhoJBEIUnmo9sZWdpjsfRIsyZpKpAKAkIukpHSaNTptKv8wR/8GMWokEQuWaaAYpNkc5LMRYsydBb4gYYu1/l7/+7fZrmaE6cJoiximhZpnBC4Afv7+/zP/9E/5PJyRLPdQLc0xqMITbMRhBBRlBGEHFnW8FwfxJwgHSMJXYqWRhwlCGKCrKgYepm5P2PlXGGoDbbXH/LTn3xGo1nhyZPHjEc9Lq/OKZebIAQ8ffqUdqvEo0ePuB4fsZy7eE5AFF7wycff5fziM3q9Aabe4KP3fpuXh1/h+SvskkCUzIg8gTTv4TgyneYaQTRCVue4ro9p32G6mKOICqJoE6UOLw7/FEUViSQwLJssl6hUGgg0MZUp/dUphrJNEg8xjZSm+Q6yHNP3rjEMDTDQFJ2SWaHT2sUL+3z6+ecI4jqTxZKi3sTSIBV1NK3M+Mrhvbc/4bx3yPVVj3LNJIlnwJI0iUiDjESA7a0NRtczNtarbD7eJ4kAyURTNEyphVYaM5nkbG61cP0Jglan3n2ErQl858O3OT35JdPFlMV1TKddJY4TSlWbXv+ColFArxtcnQ1JhYDx/JwoK9LqlIkCmd7NFYWSyMnlOZVKA0V22Fn/gOnyOb0Lid/67d/h5cvXdNrb3Fy9JFjIPLy/T9lMcRwH06zhLa8YLS6YTBKevPOIOB1TrqpMRh52ucbVxQnuKqPRkkmzmJevvsZUthCFDCHLCGOYTi9ZzH2iOODVs9eUWzqD0SHxUkdeL0Gm0GwUcJc+BTOnVimymI0x1NuVjav+hG7rLlEyZHA+4l8tHFTVplitEw6vKJklesM+gpxj6zAYnbGzu06eZ/zkJ3+GbTRpdapk+FwOztA0jVan8VcfKGeLQ9qNh6j5JooVYVkzhKxCEitsdt9mtXKRpCEfffQRp2eH6EaR8WjGfDFmOpty985DwOfTz/8UdxFgFSRsW+Sbr89otkvoWglyid7wABGT1wdPGc2OaLbKRKmDKGgEbkS3s4OzUCgWbYbDPm+99V0urg44PP2KenkbKS6yt7/NaqaTx0UaDZOr/ms8v4cxAFmDUkXi6uYAQ1dwlxqeE6AhoSg1hoMZg+A5G9sS7kwhWNh89FGXi9OARJigKVXG45T5NGWjraNbt1wzx+8zHI7RlTZ3dj7Cc/rkuYyh1ciEJWZJIctizi9f0mk12e6+w3hygWHatJtbtxX9WUS1VCZwlwyHNwioVFsFPv/yJ2iaRqVsoCg+Qhbhh5esAg9RFDm/PKXZ3qC7ZiMkEYe/7PDwznf5ydf/jM31NSgF+Ay46B9wd+8Trq4PadZTRHzu7d4nTeDV619gGA02Nz7ENgucrs5R5Ra72/eI/BhVsdFNgcnyEMPeQNNVNjZaWJbF3bv3+dnP/ozuRh1dr3E9/BwpK3J//z6eL5OFBa6dK1x3RqORETkqs3GOZEZIukoiBKyWPqghli2z8kQEWWI6n1OsqJDLNOtlBDSGfQdFSWk1KyBkyOYNUMfzekRRjKZVUFUTIW3iuksaay7rnW8zcH6JF1yRJSpxYkIU4nhjCmqMqtS5Hg2IkxAlnWLICZdXLymWypRSBXeZgrBk5Q4plXYplaA/GuAuNQJ5SKFQYjqZk2QJmrbG2fkVW1tbDM5dDDmjaChsNLbxI4c0lqlWFTTpHsvZDVEUI0vwrY8/4qvzY67O+jR8nSyNSJIMyyxgaV0GXsRi8YL9R2U++0IjiiIkKScnQpJUPHfFb/3ud5lcF/nf/id/SKPdR5UL+D87w1QNLD1HZ4paFFALAnkmYrUaHI99FBySZYTgJKSpgB8IuNEKVUrI/IxDSacoaSSZS+jP0A0Fz43J8hhFKhClHrKqIog5RbuFoRaRdBXD0snQEAUNQpVYmuCmcwbnfYQsRJdFLL2EICqoek6UqGhGTpqkJLOERPIRhAw5g0zqc6KKVIoVLMtAURTKxRKCJKPaMpuVDqK0iW7IyNLH5IkDmUIQhizcGYvZmOl4xjdnZ0SuQJQFBJGHrmkUSk0qnRYbGw3W1jaoVss0uzaG0WBf+DbVakQmWqR+iLdy/uJM9HMZZ/4N04HP/iOT+eKU1XSOF8t4kz61UpfDgxmTUR+zKCGqKmKuEsYJWS6TiAqKVUMQNcQcNNHCsk38YMpwHKBqNeIkRFZzlu6IYtVEVmLGkwm6voYspfhBgiLIqHrKZH5Os/YWcZyi6yXiTCLLU/xoiSBbnJyMiTOFJFUQZAlJ04kWc8plAUMPuLwao+tVothHlhRcR6fdrlOpBwwHK0AnwyfPc1RFwhVdavUNms0Sy+UxcaJQTEvIUsw4ntFt1+hsNHFOxyR+gGnVCYQpGRJ+GLOzv4MgTxFkizTLIMnRZYFEFJhPHP7m95+g6zGv/18+askiiuZokoGf+CRJiCjlVOtNPG+BooUIioRARJSEFDslSsUaedZgEQr8t/+t72EaMq7rov3/1I1RROB6bG1s8l/+/n/BH/6L/5ruxga6ZjLpa1xdzClVDVIMRDECMUWSRTRVRYoyHr3bYtq/hskCRVaRZJko8UnjhDCMWSxH3H/7Y5zFisdPtnj67Eu8qIWhNgijYw4OZhjlAFUukV57FOwy9/bX+eyXz2/5loGDu7ykfyZQLW8SJWMO37xE0zVkBaazMe+8VWa3+32qpSaRJ7DMnnP4JuL6+hK7VGY4uqJSbeE7ExB8pqsRvi+zXutSqZQ4P77589Z9zPODH1EuNRByHUXWaNU2WHrHXJz2KZQz3GCBVjARMxPVyPEWIcevTtnc3ue3f6PJp198Tquyh1mxsG2BfKTgT1p8+KFG7+qMwdEVxUoDTUnZ27tDGHiMehMKloRkCBQrZR7/4DGKWGWxnHF8+UPKpSabnR0sJSfyq2T5IccnA7bXH7CMXJ6+foaY5JxeSizzhJvhDXZTIDN6XA2g2t7m4PRzfvs7f4dffvOcUqVO09RwXZWNjW2+evYnNGpbOPIN40GPQqHDvd3vIBBwdnIOQogXjLi+bOLHS9zxkm6liVxfMBtMMEsWVlnk7KxPqaFg2x2SfMLV4BiocHJ2jakXcIMF6ztNFpMjIqfJWusJRydviKUrTK1LlOQMjr9Gk3LWN+7x4fY7nJ+9QBUUiGG9WyBMfVbzALHn4a4y7u+/z8h5RRgmfOvbH/L506+4t/4erXWbN4c5aSyj2RppIuKHEzTF4OLmDYpe4e69JomQohklXn/dw9Ca3L/3FkGyIMldut0OF2djXC9EN/7ypZy/dKD0/ZDh9IhWt4OQ2iynOSu3jyBnoPkIUUgarDi/zhGkjMnyG7LYRjdTxhMJPz7nwYN7zNNrJO32SS8MBb79ne/z6sVr1jc7IIaUCk0GwxtubiakecDx6QHtdhspaWKXFNzQp1QtY+oN5LzPYjBgt/kWqQeNikCvd8WXX1+xsb6FTIWX3xxTWxMxMpPh+AJbqxPFIaLgk4Z1CnaNqh7y/PjHdNt3UKotUqGPIEokYcavvv+bKIrEfDlF1AQGgwHlUotWrc14esroWuTunS6j2YBOaxfFFAlzh2U2BhRs2cJZLJkvckzdZmttH0Mt464SHu69z/lln4V7RL2wxmzeR1UzTLsEQkoYjjk6vCAOZbo7JSazkHc/fI/z83MqtQR/kVCp1NhaL7BarTCKbTzvbVLJ409+8p/x6K19JGoUOgtyQcKwiiDB2voD5vMJqjYnDU1G42v29j9BlkWGgwVnp79gf+8+tl4hcSXG82+YeQveevQR4jggClPSzCNPClxdv6Ber7O2tksQTOis61Tie/huxvXVknK5gRNOCAMPdyWhqybYK7xswUZli4OTNwjygqK1Rrl8n35vSakucX31Nbq0gxRLhGHAeldiPg/ZWKvj+wP8lY1cWCPJX1CvFrm4GNNqt5kvQlQ7IB0mt57YG4UT91Nq7YyS2WU8v6RcVCGzKVoqaRgj2i56mlPfrjEYRCh6jC3VODl7iarlFIoNsmxKkq3w0gNkdw1ZNrnofUWW5aw132Pm3tC1t3CdPnJuEfgrovyI09MalmHy6uqbWyRE5BJGVXR7QnOzwdIJ8MMZnx/8KVkg0jSbqLZCyjmiJJGkAcVajailYgolKvUC//z/+gpB0ojCDEHKkQQN1Yj5J//kJ1gFm/sPqoiKimFUUDUbRdFIhIQoivCjmOUkJkkcpJGALEpIkoIm5kiyiWFoGHpCV69BfmtZCUIHIRXIsgKiBIJw29CN0xQEgUywUFDJ85wo8kjDGH8VsZhAmuZIooylmpgFG1M2KdYKKLp6u/uXxbdImzhAzgSETCTOXYI8IPZERPG2ZSwJOlEuMgsDllGCpRuMF97ttFoW0URQVRVTN5AkBVWSkWWZYrFEt73D3TsPyfOcPM/xPI8guIWHO47DbDZjuezx/Mtjvv7052SxgCDIaKpBs9mkVCpRqVisb2xQb9b+4kyst+psbH/C7j0VRZGI428jkOH7PqHnMxiM+OQ7E5685xC4DvPFlGQmM1u9RitVmB0PmV9dIJkRghaTR03idEWqDhDiJ1SVIrEgIMpLqpENyoRy7T6WJpLmCQkioqojChGyLNPdrLByfNJUQRFjclFkGSxRBYmtNR2pkONHLoaSQOITiQmppqDJOWWpyspPCMUQC4ssi/GiiIbocf7SwZ9rRKqHlrdJ5Ig0U/BmDve23mO0/IyZt0AxNwmTENGWSRwJTYLl4pzl3CBSE6JsShqmoAiIeUQwn+OFKn4ioRkJaaaQCBJ5HKEQYeYSl1czMsVEk0Rk2UIgJ4kFiqpClkJ/GGMaCmkqkgsOkmiSRi73mm+z+3Cb44sf8r2PfpdGY4Pr3pSiZSKpGhkJWeRQVJq8fP2Cf/y//Cds33nEYjGgYsN0+kuiWAG5hJBFSDmIukXBlvG9JaYgU7GaeMYY09IRZYEovnWAF0sFwpLAtzZ+D1tX+eUvf8rO7vs0qntUa2UODw/pdLaoVUOmkzlZPqFYXOPsZESlIVKsmMwmczY3t5mNY3b3OsiKjhsaXN6M2Otu0Om0cFfvoCkRn37+GUVrnaOLHxIkKXZRoNO6h+ev8MMbkpuQUqnCzfAF5fIW9UqV5TxmPDxgc3MD3RI4OpxSKDQYL7zbQmY6Z7CYM+i7aGrGZBqxubXLYrEgCceEqcLKuUAVtoiZMhqIWGqDqTMijDMQTFIppL4Gnhcxni1Zu3ePZqPN5cUNk8kN3Y0ytVaXRw8+5uLyGEVUOXl5hlFYsJqNqJQbLOYyUh5x7Z5QNTfZ2/+Y6/4B82CIkBsYUQGzsuT48oA0gcFiQVmosfBzXN8jun6OKOX88Gc/wnNyzHKRBmUyb8VifkPB0JhOHcy6wU7xPTRF57PP/pDN9W12NvZYOALZZMGL50/pdNe5HI5oVC2KWgerErBY9hmdBDx8d4eLm2uSQEQoKFxdHNMoqdhmmSANSNG5V9vk2XKIoGkE3gpTVKiU6sz8FXqxzs69J3z545/R7WggCuSiwGA0o14rkgYzIlfGsOqk0RTTEHn26g+I4iKConJ4cUmS5ShZxGh4ha7rYHuMxwMa9TbXl3Os4hRRNVj4p7w4vmKtdp83z69ACcmcGf1xn0pVxA9crq4UKrUS52dnTCbhX32gLFSLKHKF568uaFQrdNtr+NchshFx0f+MorGBJEMSQRyZvPf4b/PTH39Bp10gX3OpVLYRMw3LaDGLzkkTA4kmlXKTt9/WCP0Cw+k3CFmMKKXUmybOcpd2Q6VcKXAzmGLpHWaDp6TBDZPY5O133+Gzr14TKg6t2gNOjn/J+sYOQbhA1218J+LevTb94ZDR7IC9zl9DkHwUOSfwJHJCNrcrjMZ9dHbQhQ4bu12OjnPef/wbnF++RtMF6o0qx1dv0FUVdyUwHR2jaiLN+g6SviQSAj764Hu8fHVIJhrk6pA48DFUnTxViaKE9rpNsLRQ5BK7O3f45vmXGLbF/Qc7fPH1kPnqmHJpC3+loJRkTKOEkJoYtRH10hqffnbAnTsPGI1m1BsV7t19SE7CN09PiZOQMA6xsiJRMiESbvj44/exbZX5fMxs6pGLSyQCzs7/FFGo3X7unsB4fML2ns7FRY8wcnlwb5ss2eH6qs+De3VmyzGD0RmyLON5Y5rlewwn18ShgG1K+I7MSDjlgw9+k//qv/xj4nRA1X5CJp2jWCkJIgvn8lYxVZ4TJRUUuYOmBayiEzqtJv2+jhBkrLUsStY6Byc/Y3vjHdqNT+j3HE7Hf8jrFw7f//7HHL25IkOjXFYRNBdN3EJVQ+7d2WUyDYiTkK++fMaH776NtzKI85hF+DmRv88yyXD9EEla3sLGLQVFbHLdOyKII0pelTAZ4c8E5rMbquUCilRkOLhBM1PSDDrtXY4OTzAtnbcev8enn35OlA7Z3Giymmcsp/D2kzZZ6uO5EpVCA11XIVO5nnyFqRcQNJ88kZgM5lQqbUY3IxrNDZZxDyHOKVfLSLJAHCfkKZycvsJotRmMFrRa22zvVbi4HFIuV/F9lUzIyAURRZEJvJDVYgmChCQNbr+75SK2bWPZJeqFKqpu3zaOhZw09gkCHzcWcHyf5cojSRLSdEWep1iWhaIo2BUbS9ewLRNdU5AFSOL4NjyFIXM3IAkjJKWIECfoxm3QUzWFKApISXGSkHCVQpaQxAE5MVKuoCs2kppj6MXbEoRooFsFrIJCnEZEsUvg+eReSkaOJMjMRQVFkjBNA7NggvXnvDpBQxBEBF1DMw1yTcBLPJa9JaIo3oZqPyDLMkBE1Wy2t1touoQqK8iKRJ7EiBKkaYzvuyxdh/54wevD16xWq784E/NcIE9vm9LNepN2u011rUy73aZerdFudrh/7yGSLECWkmUZfpoSBTHT8ZBW1WM1X+EFIzzXZTr2ScUulmAyGY5YzF0yRUAxUrxlQrGUsLun8F/9vxcoqY2h6QSBgiKkWEXQxJjT4RBRB1nWyQQFTTTJkoDddp2pc4XrRyimComAquuEqyHvPNylUlBJ/BtESSNLFFJkIMTAomrZxN4VSt5AzJaIuUiMQ8lq8/z5H2PVXZywiKblmLJMGGTIkkkeJ9zZecTTr4+QRBNJEMnEFFGOyROBu3e2EcSL22m7IpLmIeQCaQKVaoG1zQ5XswWSpJAkCapye1Xo+A5vv71DlPoslh6qUiCVEwRJBiFHzAwMW+JHP/0pG9v77N/ZZtAbYJUtciEhyQJIQwQpItck/of/k/8x3c0ishiTxhWKZZPBoE8U57c/tnGGkCuISkaex2SJQqtt44cTdLVBxbpLnl8DIEoKS2eErGyg6irfvHiOaa0xmS2QZIv5JKNcLuNHC0qlKqEvkskJk+mQ0NMQJZuytUV7t8p0NOOLz7/g+7/2d5ktz7i+DFDLHjejr1lMJLqt+yyXA8IoRu/ckCUG333/VwnTGa7jgyygVAwc18e26xjWeyydAZcnL3j7re/QHwSoRpFvXnyOpNis/AhFdRn2AjwvoFSU6a4X6fUGaBYMez5WwSSRBxSKW4jiBqII/dkJkX8L7b88v2Jz0+fmsE+r1cayWpxdXINssrWzz89+9l9Qrmo4TshqVqVeW2M+HyPLIlEy586Dtxgv+9zZ+4D+eEQaX2BrFYpVqJVMLq6uCUMBXVcYDHps7q4jKCbHx6e0Wg3uFO6hyDYFq8lkeo0fXhJECRdn55TtezRtk0SOSecChlblyb0G89GIQbzCtkISX+LXvvu3GAzPeHb4jAd7H/LoUZujNy8YXJ2zt7/PyumjiWUsq0a1ppLkHlaxxp2CzdOfH+N7Ppvdbbz5CLCQ0djrVnlzfEQuFbEtFcdTyI0Cy3BJHi8pG02c/oLtB/u4wYD+DPrzKbXSBqVaF9+9QbVEji/fYHibbO3UCZOM3uUJjU6FV6/mkHsUrAKmoDAYXrHReky7atAfvWI6kVCtOsQ5lWoZTbI5PL6gXrM4OpqzsWdg6gbkKQImfuCiSCqKbNLc3PirD5RFfZPFyiOJXKJEZjiaEcUiulUi9ixmvk+eeVSKDV4867OzrrO9uY+pQmOvgmbITKZD1tfKWGaOqmqkacx175jpZMFscYNV7RMtHhJHMJnMqJSbNFoFpKzGW49snr76Mz785GOyUEVXJQ5efcX+rkUS9+jN5jQaLXrXS6p1lW5zl2e9VwxGpzRbm9Rrv0aaZLjugnp1Eyl3mEV9PMdnPhtgF1M0IyVOF1QqJQaTU86uvkFVVbywTdlsI4sqte0mvtvH8ULSVZ2P33/E8elrjs8OKJUtslygPxTZ3n4fx/EYjs+QJQNd7pKrKavwhF98dU7g+1z+7Ix2p8Kd/S2OT3KqNZvpOL61+lRsXh99zQcfPCTBR5ZjLq6+5Fuf/A7lcplnX52gmwKjyTHVRommfZfT82dsb1cxhS1sbQsx94l8j1pdxVuCIbTp7G4yW55w+PqQte4246HJ6cmQIF1QtlqMpmdYJYMw9jl4fczalsLm+kOq1gd88/qP2N0pcHM9IUkSPv7gLeoNn9fHJ2T8CU+evMNl/1MW/iVZNicMUjbWq5QK24xHfQRZwCoI5JpDqdJm6dxg2zKVcplmO+b8+pSNTht/0ULrFrk8vqZW2eRh97cIKksG/RmzWUSja3A1OgQgDmeEN0VKps90tmR9/R32tt7ly5//iL/2177Fz776IWahhB8sMPUmWQaKZpEmAmFwO90wrCLT6x557qPpNpnQ4+7duyxHCrP5GY2mSpzBcuki5CbVSof+9DP0icyDO99CM2Mm40OiQGR3v8vKnyGLOd3u+zSbRXq9a9y0T7l9q65LRJhPfdY2FUQ9IxE9RrMp3eYGi9kcRSiR5TkgIooZSaZwcbLESW/46KO/xZMnO7x50yOx/hwrlCWIkg4kiFKCkgkoskae5yRpwHI8YDnuEaUJOSKirCPLCppuUipVsO0iqmnSqBq3RhLRxDRNZCljuZwzn7kMTm4QBIE4jsnzHF3X0XQdSVXQNAXLKmCbwm14TjOiOLzd8YxT8kxEyCUUMccsa+iqRhTFZHlOmkUkScQqWDGbzcniDEkQkUURCQVd1dANhUKhiGaoqKqMLN1OVhVRIopCfH+F664YJMnte1NlVFXFMAwsy8S2bWT5Fg+jKAq6rgGQkuP7PtPBlCy+Ve4VCgUKhQKapmGaFmubm9wxTeyKSZ7naOq/bnl7nsdqOmc1X9DvXTMdDzj59JDFbEYSZyRJhqqb1GpNLLPARneD+ppJt9vFLpbZud9GM1QESccPXQxLJFiETMcTOv+Wh+Mscfwp83GIu3BYLc74/DOPIKjixTMSN8D3RCBAFiKMcofjPxlQsNYIohWClCAKIrGYUKjVGM4DsugNabQgJ8UWNPQc4mDG6bFAGKTYRpXAn1Mq20yvrth497vsbdYRhGuSNEAsWcS+SJg5WIbEOw86vDkfQBrf7k5GGqIIWeJhmCIHB0+ZTCJkqY0gyohSShQvEQWLLBtzeXKGkBu3tppYQpRz0kgizUJWwZzZMkYQJLI8IUkEkHM0RWbl9Ti/yFh5KaKsICsyWZKRpRmymLO2USaMijx5v4sk3zrCMwSiJCBeBaiSRqNR5n/2H/yHGIpB0Sqy9CJiYYYo1JDYAvkcBOHWkiPrqLJy+x1IJ6xt7hFFNrIy5+LqBR8KMpKUEcUOYeKxmnocHXxOpaKhqgqzyZLuZpWb6yOCIKFYNplM+zhLkc2tLQ5uDljfNFlOc3JhSrNaQ5ITdu6UOLv+QxRpE8W6JoldLKNOo2qjGn1Ev0EcXxAlt7vcjVqJpy8OCSKfdmuTUrnCZ0//FMexWV/f5PrskLKl0bv6hkKhQOI7bHWbBGHGdOxSLraJIwijOaLQxnNdRCkm9GxMa8xwCLs79xAym263yZdf/xTLvsUuKUoJo+JQKBZR5DppEjJeXqKXBOYTn/HsmlK9iuv1WVt/zAfvfY/58gXHJ28o2RUcV8RxEmRVYTw7oT+cUKnodMqPEG2fUe+KZqfGH/3hC37w69/CLM5ZOjGmVaJYqpMmZRRFRdMjvnr6x5QLRXb3tkjDMiWzz81lwL2du/z+j37Ktx6/D4rHcBhTNktkToCTSwgJOCsPUbZ49HiP45NfsNP6GENZZ/2eQbXWQdNyGrUyV9dniLKIbhdZOSFJqqBaPqJvoUslqp0m4/lrtjYfMLx0sSsGyWSOahtU1TqV6jpuPqNaErl60welTsHW8cIlQizQapbonV4gCQmZ5JMsA4I0olG3eHP+lE7rDqo8R1NNltOQRnWDcktleOPRbtzF0BWOXo+5d/836HYm/PTTP+O9x58Q+FNuRjPu318j8kS6G6DqOUYmYxoWtYoBuUS31abXG+GsfP6yf395sDkGYiqwtfaQMOmR5mN0XUEViwSpiK3b1Oq7hAHs7MB01uPt93b4xc+/oHl/i35/yXIlEHghZ6cz7j/cYeVcscwcoihj5YzZv/Mxw0sbRfb56MNdfN9DkhzOz04oTyTeufMDri6HqPoMTyhQsDoM+y9o1R/w6P4jdjbe4//5n//viJMxX3z2c2q1Bo3mLlEiEkcBcRySJjKreUAYxbSaawxH15imwYO3thley1zfnKIbEqdnEwpVGVUxmEygWq+xXLiYRgNdKlLt6EhCkZOzpxSrBt5Uo7XTZbK4oFTQIXeJ4zFruwaTnsnF9XMkSWBrr8D1WUSlabKmVnGWDhdnIyy9ghCvYekxk+Uz4jii3tK4uh6zubHHaDzgV7/3HtWawqe//BLTNBmPXGQFquU1JtM+krTA1LYpl1r89Cc/4qOPPuLNyTOiyGd/14akThAPWc0lDHmbLDHIOCXyBP72f+Pv85/98/+anXsmSaRQKezx4vkxhlUjcH2U9g15prBchLSbHR4+eouz4ymlagnjxqJot3j+4muevHOfF6++ZHtrH1OvEYRXhIGCSJnN9oeM51csljPIUkyzhBP0sItNvNAgyySOzw7YvVvCtgu8OvgRW/sah4dHpKmGmJYolxV0O8BOVDxfIMqmFPUyy8Bh5fq8fPmS73yvjm5LvDq4plyrcjO5Qk5kQkWmVb6PLksMR33CMECo+ESJxP7jGCExiN0O7bX7ZKFFYvVQ1W2yFBz/ks56kdOz19TrLeqFT1gsLpGIKWY6prbJsHeGrsu31w2ApUvE4hTZhGwa4kVDqvYug8EY0wR3EbMYrXj84B0cL2e1GqIZBmfHX3NHUsnS2x/TSlnFM2Ok9BEvn39Fc01AEUwQXcTMIJcz8iRD0SCJIkRJIMcnSRMEEVQZJGwMMgQpIxNT0tQlCR1GvSGDLCcKMxAyJFFBVU1sq8DO7ibbm+vsNNZJDA3P83FWAUmSk6QCjuOwdDxc12VwuQAhwbQUSiULQcwwLJNSyUBAJM89kiSHRLxVGwoxkJNkEnGiUtXrZGqKKIEoZuS5gIhCnolEYcJ4tiQehSRZgiLdNnNNw8CyLEzTpFY30DQNVVXRVYMkSYjj24X5+c0IZ+GSpimyLGEYtyxE0zQpl4tUiiU0XcQ0zdswlGXouoiqwmI5ZjgKUc7F2+vMYvlfH56aiqporHW32drcR9d1UuH2Oj4IPGazGfPVksVixdX5JUenX/HF1wuiKGK1dImTkGJxjXa7y/pWkWq5Q6HeotteY227im4YSCpkiAhZjrNaMOs5/N7fWrJYzlksZixXLpNpjzyVSRyFOLrEW0whDyCJiSKVPPB4cuc+J1/f2r2qrQ0gIxFS7GKBb//gdxlPevjajxCkAN1QiOMETShQq8uYBZneaIpibpEGEZZWwJ1bNKoNsnDBzfWIUCyh5xlZHmGIFp4/4eFbj/C8Qzw3RVYVoiQmzT1koUoSf4Mf9JHSMggzBFREMSBNUyRJQ9FSzm/OiRITchFJAkkQESSVLMloVDVsWyD0XQRBgixCURRyUUCUPPrjOab+MYWKgm3K5LlAJqgghkiZSLvR5h/+w3/EYu7RbFRwXQnV1Mj6DncfGJydH0BukycGgpChG0U0Jb9lVioVVFWk3o65OpLYWCuQJBFJmiDLEkJaoFquUShmzCYBtfI2aXzD9eUND5+s8eMfvkLT9dsJqZAxHi3ottcJ/AGX/TOePP6YhbNAUSXMgokkmDjOnCzRMI0igRsgCRmBIzEcvqZYLBK6AmniczX/nKl7wvraLr3+iPPzUxRkKrbK+eEha90HKFqEGwywyxBGPqtgSRzK1JtdFDlDkSQiy6I/7kEWoRopSZRiN0s4zoSVM6fbsRhPLwkTH0NQKFYtLi9PqZX2WMxiapUNouQaz19QkMosV1PGExHLLLOYO5z0XtNqbvCD7/w9fvCdEovlK7746qe8OPpjXr+a8O79B1RKZVTFQdV8UkEgT5soKqxvVwiiCLuksZpFRMmAPJeAhDyxMaQW778rQmJRtOB60We920Q35sySiA8+uYc7mxI7M6bhClnfJpg4bD14gidecnzwjDSJ0OR98tjnJz/+lL/2W99jNPYIohg3TAh6fYrlJtPpFEFOWaxcVouc9d37LEcXaKbB0p1QLj/G1GXeXD+lITXwg5w8mZGlEu3SOlKocdLvs71TIXIEJsslAhmeM6NQMhhK54S+jqwZaEYRI53hRwM0XWE0GbO5vc3h8RsASqUNgvCGQslEFnSC0KdUsej1egj2BeVCE8NUWExcREni8mrK9dUBpcIadyp7RNKEq7MBubgg9Azm7RUIGc1m/a8+UPbHI6L8nHZlC11cQ1JdfDfjzfER3XUJzapjGmvUS02i4CWiKKMpZcqFbS4vL1nbqBIEOeWSxc4dm8CPIC1jFpfIYc696n38pY6k9ahV2/T6F+zubXB+6jBZvKLe/R6H530a9QqqoFIvdji/OOT+ve9zejKhWpf5//z+/56d7TsIxAy8M4bj53ieQLnSIUznqHKVNNKx6xL+ZIjjlFktBHIiLFMijWWajS6j0YB6q47jLJkMEmw75+TNU9I8QeAe1WqN128+J88n1O1dClSobFa47L1BlcskyZLz83OqLZv5zMFPhuhSEdcfcvhmTK10H1VPSSOFIBARRXD9MacnV7S7a8SJRpYWMI0uRydP0a2M3/6tv8H11Zh/ef4HmGYVSRGwCuA6LRRJJ42X1Ipdri7PUBWZDz/Z4/r6DFEqIFo3ZEINw2hzdn7K1uY+Z58+o1b5gIf3foX+4Irjl9BqN5GEmDAWCYKM7/7qrzGbesS4eP6MdmOd/bt3ODs/JIvLbO/Bqxdnt9OAMEUzQtzwkrXm+6wWY/J8jKj6jOZXNKpNLnpPsa0GYagiiQGaXkdTd1kuYkwTPHeMrOhM53NysU13e5Nnr19iGCZZPEc1XLyFjjsBRUkZDwdU11QWy5jZssf+9l1iR+XF8zdMlglGdUSnobCYr2MZAr7bJ01SzEKRX/n4E16/PMf1x9TqBfzJffI8ZzI74/zymicPfhtnFZELS4rFMvfbn/D1s59z5842hmHg+1OatTXCyCeKVyRhiCA5xFELVYtQhDXOLk7ZljqsFhqBb9JZ7xK5CuXCBuPFC0ajFWuduzx/fkJnY5f5NGI8POcdySCKAkRJIooDOu1N5tqY+bKH48958s5b1BoHRJlLlmVIioig5IRhiJiLiKJKHOVIkoZARhQGKIoPiJCJ5JmEKCrosoyg5KRpjG0LgEQUJmR5zGI+5LNf9Pj5T36KLIMsKmQIGEYJq1RFswrodoFytYJVKFDQU2y7SJQIeG6C4wbM5z4X5yMQBQQ5JE9zRCRKBRPTULEMBVlW0DWTIBAJQgeEGIGUIPDISZHyjFwQKBgmslZElm+DXeQnZElOGNwq70ZXIVEU3TIxDRVdUSkULGq1Gu1mk+KDIqqqkmW3NposS8iyjDAMiUKPPNWJfAfDMFA0ndUyIU1WgEieZmhqBgg43r/eJxIEgSRL/7wgJWFZJrqmockSsiJimjqyAt31Bo8f30dXVVI04jjEdWa4zpBRf8VkNGYxu+bseZ/RrEcQuiiKRKFQQhQKtLtt2t0KW1trFNslSk2d1vpDBCQyMSBOPBZTn5ubK77/QYnRXshitWDhzLjoTahWuvz461N+9uIKOZdYjEZkuYAyl3BnY7748qecX40gF7DUIlGwxDBkPKWAVop4c3FDmBTAEhDSlCCYo+sQpVPWuk9Qn48RRSDXyOWALAVFsjk+uuB3f+NX+NFPXhBnKZJ4C2t3Xf+WyrC44eJaRlZN/DBA00U0xWQwdrHqObIh0Ru6aGoZxJQszcgiCV3S2Nvd4PLyM3S1gSipZHmKpukMZzfc239Mu/UJgtQklzyyXELXTZI8IMaipOj8X/6P/4j59JRqaw8nuSbNCihxSBrn1Ks6v/jFGM3IEIQUERFFNBAzh8BdImQqojpnNbvDdPGSHXmDJL1dadBknSC4ZHKq3DaipYDh+BxN1rmzc4/Dl2d02tsEyRBZEqlWC1xfX9BsNKiWGqwaMYuVw/5+m/PzE5LIRtd1JHXFZPCGb935N3CCc3xXJE5cbLuIs4oplUoIYszXz55RLNRYzCJkqURtTSYKK9wMLjA0myzJaK13OTxa4K0UclzeHL6ibD2kUa1wdvkVqp6jyx10VUaxBnhzg9/4ze/zwz/5GbXKGpW6Tyb0WS1D7u5vs1q5jPo+j5/scPiNg26ucNwr6o0CZBa6YtPuZOSZjBsOKdVUdF3j4Phn3AwO+L2//m/Tae3yg+/+9/ner/wenz79U77+7E8oW9scHV1zKf6UUquDZeUcHfp0Gjv4bsDKv6JefA9BTPCCEdVy7bY9X5CZDjIUMeb08Ihu9w6eN4Esw85ianaBTBbJxSp2sGLuuNS2GqxEFzlR2NmtMRuqWAUBfdrhnV8p0J/cgBzx9FWfdz/Y4fTkmmJNYOuuwuXFFWJcZzYZIebbrG8WWQxjRFnl2atPeTv6kLtPCnz+1Rt2du+S+nMkbZdFuKCpVxHUnCgzETMXUSxSbbXI4jGaWKG+1mQ+jdAygVXuYChFknCOrElohki5avDIuM0TJ4cv6XbLXI0u0DUbSZBI8intxi5Z3ubuvogXDOl0Orw5PMMUFdrdLWrFNQQxY9h3idMlhlKm0enQG75E13UG/TF/83f+igOlUdZwJxKCOWMxGxBNa6h6jmR5JGn3dr9xEqE0cmrVJrIi8fXXz0jzjDibc34es/J6iGqVWmWfWr1Mr39CFGdIOhi6ynJ1xeNHHzLqZzTqCoqsU7Ar3N3+DUp5hFUoc3/3bV4dPmO0PGARnFGV77O1vcanX/7nvPXODiVtjz/94b9EN1xKlRJx7ONHA4J0gLMKabYqmAWDYlxlOp1iVwwuL2+YLnLatY+47PVQ9Zhe7xpb6/LeO1tMJgP82CQXM8JMRFRb5EqN3d06uVNHEwzccHHbVBcCIv92lB5ES+ptE01ZI82WKJpN5BvcOGO27+g47owkDfG9Jf3hKd31NexiTp5XWHhvWDkpa+t1xsOYBzs1XO+AYrlAq7GObVU5u/yc3f37vHnzEpkihUJOs7rGfLxCMxVK5jZZds7O/joHX0754MOYmniHL794yv27b1NqjDl86dCo7XLdf4FiBMwnCtWqTijFPHvxZ5TsNba276ErFlf9N2hGRuDHnJ+/Ymu3QX9wQaWuMJm+Yn/7WyxXA1zvlDyPGfVkTEvl/t1d5nOROF/ieB6ligFZndnCo1XboFRx8bMLMimgUm8wGDqcXH6FbduIsogsllj4MYYVY1VSJqOAYqWMXQ7IojL1loxlvYfnrLBNFzkrU2mFnFyeM5vrqEKDZmebQXJNHKWcH48gmGIb6yyXS7xVgqoIjMZzLFOh2b5DHIcUCmv0RjN29lrkscj9+3cZjyc0S99nJP0xviffhr4oRUhtapV12s0NRtNDZs41a2sbzOYjVFVGt0pkmYSfjolTBcPe4sk7jzm/OsSqVJhNHWRrTKkZ4Z1VyclJ0gxRELi4uODMOEXXDXTZxE2es76j8PLrGkZhSZoKiIKAKOjkaYrnOei6ikCOLIOqQZRECEiIogzcgrCTOEYQbq07aZZCLiGJMoosIuRgGbce4ywNkDSFMIjIUx93ds18GEOWcZHf7hqKdgFdN7ELRQrlEo1mm06twPZaB1EU8RJIsxjfcUlicPyMydzB910ECSRdQZIETF3HMmyqtTK1qoWpSYRhSOwmeKFHmNyqDe1aBVVVSZKIMA5IEwlBEBBFEUmSbnEwQcDcnTNzZqg3t9fdhn57na8oGqIoo8gmhq6TZj6SJLFauUSTGWEYs1q6zOdzsiQhEiV0XUcQ8r84ExVVplwsUCpb1MplROnPzTOKShRGeIGDphuEYch0coWh6SCkRHGMrhtkQpH2Vpe9x++jqLdoIF2TbpWhMx/P6zPseVydTTk5P+HLb37EfBADIrKgU2uWsCybQrFIpVygWlpDKu/QaQS0UgE3cNkNV6yWAT/5+ZBq6yG/+bv7hKFPTkqaiOThOqORj+tkaKR4wYDYB98VmM89UrnExShk4YSYloqgChhykcQNsWwXPy3Sv9DBj0DxyMnJlIQ09bhz7x7PX7wmTjNQkj/H7pjEzAnChCiN8QIFRWqTCCFkImmaEycuraaJ67vk2e1DRJInt5N/yWQ+HlGwdKTWNggJCAkSAkkYsbN5h4/f+YDxJCLmlIJZQzd9gshDznQqzTL/8f/mf8V3vqcRiGWev1hglWRyETJkZDWARGFwnWGWJPI0pmAWEKWYO7u3TNUf/8kB2xv3+cXP/4TdrftoQoKAgCyL5FlGqdggq0ZMpwNEsUql1WA+veCtt1r0ByoLZ0ouL4ljjXqteStZ0FJOz95QLNcQJI/Ts9fMxgLN9Qmnp9dUinusde7z7PW/oFnbI45E4iRCxMCwPF6+fM3+3j0cd46Yy8RBThiOCdI5gWuysbPOcukgCTM+/8Xx/5e2/4q1bl/TO6HfyHHmvHL68t7fzmefHKpOlSvYVbbB7XbTxlarW4AwSKgbcYElfEMTBI1o0RJcNNAY2g2Wu6tc2BVcdWrvE/Y5O385rG/lteaaOc+RExdz97GEQBhRzOs5lsZYc+k/n/W+z/N72Nw+QFFTnj495539v8t//V//+zw/+8cYekRMjx9/8IJWawOzYBBHAS+eXVGtlgijBcSbLFyfeq1AFGnkbY8wchlfaxjGjCBwSQINP1gwHYZ869vvMzpesFwENNZaOH4H0MkXSti5hP/T/+1/yp0b9ykV6+ytvYMqrnP37nsMpj1krUilVKQz6RLEHr4vIIghmi5TqdQ4Pv6IWmUHP5iR8CVf/+ZrOLOYo+PPqORvUqvVCMIliqijqEvGjsNknLCWz5EJML4MeP1mjcdHlxBMuRxeE0YilVaB4XxJKEjk83V6/UPc2RzDKpKx2mYs5zFpaKBQRVMk1tYlFuMuqnCf7c2YmWfwjb/1dX70B39EqfQaN7cjvGnMddshTL7k4O5rTLU+8dxADRZ0ZguyQMBVXdZrLabDAcupTRZ7bK7dIhBGiKlBist15ww/dZhMj8iXLQQHkmSBnNbZ2drkundB6BkEXkYn6vDmG+8RRQHD8YjL9lM0pU6xWII0pNe9wHA0NCMljU0MTeW6/5CcXiUKIVdQ/lVl4v8X2KD5CM8ROXnioWghiXCIIjeQ0yLLmc9cX9LvnzGZXRMEEfV6mZQYlBhNs6gXd+l1Kly1P8c0cjx7MuK88xNKFZMbO9/FVqscPmujyo/pXUvIqs94opMlORQ9YU3uIxQWEJhkocpp90usUsKTVw9Za97m/u3fYTJ8xsnoz7lxt4ou3ePZ4YdUaiajvsKv/tpf5/HTDwjDkOsrj0ZjHUWqMxp3MY0ifrBkNDkljRUMrcbtmybu3MLSCzjqkq36DRACklQg9E6w1AlSUiRKF+zeuMXSU/n04R+SLyo0G68hyEdkGfgOxOkQz5tgWgKVioUs1Olfjdlcr3GxvCCT57z93j0Uscls4qMaCZJQZ9wb0tqGSk3lyfNHvPu1r/Hs2TOuOg+xjToKTfqdMaWiQa1Wh9AkihZUCmu02232byjUynkuD9tUKgblfJHAtfitH/5dnh8+4emjK2SpwnS+QFKXNMtvcu3OmAymAOwfbBCHCocnH3L/jXssnD5Xlzbd3iXbux6PHw9x/B7b9jtErslkOENRSqTxGMusgunjuQnX5xlO0sZ1A/Z2qmhaxrDvECdLFGlBp3eCUU6oVdb59JNXNNYVdN3G1PM40xlhPKHZqjCZdkgiEak4p7e4xjArqDL0rztMZ6949+13uLy8RBAsNtdbFPIyaSwhJCJZnLC3s8PLlz1u7t3C8YZ0O8ccHBxw1b6gWrrBeOyTJEuErITjDdDNjJwlMRicMOkn7GzfYG+7xrOjf46i6CiSytbmAd3eKZaps3RGHB49QJIkNrdalHKbnF8cU6vnsOQiw841zUaVQBnRu47IGzVU7QkHO3ucnV/i+mU0rcFkeUUUxUiyRJLEjGaXyPkcKZAlJicvB9j5JUkSI4klMiIy/BWeR4Lbd7fpdq7oXU3JUoWMGEVfrXMVNUNRQJZVJF3+paAki0iShCQJSLMMWRRIUp84ShFFkcBJV2JUFhFk0HR1Jd6EbBVwiSGLPSaDCb22z8lLBd8LIJVQVR3LLmDaxdWEs5jHLpnU6zaaVkdRbYQkJk5CFnOPxdznerykfx1h2yalQpmikSdfrZGJGb7voigKuZyNlTORVQGBhPF4zHQ6JUkSVMNYtfJIKz9lHAWk6WqVPptPmU6nuM4qUOR4LqTSLyv48vk8xaJNvVHj4KCxmjSJMp7n4Tj/0k+URl9NOx0PD5DimIWwpGDnVoJTkYhCj5SEnG2SZgvEWEIWMrIoIE4z/NhnOesiiSZx4iGmBooaI8gKqmSyvVXhzTfuo2gyqQCx5+EuXFxvynA6YDIJuL6a8PTwCcPeB2SJSBQv8UIZSVYpFC3ypRy1ahNdl8kskWqxAUKMbolYhknm62yVIpr7EY6zZDocE7ghC3eOH+QYDgRkBWK3z3ImoCgO85FD2ox50v4TfGnVhBRlMYagY2kqoZKCuMR1lxhGDkEBBYM0TUgTjXJD4sbBAaOxT3vkYBhFVDVGSGVsS6Jc1omSJZlkIAkpYZKSphmx4KNqAmG0ZDJzESQbVU1JQ7ByOUpli5OTPkEaolsO89GEKFXIl22qefif/IP/JRU75Z2vfZff+6PnQJUskBCFjDATkCWDs/MHTGZLRLUFqYGVyxHFPlE4Q5EUbuw3MHSZG7dK9DvXBMYBsrryF8exiK7b7B1U8Z+K5OwqTnBKpbSJNy1jF2JUs0K7M2BtbQNFUVHlAmutLWaTEZNhxvpOwnQcI2lz3KXO3u4uWVSh1Vzj8fMJS2fM0u2TBXvcvlfEczPm44z5fM729j7dbhdVkUizOTubN3n06BBDLjH2fBaSQ3W9hJ43yVKFRPGp7F/zH/zH/zbTeYf1ZpFcocDBjT1sW8fzPCoNHd3UGVxcYRd0Xrx6xI2dbzMcH2IYBstFgCiK3Lp5m/OzGEXRmEwmyKnM0p/y2Zc/oVZrYhJBlmAoVU4uLigWIYlL1Mo3+OLRp1QrDX760Z/x8MVP2dvc5/vf+zu0NhMCv0utVmKxSKlVmxSLa2TCkvOLI1SljKxKqICdU3j+bMh8HLC+8TZJlNLptNFUk936AReX18Spw/vf/TrXR684Ou2yUalw2b4gEQ3C0OPG7Ru44ZSF5+ItbBRZJEsktlv3mS86yEYObxlhajpJFCAIeXxnRrFQYOEkiKLC8ckD8oUqUZJQK3yX7f3b+G5IFtq88957FNaO0ASBfCHj8cuYG2slktmc7a08iprR73Q5POxQzKlsNW9SKFo8ffiK3YMai3iEImqoQhkjV0LSY8y8xqifUjRhsTxD01pUzG1cuqjItFo1Hjz4glKlSLna5F7jWwz6I7pXQ37wg/cI3Cc4QQ9JKtBolqmUm2xsVZiNV1aoRiv/Fy8oJWlCmoxZa95DSCUuu8/Q8jq2muL7QzqDxyzmKZomUSxW6fTOmM1jGq0ck+kMRdDJ5WuUgxbH578gS1X2Nr9Lf3DJdeeMxL9iZz+H4ywx7JQffP+v8POfnNCdfMGNGzd4eNkn64vUN56RSGsEc4tq0aK2XqfT6aGbfXJ2lXL+Ju3eQ/rLAaqqstZ4l71dhZ///GOsPMSBTbFewVnGzKchfjymWi9iKE1m8z7rG3vEoYEqw8C7RFQsVLGIl55BZuC5Pm7QJWHJ4UuPSnmNpy/bPHn2EeW6gaQJBNk5upVDEQWePXvG7oGGKNYQhZhMWFDM36Bz1Wc0f8bGxl06vZju9ZIsuUZR8iy8IZbRoFwXkIQiQTggp23Q7XYplCR6XZ17b7/H5emCIFrS6U5R1CV5SwFBYTwcYJohF6cdWsV7NCu3sHM+3tQmCF06Xp8wGuB7CVs7BrNpxtHzgO1fU3j58jnf+NabeN4CdzlGVRRUpcCzp6+QhTKfff4L9vZ2mU9EwnhGoaDQ7w6wzDy94UvIDO7cucWXD3+ObVuUCmvM53P6izH1Zg4/nKxCWzeKSEKIF43Qcx45aZf+1Rk7W1Uurs7YulHnovuK/Z0Gzx+eoAwUSi0RvZjx7PCKUqWGnIlo0oJbe68zHF4xGJ+SK7XIUglLXQNNYh44NNfWmHVc5t45zfUiSeZi5iQqYgGR/KoXWwgp1kOuziPM/IDAjyjkbuDMp2h6jKHbnJ9esrW9T5oK2FaRIJwyGncgNRkMrwkCh7zdwPM8zk6vOE8d8rl1hFREthPq+TKt+l0Gwys2dxWuRx1OL/sE2U9xpiBJAh/99BPek5okaUoUZEiCQPt6SVJqgOggizp5a5OvfaPFxx88XAmbNEWQLARRIc0G/Lv/g38Lz71iPoqYDX2yLOH4YsrJ2RXtqyGBv5qSTec+cRwjKiKWViIVYxRFRJI1sixBzGIyWSKJRYQsQBIFsiwhThIkSSHLBMJVloVMEFaIHymPahRJoxjbEhH56ppgzHI2ZTrOSC4F4giQZDRZRdd17EqLaq1AzrLZbja5fesGjWqOLEkRUpHeYjXdHoxndDsjZtMlk/nKFynJMpquUC4XKZdLyIqMYSoYqkqcRcznMwikrwJMCWkqUW80kWUJSRYQhNV0CUTmM4fZbEGn1+fs7IIw9FEkEVVZCdRyrf7LM3FtbY31tTrVahlT11aCInFZzh3iOGUx84iShBQYD2cr1qRqI8sisi6gSHlSBVRDQUhBQsJNJ0ShTuK7KMhkicdyPCbLEkAmkVb3qygSW5uvs7sv8s77IlIq4DgjBsMxs8mS5XLJYNBj1JkxHV5x/vIZcQhBJCKlIXbBRtQM7IJKs7VLo1WkkKtTLpfZ2dhEES1k1WO5iDDFmL/zt7+J5y8ZLPssFyHT0ZBascygP2AyOiNzBbzEx5+bONMli+WQOFynUt7AdYbodRs5lhEVCKIC1fqc6941k6mNYZqIYkzkxyiiTJx4TCZTgniOLO2QCSAJK39kkkWohkiaxQxHE4JIJc181hvbqIZB5GWE2QTRiHDnZcJ0SW1zg8H4gv/sP/knjMcZ3/8rX+cP/8mfsujZiDmIRRExDRGiCN0U2N45wFm8oLKrkwYCohyhKTKdK4fUn3Nw6waaHhInRZpNkatnh7wjZKQJyFqG7wf4w4xCUafdfY6lVVjf2WM6buMuBbyFzM29b3J49Ao/uGR9fYtPPn5Mq76GrSsETg9ZG5BRRFVsAk8mivo8fnJBGAtYZg5Fimjt5VC1hFFfYG/vgIXTpteFt+7/Ja6uDple9MgCi/3dOv58zFrVptOT2b9xl+Ozp3hujBRbtM9PUJU81YKFJOSYTzwQglUHuiJhGVU0XcAwZZZLgUJFpz9uEwtjLPkmGxst2t3HfPrpp6iKRJj02NnZ4cmDAa3GDqo5Q9dMZotr2s9PuXvvBpsbB0hKSJT0Ob9y2d29RybMWCxNfvjDv0s+5/Phx7+PRIlyziBXiagr2xiWxlXnCdedPsVClbff/g4vXn5EqbgF6YyrzgtKxSqn53PyRYVICEgiiXbvDMcPkAyNs2en9McdPE8k03UWiUm+opIsFCIpAsHC0EbMRu2v7DU5csYalfIO7cFz5tOIkrlG6HrY9RRQEUSdemOXkTyiYBeYRg66bPHP//w/xHUVtps72OU8H/z0j2iV1okMi5GTsZMHP+4SF3WSxZJUFhHFEo1NGd1QSIMenaGGVl3guCr1+m0c7xlxOsU011F1BVld4C1H5PIqfmBhFgXcmUvJqBEIEaWCShAVmHpjkmHKxlaFKFlSrzUZ9JZEyRIhrVGpV9DUgH5/yK0bB6ShQ65l8+zFl//KglLIsuz/87uA/97//CAzpCamXGAyGnP37uucnp2RSBFhnCIJGZoZIAQbpFmfRHSIwzxIMmE6QvBzSFKGpHqrpK0iIIsFwmiJaRlUigr9/pzJNEY3LGRZIW9sE/hDFCXDUgqkQsLRYZ+vfbvKdGCQSR5B4OL5DrVmgVp+n2C6WsFN4xOcMGKr+QMOD39Ob/qc9999h9Fgjii6zEYqVkFkc20XZ5FSqVSYLeZcX1+xf7tK5zJF00VG02MMrYqcady6fYNHj5+SpjH5Yo7BsIsoaMQR6Fa4qhoMR2hGjlK5wvlpHzsvslgs2N9fw52V8IIjrq+v2d3PMRl6KGIF0pg3D+4zHCeImsgydMlkmE09VEEm5pSCvYmds/C8hFpln3K5SJJERFHGg8cfcHP/Na6un7G5eYssqtMfnpMkEeWqSuD7ZL7P3u4BDx8fUipuotsqhpmn3X3Kxx9/zFtvfZth1yUIu1QqO8wWM4J0yJtvvM/56RGVhs185nJ6fMytuzcgU/D9LovZEkUoohk6s+UZs0lEs9kkzSJOjgfk8jqiMse0S2TijCSRCAKL9fUcnjdAiFcQ7+3Ne/jhiDSTWPhz9EKC63vIoobvj6kWW7RHJ0hCiBKvo+Yzrs7GGIaFnAMtiRGEdarVJtPeKZZRZeyOKZdr5NV1QneMncvj+h5xlLL0Z3hLhZ3tFlEYM58vsXMKS2fC1to7nF0+wbRkrttd6vUmg8E5N27cotPpoKkZ41FCo1nGD1xkRSGfqzEcjhmPFgwHM/xggiDGKGKL7c2bqwCNfs1sJJIlJq+9WUSIqoThCEUWCJKYznWfdv8ZxVc5/rtfymSkSJLI/+HbOfqbEvV6E28hYxQWlEtN/v1/8Htoygai7JHFCqng484X/LW/+Q7/5l+7A15KdfsNfHeAO5/R6y9ZOKCXUkYTjcjd5tHjT7k4vabT6+M4A0gzfEdl6XoguciSST5nIMoKUTxHUSWSSEXCQhB9BClc1VCKCWm8AuBm+CSpAIKFKAeQRgipgaRGBEmGJGrIcgixQpoZxMKIOJQIQo8sFgm9kGIhR7nUQFdLrG1s0qg1qdSKJGLKIghAlEGQCJOYIPBInAWz2YzlwieMwHMjTEtnbb1GvVHkYGuTQjGHKIIgpgSeTxRFBEHwFQpIJJfLUSyUMHUNw9BQVAl36eC6Pl604lbOJqNfnomuu/ILhmGMiIxt5ckVZAzNpFAoYNt5REVGkCVkWQZS4iAm+ioB7wcuZCKapiFJ6io0pP2XAlf+6hoRkvSX6/wYYeWblWREJCI/IooC/MjHCzyEJCZKExAFFF1D10wkRV49tyDgOD6u47OYzZlOp4yHQ6aTOXEUrQYHsYhhmZTLJexSjlw+j5XPoZsGqq5h54sIQoYgCCRRhO86kGaEvruaAEcBjh+QJAnvf2uXP/vRB/iLEkkCYRgiqALd8wmvva3S2L3isx+3KJcPSMWIKHUwTZ1o4bN35wm+ozPrv0EqR6sWnCwjCRMid8IP/7LO0UsXZ75PddPG91VUVUfQAtIIhFiikDfI54v8+Yc/4hef/Bm7m/eYj0P+5l+/xeDJn/AnD3PI+RxkGiEuQghy5vH9v5Tjf/2/es7Gdh5VL2DmbYQwJs5Sll5Ky/D53f9ajS+e/lO2Dr7N7csK7//vP159VlnGf/HXqtS+9zv4/hG/+PkJv/nbf4N29wOUUCBLTCy7hZM4dHpXtNZLq5DMMubkvE2zZJMzNmlsiFxfX1Ep3+LF2R9TqTZZTCvouo8YiCydCWs7W3hRQjOp0w/6mLZCZ3ROSypQre3R9Qcs3EuCyMQJhjRyW5ilCD+cMZ7Z7LYOWPT6KBUJQamTjducT0e01upMJiMa9Q38qM+wF/NXf/tv8vDhI/b393jw6DmPDv8ZGzsN/HmZSqGJILocvTjn7Ttvs7Vd4/MvH1FvyWhKmRSRckmj1xvQHQ5Q5epXNAUJUVCI0gE5q0mSZMwnKaIakC+pXF0tQZrju1Ma1V1EDJwgRFIddLVK4DoYWhXH7eI5Ga31AqKg8fzw05WXWpcxTA3Pcwk8BdsqsrbRIg4TsixjPPQxDIPp/JpCvkqjWSbyM/qdC1pbB1z3zigaOUy9Qr1l0r88QRJtTrsjNtbqXI361BQFQW4wT8dky5BKI8dF94iyVUKxVXRRxiciXha4eecAPWuyf9DEWbR5+uhTTtv/dxyviG6tMZr7kNq8dncXfzZFEkWuri7I5XIkcYhl6fiZjz+RqTcqLFyHVnEd0zZI7AQ7UHjZfoWzmFPKr6NqAnE2pz+YswxkNndLhGORXEvGWzh4szmSnkfWNYrFEi9ePGVna5fFdA6xSbFkYeY0uoMu/9a/8R/9K9HN/9UnlMEasaCjNSUkL+Bq8JwoLiFmEnu7OSbTHmGgI2kTbL3KcLTyOfhhgqy4ZEmNar3K0+ef0WiWWNst8uXnz6mU1xhPQ2IvQJBEsnSOIKZc9y/x81PiKCVNRbbXU3a27qFqNp4b4gQjECKyVEBAJY1sRpNrUh80Q0dOtgjCJ/zi0z9ko7WJvtzg4188ZHevQhzYrG3UOb88pH35OTs7u4TCBc54jckoJPQlFEVgNu+hyqWVcAtFFksfWRWIY2EVVokCDMvDdeaIqoyu59GMAhkqqmSjauestfaYWwZ+nKDXX2GkBU6vu5ycRojoaOoE1XSoWL/D5asrfPmI2vomneGUxppGpbjPLz46ptlUcFwX09KZTC5ZzJYk6RLPTZFiC8+ZoqtV+p055bJNkFwwmlwRhJusrbcIQvB8kTff/Bqvjp8x7WZ869tbPHg0ptJIGE86eNGM7a0tnr94zmR5hEAN3/s5uqJx8bJHqSqTU00efXyGWQjIhATDMGjsJvQul+iGTKVW5eqyw3g2ZhlC7zKiWssTpjOE1CYSB6w1ayydiCD0mAwHbNW/TiQumC67hJFIRsBo4mDrLUrVIjnLZzpuk03zlOoFFo6A0/bYrbxPqvV48bzN1u4ekurh+ufs779L5+oVRmZT1DRUfKZeSLUZYuhNzo7HNFoK08ync/2M7bXbBMaMp8+OqJX3VwdUeEya1rAtgzROyJklupc+jcYO41mXjBmaso1uZMyXPfpdF1WXmS+vcDyf+XxOJsjcuCFyePolUZSRs4vI2oI0gKdPFti5c3Y2b5Ev6Pyn//ADFD1he/s+KOekaYIoSWRpRhx7aJqJhEalXGbuLnG9Cbs7a5ydeGhSRsoSSdOIQ43QETjtnqMLGp3Hn5Ir1IndmPlSoN5o0dwq4j2ZUK7dplJu0LnZYTjpMx51+OyjJ7z2fYvbr61x2b5meJ3xo3/xJblCDklSiHwdQXBAiVguA0RRQlUzZC1DVlbMVVHQ0TSZMPGREIlTEcSALDZQScmSgCwVydIEMV1gyTqhImBrRdIsQK5CGgsslkMWQp+Ts6ekLEEQEWSFIBbRDRvTtMkXTGrVEu/ceZtvvHmPtbUWhmEShjHz+ZzRaIDjrNLso16fJImwbANVNzFNm1qlyeaajWFHpF9NWyVJQlEkRFFEVnWKVRlDWYWBBEH65ZkYxzFxHJNlCa7rMp1OmToBvutx0e7gOEdEUYSIgCLJyKKIXSyt1uq5IuX8GqomkgkJUewRhjHufPUPcRhHhOFKmAlfTVGTJEHRVERBxjAMNE0nS1LiJCJOI9IshgTiOMP1XfzeBD8MybIMSVqJWkkV0DQNw9bJF9Y5uLGLoqiIGSRJQpDGOI7Dcr7AmTtcnJ/izB08ZyXAJWUlgO1CnlKlSLFcZm1jnfp6a+V5NnXy+SKyIBOGc/4r/9otoiAj9H3m0yFu4DMbubRaOcp1gUcf/THu4hzHC0kzmWvfJWdq/JXXvs6/+JNPCLIEQVQRZGkV1hISJFXgxWmA6xbRdBvHC5ElESFLkNI8paJB3jQ4PzvmP/0//1+IYo/7d99EEmwuO6f46oi/9G/8Kr//5S9QUhExmZDKGr475fVbLeIgI46XkFSxDYMUgTQVUFQFfb7Eriekco+d7a9xfPoJNfdNQCSKEyzDQBc1ltMLkCI211vsbbcYDXPMF2eEQQ29ssSZ9Njdb7GYDxl0B2xs7VPMqdQ31gjdjNPzEaXc/uqZwgb+RKFkaiiyTiwl1G9u8PCTj7nZvMXenRF7uSFrhSaV2jqyVObk1c9xvhA4nYdYVo71yi5t55p6bLHwAoRAx51eoRUMPGdI7DuEmUTeht7FnM2tLQhFhFjBkup89tnnSIJNv9vHXbi8c+dvMXf76JaApEx59vCIOwffRLFSjk7GpEJKrXSHONTZ2BX50z/9E6LY487rb3LdHhBnQ2bLlCzKY1lFsizh+eHH5IzbiLLPaOxRKq1TKDQ4P78kjAN67SG37+3g+SkXpycIInj+Idvr98jSCFGOePHycyztBlY+QpZ0VE1kKc9IrRQBlckgxs4p+FGXVPDpjxc0mnkMLWM0nuAtYtZv51gOJ9QLDSI8zq+P6PZDEAtEdCgWG4TzBVk0Y7l0MXJd8sUNLkZDlld9ri8HFO9VGbddKpVtJHmCoARcn16C3uGDTy45vf4EMdbQRJdqTuf1u7tUlyGT2TWXF4eIYYF7d2/hLQekiYCqlbFsg3n3jGpZY7LssphmTOYB1SYExxm3b36L1+69ycMHz/ASD88XUPSUxVzhV354hyePX5HEMtFA5mrQZS5cQS+HrRZJNkZkbszzl+dUSwV0WeLl4Tm/+sPvEMXL/1eS8P83QVm0bTIBktCkWLzL+dVj8lZKLlemdz1hEThUyza9zhDLiEkTlWb1DuP5iBiBi/YVhibztbe+y4PHX9JuPyFv1Bh2FjTWbMY9geGoTWujiiZWKOVjMiFhY3OfG/tv8ulHn9Dr/5RisYg713nt/l2Ojp9imAq1fIX19XV+9McPaLQk2oNTZM2k0bxBGr3EDY747ne+x89+9hN0vYwfzBiMxjRqd/AL15TKNv2uQCb2sAoeR8cvURSFNHO5d+frDLsBas3j2fMXZOJ41eogBehmDsMwuDyfsr2xS6c7xso7LN0Larm3MJQO09GYMIy5bD9GM1KkdItvf+sH/OE//QBRO8RfmuzvV/jkwZ/SPle5/94+jx5/RGtzA0sv0Ls+Rld1DMvEd1I2mzfpt5dcD34GqcHuzg0G/YA4MDl8cUqrfoN+/xWlap5aaRfPjQHQrTq16i0uL8+IMpc4DQhCh3q1geFPcB2PUkXl5OyUyeKCjZ0tlsslw+EpUqajGQleViROPUqNAsO+gygIbG3UOXk2R7YniJlJu3vJbJly0R6RKxQIIpiHQzzPJJdPyVdkvvjikBsHO3heTOjpOFGb7jk01ktcnF+TySLLpYdhwMvjMzY2ygyvF7z/xg84Pj+j133GD7/5q/QGU5ZpzHrdpCjnyVlVdMsn8lIMo8RocM3hy+e8/95vEfdGdLp9bNulWq9zctjB8TpomsFJdsZ0MsSyLDrXQ06PZtx9fZ9+d45hZXR6j3jt1jchzTObuBhGHb1ZoN8b8eDJp6xtaXzvG79Dr39Bq7VBGI7R9BKpMKF9scDK2YznzxlOrtBVA0XqcHqWUmtYPHs0Z39/n8ZGhYurl8zmm+xUSsjyjDSFTADPHzMYLhDQ6XS+ZG/3NhtrB9TrEw5fnGJYOZJURyBFtxKePzqh1Coz6HgUdZn3v/89/NmY9Y0tsmzBy5cLyvk3uWxfMppcMZ0smc5cnFmMKBncuFvmjbd3UYw5/+Af/Lf47R/8fY6PT9F0kCVAkPFcj/tv7pImGuenfUbDPoQFCuWMTJ7geQqCDKkgImY2qZKQ4SEjkKYysZCBKJNEMWQJaZohiS5JlJBEIMnCKuUtaJQrBgJlFFXCDx0EWYJMJPSWLAcO46srPv/pzzCNPIpsYtklmo1Ndra22dzaoNlc5+6tJuVyEVVfwbHnyxnT+Zz5fMp15xLHcUjijDAMV17VLCVKE0RBQpZVquUChq5i57RfnomWpWEZGrquY+VscrkCO3L8yyR5nCakSUYYxjgLl8ALmS2HLBdDOt0LXCcgjlfTR0kREUUBUzeRJAnTNLFy9iqZ/kvxmuIulzjehDAMVmGqr0QtggCSiKIaqKqKLMtkioQq66v3JRlhmiJ4Ka4TEIZD4vhf3qsoil8JTY1cLke1WWNzZ3tVMyiuwOJhGOO7Dr7r4nkeURAy6fTonV5Akq6A94aKbVYoV6vk8nlUQ8MuWSi6RKVZZt0uYFkGi9kYdwn/3v/w3yPwI1xvyXIeMRr3EbOYghWztxlwdiLiuwGu56EpKpKgYpo1+mcQRgmRcEKxXESWFuQsmVzS4Isnn3B69ozRdMLmzg3ypTssliFLZ4SupEDKLx5eIMkaUZqhpjGCqJIIEmE4pX0VoKsGQiYSJyEAiqDhB0sC2SUWLCzJ4Hja5p2b/ya5wyMkZEzbIggC9jZ3OE1cZFVHUQI6lxdUrB00wSKVMkbjId3+FflcDk1o8NYbt+lPj5AFk+GlS71RYuT3USo21z2Hze2bXLWPaeTWQczoPv0xV9c6W60bHNzSae6r+MEGUVTi8Cqje6IyXr4g1AR2mq8x914y7M5p1V7DECUi+YRvf/dtvvzoQyq1feobX8fzz/j8sE0SSqxteHzx6U94//3fwLJrpGmP+WICmch8IWJaGo57yXg0Ymdnj1Jhh/hGCsKIJGsgmG0qpsjx5UNuH9zh1cs+SaBTbxVYLpc06hs4zhJZjXjy5Anf+NqvcXp2gixs0NwU6Fy7lMoW00mbYX+AZeWQZJ8o8YkcjevOA2xjD0HpI6R1rELKcOQQRy2WC5fNgwZhPCIjxHUlihWLo1cXlItrLBY9/ChmPOmzs7PD0r+g25uRz8FF+ymFwgbBqYmaWpRLeTr9YwzLQJFNZEVkeHXG8NJnp2lhWBat/V360ynT6QgxEpj7AW+/8TaLsM+052FoDs1aRH/aw3FnRNGAYCny+s236I9dNhtfRwzmeEHIUec5eWWNTBkSCwEPXnzKu2++zpMnTxgMZ6wbB5jWbd64U+LhUZdM6+PNXOZTnftv7fPgi5+jRiWqLY0om5IRMrmW+Ma77+BN+nj9iPXXarw6PMRdOrTW97BadXxnzMlxm1yxTs2sUi6bLGdzqnWDx0+OKZa1/2c5+P/29a+88v7f/Mf/duZGHVwvIxVdCsUW03mHsr1LEgt0+yP8uM16/QaKMWY0ucJUtzBNm4dfvsCd97h77yaKWKVe3ePp84+5fWeXw8Njdnf2CSOf66sJpi1AaqBoIn4U4EUJjUYLdz4hFUfI6Q6FooUoX6GrFc7PushKhoDMjd03iZOITmdC3/lzNurfImeqOMsJrVaDheNydn5FPm+t1tP+kGp5dxVmIEUQU+bOGbJiI4sFEDzydhlTL9Htn+K5CmE8YTR7hZ0vU6/sUSqtc3YyplktkogLet3xiuWmllCViKuzAORLZMOmfTGmWLKwzDzL5TWVUpEkMNjaeI0sjAmSNrIqgmCzWPpIikejUkbJ1rnot2k16wixSLfd5+79Fr12yPpmld7ojDt33ufx44f86Ee/oFJXkESV5lqJpTOhXq9iyjt0h0+xcwq6WmUxCzGNInE2xPcDvKSHKDkMLqvsHqzz8PEzZu4Fvj/GlJu4WczGVpXx0Gdnq0T3ukfeEqk1bAY9B0FTSIURnesxtcqbKEbA6fkx04lCriiQOiq1Rh7PH+P7PqGb8v0ffJvJcEEchkhmlcXygpt3mkxnfeI4opSr4y8UvMWSlIjX73yLRfgK112iZjL9fsrmziajwZjUy2jVC2Sqx1H7kM2dTaKlRLfto5kV5pMzljO4eV9DTOuMpyN6gzFZUsf1xti5DMOwCN0ERTYJohGaZiEkJkE0IWerlEolbLNBu3uBgESv1yOKYXOnQMneJUt0YmGELJSxSzKfP/gzdLWEIta4uLpEN0TiyEcVinhhB9MorL6kw5jdzQOCxGc4GHMvSfh3fhIRRasv+v/t+zEnBWklFJSM0FO5c/cWv//7P6J9LmPkNKJkZUzPMgGFkF//G5vEmYJFwr37b31lJC+yu1ug3XaZj+oIUoFO/4pO5+qr9eecxeyS3/xru1hmnqvrQ7757bdwp3X+3n/jf0yxUCLJXMhkphOPv/fv/iX+3n//O/zpHz5FlCLaZ/Af/Qf/OYVCiyBMiVMHSZLIYoHZxEUWJGRJR5QDJBWSLKNQsAn8FFVVydKIOF6txJEi/CAiTURkOQESQESSFaIoQxRkJEFe/fw0JlWFryaIGVG0mhhGUUIar0JDqqCiazbFQoNWc4tKtUq5kmdre51Go0KhuJpYBlHCZDZlMBjQ7Q8YTabMpgva4zFJnCGk/3Lzo6kqlqpiWwa1SpFKOU+1UCFfsLBzOoqpIKvS6mwhRRZEMnQkWUSWV4Gm1frbZ7FY4DgOk/GqCnI6na7S7VH4FT9zFS4SZQ3DMsnlcqiqiiSIAERxihv4JKFDnCT4vo/vrxqBZFn5aq2uoJsrDI0oiqswlrgKZaXpKnxlIhAnCVGaIEiAJCBIIoqyehZLtNBNA8Oy0AwdUZZQDZ2MlZc3DRf4jornzhj2u4R+gJjJZLGErLnoUgNV0ym3DHSrypNnDymVNpG0GNO0sTQVIYMonJKJMF04tLttFos5SQhxlBFGC1StRpgmZMKcJIpYDmakXkD3esDdN3Xe/cYNri41XL+AG0ZI6mq1WRNEvvNrMS+euvz8o5A0L6O4IYlpsJwGvLYfsHQH/OIjid2DFTB74jgYooZsivTGXX7z/XW+957Onz36BMOoc9st8P3/4xMUVUcURP7sb2xQ/ZX3Gc0d4mh1Lv3RH/0e+zdfxzRNwshjMp7SWi8zHXtsbd/g6OQphlnGMCb0u1CtFukNzjGNbYK0y2R2TbV4Gy8KudMooJXyNGtvcvjqBVGksZz5dMbHfPzkY+6t36ayVmaePmJ/bZv53CRXbJCTM666rwCRShm8gUe1KnLedtnavs/91zf47NFTkiShVl8jjkOeP+9RrZbIF8rMnWvmExFJcchQsHIi/euMjR2F66sOd2++x1VvQqUiEnopxdqA84svMZV9JiOT66sJN27vUquXOD9r06hv4QdzlosQ1xtRr9zEDYfIckCtZnN17lKr24Txgl5vip2rsr2xS71Z4KOPfsJwcsrB/tvMFl3KxXV8V6RcURBQWSwSXr16Tj5fRDMdlkuXtdYu5UKTXv8cSU6Yz11MUycIl/hBShSI3Lx/h+NnL2i17iCpU54/+oLt9XeZOxPymo1iSShZgcXyGRN0NltrjPozFsMFmxs3uXP3Ndrtz5hNDCKni2EpnB8/wTTXuHW7wmvrBqqs8ekXDpGtrri6acjVoE2k5MjbBSbTPr4fsr5eQVeLXLeHSIZPkmU4Y5kffO3XadWmPHzVR8+LjC+mVDdaLNIRepoxHTgU7Aa6NWXe1xFln15ngSDG5BsqmWGsWKPLDD82aG3B8ctjTG0NQ4GdvRsMhhe4yYjhIKHV3OBv//X/xV/synvm95nMOqw3dnCchP6FT6VZQ9YzVEkmHfaolBuE6ZL+xRI710RXqzjLKa+9dh93PMNdTkmkGGPN5ttf+x1+/JM/xzCrKJLBdDHCj8c0S3nqlQN0rcrnj36MZYPnLag16iTxCt8gSH3cqc6k77G+WWbYXXLrxlv8+Of/mFt3brF/a5/16NdIo5Re94piUWc+DojISJKYIAgwtTI5c5MkzUCKiOMFGQli2mAx9SlVPHw/QpVCzi8+JQ3VFTLHBUPZpnPu8/zZh6xv7uE4HsNBBaSU8WiGIKbo2oK93Q3a1xdEWYe333md6yzm6FmXStUln7PIW3X0UoU4XGIWVc6eT9HUPLVajnqtRn9wycnJmGbZ5r233+P46JSr9hfI2MjSFmubedr9NrZd4OXRU8JQxsgJlKoWObvK3Dnh7KJLHOlsbHQJkzn3XvshX356zvpGi6vrQ/yoSxyaVBo5nj9xWVuH50cP+OLBc+6/uYOEwY//5Ix3v7mPrWs8776i3+6ws9MkV9E4ueyQkFGyCgx7Gba9RipMOEHgZQABAABJREFUcOcmtlbh/b+0y08+eEzkJqShQDCrUW161Pd3OX4xxvdGfO297/Do1Rdoqs3pqysMw8BZRnRPr9hobVGpB9jmNkdnp0jyAl22UHJVWq0EzxmDKHDSeYKgbiPrGa1tgyySUARoVHQSZcJ0FrK+VyAOqkymfc5OL5FlldnkGLQRQVxCnYuYtkAmXSOKkC+1ePH0GsNKMa08F+0zlouXVKtVVFliZ7eGKqwScLNZj4UzxzYrIJ/gR1vcvfV1xvMj5qOAankHxbjm4tghlQzWN/YRJQVVVfG9CNPK6J0O2dvd4EZagp98iSRBkoS4XkJaTPFDkWgpomkpj5++xMzJiFIOkEBegKeTKuAGEoPenLPBCW/ffotPP3vCW1+rc3TygKuhwlr5B9h2k+7oijhxCIKAjITxpMfWHpSKdR4/+YxqpcGjB0e8/saCjfVNxpMBsqwgiCKqKvOHv/eCX/utu0xnI9Y3amxsK6haHklSsHMZQSwQeiGu6/Pf/u/8Oqk65OGDM7xZmdRNGfYHjPtjUqHMwLvCUHJIcoZupRBr6LpCkvqrBhVJJI4TgiBCVVdTt5SALBNASCGSiZMEUQRJFMmbGpIoIokKIBJlACl+fMnLsyOiw5goyhBQIF2tsw3dolSp0lxbwzAM4jQlnyuz1lxn5+ZN0ixe8Ra/eoVhyGI2x1t6nLc7fP7gKV4EmiyhqAKGuUpZF3I2OcPE0HQkc9X7bWgqqiahK+pqGmlaVK0CjY14JZYlBUmQCcOYxXzJ9KsE+3A8Zjwec3Fx9hVPE9I4W/0tGTq5oo5hGOTKRRqmiSwr/5K9mUDgLFnO3ZWf8aupZhzHq+c3DELdwrZtbEUmTVPiJCTLUkgzAtdjESwJOyFRFKEoK5zIf/kMpmmimhK22SCXN2htvI4kyavpXyQjygFJ7LFcpszmA5zpKf/oH/7vGHVmCMTouk0gSth2nkqhTr2ZZzC5Jk5CCgULTbKQvuJ+ytI5iCKKCoVCgbfv3mVjfY3BbMCv/dZ9To9f8PiLF0iSTOBNEBKV5aiPXTB59/03+NlHf0oQ6KjJypsaBhlR4LOxucXzp0uWyyGCmCcI/dVUPk2IQx+SlPXdBlfnV3z73m8wXDxHuYYkTUl8D02X8OOA54cnaGaVG/uvcXH8mP2DHabLLqpts3QhlythaDJ6vULndMjN7RtMpmfMPJ3B5ArVTMgVqkTegjgYosY6OVkhWJ7SdhWWywHPX53S7XZZX79HRS/gu6f87b/6lzFsg6vOE7SgxPOXh+xu/DqFvMvLF89R9QrBwsMTxly153wt/2v0wz9mevGcIFZo1F/HMsvI9oAf//kDbtx4g9nikuPzR1TLW6xt6rx4dcTmxg793hQv8FGk12g1DIb9jHphh6urB6iyQjH3OoJfxI2WpFnMt7//TYbDayajkHt33qPbP6HXP0cVNymX1qk2VLr9jPk04s7tDb785AN0rYGzjLD1FrubN1h6U548PkdWJWqVm0RhiizDyckR9dom01nAeDxCFCXu3Hmd8fQCw6hgmnmurs7RpAI5u4q7TMlbCaWKztXFBFMy2LiTp98bsbZWJE5nTMdzauUbvHHru+wfbGKKEka5gCoLnD79gCdPD3EXCZ4G1d0DRCPjov2IDI1O7wtK5U1UscT9d9+md90hiBQedQWUqMursznN12zcRCenbpKrykSiSN6QyCt7kPicnl2we8eiWMyDlDIfhpSreR4d/oSC9jWm4w6yu85aw+L5iy/J1Vv4ypyJPydKNcLREtP00TWDylaLTHBZLiLCxQhRNDEUlappESwWNDeaTC4TrLLBbN4nRmI0DJkvUhqN/z805bjBDJEcdr6MZdTR9SV+NKV3eQnkydlVxKyMqk1obWwwGF4jLOYouk+psM1GYwvHmZFholgmXzz8EsGaUW6sMRx53L73Pqr6JV54wWTeI42nVGslMgJUuUwcLZnNxxQLGlfHAdvbFXKWAQlU81ucnnT45vu/xXyxoHM9QJXL5Ish+zv3WC7nFAsVMtEkSwXmsxGbzW1G3RAvecHl9RkH+7cRJRgNxgTxknQMmmbghBPcpcxycY3nuyDIlIqbROmI6XSGrF3huSlt75JCIcd4uiAIAnKWyXByQt4uo4abPHzQJol0MiFb+YXEmAdPP+bW7QNMeYurZ0N0uUi9WiVMB5ycO4wHIf3hOfq7BdJA4rOPP2X7Zkw1fwdRETk+fUm347K2toZhmfQHA0qFJu3LNq2WiazZbGxsUCnWcYIOmlrl6LBNp/+clAWlaoXRUKI9fIBZbpCJEafnVxzcXkM27qPLJrVqGWeSkDNbnB+m3L2zzXQ8QUwkHv50hJmzyZQenhsTeRY7tw6YLa4p1yXoGrx6NKFU0vAxCf2AyXjJ+toGUbBkMe9zsPsu5+fH6GlMsnQw8wqtcpOzxTnlSp5cLsd1u02rcYFdtpm2Naq1FpJgMxm/xPOmCEqejbUbtNsTEnqUHZF6eYMXz5+QZEvu3ruPGMm47pSLWQc3mZFVAtzEQizOsPMK3qmM66cslwrlhk4QeCBOEYwZV/0hrpMQBAG2qdG5XiKKATf37xJkKx9vSohhJRi2iKlvcnx6xq3Ce3jTNpaeI4kCFqMMWRERBJjMBsxHFqV6RKXcYrEYUigohNGMs8vuCjkTQZyArtfx3CW5nEXoxSwWSwRBQzVFMjEiTjwSZARJRdVC5r5A+8LDLpXxI5EscPj5z07YaO0xn/k0yhZuNAchxQsjJCXCH2UIQsadexWePn1IqVRgbb3FdDomSRK++4P7/MP/5J9RrVnEgUY+L3N8fMkf/8FL7r4t8uroGZa2ie9CsZzi+hFZqiCKKZqmcf+bG/T9pxx87YCDxneYtsfcO3iNL58+4ounj7HU3+Bf/PM/oVSqcXJyRbc3wBlEyKKCIiqkooCqyViWQZZFqJpElqWIiorvpSiit2rkSWWyRFzxQYMQAQ9RFEnVVe+2IChouoZtrdLnZDEZCbKkEwQB/cEh193nq2sEETKRNBVRFAlds8jly788E+1ilWKpsmJmVtfYPVgnU2UUUSJLwPF8ZtMFV8MZodcjS0BktUbXNA3T0NE0Bd2UsCwNQ1dIhAxFklFVDVlc+R5NTUeWZXLFPGbeprHR+mp6Ga1QOkHIfOniui7e3GPanxH4PkkSIUkSmrZqFDJ0i1LZolIqY1kWsqqsgj5xjON4eJ7H3JkxGw8IXA9V1jBUDU3TyNk2uqphl3UkSQJx1d0cRdFXwjbFCX0cJ2OYTEiSGFky0HQFO6fRqLfIMigVLcprOfx6gXv37vHxRz/jD/7gDyhVmvihjykGuP4VyWjG6emQb3/zW+xu3SIIl2xubiKbJuu1XQxdpJDLkS+X2NjZJVMUnGiEkKRMxi4bG1v863/7WyzcCcPhkMloxmTS4Lp9yY9+MifISiz8HtosJQ1nyIqB73icXirESYlv/2CdJErxQ4eUGMeNEZUYQxS5mvSQkjPspY1zqVKXopU1IYkI/AQhrWPYMvU1iaPjjxFjFdNogDKl3tqjWIqI/DnlWpGP/vwhqS9ya/ctNGuNUbvN3uY6vdFj5hOZW/u3EKkS+zOmixdkqcLPPvkT7u7eRhElahUZ3z1HE99ib62BnKqct88wTZneWcT22msE7oKTp22S2CRXLtAslnj8PEI2dHrxNfvb75O3qvzis0/ZGm5TKpUQ1ZBqbZ3r3hPmyzmV4h4CElGUYVoyr15eUasXya0VCEMfQVRRjCmjaRtDLVCvr7ZAm1tbvDj6kkK+Sa/fZTH18Lw5tcoWzfoehaLB+fklslwjcA2SOKRZu0nVfI+/8Tv3kPUpP/nJh/QGLzk8/JT926/zjW++w4cfnrG//R4XnZ+jaSaGPWc0uSBKJF6/+w2miwv6kzN0RUWWRQb9iCSBbq+NLGnkcgWyNGUxSblz6zbTeYcohLyhc9UNaNVVAgIEy6I7/oTlg59wfj6llEWkkkYm56lt7CG7CV53TMN4jCTfYCbnyRca5AyBx8/a1GyRVCqxtbGDZklc9cbcXK9y/80ZT84+YeoZvP+1PYqixWDSo1y5RbxYcHF0TaFgM527uNGEgtyiUdYJlAW6L9MPO7gJ7JVt9LzM7VsO5+dLVLvF3q7BbB6hJFUWzjW6nSNjgiSpHNx8E4OYm1t3WSxnhP6MF8cPscwyrTcN5rMeT5694uadu8h6RjBsc346/YsXlHmrhFGucNVecSarzQJZbGIbVXI5G1mNGQ9HXF+7iGIfOw/d3kPKFZMwrqGLKgt/wXX3AV7YodW4we2N27x8+RLTMpkuTQRJgriK64fMJ32atdukWchiMqTciLBUi6LdwCn20HVIY1AEFdEE3crheANGoyFb62/gOANGgyWtWpON9SqnR8+QpAKWqWKKN1AFD0nuoQoi9+/fhzRlMgqIkinlQgurkPH8+Ssa9XWidEJKCmmZKHbpdjo01i3SYJ8kgr3WTZ4/OyJQQ1p1m6U7InTzOGORb779Pg++OMK2ddY2clyfZ8j6hMRvMpg85dGjC6Q0Y29XxvUkOlcmV71jdBvSTCdfsBgOz/kXPzmnua1SLu9w2T6nP39FEE0pF97mxYtD0jRlZ2eX6TREU1UEEi6OZ5iFiPnsIYEnUyrqPH/6OboVUk4dXh5fkrfr1FobKGKRNBtQqtR4+eqcTJiRaiVUbvLamxX8VETSEi6uuhStDQ52ypSKUzw3Y9jfBmWGH7v0umP63YTN3ZQUHzNXoWzJ5PYVXjwZIMtzLk7nHNzYYm9/gwePPqFU0Xjt7dv02j2Wrs/V1Yg0FfGWAb2gg5K1uLrsoeg+zgLGoym5UrRqhREkOhd9nMmU1voa/Y5M4llk7pxcroFkNDg6O0er6jiRxIuLCZIWkgQKaTKmlG8yamdEjohhuXheAiPQtRxnp22snMD7X3+LSd/HXebIUonlLMSyCxy+GhIGc0RRpNoooJsqJydjCsUUw1b4/OEHPH14ysZ2HjETgALFcpnJ2MWSG+i5gNEoRpRHtM/alKsVPC/BmKf4foggCiiywvp6mUtfJowmNNdaXF4ukUUFy2wgMkXAQBUDUjkgDiNkRSFIUkqaxOnZIWWrghsGLGYxm2v3MdWbDBZTlq5PFCZIgkWSLlhr5SiXV1xZZ5lycd5mZ3edgrnN629fo/+jHEniIyoRYaijGTE/+/Fj9NwGu3sHqIpOLm/gLFIERSRJXWTRII49vvj557S2bvGzHz1FeL+AP/NJlgnX/Wfc2C3x3vs3+P6vFGhWtzh8dUGnN6JcXufwySlXF0ecXPuMh3OOX12jKjnGgwVRnKIoKoZpE0mr9beqqghihiDESGKGKKiASBYHkGWIokCWZETpKuSSZSBkEm66RJZl7FwBSL9anwurCSgiYpaSkbCYtX95Jo4mVxyfpIRhDJmIZVkowmpSVylVKJWrmIUc2+sNDEtH0zSIAnwvwnMjHDdgvnDpDn3iOCTJUkzFQlEkLMvAMFUkOUWUYmRFQJYyZLGAKEukgJBlq7V1mqDoMkWrQKnaIEtXneaQEscxnufh+yGL0GFyNiTLViltWVJR1dV0MZfLYVkWm2vryLJClCZkX/2OgiAgBlzfRVhMV5PLNEUzVhWjiqJgWRaWYWI2yshKhizKKLKxuofQw3EnJLHEYjxHs2SCMCHzBFI/JnFilBKEaYSU6ZiyhqGbDIZt7r75Nn//f/Tv8/DJ5yy8JfPREteZ0pnOODnrQSoQxB+CKGFaOTQrT7GoohkqsqogK0VKVYNyrYrvr3P77j363SXr9QLl3/CZTxcsgxlJ6DDqOUSJzt1791iES2RBxfM8kGJsxSSVIx5++AWtQgMkHZQizQOF4OU5sqyBrBPMh+TsmCgnc3V8TKu+jarUmXs90iCidzXFyC1w5hHhMsf9t97j5bMv6Q6GGEqEJU6o519nOWpjFEJ63RNu3ngHS5+wsW1zdpTw3/xbv0PS7zJ2F2QqDGcdRCMkCzK+ePgBtc0W40GRUjNjY6POxVWAnKtREFQUJWU2H5EvOMipyenZE+TK++w2yvjBFYORQaO5Dpgsgh6GqVIsbjGezPADkUKpiW0Vkes6G627xBxDYhAEHoZWYRxccfjsjF/79R0UfcJkPkBSQhbuEWv11wj9gFyuQK/XIU1TbFsmDjKMksXNW01Kw5Vd4tWrV+ysvcv+wU0av7vD0cljfM/BjVLc6YSq9TrVqk1/WCSOQxxHY39/m1dnX3DdPSYIPKLYxzbr9AfH1Oub2L6AKubodPqkSUTCBF3Lc3biEKUBoqBi5l1sPUerdIdvvXebf/xP/gAlUuhxTRBUuPY6CLFLKkscukfMX7X5r379a3hel0PvLVprOqenH7GcKORLLpkm0x9cosgTmMiUy0WGiwmnFz2q5XfZqeeIhBGnJ+fUijd4df4Mfxpz99YdXhw+ZKd2C6u4R/t0xs7tMp2rCUshRVZhf2Od3nJEppQZHg8ZLB1ea9Qg0YmWPq2WTbBUkeOMKPU4OT5h2p2TZlUOD1+SSRJepNMo6UhuwsXwnFrRpGwqePOQaVekWdvAC0//4gWlGKvopZiClMdZwnV3QLGQI6cWEGOJznBCLJ6h6jXqzQIXpz1eu/cGoaPxyed/zne+9etIYoqiwrvf+BpPvhzw/PkhqpowHB0xGncw7AhvYVE0D6jWLS4vjznY32ExjfHmOtVyhcUkoVKx2Kjd4/KiT5BNSbM5VsFk0I4pl8tous9wNKdWtfH9Po5nsbd3wGS65OziEIEpmrbJbLIklkLmrkcSxVj6FjcPWrx69YowVrhz8xucnh4zGqbkKjLedIZlVFcHlthC5BlClpG3RN5684BMSpnNZih6wByFu7fuMJldM593eXP3bQJnQJYtaNU3+MXPv+DmnTL7u29wcnzB/q0NpLTCZBDgLPeRzSlJEqOxTZodEUYm87lLr3eGpMRkk4TxwKVQ+gVJMidYllElndmsg22bwBLbtrH0HJs7OX72k2d0/BO8YIyRy9MddLk6dynVpySiQ+Dt07n2GY0usAtFHHfOevkWnd4JSVBAM8FLOuxtv0mttE3mhyAP0Uop4TijrO5SLg0gKpPPpfjhiPb1lEpVoapsMvd9cjmL5uYGeeUu/d6ci8sp+bzFcDjn5NWQcrHA1cWc2nrC1J0x7otkjFFEC1MTCR0XLSews1/An6XIxgIzV8TyIY7yXA1mlCtrqILDYrSgvtmgNxgS+TJWatJuX/Hm9hvMgikbWy0ePXhMNNNQ1RCznBEFEraVJ44WJHLCfL5kMS0gpHOyxCeKVWZTn3zBYjKHQmGFd8nbBbqDLmEQE8YReiHBVm1msxlb+wWSKCUjh2EmPH12TC7fRNFDwnhKLDiMRwk37t5kOBwTRR6WbSJLIVkWk2Uyo0EPu2GTy93g1csLbNPkun/Gzep9cmbEwnXQVQ2fBBWLJPGJIhlDNph4c8xaGdnq4S9nlIoVJosBXjQjTDySJCUIQgRxyr03qqSJSLlSpNXap991+PDPP+ET/QFf/85N1rYkhp0qyH1EEWyzwtPHfX77r71HrmDSv0pxPBeroJIkJoK0WqWGYcjtG2+iaQ739t5Alco0D4pcXHWobe/Q6zl88VGfnKkzu76mvlZjrbVBt3/J+9+o861v1BnGU8hUKoU1HnzxgjiSmE1CLs4GtK+GRKnM8dEFo+EYUdLIWKFBDENDElb96oKYIckQxyGCmIGQkWUCSZIhiApBHJMkAZAiSgICImmcIcsqgSiumoYU+5dnoi6JKKKEKECaxpCkpGmAv/C4ng65OHxJEAQkZMjqCpyu2w3yeZtyrUyhUqKxXsawTCRFIxMgdXxc12fpekxmS4Jglfj+L0WuLM2QlZVVQjdULENHVyUERUDMQEgDhAwQ0tWqWkjQDQXT0lZpbvilX1LMVmGfIAiZzoYMR91VpzurVqNVuEdEliXyha+8l7aBpeu/xBgFwcqnufAdZss5SadPxmo6KSmrfu1qeRPD1tBzOvl8jgyJKI0Q9Yz3vn6Pf/bP/oAw8chEgSxzAQ3Pd6kUivz5v/gTdjZvMp7PMOwcMhJ6XkQtFsm1LGQlIw0SSAQC3yGYTzg/XhAHMWkWEESgihqCEKBRIlRDLFumZufwTIta0cSXdhHCJcI9gSQVcBdzMlRSHCyrgGrGbNSavDp9RXt5gSS8znjRw/eWqLbIHvoqTJaGqLKKrkbMZzFCoDNoX7NzUKc/aFO06uzvlPnFL16gqAapPUbPDIbOJfJUQUwXbK3d4bp3SCFXZDpZUDMbuOMJk0Gfpt3km/fXaN56l1mxx3e2vk4Wzvj4yU/Z3lrnkw/G9AwP08pz5RyxXTigfz2gXCwQxEXm4wF2qcpsOQFxne1mHlkK0cwCjw+f0Ki1qNZyjKYdCgWN3uCC9Y0qQtYkVxiRhU2yVCRwDQI3JUkWPH35gHs3fsBo1CFNx6RJhpZbcHb1hFKpwOnZEaKQYBtlIt8hSQMQVDx/iG2rvDq+oFotous6V9eHTCcOkpCj3f4jEs75oz9ZkkUGG9smm1sFdEFCEwwE8RkvHs0p1Wu8PP2IODLpXPusrdc5efWCzc0dbKHKbHHJ0pkRxUs0pcbYP8Y0G0ynU1Q9Y3ftJv3emDAaUy1XGE99yqbMuD+gYFWpWWUcLyPfuouazGlad1guVEQkRBFG9+o8MjXC2S18hvQen1DQN9i/tcHV0TlpcUkWZ8RSSpy94uI8T75gkq9uUF8rMFoc4k1kNKGKkE4I5j5bzTc4vfiSenWdgiHhLhYEfpejp0u82CNfa/LwkyNydpFZOCMLOwwjD8NUGARLlEBBUGZcnI1JI5m5sETQM+zSOoqV4PrX9CKXaBGR1zdwsTkf9thv7RF2F5TlTS6OB9SbNnZR4fHTzl+8oNzY2ub07BpJqZG3BCJPI/ZCzqZXuMGMzFd452vvMB35jK8P8ccS3tzFW4jcu/kuF0dd3v/GO1xf9zl9uaBg5DDMHM5CQrVqVOo5vKWEK40xLIdxNyRvl4jiEY2mQhxXOTrqUKrYyFmV6/Epg/kSRVugKhbDYR/VkFFUg/6kzcaBgTuXsC2RwWDJVFQR5IBi4Q5WLmE8cjBKZRzPRY7BsGM8f4RlbHLn9hsslkMMUyMOFfYPtrh1cIPDlye88cbriJrDg8cvaIk72JZElqZEcYRqDOgcDtBNMHWNyfia1lqF2nqeWLqi305Z26gSeC7f/e679IanXHQuMCrgLmKSaEyQzdi6ZaKqW1xe9JksX6JmORRZZjptU6nZJLGIqMyJIxVR9lFUm2+890P+0X/2T9jeN0mULqqxTUPdxI3PGS8dWts11ktVfvLpY86OL9i/fwfFlOj2rtm+YdGZdbE3DN65/T5HT04ZzDtct5+SBBnb+2s8edlho1nHVizax6/oz86xinmkVETWzzDtMpnUZOpeoRl5JtM+zeoegZ9ycTag0txmOQtplg6ol9dw5gv2tzaZTOcodRU5DZnNA6y8xrSXIZsGmbggWGgYikIiy0REEPh0u12W84y79/aRpBmVRh4v7JGFLp5rM/VDVHNCQ2qsvHCJw2V3hK7UKeaKSJLHmuKhtlJ6S4duajPrBShGhiYGzEdLhGaJQrHMcqDT7S/QTB2ZGAGFJAvJ0oBgKROnAsPZCdVSCz2XMVp2WLgaZ1dnZIJLwSgxdyOSOECWS9y6/Sau66+QV2JGvtBif3eTaCmjymM2NgtURtmqxg2ZKErQ9Tyt+i0ePvoZ+Vye0JeoFEskYYqqhYhBSiKGyJlMQoYgQOynCKKKLOVWCCNZYDnNSLI8k4lH6C5ZBh6LyCcKIPIXLN05CBUq1QJ+sKDWlChelPjk56+4c3+H2/eb/PNXL6jVi4TRqmtbROD5wxHf+e4BF4d9FosZpr1NlrmImUiUuFhWCSd1WPod3v7ON3nxZIQ/P2Xsn9KyvsN6w6BRsgljj1enV8hli4vnR1RrCuNBRBLoeNKM/rjD1oZMIibcf/cuihLhO/uIgkqYJgwHDrpm8eGHHzLsO5BaHB9eIoowmTo4TsJi7qDKGrJikKYZmi4hSRmqGmMoGoIg4Ps+ksBXDUQrVJDEEjINMVV/eSYK0ioAlMEv0TyiJCEKKpoiEkUJpqQRRi5pIiOICcHsgo4T0D6JyBKZgBRJ0MhbVbS8im7YVCpl7FKOSsXCsIuIgoosK8QRuL6H7/vMZ0uuLodffd6r9bMgCJi6jGqsfJSWpmJqBhIZqiyTpCnIBooWkMQuSSYgIKHJKlbBhExElWTEr4yiK+9lirNY4sw9hr0pQRiv8ESAKIq/bBeyLANd19DNHEImIcsiomiSJSnjSQ8mEAQBQiwgqhqpEJHPWyiCimKqZJmAEEkIskKUOCvAvqrQ6Y5xXZelM+Wye0LqynhBgB9GkGao6ld2gFwO3TDQCyKGZaBZOoZZRlFW9xLHKVGUoPgCi0nINJqSpg7L5ZQbN++h5FoEsYOISL1awwt8AFRFhCziRz/6gH/6e7/P19+8z8HNJl98/hQxsVEEk9F0AmlMGidohs1svCTc0Cjt5jl9MsJadNlr7jMfXPPxj4+JsgTNNpBkm0lHwS5V8dWEolhnMfKoVW7hKSPOJz6K7aNYKuuNLX724HM6ccDo6QOSOdTzX2IFE06OnnK0/T5yK0EmjxLqvHnzTXr9OZXyJk8fH/HeN++TMSYMF5imiCyK6ILBra3bpHHKbKbyjXf+MrI54+T0HNMuUalUiBwB3QrZqnyPpy8+BTWPpVk8efpTbt+5z2//+r/DBx/8CeWKRRjPiFOFFje5uj7ECTcoF9aYh1/ipiLT8xmN4jaL4ZBKKU8cLtjbWadQLnFy8pDtyvco5K4wcjX6kzxn7Qk797Zpbhp0230ePHuMqtmkYkQqOjjLIbs3f4CSnRMIz/DDGdOjMq36JrVCnX7XZTHvIwgpghCThD5eoLNRtdms5rm1812ePX6Aist5p8/62h6b5Sonr0Y0GjmenJyQWjXSSKacL3N4dMlkYrCzscH5cZtiNWA6aDOfJmRZTKW8gWFUSEUNd6aRag6jgYKZkxhPekRxQq1URDFk8tYaohIxG+ts7eSYTRecnR+SOXDhfsbm3fvEkxnDYQ9JUFhOOoiFdWytyrw/QrMhFpbokoIcWxTsEG8psZmLsWo6jjvlxZcBlQ0bbzZG9Mo4wQhTbxE4IaosY+VtlosZ1kLlh9/8HmauzpOHf0i/P6N5J0/OrBBGCQf7b//FC8pXL+cMJiMO7hjYuSKT2Zyryw6aGSAisXdzVfmkiDkso4qx65OlKuNpGz1nEoQRjx49pVpp0O68ZGt9h8BVqNVMZLHM4xcP+K1f/x0mowWPHn/I67e+gWkbPHj8BbKSo1o02dzYQpJhuQyZzefoqsXCGVJeX2c2VckXBZbLJd5S4XjmMZudMh2nvPnGW8RSj8SFNGVV3ZS55Ap5dK1BuZ7w8MEzarV1slThqv2YNFEw1Aa/+7u/xsNHX/LZg0+5fXed/tBh4v2URrPB8fMzsrrOa3e+zh//6T9nY6fA3s4+3f4pO3sGs75Nt3NF3iyyuWfhOAvm7oiN9SqT4YBWbZ2Cvc319YxXp0dUyyXiOMa0FEaTHooaceOmydOHPQ7u5jGsNSQlYjDoIysJa1s2mrjDdf8Jy/AJqTCmWKqRIaHKZRTDYz675MMPuqxt30BKpwTxkGKhSn8wJg1tpKhG94WDVIjZ2m0y7HY4PTrlzXe+TRhMiF2F4fCa3dsyi/6Y8VgnSmPW1m6g5wNCL2Cz+W1yuSYf/ewXbO3c5OLiHFvfolQq0R3MURDo9B4z6SwRBIWT3udAShr0mY08kCAqSyhWxHA4ZDkLMfMSfhShmAlkBotFB1MtsBgpqKKMpcscPjmn2Sxy584eSaKw9Lokvoqs6QSeyXnnlMhTUZUCZAmCInJycoZtmHzW6eAGC8KsgJ9CoWihmTmuz3vk8lVylsF8rNLakDHzLU4uzwlCn2LJIIhCZLmKE0xXfdCqTZIoLJyAwBPpej28wMVQTfRinXnWxdAltjbXePriIQvHpbXWRFEU3MWSTqeDqli01tcRlRRdtQnDAbqhIaAgizaJFHFw84BMnHB66mDbeep1mdyJSG8cYSoF0ixBVjJETFxngbMMODjYpdFoEIQeFydLxmOfLO0jJj7LICAIA4b9Hrv7OX74G3s8fXTFeHrFt7/+u3z5xcd87Zs71Bo5LK3Ab/7mO/z4j4+IohhSDUHKkBSPJ4/OSJLvs71bxzT1FeAagTiOMQyDxdjBcYcUKwqHh8/JVwXCEHLmJrNJgLNIKZWnLNwUsoBZPyBvFRlPjtDlTabxK2ShQJT49AbHrK9v8vEvvsAwZSrlPN1uF1mVuXnwNnEace+NKmauzPr6OnnzV+l2RiyXLodnJ4zHLv48jzcKGfT6HJ9f4AQK426AgE9GhG5IkK1wQIYJUeSjKiaCKJNmwS/PxCzLkLGI0giIEFJtVTcr+ASxRJqExAnEcYooRmQpyIZMkhnoeQNFFglJETIQmTNxEuT5Fd1rgSDKUDWTNFl5uU3TxDYtarU6uUKeVrnI9loDw7RJBRE/CHADn8VshB/GDPpTLtwAEhCFDN3UUBQZRdcxTBHbNLBNC1PTEKUYQUy/wguxSounACKyLKMV8liSQFUSEYXolxWWq1W/ROBHhFHEYjQlPL8giSEVUgRJRJA0TDtHoZRHM2xKOYFMTBElizjOKBZtLDOH54UYhrGCtosKWZagqirTicPCmbO+tkmdBqqs4AU+cZSQxDGBG7Kcz3H9BZ3ONdGZhiSDLIOi6WiqtVrr2waGqVHKm1jrReI0QVJ00iwjTX2iJEZIFQQ5wwt8VFlBUSUuzk/4g9/7L7g8PSOLY6xKi0QJUeQqBDHxfEyYLQjTGFGCKHAoFvaYzlLOJksamzVCb8xIrEMSYxYqvHXzHQ7bnzOZDvje177P8yPwkyFN22IychFth5PDMzZ3i1ydd9HUMrOJQ7km0zt5hiqtYdam/OLBR9zZ+g7f/1t/g/bRpzx6OOX+mzvIus6L5z2sksB0MUaz4fj4Fdf9Y0y1wmuv36J9tbI+zOdLDCNaoYom18zPx5i2TTHXQExMFEEnSke8OHyMkRNxEw93Bj9447eRFx4vzwcEi4DAECmX7zGaXeAtT6gUy4R4zMMplfwGl2cTKuUNRLFPmIxZ27zLi6efEDghWSrjzkBsDOi0YypFga3WLR49eYiirBO518w6S/a3bjBxr5HkHIJqcnL+Iz7/osTbb/0K51cVRH1Jp3tCGis8fvCS/f19ms13efriAeXcTaazHnvl23S6x3znW79FrvE6f+Mbv8oXP/3PGXTmHPd6VPQy1WqefK7C5eUlxZJNoC6QJYMssdiq3WHhXtGdv6C29i5/9dd+lzDq8qc/+X0SPKIAkkzFMDK8doYoOgSBR787ptHKM5u1GQ5MvvH+24TRAm+uMewPGA2XFO1NJnGXNMuwxAa5hkqQOUxGE27cfYN+LyDMIhrNTa7bfVRNIE1iKpUKjqOSeGMuT0NGT55ye7uJLrkspx2SQEO2NCq5BmNviWaaNGprIA7oXj+koFv8/Bc/JkoNJr1zahuvISs5Xh1d8N6bX2c8/vQvXlDKWkoqRjiux2wxZzob89Ybb9C7dtm7v4Giyjx8OCTVr5gOfLa2dghDl3qjgG2r+E7Iwe1tAtcA0ScTXObzDNUQcJyXbO9WePD4AbPlJZJW4eTimDiOKeRq5Aoio9Gc5WJIPl/GMvMImUWn18FxpziLI3a27pGlS0bDCaZRwPcd1lrbFCwFP3JZjBxeu/M2R8cvCOIlfjTHSmtIis/ZxRGqluGHE657U1x3SrNV5MEnVwTBBMsw6VxfcOfmPYolg16/jivAjb077N3SCR2FvZ1d7r52l6UTUqzM0eU81laApd/BUNc4PTxkvZVjOg+QRAXPc9hau0WaxVilESWlwHwxoF6v8OLFQxqtIobdIA5s1rcVcmYRdz5iPp8RRxmRb3Nj/x5BNGfs+BwdnfHW23dxPBdDF5DUkIvLU8aLAZvbVWbTc/zcOjf29+kM59y/vU1pfY/LRx1+8eUfYwsJ5y9TNEnmN3/7PsOhRsHeQF9TOOuOqZQMJL9IFC4oFnTG8ymyt8Hedo1Xz04ZTS7IGTqpMOWdb9zl/OkpoS8ycxzmgyWldYPGhk5/eI1s+dTqNvORi2XlCdIpUVoi9fI4iyWiAgilVRd6NAAScsYWkqSgmUP6fQ/TWpIvSnQ7CtPJp+zcqpJECuPlJSVlDTnfRkkj1krfZTHxuVbmzGdjRuMRj64nvPf176EXN8i8gF7nEfdf/y1UKyR0fRq1XS46j3H8GEUq4wZTSD10E9Ikh6wlGGbMpO+gy2XS2FrRDXpzKpUVC1EWZIp2hbVaC8uIEeIck8mMjbUNojhlPp9z+849xqM5V+0zylWJMAxXuKJlmSRN8f0AURRAiri+ajMPPeJ0hqbqaKrB5SDAKFlIRy6ilJDGMYgJgqDhLGN03SBNJPq9GaomYtktJlMfQRCIA5fRbIyii3S7XX7lh7cY9mbcvvEWxycvcJwRqqriOV91YTsLtm9t8fWvH/Dhnx9RrpYIIx9NFzk56vDkYZvtPZs4CUjiFFGSSZKAMIxXkzuKqKrLs2ddbt4tYOklBm6PdvsL3n/vuzz8vI2sOjTLe1y2P2N9c4s4MhDyC4KZiiJYJJHK+fk5oigjiDJBEFFvbOI4BnYh5ujsS6q1PPlijsAVyMI6g4XHZBRRqG9iWCe8u9vCWyrcu7/J44dH3L3xd7g6mRNJI85Ohhy97DPouYRBxsXFBcOhS+BnpIlEvmAg8C/B5rIiIqkTdK2ALNmE0YxEUCEDQUyQFYEsBVGU8QNnJUIJEQQRBB1vmSIqIpmQEkcehi6hKmUyEkqqQJKuoORpvCQJXEaLHldXz8m+QhdlooCum2iqTqlUo1QqUaxuksvLGGsGuq6TkRD4EUvXIYgSFrMZYyfi2hsRBoAEhilhWBqGbmOawipIY5srkzoxAiKRnxEmCREqgiAQRV/ZBoixcxYFTUJVZVRZJ80CklggiRL8wMFZeiync2bjAdehRBxJCJJAEvs0a3V0LUcQ9kmSr1L7QJpJpMnKSvD555/w+lvvMBiNUTMZJBHEbAWel0VEVaRWrrJurGGqFopiIKHguQ5x6LJcLgm8Bb3hkC/aHRqNGvffvkPiJl8xQFNkWScWfNIsI5+3WUxn/Okf/ykffvCnxEFIuVjCc1yaNZswmCBKBnu3C7w8/ohbu28jSR+SJgkZGdeTV1hbbzOfhatNg28iMCVnV9hoViH18BYz8maF2eIa26xQzbeY9o4p1m+zDF9i5XTSzGN9bYdXL16yubbLwa0WLx9+gWJ6NI1NWr++xbTf58tffIFaUhAKQ54fzhEli9fffpOff/IB29vbrG00OT8/ZX19l0qxwmzqoms2hp7j6rLLnbs7DLoBxYpIFhRQihEff/Qlt+5scnR8iZ3TEaUIWSzSWC9hJ3usV2I+/PCPmS5Ebr5Tx7ZvMJm0EVIZU1bZ2rzF2XiCM+9w9nxOrV4hZ2d0+21qtRtcdi+IkjyNxhtk1hGVZgHBAFGf4roR7ZMeG60dVDlD9rf4je+v8+Xjn+I5HSSlxKAz596dr+EEc54f/pyJc06tdJM3bn+Tzz77jN2dPSqlKoIck7NNQKCQa2E3YuTiBqevDvnoz/6ISnGX22/c5ua7r3PaeYkoRWSJSZoumS3aFEsHRGmP+bxMrdpAUDKk0GC7tUYSwNbONp8/PESmwXatQic7R5PzLJbn5IwyS7fPeNKHLCYJDSChWtgm8ie4wRmNuoYmV0n0AqWqRJrC/tYOotBmOHQR9TqqvuDhs0fYSoONnT0yyafZWGcyvWBrbZcoO+P8bIRlahimwX7xJqOLLte9EDNfQhBD4iCiUfQomi18N6Z9/Yzt+jtsN1UEa8CLZ0t2d3d4e6PB17/3rxErVf6vr/5nPPrsQ9ww+osXlFfXPb73Kz/k1fER49GQm7vfIg5iwmDCq6MHqOI6vqMihhFbW1sM+w75coAq7bCciEjygM31m/zBP/0jai0RTajTuFXCUBuct4/pjwbYOZ3BGHRzQBzPKdlF5vOIvF1DkmOQPbIsIgod0kSk2aggKVXIJBZej9gdoqtVqpU6o+GSxWJMrmQyX4zQtT2Gg4RSaZ3Ls94qMWnk+ezTL2ltWjRbBppqc9U+xVBriILEi+OfMItSXnym8iu/scbd21s8+OIVse8g5RWqjQLeskrMiHt3Duj3r0kTkc3668wWfRqtPMNewNK7ZG97i5H7ku3dJt1uF7uQ8eLo5ySRw+ZWnc6wR8IM98KjXK7jeROEZEAaz9nc3CRNEwR0bu//kOHoklqtxsV5DzPvsLO7jrP4f9D2XzGW7Xt+H/ZZOeycQ+Wqrq7OffK556Zz753E4QTNkLTlAUmDomVJBiTYsvViwAYMA7KfbMmAzQdBQ0o0RdEUh5wZkhNunBvOPaH7dJ/O1ZWrdu2c9157r7yWH+roEvATH8YFFApVKKCwdy381nf9f9/v5xugJobk9TxxmEQ3BYykxmriDQRJJS0vUIUJesFhdBnRP3d5+uR7lIsVdm9tslfdBUVgc/UOL/f/nNl8SMK4SRAETCYumnZVuTlfdEjY0B9A6naNP//Bn1MvryLELoVCASOd5NXBSwLLwkgl2b6+woPma2w3RiOgWk2CoCDFFpVKFmtukzSSzCYCTthGNiGRKDGdOwiShOfFpLI6luthWWdoqsncdZAMg1jK07jssbZepX1hMZ2FuLHIbNInn60hSn1ur1aYDb+AuMO1SpWtQoLEh+9z0ZriWCOyyRB97Q69wRDT9UFIX2F6QpNsTkWXiyz9NrpqQKyxtp2nNWzz4vUpe+s7xL6BbkhEPlTLNURpgYRHqZSjUlhhNpkQRgHrK3n2D15DBOl0lvWNmzx9+gUiEumUie9M8WMVPx4zm3tI4hWcOghc5tYCOW0iCyrzqYSqKUymNlq6gJGyEaUIQglJlPBdB01W0NQEgmRzctLCdUK2tvfo92Uk5Wol63sOlj3D7i9JpXWQBgjBbTrtIb/167/Np599QrmYJ5E2mcya5LIF5oOQG7fz/PD7An5oQayj6xmm8Ygnnze49/ab6IaMEMdXwG4pJvR8wugKt9PttdjZucnC6nFr9zaaUsB++QmOa7FzY5V25ykLN2D35k0sa8pgAJlchWQiQpUlgiDi9s17FPJFFMWg3W3QaB2TSq0iyQus5ZCwAxvrO7jOkMlkTr83pFZP0mmcEToqXjQhmcnw+c/3OT+2EIJX5PJl8rkkot7gnQ/uYU00ynWZw4NjPDuNF10yGxq8PnhJuzn7xUy0JhqDfsRkMmJhuRh6lkDoIMsqpppBFjXiOCaOAxJGAggJbQnkBaIYEgYhghggRBqykCPw5njqiCgCxxOJRQlJVBBi8arpRhfIJK5W2wBxGCGKMb47o9OY0DjxQYYgEhFEDV1LYRpJCoUSZsoklUlSWy2jqhoJLYMQS18GZkZMpiM8e053dPUQEEYCESLqlzWUhqliGBqmGqJqCpIuo+omoe/jRQLLuYMogh9NEUIZ318QBz6anEJVEuSzIIsJ0K6uV9eB0EtjaCK1epXeFwPUtHiVGEdAFL5sJhJDFss5tm2jKgaSFl49aMkSkizj2j6B52NbAWEYokoiogSmqZMt5NFNk3J9BUEC0zRZug4CCpK4xHUiRFEmEq64xIm0hiaIPPzsAX/+r/8V0/EQRQDV0AiCAF3XGXbPGPR8MsUZjfYQL0rQbV4iCwp+HONFNqosMHLbpJI+caRTKG8R2Psc7L9EGyVZTEJSlSWJxDqvXj6j2XG59dYa01FILEyREwnchcrWyn0S6ZDFwsZxpzh2mXFkU0Jj4fpM+8dYbZ3ClkHj9AvGgU9GSCLGEeenPfZ236bX6+A5S4yUgOfmsK0FhpGgWjE5axzxrW9+k8vLM6zFOZpeIFbnnJyMWdu4wXTsIEoB9ZUCx8fn5AspLht9JuNLQrnC9rvX6A7miGYZ2YypmDkODrqUCrfod7pMh0s2VnbYfmOHV0cfEzFHkTbZqBV49eyAXKnAzDtnaU3IpmpMpz7Lmc8yGqMqCRRFwQ8sfvL5R5wc3sHM+owGMdXVFNm8Q7s7YmN9h97wcxJqHiGMCF2dfLKMSMRkOOWy9wxFLhNGC3Q1j2BrFPIqdzeuY72Z5/DynGnn+6yJAYlEQEdyGPdD1jayrK4WKeRTSEqBfr+F7/uU6nn8Ycz9W9/g/PyMf/ndf8Czlw8pJLawpxKGWsN1fc5eXPLmvW9x69Ydfv7JDxBJsFIrc9EcMHUP+OTBAdXyNQr5JJfNYwrFLNOpRUKWkOUl7UaHZGIFdBd/rpLN5knGAa61QE/4zKYd8skVHGdCImNy53YBd+kjyx6HL/ZJKiW+/uE9Muk8+yfPETSFRbuHnnHZKOToTya0L47ZWKszsqeoQkjv/AtCsUL06Mf0l002tnMsHIeXH3/6ly8od28nePDZcwp1j2xymxdPjti5bpBKyOhmHse2yOZc/CCNIMRXN0QhBchM5kdc373FwetzMqkqS6tLIASkkhIXjSNavUO0fJrZokexUEdVYjzvqks3UEIWksfcnpBMmoRezNSZcPvme1xcHhJGV8bvfDGFqq4zGQ0YjlqsrK2zWKSYTLuY+ha1Spmj48fkCyli0adY3GbUt9naqaIaS5ZWSGfaIVcooBZ8zk4b3L6fZ3s3zy99uM1kIPL68DELS8J2ZuTSt3A9i5l1hGJGdAYxC3uELtcZTzuMJl30xA6ymkJSXMbeOZ5qcdQYMB112KyvcH/n65TSdQIv4OXBIcgLXp/sIxSmCFICWdXQMxFHnSMKRhLbbbJc1KhXV3h58JBsIca3y+h6ge7smEU3oL6W5vmz11SrWTTDxzRVJoMBCTWBKxrk00nENRcpnHBro8CGrbBTvs9HwwELp8+nn07ZWnuTnetlnj9/iSSPQA6p5NbRWHJ4YjMczJAVjeG4RzpbZDxzSYge13dWafWnSLKAYa4QikOWzpRf+Svv8/nBI8J5mjCW2NzYIWLBeHSJmVU5OfKp13P0Lp+SyeZAVLC9LpIg4y4TLPQ57daA7Z0VNC1Bva4ghhLN8wkIOpO5S7exJFdJI4oZvGiCQBZnnuHRw1Pqq2WSUkhn0CaX3oNgnYX1F6xWd1ANm/NGm6VvES9kOn2L7vgQw0gQujKpnMPgpEM+VcCxA37+0UuqWxnqKwVERWY+76MERZJmiiBaUq5kEQKRjY0NlnOPwaCHmVB4/vwlu3vX6PeHjIYTAl8gkyzR7jTx/YDt9VVmU5uUXibtR4giBIGPIIhIisrcD9EikEOHzggUNUZzmuhyjkhTkSSZiABVTCCKAQvLQxZV9m5s0+l0ieOYySS6qswMlrh+SCwIjPsz3n2nRjqbwnaXHJ88xVks2LtV4fTEwvOgXN7GXraRliG/9bu/xD/6b56gqjKOE2AvRcyEwkc//YLf/b0byLJAHAaIiIR+gKoZSFKEKJg8f9LkK++XKNdUfvLjT9lY36VcWaHd7TAePebtd98ijiQkJYUopkilz0gndDLJTYazEXdu3adaXKF92UdKxqSTeVqdC2o3thnPB2RSNd5+8+v87KMfkcumaTdH1FeL+N6ShObyzoe/xtOXD7DtiPfe+TpbdYdEXuDFwRc0GjJbW3fZf7lPIhVy/PGEne1bRGkbgVVu3EwRyKf89b/1xi9mYhwJnB73KJfWuGx0ef3qgsmggCAIHL3qspgP8dwI35ORZZEwkAnxyGZ1PP8q7COKAUgBguqgiBGRpyEBoRARBTGxEmO7yy89iQKapyLLApEAXuAjfonwUVM6hpRAiQWCyCcWYuJ4hmsPuLw4w7F9YgHiUEdRFIxEBsMwSWfT5IoZkukcuUwBDBnDuAo1eZ53hSJauiwtl2F/SOBE+NEVg1IUxV8A1hNJA1mWSGomRkIhk8pfnSJK/pfBHx9XmBHMPMJQQMBHFCEWEqTzeWLRRSBJLNiIyIRBiKzLJBIJms0GK+s79PoTNAwERQZRQNauUuqSJKDJEnrCQDM0DF1DFEUWjs9s7vLq1SFbWxvouoqsalgLDx8Xx/eukv2agRgJnB2d89Mf/gUvnz1HICKTSSHq+lUTkSgQxhGoSbY33+PH3/0jzpuH3Lp/k6V9gee6qKaOFqsMey2miSTlUoHu/IizRoeNtSxKMYVIir31BL2exWg0o1zcplpdctk8Zzoa0By4FLN7lPNZbMtm1F+QNEucTZ/QG1TZXPkGCUUgspasrNwhc6vESf812/p9ZmMLI5VAUHyanRZ+nILY4I23djg+FYiiq1NI3xGJoggBj+PDSwJXJV+Q8Swor5dwHBDUGZ99/JBf+6Xv0Lzs0bzsEAQe67ktqltZItnms89O2d3ZRJcC4uWUYiXDM9cnuVpDjQPmswHVjIGqxWTLJWJvRmGzztt7q8T9S1xV5KDdpFZe4eTlEStr18jnDYq1Aq9eXTKe9LAsjxtvbzHsNFhOFyRzGeb2FEHzkZWY1uUZmmFgL0RquRTN1hnrG6skTImTgxmKorO38y6XnS8IQpf+aM5iEHN+eoAQTpgup9QSAmEGLhYl6tUSQtHGcyQ0ucRisWA5l5HlCEUqUsgUWVhNzi8H5IpZ2q0xt3ZuEwsRzw73+fY3P2Q0nGBbIv3JBYJ4k/fe/SoXF5d4tspGrYZrK+x3G/y7v/Er7B/9hHKqQvu8wTtv3+bg/JCLizOCpcOwv2RoL6lkUiSKEv54yXR+QS6vUMxkWCzmdLpjytUUi4lNtZwnk8iyXp5T2Nxk//QZplbGEx2k2ZS1SgXbVDk5O2BlZYdCLWA+aaBSZG/NwJA8zhseJ+fHaMkl3Q5s7q5z6+47f/mCUgiLJFNdTHkdyQgxDAFTXgW9ie8tMc2YlLBFKl1gOJhDOCBhpJlNLUr5HSQhQ6v/EFkpIbo12t0mrre8AqMrJYxAZjS75O339vBcnScvT1iMbVJpg8PHp6xk6iSTOSQxxfbOLpPZnKUbIYgC/WGHretFxkOB80aTldUCP/3p5+xeu81ioZJRQxrNLl7oMVtMCEWH8zONTLpKLI35+7//z/iN3/4OASLjsYOATD6/TkzAsO9jqS1y+Q0m0zSyGVKQVnGdkNlEI1KPMZQyQrLI0vUImGAvbBI5g1Bw8UKX9mUDLwgpVHUMrYBvQONiwp2VbS7PHfrDfbKVFN/93uf0RhdU1zWKxSKCHKEsDazZEimdpFbZw3ZmKM4MVfWZjH1qpRwLu0e5mmI6T3LZPMcJJ3S7AtlcgqOjI0oVk/lsiRan6Z6d8cbde6ila3THLtr1FX788U8YMKJYLZCqw3nnEGMxI5GUUeQ8tVhlubSIBZVisc75ZUwsu7x40WJ7Z52l67AcuLRSfZ69esq9N/YIhQyCkqZeStA8u2RjV6aa3iKXznHRajGd2MSaRdYoo1zO6Q0HJNU6i/kCP5hgKAUiN2ClVEdJegw7Ns5cxrPnaBmd5XROvZZBVMscnZygJQTmHhAlkJSYqf2aSv4OH3/0CXuLddYrq1iBS+PyFZVFFzMh8/TwJdW1GFGRUNwyJwdH2I6Nmc4T+wKyrjKbeciU0ZQEvjejVDbwliLD0YD8VpJapU4YgWtZKJLP3vU9nj06ZDYd4ocesuqSyaYRxJCZ1cWLHATJQdVifB/SGQVNUZAlCddZ8OGH32HxyReIYg9RuKqXCwOL6dBltVxFFQWiYIRiGKzXNlCkJP7PL4mkiFj0r4IykoQuJwg8EzEWSCcSjGYuXqgi+zGEDmEUIgsSmqJSXRX52Sef8s33f5d33vk6pdwG/dERC6fLMnqNqRcwEgXOWvtcv7/Nt3/lHf7FP/sxpWqGwBfQVJ1ef8LTJ+eYpsls4l0BxeOrlpj5fM6wb/GND++jUMGzNCLxCC9YsFb+Dq3+KxTZ5PS0QTZX4OLsgDt37rGxneP4cJ/tzU2EWGRu2bzo7JPP5nj85IesrV0jlUhhzef4ro8oRLx6eUg2VSaREFAUF0ny8O0MSaPC5XkLkxKrm5sIooKghey/uqBer6BpWSbjGWn9FobZR1USuIs0x2f75AtZXh8ekTFv4Vurv5iJR6efYSSgUCigqQlSaRkhFtm7/gbNy2OSWhnLcnj58imCmOaiecJ4ENM6mZPP5el1R8yn4IUWomohiwYJzb1q0pEiNEVAFOMrDmckIIkKkRDhRSGCGCPLMoqm4Ps+8Zdd3r4UECBesThjBVnSkBQFQ48QJYlQXAIRQjTGXg6ZTQPOT0LiQEXEIBZdEonkFe6rWMRIpUmkkmyslkkmtxCiGN/3cYOA6czC+ZKLOhr0WC6XBL6IqhgoKsi6iGmkSZoqyZRBKplHN0CW/Kv0fxggijGrG+uoWgbXWyIpMnF09driMCLwI1zbIZkwSKUy2MGCMI4IgysPa2BbhC7Y8xmu6xLFLoaRoL6yTrVaIlXU2NxdxfV9ohAIHFQ5JGfmCEOffrfNF48+5/z0NQevX+HaHpqmIcsKC8tCViXCKEKRBCLga994g+lgRiZ5FzVtMw/6XK/soWqHX/pOBaSUiyxepe1v3dzis49e0eoKlCoFvKnMqB2i6grN9mtyyRyZRJZMekY2Vebg9ABJnBIGJl88PCdXNkgnN0kIdzFkmWis4aamLGdDPMWg0XuAEJcwsytUa31sx0dQbBJahvW1Ikenj2h3QiaTDlEoY6oql6cj1tc2WSt9HT2xpNV6jWdHzMYOetZAQMXQkrz7zl16vS6Bo/HOO28xt2Y0Fz2CtMh8OkHSDUJJIlZ0mp1Dfvb4AW+98TaR7KCJGWqlEi+fPSTR30BJG6xkN2l2n/KHf3DE9a0yB4cDnFhAED3u3L/H2toa+/sPefjgGaVqDcfus/RCUt4aln+B4mUoVPO48Zz5wmVjbY12s03G2EWOmnQ6LUrVMpPxEt8LOW8c8da7tzg5e4FtB2zvJOkNTul021Rra1Sqm/Qbz2h5qyyDAo4yYzTuI5EjjHzOzxvcv38XRcoSRHOG/avr21uozIcd7LlHuVpjOp+RTOdxHZGJNURWMwhqm2y+zKOnH3Hr9jrd4RmKsEomKZAz17hxTeL5k0+IRIfLiy5v3L/Pw08+RiulWIwXvHHvGocvztjaeI+bNzN8+vQTJLPEdHmKPUuT1mqMp49JprYYj210qU4iK9BpnTOZzDh+fMhbb73D2OqCMMJYGjhRxHI+Iy0ZTNo90rUiy1mILMzY3Nxk2huztmZz985N/vRPfsDOtTUax+2rDdZftqBcLM8IoylHBx6pJBTKGs3mKxQ5wdrGGu12G0VMUCqB51uUKwVarXNqtRoiKc5PRxipbaazGY59zO71Gwz6YzRVpl6rMB1ekDJLnB6G2NEZXqCQzWpE/pxiViWTztBsjFCUmKk1plbfJBIV/MBmGcw42F8goOK4Pn4YUq7cxg1nhJHPcHTlBxv2HTLZLIE7RUvZWP4DXMvn/fe+xnQ6pZi9yXD2EkWRWI4NUslVdvaStHsHhMGSRusQ1VDImDn0lEZ7dMaNG28iKEl+8ukfokhLyuYdNDlD4Or0Bn1ir0gQQNY0sXoBXjClVC4Tp6v8vf/m/0bCFHnz7V36+xdcu65QnOzxYv8J925/E8ueYiajKzGoqWRSVVzNxveXZFI1vnj2kGTqJWJcpdOaMneHaCkVY2miiTlGkxGykOfdN79FYF/V0Q1Gcx487DANOnRCi+86E27VtqjVcwznLhI+jhMzXJxSq2RpnI9QkzBYzJhMZti2hplIM+z76KpIo3FKKq3jLnx++tEjytUU7caIlbLOeDzmonNE43xAedsknp4wVjJIShJdTGFZBcaLGD0pIQYygRcwHs7xIht3qpNOBBTXSrx4fUGprCNIMxazgISiYC8CynkZUXWp1+ugWfh2lkF7ScQc2V/DWAn52lfvcXj8HMEN6M97GCmD29s3uLy84N6dBBCT14scXnTIJEQ2dqp88nGDG9e2EUWL4cimmM8wGPYIQ59YcfHjOQk9jTsHOzohCEx213YZjUbY8wmmKeKFMwqFHPOlzmA8gljG8TxGo9FV2lYYIcYqvu9z784d0mmJSj2HHzqYRoqYDsQRkiiyvlLDUudcdpoUSxky6YDRaIbtOSQqCroUgKB86d2DKBSJWWLPEghFl1RSpzfMXokRN0QIfUIxxHNdJNlBlJdsbLzPaDZnYbtMhhGC6JDKJDETNcJAZjKzkJMCH3/6kJv31vkXf3C1eg0CD00zcOwF7UZIrbZCp/WadKqIgIJt26RSaVrnDpmCgKl7xNGC23sf8vz5UxQlw0XrEbVVg1rqTVrNM6JwgiIoqGKBzc0KnX6DyaJLjEc2XSUWQxTDIZZ8cukig0EbTakztZ4R6iLDbsyN2ysEjkhv7rN3o8ikd8lZq8n61i6nnZdkjTyXrdf4UcDcTxNLLtNFG1lYZTya47o+pWyCO7ffYTI9JZevIVNFFJe/mInJhM5K7RqNxinPXzzjg3d+l9miwScfP2R1NYuiG7SOznj/63dp987ZvJeilN5hp3qb45MzhgOLZw96IGrM/CWjzpJGY06/3yfwoy/f3wBFvfJtJpNJYim4qmAUZaIgJA4jZIGrLm8ifFdBvKo6R4g9RASi2CFGxPdiCDVkJSIWQmRZQpF1YvHKSxgLIoKvEkVLRv0RneYLguAqJHO1ZhfRk2WS6SSpbIZCqUgykaKyUUPVFGQZlnFEFAXYS5/5zGFpWfQ6U85PQoToCM1IEkURelJET2gsl0tyaR1JiRC+FIl+HKB+SThI6AlmCwvbslB0HWu8QDMTBFFIGIeIkghShKrLpIpJFpaHIEYsnBmNlkd8EWGaSVIpk3whiZlPkBY0Xnz+mpfPf8rr18/odRb4oUPgCRRzOn54hVWSFYUgDJFlBUmSkASBYOmRjFMEYZN33/kqrdNXTEcufugjSiK252A7q/zP/86/xz/8B38PTbtFLXsTJZnizo03GHcPePn5FyjGKqqSYukKdA+PCeKARHrJG7e/ytJu0+v0KeYr5PIRo/4AU1ghq+e57D1ntbqFrFSIooj2yQVvfmWbxvEzVm9c4961Tf7wX/9/0BIi+8+nVGvbjPsOmys7tPtnrNbuEAYvyeY1suZ9vvHV9zg8+SnPnz5jc2XOkxdP8b00U1UmDDNsb+4iKGNOTjpUaiukKmOePdrHc2zSqRSlQo3Xr/cpFCv0exb7+y9YW6+xvvomM2WJmS2SyxUIxIDJeIksZ1FrJp8feehJmZW0z3g2Z6kkMacztq5tcHbRZzn1CYQFAjrd7kt0MUUQx8iSxuXlBSsrNzGVDOv1PDDm+HDK3btfJ5VTGHRnDPsjipUknz38Kfn8OvXaHgeHz0hkK9y9sY47HoOj4tk5Pnz3bS4PTqmvVFjaPgolkgkPxAWSqFFdEfiLHz1mb+8GXmihx1mQHBZLj8lwxtjrMbciVisr7B9fsrJqEAsw7IoUKzKXlz2KxV1CP6a7vGDuupSyGY4a57z7lQ/xpXNOumdIaYVsKok1aPP0yYzd1R0Q+7TOfNYKu7S6A8rFPZaLJo43wFQ3caOYfCVBUnNonC8QfY/SRg67L7J0BkSCw3KuIksevXnIB/f2aD58znTuouSW6L5IQtPptk7QlAyhanDUtnjvO+9zdvIKXc8QBsFfvqDUpDzNdkC+oFGvbV41EnT3ubZbZjz0kUUBw1CwJgbDwZi1DYGEmabTaVEu1HG9CZIroKrmVWfnTKDTPcdcZGl1H5MsSaxX3uO88YQocPFdB2eQZG/vHpHUYDlbsLN7ncOjFrZlkfEmWMsZl+193nrrOnKwhuM4CPIJ/b7D7p5Ar5EmlRmjiAVk1SSXX7BY2HRaU8KyQuhblPK7SPUJ44lEt3+G69tIckjgLyiWK5ydjsnmqkxGE1YqawzHS4rlPOeXbY7aLzmzD7EjqFVFTKlEiMTCmTOdDLFmPorUZX1lE8QUi+CEVCbFZbNDvVzirXe3CTybXEGnddkjpSdZ3cxx+61fxrWvnrbFQGJ7dR2fESfnj1guQrK5FL6jsbG2jmmk6VzOUbQA0VeZTR0cd4EviORyCoPugmcvnzO8tIj1LrYMqhtjGmmUVptvfOs+ehTx9NUzMvkCceiQSYuMZ6AnWohagOPkkDUVWS0RLidX3amlLEsrz+HpC5KpDZxlhONGdFs2M80ik8oTCROu75XYuZHne997gVopkcgEFJMaZ+0epcwW4+UFCTNFykwxHHQw5ynUBMTOHFXUmE0GhFGfMM6QTa0gCH2CyCeTzaLqSSYzHTeYUUyv0h0MSSgaK7Vb7F5b5/z8nKVjs716Ay9eIi990qaOLw6prVe4aD1l3jUYJ6fYwZSkbuDaSz785SrNgwHOMsRImQwHTRwnoJAv4UURQiggyx6SIJDNVEHyyaR0kuYGi7lHJlVH0UNevT7EsiwEISaRSDOcjJEUBQmBxWKKoWaJYpdXr56Tyecw9TJzq4nW6iGJ0pcnHiHz+YDEWorN+g7jyRnzic1iKiErPRTNI5HKELEgCBWIBGTVQNVsTo46XLu+webaPT759BA9K+BZHppgEOHhuEPW6jpxHCLKIqcXp9y+k2S+tGg3Ar724Q1mU4HxvEO+UuKi2aJ1dsIbtz4knUkShQKKol1511SVJ49OcV0HRZGuvIORhKJozKcuZnpBsbDKdCSQLU05Pv2CykqGi8tDMgWT09M+xo7Fta07DPsm+WQZI6EzW87wo01QBF4fPMXU1ynk16lXfTqXHqZi4rjnLJcBxHmE2CSRWvDi+StMLU8i7fP6oEupsoPtWRy9foFq2BR3tgjCDDtb11i6PS7OusSCh293KORWEcQu3dETvr73HWbzBnJcRZJDls7lv5mJWshoNEUUNNZWq3R7B8hylvpGgn5vwHDUp1LdotOU6QxnbO1eo9dOYQ/bPNv/nJ3tPd58Z49CrYKvT5AEF4QMx8fHFHNFXu+fIMUJjg+aTEYezcs+y0nIfH4VzEOU/w1cXJPRNBVdiREkiTCOiSIREeHqNFqW8ePgapUuB1ctOvFVwCYKIkQpJoo8iCWiOEI2TdRk8hcQdM9zkAWRwGszGcYMB3B0IBBFCqJ4hQ7SNA1Nz5LJZcnk0pipBNVaAW3buBK5kYAQXzXrTKcLppZNRMhlp00YSoiRB6JyZZv48pTS930UUeH05Ija+ipz28blatWfTCZRZJnA9XBdFy8OUVUJTTUQYxEJgUiKsT0Lf+pw0bxkMWjw4MFjfv1Xf42/9XfexfVWOHo2wrYnnBye89NPeiiKhut5CJKIqqp4nnfVGy+I+EKfRXfJTnWDbCFB1bqJ1TxHM1VAQFVl8imV73//J/zVX/0qn312iZnN0esf8OrUZCWfYuvaffrjJeVUASmaUqsmMM0kh68bLKWQ23ffwLV/Dn4dd9Hl8uKM995Yo3NxyJ17X2Huj9gorPDZy79gZe022UQNddflyRffwx18wNe++j5PD37KYrFg0s9z585dXh48wl9q9HsTNE2j3+/TbPw+o9ETPv3pOf/pf/rvMbMfMOlJpPMiQZhj6bc5P5xx/U4dRbXwPBFB9IgDh5QK+89ekU6ski8aTDoz7mzt4MgOZ8dnaMoqsSYiGCrTZZswDNEig6SZpDE5omO53CllCSYCpWqBWNC4PL1EM3Tu3qnz8kWPm/e/xuNHH7NRXSGXh8lIYzaZkkmZZDIKppbEyCocn0xJ59JYyzm98ZyVeonpUCKTXmMyn5FK65w1nhL4Inv5LGIIQxuKtTK//NYGk+kJHdfmXvo+mQRcts6R5BTZTJ7LRpuqEFEo5NA1E50Aa3nBYh5w5+abHJ1dYmoqyVzASfOSRELDW8LOzg4Hr08RRAUjAdPxhLW1DZZen875OUXlAxKyztPPXlCoyjRPWtzcfQNDdjE1gYk3Yb6osHmtgNWPmPcGDAdnzCOVhKayWIAgiUiSQLMxolrOEPopBGGCLybZWE3h+yMefPGEveu3eHLwBZFSYNu9D1qSQhJiU8FQshhqijgVMpyGrBlperMpzZmFIdQp5VOc9w7+8gVls3HCxmoNIdbxrYhM1qBUKpFIpBj2JsReTKaq0OxcghzhByr1tRLdvs/h+Wsq62/Q6ZyxWkryxWfHDPqv0Co+RadC4AUMOi6TzkvKNY1Q1mi3Rnz4jbeYjGdoRoTk69Tymzx3nuJ7S+YjncblGavrAs5AxA7PyGRyvPfOO7x49gxvZpHNgR8JDMcjKiWVIIiIQgGZFOVsFt9PMOyPyJV04nhKcSfk5GLGSmaXlJqm1eySzGqI4wrZVBl0CVFucXE0QEnabJY3WC7meKJA1Cxy1o8pVJYspmNUUaKWrlAqbuIGbWZ+RDa7gaYG2IkZC7sLokgsyjiuRXFFw12KHB1fshnX8L2IZLLE8dljViqbpHIbvPvuG3g2XDQO6Q+7rK1Xef6ySSyBkTPIZkyYDuiOxqiqxXSSQ5EkPv/s53z1a++i6Cs8e35CoVbAmpoYuoo1CyGrcW3nNoayhmWPOT37gmwqCe4W+bzK8dkD8toeseOSTKkcH80RIovRcAGxxOHrIRnDRFSiK9+jJjHonyIpAVnzOi9OzyhumkQL2FjfJJ3a4/nzPyTweiRyWVxnQLvVY27PSJcT9HoehXKNeq1M73LEVvUGHhb+zCGZyoOroYRzNlYrGLM5Z40JYTShXC0wnDlsXdujOW6TTHtEkk7jso8qL5lPQ+rbOgvBoX3eQIh93vnGFrIS8/SzCZWbKc7OLTRylKsKnYlFFFncWL9JqlKm3bxgGaq0+wMUzyCZSuPbNsVMmq41Ilw67G1s0u12KBY3WVrgOBIQMpu1EQSBZCpFGOZQVJ2pM8UKfWzb5+zsCH/eYq1WYnvusbCvQg6iKDJZjlnIEc2Bx2A4JJFykFNFgthDV3QMdc7IFlEQiOIIX1jgEnLv+g3efGuLRx8PkFST2BcQCAmjmFgIsRcOuZKEqEe0jh8znAZEwQq5tIyoejx+csbu9RtkkjqLcZ/WUR9NKXJ88RG6LjOzHFQ9RMDEMEROjxrIqnLVtY1NDIiiQRAMSWXWuXf3G/zRH/13GIk1TEPgxfPHqGoWw09gqgrzRZNr27d59OQz5s4DVmubBP6cbttCTzvsbO2RMHVOzp5iO3MWnsXBcRtNB99bUK1WCcIRjfMR9TWNKBxzcHhOLlsml6mztbLH93/wr9lev0tD+pRKqUa3f4pkSty6eZvPPvsJe3urdDonpBJlDN3jxx/9U/L5PCfd5+xsXcO3o1/MxGJmk5m15LJj8cH7f5uzzqeE8whDLpMwYgr5MlOrTzpXxieHZzmsra8gCGluKl8hCpdk8gYX5+eIqsvMOUMXqxSKJYJwwdtvr1JNrRP9cpJGe0Qo3CKlZviTH3yPdDKPHm3y6PkzXCtk2LYYz2yEUGLuTFCFFJqRRtYsFJL4gocpq4Q4SKJBEGogRISRfdU4FUkQ8eWDQAiCA3GSwL+yEkhCQBjqiLJGQkyAHOBKIiESCh5yHOF7HrPZGeORSxToBKGLKhpEAhhaHjOlki0myaRzJBIpauk0QiaDpq3yLPQ5Oj7AkIFYvHqgCmwEQUU3F7Rar3j59AlGQkWUQkAmlUyjm2kU1URSTHRDIlYNMlmBKDCIgxjbmTDsDxiN2iymNvZ8QDZd4b1vCqysGmQT32Gt/pTjgw6//Tu/ycv/4L+k0x2SMNP4YYDvuyiKTBhHKKpJKp3HztkcHu2zEt6lXk9QnCZYWEuEWEZWBHKpLD8/a1Iqlbjz5m264yG3dn+Nh6ef8OQkzf13tjCyEZcNhWUoUNuuoQox1ZUcph7w/NkZkVBG1Rc4kwVff+cN6lsVZpHN2PbBEhiHM6JlSLGYo9F4QlLN87U3v8I/+u/+jN17OyDFZPUaCENef/GAzb23mdldZE2gddKmtHqPG3feYHB5wtZbDg/3f8JousRIyuiFDO2LFmvVLMmEw9gaUaqlWFhDJEehXDepROtcz+bpuxYJPcs0HuGjILkqlWKGg/0f8Z1v/ruM1XMsy2K0OGW5UMnk0ySWWTbKHoq8Q3ZdY+E0SBt1bGNOu3tM4F7n3bfrxJJHuVohNiK8MI+uxxRyWQbDMYloE4IWo7FBsWjiRyWiyGOlWMGMBcqlAloiZDrN0Lv0qK3ukjMkTNOk0WiSTGRZLJoEbgVFT3J77xr1/Aqz8Qndbh/bjyhm1qhWSgihTJjp0zg9Y54pks1WGY6/4M9/9Jid9TLSPMHEFjGzeSQvpLqqISgxYqxwc2OTRy8ekC6vMJp2cJcx99+5x6unn/PWvffx3Bhd9Hj71ruQczk7apNOvclqJWa5EBiNVBauT+12hS/+7GckkhHDQZLavdv4gUWjeYlMjUyuSpz1Efw1pv0mA8+nvGVSVtbZWN/BnrUZjYZ89LM/5L3b93BDn+Xco5CB0djFH+jkMgk0oU+ARORbFFIJlvM5gmT+5QvKTKqGqhRIpkzm8zYvXo4wkxL7Rz2cpcDaSh4jFTF61aO6KtFttrAmIaU1lcVkh6/c+Br/5Pkj2rFO5XqN3EaSe+/eYHDscnrxBNubEYRjnn/RYrV+g42V6wioiPIAe6kieCp/8fM/4q03v0K7dY4oiNy5fZeF5bN0bETdZRFMeXU4J1taZdjpUa6HpMQiGWOF4bRPp9elWE6ws7fKZDZBV1WyhSSSIpLMFhmNB6yt7WAKaTonFsmMxHg8RUhm0BI6R+efU6lmkRPJqzWREpEr5Xj66hmiNkRK5FnGEZWtG0RuRCFbptl8ydwZIoZLFsY5ncuI27euISsS1rxLOhshyBHl4k2Gk1OyhRWEWCeZ1BmNpmxtvMX29jb95oz2qUXIFNtd4As9Pvlkgp4xaLYGeK/7rGwUUUWVYjaHqsqcHZ9TKWXZu/Y+pdwNWu0+Ai1GfZ/JuIuqC7x6+RJVTVIvlrGdMV40xDBFFEXDccZIkwLl4gaLRYyZUJhaDptrVURJww/mVNczTHoC4OF6MxQUFlOLoahTL+zw/T97RnEromLK+L5JGCW5uLRwBA+ZCbNpFq2gUkMkryeYLpas5tNsbNbY2lhl34tIpxJIiQKvz56RKmSpF9dpvHjOy/0G6YrI6qrJxckSQwmorSb48YOfMA/HfPjuNmkh5q/93W3u3dDRpwle7p/y2XGMLRTJpEUySpJ55JHM5Vn6KbIFBWs8ZRFNyOXBiOvYkYu7bNEbjLH9JTE+Lj7t/gDP1xhaFrLgkTGTtHszhpMlQrJLJq8xOuhCqLL0A9LpNAmjzsIdcd44o76yRjV3m/PmAbavYKg+CD7D0YQYlViIrsSfpBN7aRz/EffuX+fyrIcrNlDlmxwfXuI6CmKkgeQiyRqiIGFbY1Zrde7cu8UP/vSnJM00lj/F0EUCO0CSRDRJZnu7Snd0CYqJR4dOr0e7HXD7Xo2T85foepo4dskXUyx9k43dTfqTF0jpBbKbwtRinIVHDASRfyVM4BeswjAMAZiPF3zy4M9IpjOIJGi0n5AvKfiuxNKa4XoBqurywx//OavVuzx89DMK2RW2Nt5Ckp4RYLP/6hBV7WMtHE7PjikWqty4uUK7c0L7MqBc3KI/esVlq0O2cJ/JtE02V2Bjc40nh19giKvkSmkiuYOu3WX/YJ+1jS2GvUNca4QqZzg5fUUcz0klyjhWCkSB6kqB8axH4Ptkiv8GbL7/4hJfXbL9rsO/+Pi/4PzY4xsfVMmnPyCaXnErC/lrNHufEggaaXODmWWxtObsH/2Y2zfeZmHb+FIfXRVRwwqSqEOkk82ZHB28wKkGhK6NHc0RZBF76bK7l2elfIsw1KjcKZE1S8RTAzHU6E47yFqa89cTnr86YT7TmQ0s3EVMfzFH0kyCaIyqiZhGEkGQEOUASVThS3ampukEoYLn20gYBOEUVc4QRjahoOFES3TJhMBFFgXwDXyCK7i+YKJrGeLIRzMUXFtGEH1cb8TckhmMB8TBMWEYIkogiqDrGpIskDBkYkEgCB0kWUOQRGRBxnNj0lmV3/hr21y7IbH/ooM3WeObv1qiP7ng8IVFpmhxfNwkEa7w0R8fIEoJPN+6ug59CRCJsckVqtgLiQc/m1CpZGifXTCbLrix9x6dwSkLe4mq/pv/sSiKV+imKLqaia5P92TA9dVbDJotZv4GdaVOwjgkJiSMIkqlIvr8JS8P97l9810kdYVY9tE8lcw1Ddcaks1nmFgtbl7f4vDgFCHOUC5FvD5oUVuvI4oSmi6Sz9aZTxyePHmGj4CRPWCjvsVsMOL2zV9GFpfsFd5CkEuMzl/zvT/7+4S5LKuZiPyWjGDkWHan/OzzTyjVJfy2hKKVWHZOOVt6JFZ0FE9i0nrF0LJ56842l+cnTAZDnMklZnYdQQm5PN8nUzQJ1DSeJSKkQwbTkLvvfJvmxTm1soJvWFhji37fZ2fnOhfNp6hy5iqhL6fJVwukEjHZ1HVevjhlufBw7Igbd+7x6WfP+eCDb1Bd+TmDTpNeP0GpVuf67k0+/vlD1lc07t9b5dMfXKKnDObeBb3OGSm9xt72Kh9/9IitaxssFxK+EHPW6PFb/853yCQEnrw4ZXNzk0lrQPtywdp6jZPzU7LaLYbjS9RlBrIT9o8+YWHZ7G4VuWh0MTdMhstLrImO47ZRFAkzIdEbjKiub5IpWuixjOAXSJohghlzeH5OYmoQRg6d9gmJhEWhtoJi6Mx7TVxbQBBSrG9s0u1NuH79BjJzzo4vGAzHjMYh1TWf88aAw1d9drfKlMp5ZsMiq6UbDEZzrl03abVfIwlFTLWIbmicXz6DWGG7tkp+Y4XLx0+Jzkrcu30NazjEUMsY2pCILGNniTWwcZc+07HEaDFCSVYgpXF+ZnE2crm9u8fYGbO2VoDxv72HUvy3/cV0LkUyVePp831GowWpTBrXBc9zWQYdnr/a5y9+/DO6/T7LmYnvzlk6HRpHAb1ejz/97n/Lu7/yNpKag8hBLOqcHQ85OXtBsaoTLxOIUcTt3TfJpA1qqwa+F7K+vkYyLZIsTKhtVGmNHpEviSSMCloCZC3Gjac4rsVs7nDZO+Ok8xFnnRd02nNaF3OWMwcI0QzQEjKyYpJI1BHkNIulQBgarFW2CBcyzsigcTZkshzgi0l68w5N6wUXg2f4vkejdYgVn6GnRGzXYTDrUcxnub2+RzSL0YIM047LoN2nXiijCCJFo8J8NqRez/P2Gx8giirD/gJnKdDvLzg9mXB6cUSvP6DbnTMaz+gOWjiuSzqTJ4hdOtMTJm6L568PCeIFaWOPREIhYcLmus713U1Et44YZDHFMqZaYaW+DeQo1tN89PPvc3L6isBN4nk+pqnTOXd5781fQQxz2MuIlcoqURCysEcMh0MiZlxcvkLXMiAKLLwBCSOJCCiqQOCLEEpsrlXZWi+xt32d9XqFvZVVLtsjOoMGm1tJIt+nltpEEARy6QzlRJaUmWLz1g6pvEE5n6ayXkXRV9m6neY3/1ad3KrPDz/9E9SSQ3EXGr0HbG1tkEoHzEcWkafxy7/0Aa4Ds/EYNW0g6Wm2t9ZIG2nu7XzAfBBiWTN6fYO/948+pTmboBhZUhWB7HoAusF5W6C39EmU8zx9/hmFsgSajLuMiQOP8WxIf9BkeNLD9nychc16soy4lBGVJG4Qctm6wPV9upMZR60OQ3vJs8N9Ti5PSOV0SmWNra1NUokyX3nvOxRyG/yNv/6bFPMlup0RoaNTz9ZYLezSOnEo5rKYuoYoxohSSDaZQxUUTKnAtBdy68YNfu3Xvk25bPDNb91E0fyrAIwYE4sScQy6mkAg5PD4mP4kIpMyUcQQSYwwDBUhiCiVZbKlJaIyYRYfkM1niOUZ1R2Fl4eviCWPRv8Rr0+f8ezoAXrG4tnLTxAildu3tnEtj9APEGMdQfz/HSVX34dhiCRJHB69YjLtMhosmEyXzOYOvnvlpxtNj9A0hSgwuWye8/GD79MftWh3W/zT/+Gfc3x6QetyzHTsIckGuXwaRfMZTzoM+lMCz6RYLGJNDFoNmXtv3KJxeYw198gX6nzx4idE4pKXBz/GW8hMhhEPnnzEdDmm2d3Hdx2GPYHta1WmsyFJs8ZkMmJlvUQYRjx9+oREQrhiU544v/hM50w2NvfQEmne+MaSN7+hkSrCwlmgqiquE+GFJximhpGASBozm1oMhk3q9XUSySztwRmJrEi6oON4S2L1Etno0em+plBM4sZzps6EcnEHe+oz646QAomTi+dcHAfI01V6Zz6XozFCIYftmKysbvJ3/8Pf5e/8R+/yn/xnX+U/+z98jf/6H/5v+N/+77/G7/zNCu99tcbKapIIm8VyynS2pNWaMBwtsKbyFbfR9VFEA02TSBp5YuGKXaioAoom4UcLIh+EwEHXPBRFAsVHFH3CcEHgx3i2ThjIBIGCIhvICiRMSKRECgWDbEYnm0kiS1de0TgWiKKrlihBiBFFmTAKUOQEi3nIb//u1zETVX79dz7gV/56mVv376LpSX75N26wtVviP/iP/wqpgsDEsvFDh1CI8IWYUBWIdYVYV9B1nShe8slH+9RqNfrTV7z13l06wxNS2RSuc0VW8MMr39i/6XQPkWWRb33tHikjSWswY2f3PnEYQOwjIhKEAUEUsnRD0lqdN++9y6tH+yRVnbPWCNPIMGz6OHbM4cElW1tbvHr1jBs3bvL+B++wv39KWk0Seza9zjmGmSJbuI6cUvD0Afl6ha3Vu5g5gdnS5vadN3AjAZQNGsPX/MH3/x/8n/6vf4v/+//lP+f+tfdpnLcYNId8/cO/Rj0pIAClWkQ+LBA6Pkfj56ixwrjTJJQybJQNOm2falpBEEMsIcYLT5lNRhRqK1ycn9FuhRRLa1jClEA3mdln3LxfJ19J02r2ccOATEnDXaYRVY/x7IJKaR3fMdneqjIeDjg7bvHmvXfJZiTq9TKj0YxKvcBgYDEZprhsTLi88JFEk073mGu3ba69/ZK5/4BRMEdNySiigIGKGeeJnJj3v/ptlixYzHUEwWfSn7L/fMR85qDKLq/3H9CfvcAPl/R746u1/+SIycQjDERsz8X1Jrw6fIruJ7mxleH1iz4RHs3uIa5n4Nkes0mLy4shJ6cW1iRNqzskXytybfc+K9UtTCnJZeclczfmzbd+FW8uEVgLFtYMJ4wJQpd2u0k2XaM1fs5njx4jKUm2r6+xs7PHN7/2FouxzWwQ8Ou/+TXmyyFje0CjP6Gwucr6ep35NMBeRJgJGdWMUFWV1eodlssl7bMxSB7FdJprWztgxoxnU+JIppa/RpxaMhm5bG+tEMtLBD/P9dU9TMVmPlMIHIetNQnfXrC6e4fhQuDl0SX/th//1ieUetri9OwRktHH9opsXi9ydhwgyQoLq0+uoOLHA6QUHBwNqOWv0+00WN8uEYsLquk9qvIegv6IceuUSJCYtAPqW1u0Fg22bxRZX/ufsljMGU9nhJHDMnxNJn+HpauyWC6RhCKZRJ58epUpPQ6aX6DqGqEM7tym2w5J5RPUK1VmcR/bCkgXHdKZFM8/fUS5niOOBeZLi06zjWle9f2OGh5e9ha+bbG2UgE3ia97zCcCxXKVs/MWL7+4wFQM1q4niWSV/mlIsbrBwckXbK5sIMsVdvbytIfPmE76ZNM5glghCGWSyRQ7GzcQ3AzD/gBR6+EFI4QoS716hzBacn7ewPcF4niEpotkMwUK5QqNzms+f94glayDsECQdBonIabpUczl6fZH6HoGy1IQRAElMSdYWuCmsO0YiPjz7/6Qvd0ypp6m2RqRz5jkkhvUK3VG/QGD0SGymGc6yxK5Vd7+ypt88fAFqpxH1+acn11iJBOE+IysJUJoIAgR5VyaducMy7TJJWTytQrdZp8b22XeKW9R1mtYYwuRBNPZGDGS8aUZ+92HFIppxFCgUpU42G9SKSZI52I8b8i4ncedJdlY2SWbjxiMGuSKEqPWOeV6gWQ14lvvfpNWc0i1mGUwDljaHmsbRR4/fMyNaxusVmt0Lk0CZcIPP9/HtvP8P//wAs8CUQ+IhYjrNxL0BgPc4ZLL9pjb128yG0M6XaaQqXN6fICRSOHYHgs7RM0YWI7NfBkSaSZ9a04QzslU8nQnE9JmFWcaEfFlf3Oc5tq9O0ReFy90WKnf4PXhYwgUnjw+ote/xEjG7OyuspxDFFhkcwKr9Rzuo9ZV7R8ihhIRug6V3BqhMGEx9fAcj9KKTTono2gq8TxEkQ1s17s68UEgjgIGAx8/1jHlkETCIAwC0okkvdaAjW0Fyx4hiiLpTI3O6YxiOYETjElkk0yHS2JfJBJVzpsddCVAtNOs3lEwb2T52b88RiBFKHqIgvALv50oXvEEhS9/JkoS85lDGOm4UUBzcISZyjOzQlTNw16INKwRitajVFhn2J1RruT44tknvHXv6xh6nqOjR2TzOcIAer0+CWON8XDE69fnvPXOHq9fNImiIwZtn52tVSJ/wr17d3jw8SNS2QKXwzFqbPDu/Ts8ev45ZlokU84yGzlsrmwTRwpiuIk9PeZg/pi33/4QP5xRqRSx3Qmx5HHv7tf46c/++BczsVzKMJ2PmJ+mSKV/mWriJ0RLl+FgTL2Sp3vZQFTHrKxtc3zcxpUtUpk0q2spVNax5z2yeYUnzw7Y2dhF11VmkwACF9cfEGkVisVtHKfJdAKr5XXidImXT/6IWr3E5ekxcvgu2xsaSzzWSmXOEs/47MknzOZ9LpqX7GxtUyzn+PnDH7K3m2V95z7++xGeP8W2DQTSHBy/JJ8r82L/BeOuw3Sk0x9MWcw9HMdCxCCMIxLJABAIlRBJLGAmQCIm8FQQbTTBIIgiFCXCDRzCKEZSIuI4JAplEAIEUSUKAkKurg/LshBFEU03v+RISgiCTBj5CAgIwpWwc5c6Tx83Wd8NGXfACxyO9l+STqjY8xnr5V1W85u8ePoDVEUBUUIkgSxEiHJ81bUey0CEmYw5PHrJo0/brKyWePhwHy09w531kCQFP7i6L8iy/OXJpAJESLKAE/hsXdvg8ckD2pN1qrsS3Qcj3lAkvIV7hSVy+rxqNSisbLB3e4Pp+Dk/e/g5d+u3yaZciNO8/dY9fvjD7/POu7/Htz/8HTzvmBcvv6DXarO7+R7ZNYPnT57w/r3fwnVCFvMuMgatVou9tVU6kyb/xX/9v+La+iZffPKIcafHvfffYThs8v/+h/+E23vvUXQ6RNoqB1aXzY09Fp1jGod94nTItZVV5v0BorHGWt1A0W1Co05ZqxLZbZaeRGs0RNNXiIQ5zaaEkgjZqBkcHc9I5kKmiznVWpnm8IJuv8V4PObtd96n2Tomm9cgylOrGCzdPrf27vD5w8eMx1MqpS0k2SCIrwS4H0MUCpydH7FSL7Czs8vh4SGNizQT55R05ZJXz2POXl7y5nu/BgwwE+8gKBEvHx9jlm7geFPsaUBuxUCKxqRyPmbGR6bIRjXB6eUhq+t38MMl540zqrVNOoMjFGmHu/eu8/jhY3I5g2s7d6gXUnz69AvWdncRgzTlrEsk+JRLKs3DKYE3IpdbI3Q7VMu3Ec2Q7/305xSrMR5LYt/Em6qsvWnSPdPJl0ymzpiMZiJJXUbDLuv1GqlEGUmdYnkdGkdNZKNC+X6Z6DLF1vUSF5cNCvlVlos+xYrJ1uoWp/ZTUtkNDOM6F+dTarUsshLS707Y3LhG2OsRzbJsVZKctV/QGnjcvHObeWdMsZDG7xS5eT3HabPB3vV3MXQZazFFtEJkVcXIJvHjGCPh8rr7EZ2zmOu7G3/5gvLsvM/5eYe33trBmqgMhiMGgwGoPWrrOc5PxmSzeUwz5O0PVCQ3QzabZ+92ntBtceveHX7+F39OxIJ8fp18IsPOrQKjuMdm+hbHizGPXv4IgJ1rm8ymLhm1xvllA2scECJjGFNq+Tv4jsfSGWKoq4haAzFOk8/XWC3mOG8d0DzoUEisUsya+NGEg/PH3L19h0CI8b0Z1nyBIicQpRlB4FMp11GTAUX2mC377N3e5PjsBE0MefX6nN/6pV9la+MualCh3e/xvZ/8MWldxHNFRKHC+cUpjnhCo3vO6maVmady8eqQWPgTKjWR5cIhdsD1lkzmY8rqNfKZFMNRm4XdJZnIYehpTo8PWF9fRxIjNF3h4Pgzdq9vk058jecPL5FXfbIZlf2zKRPZJrtcYTJdMrIuKGVXGU+GbGp5UqaAbTtM+mOy2Sz//t/8j/nDP/n7xIWYXFHG9+YcHR3w7vvbXDTarNbqLOc+w9lzVjfeYP/VMTEum9cSfPTRPkmzQi6fZDSeEkYe1sxFRCRh2ISRRy61QuAMmVgz5vGS/nDE6u1reMMFYWBjOTOeniz4G197h+eff0TH5oql+fIVt2/tUM8kmS86ZNJriPYW/caM8bhLNruNvxximnmKieuQ7oICzUmbh0d9yukaO3sFRo8m3KlmGVtn5LMZhosenafHvHHn12l1J2TSZSoJAV8LsaMWQlBkORtzcPAKM1Wh3ZiTSGc4OJ5gkiZbnjP3JlRqW6TNLPNlEyeYY4/HpPU8jXkLxdCIrZBqPst+Y0w+B9ZiAoGIaYpUCtmrNC4BRibN5f6AN++vYuouYeTy05+12L1R46233uDTj07Yu1/EWXjISoh7dEgUh+iaRhiCqIkkzCzEKpXVHBcXFyDIyHESPxyQMJMMxQVxZCLLMQgRnudgJg18N0MszwniCFkyQFigmhJCFFOpK8RRmXE/IlEUuH69yvOnTxF1nUw2iSQukASZo8NzUsUcfhyzVilxejIikUiRzBssXRFREa8St8JVGAf4hbj8H7+GsUCn1yaMDErVAqPhlMn4yjZjJjZptJ6SyxZx7BgnWOKPNYJA4+PPfsy1rXcZT0d0B02Wy4j6ag6ENKI85Zd+6ZdoXD4HwcX2L3j7vVsMBk3WVuuUqgZvvr2H74mcXxzwra9+HVyNjY0UdhxTLtfIaBLO0iaZyEMs8Nbbd7Fcle31t3n85GNEbYCiKPSHCp8/+9dMZ4NfzMTFYoEgeWTSJVSxTE38dQ7bn7G1YTAaDjEyCm605PPHn2GoGar1G1jegNcvXnNzq4qR1jES66zUfeYzj72dG3QnHYKljmFmseYBybRHKllHigLGgwiWNr/5V36Px08PydYuaHSeoOXfQtYinn7+BEGyUPUsi8WSSnmd4ajPdChRKlUYj2xuXNvhsnnC0vLwHINf/eXfQlYkhsvXvP9hnmp2l2QW8Nfodi65PG8z6FsMh0suLxb4VsB0sWRqjVksQUQiZoSqKohhDtlwIZJJpVUCP0bA+HLlvUQkhRt5XwL7A2T5qoXJ80PiGMQvKxfDMAIhRhAjiAECwlDi4tjh2s0cgelgjyIUzaQ7PiSbrtAfX/LzTz7j9bMp6ayO60XEUXwlYmMFMY7QdR0/cFksHIIg4g//+c/4m//+Do4vc339OhetlyBe9aJH0dW6O4qiX6y9TdNk3G3ihhYb195DkjUeP/se72h7V3WsioyiyFhWwK07m0Txgma7jYjBr//mX+fi4xe8+8G3eXV2zNHBIXGQ5vj8AT/4P/8P/C/+9v+O3/ub/xH/4B/857zaP2FjN4GipDg9PyaZUNms3+Gy/RxTvIkViCSKFXZ2r5E3sozyHVrdHjfe/RqnRw0ET0Qxl9y7dZvDJ0+xfnbO+ld/DTedo3rDQk9m6Hdc3ljfQnHnrOyV+enjj3At6MkjzIJKJMq8c+d9Ju4lwdhEcoYka5uYhkwqHRDHMclEDk3O0+nsc3a+4M6928znM3Qtc8UBXfiM7SHLhU05f4vv/Mp3+OLZQxoXLc47KkHskM0bjKY2iiywec3g+RcvWd+oU6mUaDZb7Ny4gyLXOe8OqFZEGq1zfC9innvK6eEJN6/f5Pmrh1Rqa9Tq68ytEelSDo+IZy8e8saNt7Gc5VXByWhMvpRmZ2uL0TCkmN3B9x167QGBo7G9WaNvzfmzv/gzUsUKzX6X2BqwUl5HTS+YjxRWt3WEoUQul0dCYNie4TkZMiWXhTsnnSoReS6KrNDt7FMsSczmQ9KZIrNBm1rxDqXSaw4PXrF38xoBEp999pDt+i5aSuXZ0wOkWKG2Wme271Bf0Wi0F7TbC9LJDtmixv7zMbX6dXZ2q4wmU0ZDCyMpk8j6LOYp7FnMe29ucWM84Qdnhxz+/HOK9TTPjkW2V6tIUoHiSpY3bqT57ne/y9b9XydTyfPRTx+xfaNMTq3SH7dpLwYktdxVa9ZftqCUJJntayVsN6RcTUFQZHWlhCdGTEYWtbqJJtcQRI/xYEzonlMtXuPodQdFm/CH3/8jyrkElWIZx8rwyaMD7rxp8/y0x7tv3mM4vMRMCWjiJsN+hKqoeMsko+GERCKBpC6ZzWY86j7k2kaF0NcoZbcYzByKxn22qrd4+voPyWdkVleu4y6T9Ef7bF67TjIl4i9CjLTAYrzA9ZcgzUln03RbV8iHxWhAWs8hSzWmY5/mhYUqnrBSvcFskuXHjU9Zq+xxdHpEd9QkLKRxrCmj2ZTZ2EIQVcqFNdJ6itD06FohrbMJg9YcUzdZLEO+/eGHVGoew46F5zukzA3Gwylnx2csFhaCINC87CEoMY4roesrfPbzYyShi2EkOLtsEx7JBOESSUziuRdY9hxZSmPPfFzb4eK0RyKpYRohX33vPv/B3/1fc231G/zJH3/Gct7ASKQJXJvqisTBfoNicRVFcZEli/FEw3EuqNWSeG6WJ48vubZzm6k1wbJ9ppMhxVyKdDXPeDkjW04wGI0ZdwZcv7XCo+P9q/WrqtI971HMidy4W+bpS4+3dnL4QkyUKPDVNZNf3vPJ/Dvv8gefWYh6gfGpy8QS8aMZkuMhGCFH7Y9ZW9tgPjLIGC3ySp1QsRjZAWrUx+ot6E02KJYjeo0ZljWjnF9Bzqa4sPb56Rc/xBQjMmmDRisikdWZTSOm03PKK2mW3ia9ywtCIUYXdLLZLPVimURW4OLS5+zihHyhihcOiYmxZw6SGZFNZ7CsOSk9T6lYxA5DphMLXZWJEcnIWTYrNbxogjVbIMVVbt1MEdKn3ZmQTBS4d7/G3t4uz54esLtnoCcMomDKfD5moyCjfpmU1nWFTqNL+Zc2yKSrDIYt0lkJMEgYRUS5g7UcEgsaQgyKJBADqqHSHU6YudKXSBgfIoFYkNC1FFF0wrXdGs9fnzJfLLGxsRUbI5FC0iQS6QBBCOgPR1RLVWQ9JghNGsNX2OcaX//mfUo7CV49GZJPSPiOQCwKv7gB/4+iEq5OLCNidENiaQecXxxBLJFMpxn0W/jeFMNYYbY4vhIRUZqFfbWWOjvusbQeoOgucRyjqEl64w7JREiplqXRukDVEuQKJkn9JvlsDdcfUizqtJoXVMrbhKFAqAwYLA9YDk3sYMzG9iaj1phUKmQ2n6GqedqDxyQSCYQox8XlIW/d/w7P9v+MIL4gCJbYiyR7N2/8Yib2+11CBCJpweXwAE3R8N0M66tvM7M+ZRF2UI1t9m4nse0hC8dBiHNs1t/jK+/d4A/+5T9mdesG2QJM+gHdS4vMisA8nqErdeT8hPPmX1Au7LBaXaPfaSDOLR58so6gGaytXsN3noHU4nJ4wsLJkkzZ5LJJBF9E1/OYmQmxK5DKl5hN+kxnNrqexzDTdHpDXrx+zM7OTU5+8oy763fpdaY02hNSyYD5pEN9Lc36Rp18qcjQOkKKbKKoiGGE/PDPXiJSYWr18e00g1aLxSJDq91GkiSCwMTzrKvaT1MjCCMUU0YQBGRZxPd9NF1GVsD1LFRNJgyuxJyqcYXpihQUFRAXiBJIusy47WMHS8LxmErlJumMihfMCF4mCYUQUVURowgIECWBKLj6m7qhEMdgmgaGqXB60kSSrtOffs7JWYw1E3FsF13XgSv8kSBIX67iI9LJBMlSnWD0Gs1RmDinjDowS44JQucKFi4oJLMZFqJAOVFk5gu8PjtF7RW4/+ZbvDw4xA9njN0e2WqFhBlRWfX54x/8Pl996/f4W3/j/8h3f/L72DOd67vXePDoe5TzK2hGhBeWuLai8aD5B1c2pOUdFoZFuaLSsg45+ecz7t/4KqIw5J//0Sdsb9/hP/mr93nyTKFUKPD9k884OrZI6hppRYNOzPa1a0zcEfn0GlrBod/qkC/epVIs8+jTB6TzBd558zpPH76m2z1nOQpJ61lKpTSDyYzji3MSZshaOc1sGuE7NlHsXYV0k3kce4IgSBydPObJM5uVtTrJlMbSHiPJMa/2nxJEAnu7bxDGUyIW2LZFqbSFqs1Ja3WaTZnJsIfjjVm9do+bt65xcbbP9Z1bmFqKej3ixf4Ztc00iujRaGncvf917GmDbvs1cnqNIEyQlj167REbG7ch16XZaVDKb9HtdtnZS9EfTgjEJLYaUCwXyBRVRqcBJxf73H/7W8iVJb32GYpeYuFMqWSrZIpdRv0LDEVko/JLOM4R/UGKte0ya7V1/vC//WckshukV0wCJ4th6CjyXWbKASpJwtihWqkzGk1YzRWJXY9yYQV8UKUyqYxG0L6gslomW8xj9UXu3dvCcQXKqwJjZ0T/soMsyxDfpFoPaS4umZlVlEmNXOKA9Np1cisynlBib/ct9HCAktjAnR+Ty1TxFkN8d0q9ViAhewSWjS5IqEGaYm6Dhf3/B2zQte27pLMK//0/+YfcunGNuzdWabVfMrVGFAt1xsM2vfBjKrUig57HzvptLGvBdLJgfX2TSkkmUTKRfInxYEouCLicHLGTTzPqNsGPiR0DMX1Kv+MhkqZUscnnMoTREt8XibAQNR9BSjEadzEijWp2j3xGxnIPKdXKZLK3GM3bBLLFux98DWsq0m4eYrlN1nPrFAplAneBJOWZjxeUSmUWXo/J0ELNhcjxOoLhUapWECKRIPb52aN/RfNyjuP8OaI0J5PVOX3aIpcukVTTzOc+guwzmsVYVhYv8CgWt+h0J+i6SRRKaLrED378AN8F3/HY2MwyGbXZ3avh+yGRrDKex0hxiuVkiu/3iEKJRCKJoKq0BhM0JcFw2EdTIxRBIPRdEgkRz1KYhxeUV5IYRgHHG9MbTaiXvoKmqbw8/mM+/JUtukMYDEbYSxtF07EsD1UrEAgXGFmLWNTpDwcEYZKlJeE6EW7QZjydoEoOd27doHHRIRTmSKZLbxiiyGl+6dsfcHD6HJkYJZZI51XSpshkHhB2QnTFxHYChr2QkhGzxnO29Qrf/ckl3fEmb3/92xwd/mN8v0kikaZ15iHIIYgyp6/7bO5ssrCX5NIx/cEQ0Va4d+sDjk5OmC4sNlMVtNWYdjvGcy2m/SkpPc3B81NyRhE9ZbOQhvQaKvlsif/Z336Xn378Me2zCZV6CUnK40UTYkJG9oSz1oxcqoBqwnHrHDnwsbwII1+g1WuTjZNInk5gSrxud0jpAcmEiaaKCAkFN1gSigGJlEyvP6a+dh3HWfL8WZ+lfYHtDlhf2+XlsxbzWZ+bN9/kwUfPMdMxq9X7ZIYNNO0IVbk6UanWMoSKy2D2HB+f1epXKdcMXhw8ICmskM8v6fVt5ESM54VImo5q6uwfXrC1mf0y2GARuBJ61mQxD4gji8uLE2TDIZl1Wfghi2COoGawFzbJvI8viihmkkJWwRnqaFKWUjXLJ5894vHTJ1Sr65w8cxFDB0H4kksZgS5f1eZd+ScFZEXk4mTAcBCRTqdpt4/JpPMMujPSqTwz5xIEG1FIMh6PSZohnr9AU1O89ebXaHePGY5npDIpXG+ELpoMxxNy+RssnCmyHmNHTTJGmWUwoFBNkkqB6hicnh1Tr+5SylU4OHzO1naRjFTh7KjBG3e/yWj+DNcPqa2rPHr4jPE4Qy5XwAtG/Os//adcu15jtkiRzybImBkkSfjFTMzk0hyev6bda/PeO99GFkQSgcJHD79PLpci9j1mA5HVe2m6DR0rOEFTTFZWdjg8eMW1jTeQUgLWFG7c3EInw9HFJbmihDMPGI/b+LaCY12tbwXTpbSxBs4ak+4FnYsm5coag8UjrLFHIZMhkVnlxcuX/Id/+/f4x//kn1LfCCnn6oymFp4DRycdKnWDxXzMq9evkNUUy+iSN99b5/mzE0xdpVgLaDVeIUVJnr/Y5+7ebzGbLhkMQ0L5lHKqRlq6xfv3I/KrCo/359y/8z6lZIXpfMKDz39MbWWFTz8+R1Elzk/7DDtLfDtgPArwPO/KyxhFzMcOiaSBJOsQCiiSSCD4CIJMHAeoypXANAyFTz95wLf/yq8RBwPqpTzV0i0ajXOa4wm1VYOPf3iKYUS4S//qhFGKEGIZohhZVpBlmdnURpADZFnk7KTL8QuVD77yLfqDES/3H+K7AbrOL07a4SqQI8QQE2K5h7SbU3JlEz902Luepf95hziOUHWVwHeQpQjkFLFkcT44YOPmCqVcipODhwhpA91UiUKV/bMX3Ln2BqX8DfqTc/5ff/9/ye1bv4GWHHG0f8zX3v+rfOfbv8aLFx/hLXNsrq7xuPkFKaVKOjTJ5HLAkvHrDnvx1xA3JObLc7JGntv1NT54Y5U/+ocvWd54h7uTGdbpjGx+zFbhHWJvxmi4wHKPePU6xXw6or6bw3JdRNekuJ7hvfff5Gff+y7TtRKJqs4e73PWeUYQCcw6AkIc88WTH+E4Hb71/l/lcnzB8eFzUvpNbt3TkSUbQgHPX2AkNXLFBOPhhPffu0+3O+DV6wckU2lSyTIPPv8JmqZRrW5jLfpsZ3VarTZRcEi5XCaT+iYrq3n6zSnTXpNW02Zjx2S4cHCWIRu1CvV8lhcvT7hzu8Zs2cd1bRJGmqXdoDfo0x0X2bt7Eztss3TmxKGBrqWp3UpwcnKBgAqCR7pYIZHOoYoGsrlPzlR48vxHpLMpRv0R6yt1iqUKk6nH0lHY2Mixv99FiI5JFfK88ZbBJz//mNb+Fr/+27/Hn/7ohxjJAkVP4uXhE2SyvPfGN+mMvuCi41HJl1jbrdAcd/BjiWw1wWePPseZFzDVCoVUBaIeZy9neP4IUVpjOoNELgOBRLWSYjGHYa/PpNekaXU57p1DmGZ1U2VstXlTfI9iMubk85+gazYT+yVOKmIkxiQnA+IY1qqb2H6fQslksZToDubEiwmHlxP4nb9kQdlsNhgN89zafRMCjecvHnPz9gY//lEXPwWr9Q0KtXW6wwGZnEokLcnnV5AEk0w6gaa4XLZ6tHqv0XMq+WrIxMpTzuc4aUwpZhU8T0XydUytj6xApbiGNVnSHfRI5bJE4QLfUxgPPSQ5Ymdzg/1nfRaTEarp4YsilycXRILCchITxTqGotI4u6S8XubjnzVY25GwFl0qhVtIqkKmFDM6jRB0BctxWc4fE8k61VqZxXRMv9+gnC+gp1QkPcAelon8JKNxk/G4z8zxMVMZ5vMlRkrh9LiDkfTx7Bm6ISDqKt22RSoOGUxnFLMrhKHPi/1TTDXN82eHFIopoihDqaoyHMzRTAVR1tBNneOTIxJaGkU3yK6CokVMhwIbWwaqqePYMYg2speiO1iyseswHEpksmleXPyM//Lv2Tx78gmb10wqlWsc7ne5dfsGjz8/4ubtdXqjJ7heSDadJhD6KJpKq9skX8iSS62zcIbY1pzaWgpFyhIyorIl4zoqSrSJn7L5+OFP6XQ6oKQpmik6zQVaZZfOoEk60aGQVmk6Nu/uTfEvRnSaDn8ezfjsIKYbOfzoRz8iCAWiUGE6Ctm4ptPtTFCkVRZuh3ZjgpGoIis97t1Zw/STdC6nFJIr9OIOi9EqrnvKwjLp986w4hHJqEpCE9DUkKdfjNGMJGHkEaFx+nqJsKxTrXscHIzJF0VGkyaxqGIqIsWUjC9PidUl2ZRMIVOk0e+jhCJaPYszFRDCgMidkitfZzrsUSyqmAmVbn+MmUnRn1nkVBklYXB6+ZLtjXXWN/Y4OzVZLNq4fh8/7qKoMksL/trf+FUuLvocnLwibsyRZZMgiAAJ1IBs5hqvjn6IoiyRZY0f/PBfkUyXQW4QBD4IGnHsIAkqcQCyqjMeLyjmHVxfBDFGFUSEKKTd6/PO23fYu6by0yc/plpZ4eXJCUEIqaRN0pCYL0Q8wSMQJ1y2ZdYLORKpgPtvvstseIkoJPj617/Kp9/9fRQhgSeEEItEUfiLmRFFAZIkIYpXXdSzicV0ZBMtEyjpNPVqivPzcxzPp1DWCZwE48EMghhrGkDgMxtecv3GdZbHHoaaY2o3sBYu2ULAy1fPeeudGwgY1FZruE6TV69esb6VRdEziEIRQZ4wdy7wlgEr9TpJ7RaKuqBa15lbArN5go3dDU5PGpTraVRJpz84ZGalefOtW0RxQBSXqFdrCFGBs8bhL15fbaXKb//6N/nv/+D3+eyTn/Htr/8Oc7uNJA1Zq3yDl69sCmmT5UhgMn+GKtaZW1NMuUlaMxlOmkhKk3TmLQx5l1JOYzSZY8+XFPIVDENHjDLM53Mcb4QoKVhnLoUVCUFTkMhwfDmm33T49gd/hUXU5ux8yMb2XfaPHyIkzjk8CWkaHVLlAo5l0ur9Kfm1X2Vt4x6TuUd/3CQry+w//wIhXEdPR5RKZT7/uMd6PYEQ2wiSg6wpjKanmKkbBLHH5dkxhpihfX5JJmvweP8HvLn7NS7Oelzfu0UoLnnj3Q3yhTTvu0viaMlq7Rbf/+6/RFUSnB73IEwyn8cMeguGowVLy0M301c4o0C9Wj3HHqKoIAoavgujAUTeKqsrm1w2+qTSWY6fvyKUErS7FvlCEt9LAiGRwNXaXXAQZAgF4UsSRQpJhDByefjZK77+nV/n8eOHJNMCsiwjcCUor1bfMUTxL/ye6VwFST3gWeMRebNGMbXGlEPiSCaOIsLQZ7ZYEBYjzk/a7G3tMpzbPHr8F2wUtvBsge5UpLKa47T5jM9f/4R6qc5qZY233r2HKjU4vzxG0xWCeMDnj05xXI1kOoFeKLAp3LpC8hWr2IMz9Ow6k4SI6RhEskdJTNA9GWDmQ/6r7/4Bd+98h51tgY8/fk25tsU/+fEB/vaE7Zs6HgrLvoXXuUQO+vjzMaqS5eGzB/xK5bfRxBXWN7foTZqglWlN9tEkE8lIIPsCupfm5madV2czLs5P2Nq6Se7dMkEQUSuXePnyFZ6/IAhtwtBlOBmzW72DFEksRhF7O7f45MFnbO+k6Q9P8TyFr35lndlsxNlJ+8u0fcx85pDNXuF+nMWApTeiWhWRBeiPzlmr7aCpSQajLt/+5rc4f/mck5MLEqs6nt2iltsmbZYQtmNkaU4iUSKbXSWRbDAYnFNevUUiWaLVPkeOm9Tru0TenFarQzq1ynh2ymI6oZCtkTQmTPpjNDVFrNnUizcp5Ets71o8fX5ChTX6wz7L2MKaXNA4h7VamWx6BTmbhPSYk5cLzk4b7Ozdpjv9lPPLF1Ty26hqlkGzTTVfRIohFNocngx474OvcNF+hoDIWn2Tk6MLdq5vcX7SIIwjXN9hPAy4cz9Jo+nx9ZU3OXn2iqhmkNE3MeMl/nJKfxYg5VL0Axlr2mF4ZrO1tcL56QVS0WTaHnD31hbHL/tUKlV+41e/wtnLj7l169a/rUz8t095r1Q20aUCb9x/H9NM8s6bv0yzMaJeLZM0FSRRJ/RivFGFlFrFsWzcMKYzGHPabOB6ZaqrK5xcTFkuVcq5t9mtbOK4JgklxNQypLMukhIwHllkEglOjp8ynA2wgxSnzWPmIxGNq47TrbUVLg6buM4l1VoRkQTWQkDRciynDiklTeiFLG2fciWHovQRRYtIUkjIBRr9YzrTc3702c+R1TnL8ZyJvWQq9XGkc5bOgmarSyaToVjIYV/4iNMEorfEGy+4vlIho1RZX9PQxSymXmBpxaSSOnEkYiQVPB9mE5diLotju0hikZklctnrM1kGLEOVy67LaCpzdjxAlDTMZAZRTGIt59juHEHTmDJjMhtjzRTKxU1KtRyLxRyRq3q2Yj4FsgCCzMGLLrPxBFFQWV/ZwvPHrG3oVIvbzMcOv/c/+RvMBh53b+2QUJMkFYl3rv8qurBJXrtOWkkz7gRYC5P98xaBqpGtlWgNRzSHZ6TSGpOORWx7LManjAcXzJdz0jWdlapErValkE8yXXSQJRD1Tc6GoEoC+VDG7nbojg1enPu8Pu3RnV6yf3hAtpTDKKyQScpMp1N8ISSd0ZhbArFgkcJm0e5w8LzDwlXYuLaB7MJ2apXhrIfnZxEiG1F0wDYZWkuQDWRDYtS1CGOYjG1arUt++L1XHJye0+m6+H5IsIwJfAd3biPLKm4kMuh55NUqwTJAk5PElkpSv0pfZ0oGakUjXdogJkRJRsyCOZ1Jj1hSkLUsxxcTnr0+YepGHDV7tNttNFkn+eW6TZRCMpk6qVQJaznn/GQfe9nl9s06+UwR4hhN1/B9yGVLHJ98im9n0dV1Hjz5GdlSgdF4TDKTxHL7yGJMFBvEokwUBcTRklhSWHo+nr8gjGIm9pgoEHFnCwp1n8vLCd5SQdJlwhgib0Y095BcD6dvIVngWRH5Wpn62jV6c5vm+AUfvPUO19e3KdazuDL4gYCqKEiygCDGhOFVQlcUVaLoCiEUxhb1ep1Wq8vdm9/h/0vbnwXZlqbnediz5rX2PO/cOY8n88xDTaequrq7egLQAEHCECyQhEmTEi3bCodtORRy2LKvFHZYEZLCNmmHJcumZDIEmqIwkWgADTTQ3dU115mHzDw5T3se19prHnyRhYIdDkfgAt4Rf+S6yIx9s/L/3//7vvd5/42/8+/T7JySqAmRqJHPLCHrMeNozNgKqM7PEKVc1KxKs9OnPlvB84ZU0jO4ps3ZkQ2ChCHV6bV3iEODQc8lk5aY9AP6/YBETLj/xi+g6xqptESrNeaDD3+bZ08fYrsJXtiiaGh0Lh6SkQSEKKHVmnLrxjtoWkAxewU/CBiOTxmMhuzu7KPr+lfr9GzAWfOM97/982CYfPzwIwpaiVwmi2mPKJauICkwMs+4e/s2kmhz98b7rKy/RnMyIFfPoAsrDE5tFDHF/vE2mbSAFGkIiUKrvYekRmj5Cc/3f4ztaiytX+Ho1e+wSIf3N97gN77/N/j5t/4e7dYxS5U32FqpsmDEmOYpfpzQKKRxehbDlsfR4T71ylVODy949OlDQtthZWkJaxIxHaZxgmOILnj8tMOtjfdoaFXKaolp4NC7eEil1ED0QE8qRIlEoHmMXB/HGxEEQ3w5JBF0ZCnLH/3gYyZth5ePTjhtnXGy7xEmOm+/X+Ctb73Lv/lv/33eeX+F/+n/6j3+o3/0b/Hv/Hvf5W//929QyV3OK4bClCQSkRKFJBaQdBfHBlUt8M4793CnA1x3wKuDc26/ucbu9hTHDpHUPEgJSCKqJqKoMoKgYOhgm/alaUyI8D2bVFbm409e8Ec/+hm+bKGnVgjjAL50dkdRRJIIiLKA78HcfJlu8xAvylFIZSiWqvTNMQvVRTTNwDJtEsD3m5zvPwADWlMP+/AT9NM9TAeqixVm5j1OTp+yNL/J7Y1vcvPG61hBB9M6w/J97t3+PltXXsN2e2xuLqBqHhenO5wcHJPW5+h323QGF4zDMqZpUlRTPH7+OROzS+jbbFy/iiwUyIgGqxs1uid90hkYjFt865tXCeggSJBRIrykzts/9zW2Xt8iElXSlYBSxeAPfvBfMA1baIUqO7tdBM/FH1sUMyUaN2YYqT5JTaZr7pNJa2SrOvu7n4AlctLc5uHjI25c3cAdjQldgbnZZYQg4OB0h50jiziwqEgN7t/7Ht7UYGt+i2IqYTKZXCY2yQGaEqMJAnIyIPJs8kURZ9qipKVpFOdQ1Rlu3X0dQ1KJ3ITZjSV2Dl6RKs9y9eZNaoVFWi2RaaQT+BOivo87sXn1cIfm2Svap03q5TxffPwMe9Ljar5GMClSKNVxA5d284x+8xQSDS9WKRaLVMoGimKQTefJyhpEUx4++5ipo3LlyjXKeZCiFMWMQiETY4Uj9i9e8bOP/oyfffoxQiSRz6VIVBFFL1PO1biyscXuwYfYgxPsqMfDB/tszNd57eoN3rh/lQdPfkpGM8iqBRqb+xjlR1jWHhNzn1p5Fj/yyVcVgqBKQV+hbwfUb2+yMrdOo9GgUKxenqVOj2cvnmMNHWQjg6aqnJ+f4vsuupVAogA5IiPidHjKJ4+e0I1LPOv0/+oFpTlMaLW3efTit0mkNk+ePmUwGiHrCelMDdeWLlV7ElIuzCAJOcJgQrEoIErw0ZOf8OD5F9y8c5Ura7doHjuc7DU5PHiMIkbsH+2SMdYwxxmWlpYoFivMzjSYry8Sey4aea5urlymk5QLHB41KZQKlMsL9PsO6WKDvcMvEOI01nTIzvE2n798ykm3RaFeRquG1Bdz4FXpTWwCLWSaWKxsVGmfBzgq9MM+niBzdDZm/+CIqxuvc3RwyOzSBb/6Gxv4vo3nyDQWZ3E8mE5NfCtFtz+i1+uQ1jU836VSniWJNFwnIiGi1zshDmQ0RQdhgoSGN5Ug9kiikMm4Q4TH8UEfc+BjmSaKnAZRQJMllKhMrV5k6vQQFR8jrZEIEuOJz/lFn/NmD02L0FSBtJFCIKCYnWc0CDk43kbTigzGHQaDEc9efMbrb8+ysbGGbbvcuf0Wd26+T33ORpJjJFHjyuY6oeciiT7d5gXjfg838Bj2xgxaNhm5TE5ZZTKx8AUXKZXi1tXr3L33DlPXQZMy3LvxPSRBpzyTYqZQBTPPZ4c+Z1KZvhRxZKbpaTPs9boIgYQ1Fun1OhipIrO1G0RBionZo1QSEQWVtnVBKAWcd1o8fP6Cz5+fka7W6HYmiLaMIipkjRS6WkTQFVwCxmOPo/NDGkt5BCUklSqhpwTieEqjtIYq6JAomN6ITH6RbKnGZGpjuxGTscPO7jFJmGLUd7GmMPVtHM9mOpWJApUwGHNx/grfigmdkIyeJaeXmQx6zNYKCJHK9rMjphOT44s+v//DH/HJZ59TzM/QaluMJz5jy2Qw6fP8cMrBxZjRoIqsSvhBgGU6qKrA4FggSbKkMyKdlksup5IkCVEcEPsFVLGBKP05+/GysoJ0eTD7gYvnO1iWSxB4OI6N6RwytJ6B2GJrfRU5CVhZULj/xk2WFlbxfBnT6ZOEKnOVWaKpQycyqS02ePD5mIEt0Bp0OdlpcmV+lsB3iaLoyxmyy8NYkqSvnkVRJLBlWic+/87/6D/k7/39f4v/5B/+b7A9G1FMyGQiTKeDqmlUqmkKxTTmuE86m1CtFQgSk0xBJRYlhqZNOlNjZq6CIEc8efkJUrzO1IS0vkCtvE4hXyVjzNMftXi+/Zy9o22K1TxXl1/jO9/9OTqTc4rZCuVKlvxskULmCk4yxYm7lEolxm1YKr/Pq1dPEeMsYlKm7XRQ8ypxYn+1lhcy/PgP/4DWjs2vff/voak9VhbvktHzfP7px0ThhDi20KQ83abNYHzA/vGnnJ3tUSzMcHI8QKKKgEIU+sS+RG80xE7OaPaeomlZLGuKgMfVjdeo5hcQ1RHf3lqhFHVo/vg/Y+7it3lv6SVvlBK2hId8eyUimLi0Oh7lShZBXqPUKEA45NbGfd67/99mYg45PPuCQkXGiyak9DJLizNEjsInH1h4oxDHbYEE6WzEzEyPnmtRV1ao5FbZO9ihMz1mr/0pkqyRjl8jm6wyPGwyHR6hSDk2b8xSqkXkFwWurzcQ9CKh2SMKa/T2xhzsNYkiONkeY09EsnqGe28ZIHtEoYSAgiQnBIlHlIAipxhPeojJJUez1xswP7NErZxheeYqjz/qIYsKvu8Thj6u6zO1PAQhQlVlZClNELpoao44EhGSDCmtQLc9ZtTTWJq/SrdtI4pfBmDEfFldh4QYhJhMNsXZaQtr4lLKLHJ6+ArPcpkMA1zHwTAyiILKldLbXHvvbZrdAcNWh2/+d/7n/Jv/43+PrHOEa5ocH32MOwn59V/+X/Mbv/4/4eEXu3S7HVSljKrKvNp/gmVN8WyFne0DpvaYxmKKQtXi2as/pVDJksQqiubwfOcRulpkcf4qpjVk9+SETr+P43iszC8zPG4j+BrvvvcdyoUa46aPLmkc7h8xdabE+h5/8MkfsN93UKMCulpmaXmW9fUrTCYhb91/h/XNRXwsMvksZ+19zIHJ1BwxnUKhkkbRFM47Z7QmZ5x620i6SK4aMDR17t69Sy5OOD/cJtIqzC5tESUtxpHAB9v7nHQvWF5eJt+Y48JzGQ1OcawRu68eMhqNcD2TXt/k6PgAzxW5tXAPLcoRTiPEMKKUqWHZPYKkRfvsiI2NBSxvQrG2xsLKMtev3kCXJTrnGrnqHFM3YuvmMqEZsLT1BnYUkC7P8O7XlqF0wutv1Dk+e8DJ+Yir95ZZv1Pnys1VMkURx/GYDkOmUwHTNTGMOQ5f7VHKiTgmzMzW2Ttp4XpjoiBFrIbsHG9z0drDD7pYQZuzfgfT87h79ybPn3+APUlTrs+xcWsLvTDPN77+bfrmIefnGqqu4k49rm+t4JshK6vw6vkLommafDaL7/ogjVGkOuXyHJY9pjZbBMGg306QJJl0Oo0oZXn67BXdwYRyPo8QO/jelFJVxHNUFDlDKpuhUCrx0Scf0uycEYY6hCUk0cAx/b96QTk0dynkK2Tk60jxPJ4bYzsxtqugpSqoUo3dlx1q9SK5QpZSqUan3/uKgaSqNpGtYg1NXNOj1X5BuuKRK80ytHVS2SKTqUU651GfrXFx3iKJY0LPoZgqktE1nj/dZmf7mLPmNoeHDlPHZWI5vNo/5ORoj6X6a0wHY3LZNPXlEnotg6sHPD56zm/9zjN6oxFu8pSVKyqRFXP8bMSzR0c8PzknEmwkWWTQDQm9OlGS5ri7i6hl+PGfevzoRw84eOGRzZe4GO5hiq8QtCGNxZh8WUQSRSYTC1WVGU/65As6jdkKIhKRd4nV7Xf6hJ4BQkI+92UebxKiqClEMUMUReQLKdJpHUWS0eQ01XIDVRBIkhBZiTk53Wc86eOHIY4bkUqlCYIAZyqiCDrOdIIiw87LXY4Oz2m1xxwdm/T7EaFg0u7tkjEWePzZOUvLDexpyD//vf+AduuQXmuM64+Zb8wy6rsknkBOmaWUyaGHBoacATni1d6IzuScbL5BuVymWmwwbIVMpyArOqtL11hbuk4YR7x4/in2sM/K4gbNgc04ydCKKnxx4hKoU4qZNUYTif5oH0HqMzu/wHgYMDNTo1xcRPBnmYxCRq5NJI8ZDRwmox6KLNIbjDjo7uAZHg+efkp/0qM/kBFFnTBKGA5HBLaMKueZjgfEicOrvRa1RoFiXqdcMMjlYjJ5GavXZthskhKzxL7IaDxmOPUxA4G2OUbTHFwLfE/CNE18L8H2PFBslLRAKqWzv3tKEstEjLA9E0VOkS+k0fUU7YlLZzAkm82z9+qcTttF0vPImk5v0CWXK+LboCkCSWwQhBGCnGBaU84HRzzf2aPZdrHdCeNhgGuJ3Lj2Gl98coxtg/DnIlK8NMdczvolhGGA57k4jgmJQfuixfrmDNc23mX3RZvmSZe8UaCSyhFPNQSxjyCJrKwsMb9YQiJADSWOdp7hWoesr6bZP3zK7GKWQt2n2FCJEhVRkC8DpBPxK2Ebx5epMqIo4noT/s7f/h/yjW98k//Zv/sPcP0e6VQO3xOZWiF9d4SWzlPKzNMoSrx1ZwVD1ehNd4kyfVqtEY35KnLKwvFNVMOiWJbJ5GAaNJmbXaZWrjG1QhA1kHyGkxFn7WcUc4tMhj4Xwz2arWNeu/MGnXGTTx7+mNZFh9PeEbatc9EckskGTKxjMqkUmhEzdnq0eybpqIRtvcKNd75aDx6ckM6mUNIOhy8S7l2/z8TZwbUU7t3bwrbbFDLztJtdzs4uKOZnqBdv0usO8MMhI+uA7uCUufkK40mXRGrTH57RaU0plSrUZ0qcnrQ5OxnQ6fT47Ivf50//6AGnVoEXikLmzVX+6E8/4A9/+FtsDz7ih7t/wA9+8Pu8PD8nTvJ0d0OMtEuuOMPmyk3Ozl6w+2LA7Zv3EYQsrd6Yk4ttEgLmGjNEjkDgpri2eZVnL19gixYL69dJhgGrK8ucdI+wJy6VUh0xfUG/JzCetFhe0biytoCm5pDSIXsHTylmMuy/fEwmriL5eRKpx3m3TRyuoQltbPcQRYVh75xeb4/x8IjWWUC5miJOEkQR4kT4Et0T4gcRekrhow8/o9u3qDSWqDQqtJrn+FaB06MxWtrFcy9TdCRRR0DFdX2iKMGe+iSxiOc5XybtXJp+UhmVF48uWJxdYmV5Edf2UBQNEP8/iAUQY5pj5mo3qVVLvHiyixjnIJaJ44gw8i/B5sBR/yU7H+4xU6mSEqf8k//rf4rY+DnWrl5l/+Bzbm78MvO1O0Rumt/5g/+Ebv+Q4+MeklAga6wwddpMpsdY0zGl0jJvv/GLlAtrPH16Tr26ga6l6U+OOW29IJepc9J8RUCfer2BbGg8+OwzRNIoYZ6DJ3s0O7v87o9+k48/+QDDL+BYA4IkZBxaqN4837/3NQxDI9A0Ou0xn372AU9fvELQbX7rd3/GxAnZvH6LYq3A05fPePTpJ5QKRX74Zz9EUeuIQhXXTnPnnbewQ5/Y14hcib2THUauzI03b5PNSHjTCXNLs9h9l8hyWJmfp2zU2H7ygNFJl/dW3mJpLgeByExxDc8F1wvRU2lkFTrdY168OGPn6IwXh2eEks1PP/8TXFFEkGaoFOYYThzG9pSXBz/j4mxCY2aJai1HQJ8w0Snksjx9+Zh+ElLN55F8k9SrQwq2yPr6N3nrap5vLUl8bzVC92M6HZkf/uCnCJZKPV/h29/611jbnEVPZ7GmHvNzi7x8vM9gsMfjh9tIioAkguMMGY4clpc2uXV9DXM4QYhEAl9kPLE46Rxy1gxYWJ3j5bOnWB2HzeUcp/svyepVUoWQVqfD8+fbFFOzTKwmj54c0Wte4/7X38GLA9568xsUihnu3FygUY8JnSGJkMMLHdAu+MEf/gGtizGK7uCFIoXyDLP1WQylwKg/oH3mo2kBjXmDTruPpPgsr14hm8sxMbsUKhkmY49CofBXLyh7gy7d4TGd7gDPTZCNLjOzCY4z5ej0EYnexQ1jYjHHy50zDs4u0FN1hiOZ0/MxuWKBwbBPMVfHtqbkszm8aQMlVSAza7KyUaff8+n0fI73YzL5GoN+gKxqxImHoZWpVzbQUgLZzAyzS0WsaYAbdtncWsEKdfqOQ9t6geeHEMQ4owvahwfEk8th771nJq+em5wcuCiBTNWoUktf4erW1znvjuhPeiwszxGFHoWiTOv8Ansgk5Yb1Arf4mvfuMXPffe/xXvv/wKxArVaDV0rcvetRbauXxLtJVHH9wTiGKLYxXGm6LqOZqQQZRnfi4AYPaMSRDFGKofnicTxFFlVkJQMhXyVWrFGKVPg3q0bLCxWSBtZZFllPHIJg0uYaej7TMwhpXKeKIoZ9KdUSjWUuIBvJ5jWgHyuhhvY9HoWjpnBt7McHpySLnWJhVNkWcTQ8yhihpu3GyzOzSIKEVevFslqBoo8ICXVkaMU+bzEeBiwfqNAFGZwvAGd8wApiXEch/39R2i6yOHRCSdnL4m8BCnMoUkGY+uQfCpHKb2ArlVYXK7RyBbZmq2ydX0VQo1irszOducykWeyT6d7wWh0ysJyhoX5Cu1uSNfr0Xd7/Oyjn7K/84hyKs2gd4qSLfD86JThdMrZQQ/BiVlfvIYUZ9h7ckIlN4uiBFy5eosgEfnokw+JooD5uWWmVoBeUMgV66TUBcSoiGcnpMUs1fQcSqAyMn0ifBw3RtFCBLWFKCboYoVKKYesqSyszaCm0lhOQLt/xmAyJEEkSGxULY3juFiWzavdQ6Z2wO6rPU4u2kymMa3zAyyzT6t9znBiEiUeMR6xAFYUsbp+nWfPX9LtW4iyQK8l0b7wSGUNTGv01ayiIFy+e0kifIk7iYiiACF2SMIp49GQ+dkq8wtLrK5uMVNbuuQJhlXiqE+t3GBzc5ZsJoXnWhCpGLrKcuYK4Simc77D5mKD05Mjfvb4D3CEAX4Yf/Xdl8ac+KuD+KtDOYKj/QH/5//sPyCQztBSIr7vY5sOSSgSjSTOdn0UpYgbpRhOJVbXr/PWa99kY2EJSfHxfR/NuKwcSaIIScyg51OuZNjd3cWyWwwnB5RrErI2wfcDltbSaLk+iqQSRBOSxGbcEcllGpQbEpPJmGKliiKL1CsNiLPcvv0GJ50PabUmhJHP5tUsODIZeZFBN/pqSVpMvjRHd3BOqZEwNdPs7T+nmJ+n2TqiXi8gyh61uRhBjJCFGudnu/R7J5wetpDQcGyTXq/Hs0ctLk58zo+Pub5+j8kgod1uc/3GOo7n8mp/j1iI0dMl/nh7l4+et/ng0Qzf++/+L5hfuorqLTBtebTlPGv3N7h2q8Zy7R3m5maQtSqPXhxQmanz/OWf0GoOyeVnyBU1TLdJb/SSD3/yY4YXAf/+//I3aJ35vPnaOwxHCSfnNu5xhLt9xvbZCd3ec0z7OZNRm5X5TXTVQFVzPNl5SGsyYdQVqaYFHGfI7OYm+VyGsR2xUb6Kpkv8+Cc/JGusU8usU86tcvvuG/SbfVRFoFxYY3G5jmkN0QwVAQXfi5HkP4+7FGl2QrKVCs/39zhte3zj21/jv/ntHzC2EiIxj6xKREmIpIgkYoAsGRgpic3rRRw7IJ1RCUIHRJcEH12Dzz95RLs14uWLB5d//2W605+/v5J06fTOZDIo6SHbu18QRQm18gLlYomzswNkWSZJYmQxYeL0SZVnkbUcajaLkXf4nd/6x9z9+vcQyfHowSuKhSV+8Gf/O/aPP+bK1gKv3b0UdZEnM1O9xurSHQqFHAgOL14+4/ioxdL8LerVEsWKQGNmmUqlxOJaETewkJUIQWsTmQE3b11HxKeYFxmOWxSrM+gpjbVr86hlqFUWCHtj5vMzfPPtv4bTNXDHIwwxplw2mJ9bpT5T4un+v0DOdEnlE37ywU8ZDmLe/tp7lPMFbt98h7//P/i7/MmffUSzfU5tXuT43Mc8T7CcEZqRZ66e5eWLI2KjQX7mNrV8jU8++Qn37n+freu3OTt9SiFf4hd/5a9TqlSRfInW8YTQCdjauEEiOHixx+L6DCmjTLV4izPnFKEWkJ/PMnUmNGYWkVSJs/YZru3wbO8VghxgmS2GvR4Pnv2Qn3z8Kbfv3eD84iGmPSSJVdaqq3zy6AvKjXWS7JTOywfs/fB3+c//o/8KoRcwlx2TSmR8J+C9r7/J3/xbf4vbb98lkTIsLb4OwMHxS4xMls0bW2zeqlGqD4nDCNt2mU5NehcRG8s3sIcp5uubbK4uknbg7eurHL78HHsyxTLbhJGDZmQ4PW2j+iFyDKLs0h1csH5llpe7L0nl60S6z8gZ8vhBxGiY4pPPLlmqrVaL3oVP67TLy1c/YefVK1xbY211CS8Y8fLFHrOzc4Q4fPzpT3n6/BlT2yQIXKJQ4MXTU2QxYffFIZ5vkUQiE3PAk+c/YThpkk5n/+oF5cQ7oW+9xEmOuWjtMBgf0OocQiSjy9AcXCBqeU4vppy2u+ycfsbLw0ectJoMrCnHTYFXJy1enT3l937vR1imS6v7Oednx3hmnsE45K1332R5eZnD4xadYZeJN+TF3iMSfUq+uMJF65Sl1UVMC0Rlgh9YEAsc7D8nCA5RjAlJUmeh/hqvbd3nylyDd+6+SSat8+aNt/mVX7vBzWu36FkCjw/3cTWXcnaeTDLAaifoQQWzPSVvpPn4z55zc+tbpFI6RtZmYlkgTvkXv/t/4/MvPkKTqxClyaRL2FOfMDYpltJkczqpVArXdfE9h6yRQ1V1ghAkFYxcSK22TBzmL7NiEwE7cFG1hJWNZcb+EYF4SqEAumrje6fkSyKT0YDpaEK5UEcWVVzbJiFiMhkwHdtUSllkSSCtG5QrOUrlNKomoBkykixyfn7OyGwzHNt89NFHZPNpRgMVzYCFxTqv3XmbciXLdJwhUY7xbZNvf+M9SsYGmxs1YiOi07FZWZohsUMGF10Er0S1XCQWZErVAr1Wm9FgQn884ZOHf8ZsfYaZSoMgGZPVbRw3zVlnn87pIeN2iygWUAvnCMGIb37te+jiEpFgMRk7lLJXcdwJ5UINa2Tgdpqk3DopBGqlHCQBiZDCTTRmVzaYrV9DUvOUZ0RKqTzrjQYLs8s4wZQrW3Pgalzd3CSVtak0Snz/V36J1SsNptM+S4sl5pfmSadFgmByyRETBAxDIA4dGvVZMsUqgeijplQq1Vl0LYcsiohCwqBl4VhTPDvENidoskbipzAUnTC0UIQUhUyaUqGI5wbMLy0SCzHHZ6e0OkM8V2H3qMdB02XnosPEDkhiCc+NUEQN/ALPX+7QmF0hly3iuxKyovH42XNarQuMtPzVAXj58BfVFSFJEJKIyEvwrBgJiYPTx/zW7/9X+NgousZw6KCkHSxTZX/vGVLkIwQS+XSZ5cVN3DAiJMCcpPCsIl4QE4YqzkQncKfk0vKX82bJV/uFKIqXKIsvP4aR5o//+I85az0kX8gw7Hq0LobkczWSKMZxJTq9Y5oX+8iJTk6bwzZ9dnf2qNXWWJ6/THHSpBSFrI4WN2id9JmfqTMeDui2zxmPx6yubBCFCUeHbRRBYzos0OuMmJhTVmc2CSebJFLIxdFj5KBKpE4oZWpoKYFCeQbbl/nk4w7tnsny8iJaLBB0dfIzebSsQb8tf7Vmq4sIwRQ5BlmQOT47QpIC+qN9iuUU44lNGKpMnQluMGT/4JTNzQ08L2Cutsm1tW8ghAVyORVBHhPFNlev3ePJzp8ymvTwvJBX+w+wpk3eu/9NvnP/7zJTz3Fr7Qqzhs6Dz1/xL/50wMcHEXO33uPIS+NRoH8h8Pj5h6zcy9EfqiCJXH9tnmq9Qr1hcHK6RyoTcnFxQevQoJxbZWXhNreuvc6Lzw8JApPmXo/GTBrf9AhinRkpxTuv/xLN1gW+JzFo6xSqLqoh8uzpOZ2zDBtra9y8/R7Xt+oUMhmghqZmwZ2iSSJ+CHeu30XSVklrPrl0lleHfeJkyFx9iUr2Bq/f30KU4kvyRZx8KQyFS8SQptDttbhodjG9Qz76/LfxIotPfrZ3SROQZYLQQRASgvBSZJkTh+WlEr/2G68TBh6+HyGL6S8zy2UURcNxQk4PLMqFOmEQoCgSkvTnFzKBwA8vK++RSLO9z8bGbfK5ErZ3wXA4YHHhGoqqgJggIpLPVKleWUCRXcZjj0gX2X7yE9odjW9/769RnU9h+S0OTnfRtDxhPCSTVnn25CmHZx+STVepltcIgoBqrcj5+Qnzi0UKlYDdvYfs7u5hjhQsd0xnsM/83DKeIzNoZdhc3aQzOiNIbGRdpL5aRpDHmN0Bs9m30MSYSA/5lb/+t3jn9mtUZxRmNysUijWGLQfXicmmGxSKGTxHJghdAq+Mkc5cwubFDPdu3eGjz3+f9kWab33jOzRmq/iTOZZm3+I3/vW/y+L6FXrjKW/dvM/3v1OgO/wRhxcPaKyWSAY2P/rRj8jm57nz+hU+f/mn/OCPfoooZ4hLCndu3SOKTDqdDrdvvEe1ukl/MMLIRTQ7z7h6bQ3XyqImKmVdxG/JzJYXCMMzTg7OKRRUwsgibdQxchNGo8Fllvtoyq3XrlDQMmSLJca+wxtvv0l7YKFW5/jtn+3y6aHP3De+y++eC/yjn5jsNiVuX7vCb/0//yX/+J/83/nD3/1D/svf+j/xf/iH/ymjwZA4cfj86eeYkUUqncWLDrh1e5a7r3+b2fk32Nxa5tmLz2ibRzw9+H2OD08ozaY5OWuiUeLN19OEbp/FxhL2tINhZCkWy4i6ycbGGu+/9+uoco7N9Q1mqxtkjWWmQwchUkgZIsvLy8wvrxIlGS66DrXZK6wu3GJzY4N6rUSpVEDXU8wvVDk7veDiyGJhZRktpWIYNYLQw55GzDWqBG7ElZVNRv0L+t0+GWOOSrWMH/XY3jn8SwvKv7TLWxHnyGUj3KDN0ekxq8vXuX3jLZ4+vOD44JTa4hymNaDTO6M7OKdYjej2BtiOTtxL2N4+QhRimhc2jYUKERq9QYRqdDGPTTxPx/N9BoMm5ZkyrfaAWBjR6zisry2SLR0z8dp4pwGylJAr1Dg5aJLWC7zz9dfoN6d88fwZy+sFLprPSYIGhCUePXtJtuCRVme5aJ0xjQeMbdi6tkLkqQhyyGg4YX0+RSFbhUgBwefOjSUSMSBd1oAyjx8/Q1ZtljeKDEyLOBpSrld5tX9AkrIolOd58fwVc/MVwijCc0VSRp5ISpHKJpx1jhATEIUyrhPQH43JZCVEJSQMfFS9zM7uPq4/5NrmEr3hiMXGAufNM8LY4/q1NU5OLgh8FTe4hANrWhpVkpFEjdG4y7XrG6QNhWc7n1IozqCmGownbaJQIZNWiWKPVCrFdGozmjj0Oibj8ZDV5U2ePNljbXOGjWsxjx5ZXFm7zrWrmxDpaBmXv/nrd/nB7/835PJpDg+7pLMi7771LV6++IIwEmi2z4n8NL6l0e62qJZVps0OfjymVouJR2WG7imFWo3JdEQuG6EVJY7PHUraiPPzV1jjMUvrJbyRhJZOYbsqf/07v8r/9j/+36PndFbqEXmjDFaGUe+ApbUqvVYLVcwzaPdIKQlrS/NM5AHXr73O88NdGosNYikkGNi82ulSblSxbYf+wKaQVzCnEwajLJJsoogBttOhWpmhNr/BZDqm3TvFTcqkFIHxMKbWKKJrWVqtCZPJgHwxxvNkCoJKqaQx6k2JHQ0hDJmOJiBElOdyJJ6LoijECPRHQ2zPJq1niSOBMEyww5hY6HB0bpO2Y0RBx/cgkX2SOEZRMpfmCbtLtbBOu/cIxAR/CIGf+QrCLCAjSSJCEhKG/mVmspAgSx5BIJIofZaulBkM+qTzBS72XiBKEZqsEIRTdLnB4asm9dkiQZCwvfOUdKrAbuspS/k7XLu6zvOTh5hegirIbK1dZe/JNmnjL+C3SZIQfykq/7zaEyciO6+eUFrKMu7IeFaCphURkxRR3CSaTFBkneFIxrMOGfnn5MoKGdnAbpmMuwGBNyQJ8xzuvWJ5PsXG0ha9zil6SsCaBNRuVRmNRgyGAdliSLPZpOiXuLH2Tc5OTokiDS09ZnNlg1Y7RRB3CaOQTnuEKw/IqinOzz9mufR1FmrrOP4eaV1AjeY43H6JoMgUCn+RaxsEDoViEVkTaLZGrK6uIaCTSZcZjbu8OvqQnLFAKb9OKeeSMxKGvZhf/ZVf5aMP/5hifp7XbnyTn3z0e7zx+ptkcym2jz/H9qdksBmMzrm1+TXESOWLDx4zbr/k67/8dZ4//IRi/Ra/9hu3yEkGp8cJx/tdNPUKs3NXcTybVm/E072HaIGHKqVo9i3C+JSYkF/+pb/O3t4uhhXwzlt32Xl+wObKXW58t8SPfnzA4mKZP9v+fZRmmbX11/jiwQ5iqsbBy6cEXoFCbo0Pf3rB2lWLgCFXV7/Gynyd3uEx2dkVZOo0D56QLgf0+he8vbzF73z0hKWNBro6oT1+Ska1OD44IF+fJSDkvDnAvPhDBClE01QEdBAdEAKSWANiND1m3Pco50WOjyN+4f2/iSR4tJsd8tkMUdAFqYAYK8RJgKaqJHHA6nqD+YU0VzZXaF70UFWVBBcEmSgGSPjZj18yv5RHEMHznC/5kwmCIKKqGmZsUigUiVyNbCHLvddu0+rsUMjWuT+/Qfyv/glRGKDoOqVani8++Ji1tTkULcbvd5GNmJ9+/luErk6QDMmn6hQKZRKhjWvOMxRe4YU96nN38AOHFy+/wLJcGjMa128tUpuVeL7zmKmVYqFYxwmfMRr3KKpvUq9scHpygRtM8EO4eed1xt0R2wevmLg2K0s1Qi1k0mvhiTqpFJwcTrCn+/RebhMHFge7Ld66eR8tb3O4f8bcXJ1CapXm+QEjfcRbb95n2B9BouD0XTav3OCkuYPT8ymXFnHdEa+2H+HM5Dk/m5DNxXzx9COqVYGpFbJQvYY38FlevoZWMAgmh1ycjVlevoboOmREj74d8aK9z9bVG/RGHUrlKoVCjoefDfn5n/sG3fQ5vt1haSZEU3QKmkbX3mN3O8Xq2g3y2RRCLDGwBlxMHHKywFxlg+9/Z40gDDk+O+Paa7donhzw4MEeV5c3KRQNnh2dULx7g3p1lrE1ZRolaNkqi1eKNEdPufHGNSzfYrd7TrUyy9tvrDDq9xAVCzWtctLq8cUXfdJyzHj5OftHP2P1WobVKzoTs8/X7rmMmvMMj7JMIo22f0gxO4OcWqWQ1tjb+xzPnjLtm9TnVoniLI8e9kHeZ3DRpViSyWVmiL2A+kKVo+YD7lz7RWTd5sXuAyrVLIYO2XTAoD9BUxQ0TaDb7pHNVDg5O6HVHLN5fZYgvBxL040cqpYBbExriOeIBIGDphQRBBtVE9nfO6A+s0jM/x9mKEuFMs1zEzVa48bmd2lUb/F7/+pfMnZfkC+UiOI+veEemuqT1vN0ziUujkVGfYu93WPkRELG4Oigw9FJm4ePdzk4OWYwdnECFynb5KPP/4xOe8RodE6r1QR0bt64zcnZIY+efsB45DIc9DDNMc+ebqNqRdS0Qacb0Ro6IKlMzDa6lscwioxGJmnZYS47g++d8OHHzzEnCY05l8mpS7/d5NnOMwaORX1xFdMJWV66SzFbo1TUmK0scvjMQYtm0Q2JIDQ43feIbHDaJfptC9eLiAKdiWlSKV8iEwgUEl+l153QbDYx7S5zcw1yuQyuk+CHY3KFS66ZIAaomoGYHyGoDtE0xcNPznBjl7PBMaVKmVxaJQhdvMAindHRDQXLsphOXdK5NGESIYkQRi79UZ9MJkO1UkORMqQ0nVzW4fU3r1IrXQNRYm4lRbvfQ9F0FMVgMDxH0sf0+ucEsc/WxnXOzl/Qau1xZatB62JI+/SI9fV5DEND13JUZtI8ePwB9tRk+9k2qbTC1Zt3eLG7hzmaMhqaHJ91iSWDwE9hFAp8573v4AwkGisBpVKBg1cjxpOQtFqm3xpQqyW4tkgmq9Du7LO18RYf/PSPLtu25gQx1FGUCk92XrJ+dxUpY+DYGazpMU5wSkqJiVwZrSQyCSyCYEw2JZMt5Zjf0mnMa/j2lHG3T6/f5uXuOVMvIApTTGMFJ9Cozi6yvLaK7cactyakMzWKxVlm65tUZ+p0+z2anWMyuZhcsYQf6tiujRcG2I6AYogkgv0VSFnXDCZjH8e2OT6/wIpCWuMBqpYml6ogxSKxbyGTELsirjkmcH1czyEmIE5E+pMhQiyip0MsU8DyL4gFEccfY04cJDF16aoWZERRQhQlQIBEIIlikigmilW8cEqukMYLNMwpPHrUwpqkkMUc7XMLQoM4UMkVsrTOPRy7A7GPTMJicYWikieyfHxbxtCLlweuBGpa+8qI8+dQ8/j/re0NIIgGrgenhx6d80vIeqNWpds5ZGVlBdPMYo6gVtFISxJamGF98R47L7p0xjvY0y6L83WUuMrNrdfRNYn52WUmgwRFqDNXeQN3ql3OsXkaQqhz7/o7hP6E6ciiUV8lX90giCacHr+gVClydjbENTukq2PsSURgScxVtmhU51BlFVWeZ37uLfSiwK9++28zk5/h6vrdr1YsTukM27QuTKZWD9fpoSVV9nefEvpQK9xiMBhRr2e5trlFLi3SupgwHRu40wRVVLl19Trvvf1twkDCnOgcvjpEDAtk0wWq5QpiUmJ96V1mauu8842bPHl2QjDVmE6nfPHwJ4ShiZyWCBG4tXgFUdAwUgpxYFHOLNI9dcloaWRRZKlxlzdf+wWSwMAcRtTzt9HEiNmFkKfPP+XJgwEoJsf726ytLLNYv8uPPvqndB2HYEZEEkZc2Vrh6++8y/UrNU5fCNy48k26o2Oev3jClZU5JG/IycE5lblFFHUWkgLn3X1GkzbhuINtR8w0RHaOz8gV6jh2l3v33yRbTKMrPpmMSKGYwnMjVFVFEBNIJGT58sIkKxH98zQzmXdQEp1PPjiiN3ARJRDjNFEgEiWXhjDPETFSEnOLWdLpNPVZBdM0iRMXQUiIYpcoCkhndQ4PO5yfjEgSvmxx/8UMsCBctr49z+P2zZs8fPAp9jRgaf42xWKR0bhFGEQosk4URQybIxYX8gx6DtdubFAsbnJl/nUUygwGe0R2ikef73Dn7lUsu4+aNgkjgWK5yEXrFa3eUxKlS7VaZjqd0m5f8PnHezRKt5ht1BAFGdPqMbUcQpqM7W38wEWWQwQ54MEnH2AOfHLFDd577+cRZZehfczAamGZh+SMAsfmOY9Pn/KTP/kBjdQSf//v/gryzPTSzDQdYw5disYqv/y9f8CNazewJy4zMwXW5kp4pk2vPaLd3qOQT6Mb6S9RPBIT+5xSPsXiXI2ji22e7PQZWwLmxGXvxR6vxi948OQlP/zxD1BTMq4XUqkv8upoDzmyKNfrFCtZSuUsw+kxdrDPxtYMspIwtk7Z2dtDM1LIukq2vIKo1SnXlqnN1Pj44xc4Ix93nLC0OI9jaczP1dl9sceTx4/Z23/J8WGXMFKZqxZ4cvgCdxSwtHYLIchysr+LhMy1pRmWizkqhTy7212KhSxC4JFL+4SexXDY4vikSacZglelmM+yvnIFPZXj2ZM2Syt5zs6e8eGH+xweOjz9vI07KvL9v/YtSpJOI5/j/GKPwxObpatbpMozqMYqgRSzffqMUBpSrnmIScI7X98gnTZ4ufMZIg6ra9e5ffObmO4ZpxfPaFSWyRlZhu0pKiqVSoWL5jEnx2OyRZVm7yV6Wubm3XUaM6uXsbVZgWrdIFtI8HwQk1Vqs1nO2z1sJ6BYzlMoaly5soVMlrQR/X8Lwv8fn790hXL7xUdEoUgptYKacXnw6GeUCnMUqwKnJ+cYWYuUXKbXjXBsEAKD2I2IBZdKYYahbWLbNkKYwfbHlIopwiBicDHGSZv03REz5TrZTA5Z1MjqOYKpymg4ZdR3ib1VXn99ncdPPqVSKnE2vcCNDnCGOT779AWVGZH5VZWjvQG5dEJMgqTnmE7OkeIyklPmtbtjei2P8/2IahGGXRG71yWja0R2jDNWCCcPqJQNDk4O0LMy3/1r77H98pAoiihV0vhTge7pENv3SDyFdDomtEVKs2Wa5ydksyKuGxKEAfVqnuMTkyRWmfQSZE2hXE8QyZLPFxlM2pyedsnlU+jFFFIMqzcapLPgSxOa7Rau6yF7Au3xGTONKuPxFHM6QdE0+oMRbjAkZWjk1VV2t0+pNVRUXWdij9nba5JNp7l1t0Ts5ND0AVEgkS8Dss2o1aOQukIcihQqCkmYQ5Qd8FPUK3WePHyGorWQdYORe0QcJWyubVLLzPHFox8DMUZBQEglNGornJ9NyBeraEJMLiPjSynG1pTQcslni7x6dIQYdXB6E5ZmZyjrMygZiUalwNj28aY+6RIIyIyHLoguve4Fd1/fYGy6xLLM/umAtSuLpONZtj99yexMHS2TI5ba1HOLzCzMIebW+fzjn2EIVRJ2uXLlNqYpcvDqKYETkJZrSLFAb2Qi6CpG2kQPBZyRwbgb8Vn3CxJZ58qtN1F1jclkTGIoZIw8HbtDd9hnrlYmNAMqBR1RFWlPx4ycMaqioaehUE7R6TQp5OapVmuMJx30dIZQllHSaS5afWoZGUM3MM1zPFIoiUG5nCc5HSOIl7NckgBx6DAcXyCRQ9E8hiMT21Qp1bIIkY7ZTZDkyxadKEpfmmKErwRdGIYkMkS2RhAHtDpNpr6NKqoguJhDiUJOZjya4rljypV5+uOXEK/QqCfY4wkZQWUwfM6zp1Mqy2U8NyZKDPpWCz2EgARZ/gtXtyAICF9G1sVxTCJYeH5Mr+WiSjK6KtHtdFhcmuHiyKJcaDCzIBCJHmJa49rmLTrnI379X//bPH7+I0RhwKBnkysW+No33uDksMWTx8/YuraEoNg41iG7uyK64ZHLzVLOXqd5esbWyjfw7CG+OmQ4ajN1JozHBn74kttXv8Oz3U949WKPYnYes9dD8NMYdptEH1Ksb/D8+JDAcUE+YOPaHXrDi6/2xErlFucnbTqtc5bnV+iOthk5XdRolsSXkeUQRZF4+uRjivoaohCTK4W4U5cbm++QMTI8fPA5I2uI5U3pj7f5pW/8Ko5v8vDZI47PRvzk4vf4B/+9v8fK5puUi2WGow8pLM5RzBV5sfOM8/YZZ/0JhSsxrttAC4YMJybXlt6goN5gdrnA87PPyOd0js5fMLA1UuIWpVyaxcYCtuPw7MEnZPMyB+cPiN0J1VwGc2Axvuhw5+Y38SyTuaVl2ufbNMpv88Pf/22uL91GMXR2nj5nNA2oFVYIxSIxE7wQHM9GUUZcnXmDSeRRqpmksiU2btfoHJwh5wPq9SWOTzuc9A+REoMrC99jr/NDKrUsJ8dc4qeSy0tKEE3J6hm6I5uHj5/y7tcXEI0L/tlv/QAlnSNWpMuIRdG/NDDqYI9c1tYr3H5tjrOzE+7dr/PhT/YRhJggSlAUhSgEWVLpdaZ4nksqlSaKEpJEQBCkry5HSRJTKhXptI95941fpFBSeLn9jLRRIWtZiAKIIsiSAimfTK3IyasX7G0Dmo8giBRLYzoXLmmtyL17OQ4PdikVGnQ6XYLQJpeZYeKeIaQiRtYegeIzsTIU8/Pkc0ts7/8IVaiSKyhoch1VVoiihOdPTokSFzFOMzubw4hmsF0RJasxHA6xxmNKSzOc7o3IVWrExJRK8/yNb3+bSuEaE8ekM9nn9tIS2vUU+VSGo8NDFuZXcKdDht0mmXQRcyhyfHjAzatXedT+HF1VsacTztsd1taqBNEp7X2F4lLAzm6Tt7a+ycsXT7h+4x7Pd/eRazLNkyO2rt+hVL7NuOsjRxq1+gJPdn/CuG9yZ+Eq5XrA2dmEcn2WleUa7XYXa9pnNDQplld45713+eEPf8hJ6wLTv2AutUwwCri6vMbpxSE3rr3G8dljhCiDbYtU51Yox0OUI4EHTz7hztqb1JbmsJwhlfIcn3z4p3z8+RH/7r/7b3O4/4iDziGzjTscnu0zP7OAoJsszS7SvhhjuT0GXYvllTW8uE37dMj9dzfZfnHK/MIWsnzGauUW075PIqaIbZlG8Q69izY/+dOPqaWWeP3W91lbOePg6IztxzvkM1XMYZ+1K7eQBIvTkyNETUASFPr9hEdPDrlx4zUqMw6n+7sUs1dYXavTHyWMpxMcx6A+X6NvdVDdAjONRRBlBqMmKb2IOe3RqFc4OTmiWprFC3p89tkDllcaKHKKVFYhETXyFdg72CGVMSgWlpmp13n16hWS8JeuO/7lBeX3vvtrXJy1GfRajEyT9atzTEYKK3NX0MIzHu1+wvpGFWiRMiJiV6SQAd3YxHT6pAwJ2wpJ5x0UOUMx36Db6iNpU4RYBjuHXNMZTjwyisbJ0QH1hQbb+9vcuXmXKMzQHbaYW1zFdce8ef97vHy5je3YyLLMRV9h4k8J7CnNVp9IDnn9/irNE5FPv3h6GUeYFXFDj5yRxh06GJFGbmEOTRBwxiGykGFs+pz0d7k4GyJoBVx7n3a7ja74MJonikWm+ORmKnieh6GoKJLMyd7FZTatpmPJI3L5HLIMczMFAFr2mEY6x0WzRVqzkEIJL7GYLefJhGncFy7ZsoE17mPGIRIBYaTSvIgoFQOW1xtYXZ9iqkBKkynlC+wdHWJkDAo5kWFzQrmWZTjoE6sK3rjP/MISiRPROhPp9D9ldfUKldIMB9uvkGWHWuUammwyslpMvBqufUS9OoOcFHG9Mn3rnOXlLK7rMxpfICoyprfO0dkx73zze3z89DG94Utu3Fjj0aMHFKozLK9nkLw0giyzOpPjgz/7DCdyqFWbhGmNfK7MyURCjlX6zgnfemsVwfbADEmkPKVCnr2DQ+JMwsHZAY21WZrdKbVGDsuBTAQLs3N8/MEjZtcKyDmLfm/Mna3XOGy3yHohZ5/sMhqNyC/MslS5zdH+GaEUYsegZos4noWapJibr9LpnKEFDgF1FpcbuILPeBgxHIzZfbmLY1qMrC7lNLgyZJQs8+V5StkcyzWNsWfhjiR0TDxHRJElJqZFjEq1ukxOT2F1LxhaUyRJYqZcIJRKnPTHyFkf0wmwfRFBcJA0gZGtY7gQhZexb2EQgphnOLDIpFRsOyJtGOTTEYmtEQUaIQmyKBOEIarI5SyZF1waZMKQRIAoEYkSk3xGIafkiL02UydgonRZmF1m99kes40iulqmPzzi2tYbZNN1DnYf8PqN+9hhh9YgxJFjrEl0KRAjHS3rI6ZCBEf+MqIu+dKZG5MIMZJ0yfNTJRVv6jK/uEqh4vHJT464e+86YiyROD3CYEzkG6iawrU71xlNhohRxMnRDkkQk8/PMjdbxbEEAkdCEGWuXn2T/ngbezolm5qj3+mwtjGH73oEU4GjV/vI7gwLKyphd8y5+YqZK3WK3m2ePf8ZFd1lPTXHg4FFIqfIpbNYnkdtIceTpyaK3kIRxxRKVZ7vHnNbk6kUil/tiX4k4RJRXp4nV1E5Py4QeVNWbudw7DH9sUsxNc+w10TRxmwt3SSZgpSInDVPWV2eJ/JGTEavGAchxZl1FlZe549/8LtUjbe49l6W9niXfueUxbm3kZmyvlUkJVQ5n3S4uf42P/7wn7E5u0KqU0ZPC5QrBVxRIJwGzOXzWKNjbizdpd++YHbxCue9Zwwmz7h77R1kyeVnH/8OGwtzFGslOn3Q8zUCOyQ/u8z68hK7xw+wogrJWCKJV4iVEUp2luP+PnOlGlqcJ+106Y+38a1FklChPepRr83hJRat1nPGYY70ZAbjTsLO0yYZY47FQh497XB+YVIoruIQEKfyLCzeoFTqcbQTkhg2gS+iiBGylMLHRRQundyxdMpv/td/QvdUpJiukUQBseihiTmceIxMHeIxy+s5jlpP8G2Bd++/z3+R/hmhD6Ik4/kgyQkIEMU2oV+8NAAJEZKoARDEAaASRiGFqoynjpGGGs9fnTMYd5mplNGdCEkRESUFwhhnIHJ8fMzitZsM2j3iZMgwbRIeijh6i8TKUSnXSYRTxHiJUtbDnLZZWFhi99WY6Sgkm5snnc3huTHtdof9/QNyeZ18NcvRUZvIT7FxdQ1JFAndhHx2jma7w8Nnu+SMAoRpInuIH8L+SYd33ngf9eoZi/NFjveeMpvT2G8e8PziT7g4HnBz6z6TeEoqzqKk2wSOSb97TiDKzC6uEjs68/U8WujiYjK1ZVbWN5DDNJnCkMiLcL0QMesTTAusN94klD3GnkW33SOjG1RqKxweHjK2dxFYJZMtIxsT/uzjH5LLzWEJAw6aB3TNFPkZjTC2OWsecXo05vXXKtx57QbPtw/5zX/2T2mUy4ycJnMrdS7OnuMGMoWSwtjs0+2eUm3M8+Txc9Ya8+y9PEXMGhiFWZZ0j0AewziNRpZnT3cI4iLf+maDILTJ5isYE5PxoEO1UqfZ6zAcnGFIC6SUHJHk4BkCVzbz5Iv3eHHY4vx0wPriHXKFNhlnQOIpFHOblLM5xpM+mpajWEkxaHU5sT5iaN6iuqAzO1OhM+xQzhS4cmUW1w9JZJnz0x4oKfzAZTxxuPPWbfJ6HtEuIjEgjE1eHQwRtAEXp5fVdiMlI6GRyBOkJEvzYoeZ2gaeP8UPppxdtBAlaHY6rKwsk8pUsd0+RgpSmYhhX6XdP2Fr4zZiXCGXD9k7fMrEcllZWflLC8q/tPT0TYWluTd56/73uX7123imRiGTpdPpMo2mLM3NIGJSKkY4lkNjpkQ2J1NtaCi6gec5NObyiIJGbSaNbYORMfADg2y+wcLaGpGo0xm1MJ0xWlrEj4ak9BKnR13OWzsMhxM+/+QlklBgb38b0+pxcX5EEgfIUY9Ja4RnylTL87TaI/75P/sRMQYb15dI5fI4Y4WUPIuaK1NbXuLa5hYZtYDjpEmnGozMDietF1iWRa02w6u9FxwdHUCgQWLQ7/bwPZPGTJV264zRsAsInJ/2GE9sZEnHsUMcS2I8CGm2ekRigmyoSHh4oUdDr5H4Mb44pJrWkdMGQkOgsJgwuzxD4lcYNT0MCuBGNOYgny/S65u0/Q6tcYfl9TUuzDHLW0VW54sU8hXuvFElDKYIUoFydoHNG7eo1UqYwRQ/9PCFPK1Oh/3DJ5QKBovlu2hylkJpmSS8ytQUyZbSTOOA88FzmoNDjHSEG/d4tPMhVuyQyaR4/NkOT5895sNPH3DeO0Okim9HZLIlZE8kr+TR0x7ZtMynP/gjNuoZvv7+NUIvQRdSPPz0gHqxhBY1WMwWMAchpmOQKCpnF+ecn7bonfWp6WUEK8bqTFEjkbMXY/ZfPWA07fPi5JgoFzFxRCRvloKSwnclXj5/zPbLL5haMaVSheGohTmGlFpEUfKUU2nSsU08HaATokUJQiQxDBJc08byO5i9C6Z9n1q6SFEIKedSzOUbeHYe1S4S9FX2D854uvOS49MLWq0RA2eEY0psrCyiKwkrc5t4doLrjhgM2yRJmsbMArXqLEmo0m5fkE7nGI9sxmYbYgFRMOh3PWzbxPMnxPFlyxpifFvCizX6PYuMUmTSE+iPTSZOiISOgUaSXPL24iTB932QxC8rhZct57QfI/oBuiYyHk5xRxGiIzHqOOy82iNOJAK/hB/EpPQqjuVwuP85tVkJLRcxGUmcn7nUKousrC5iGCp6ZoiuSmRzlzNukiR81R4UBOESI4SIIEiIUkwUJlRLVYr5NHdeW2XQnVKtZ7iyOcetGxukU7A8t8LhwR612iyrK5uYzgW5fBFDzTHsTxDUCwbDDl4wYGi/xMgF3HvtayzU7/KNb97n3t3bHL60efxZkzs33iKfKyEkBrnUGkuVd7C7Vfb3XjI7s4FmpEA2MQyHRukKgj1LvVDHGnaRcdHiOcb9PuNewNfff496YwMjXf9qdQd9FDFFtZCmc+qwurTM62/8HKXqPNuHJ6xdWWIw7BJFadLiEnmjSqUc0+0/Y7a+xbMX21h2k6XZVWqpHGUMzo9PSGXS3HtjmUIly831FeaKeQoZi0SJONm9IFu/QSadx5fTvP7+m3R7OlsbswgMKBY3CCcesqyi1SRm1pZ4+637yLJFpZzn9rW3makZZEsSvbbJW69/m1JljV47RFF1IsnB9yLmGzc4O25SSs+xvnifD376IZKQpawVub5SJqtpqEqa1eUVbm3dIi1q6KkcKUGnUNARJI/BqQXVHI3lMlv3F+kdtKnPlBibHYbWCQeHu9y6t0Ci9hiMThhbx9imQTan4HoTZCl12eoW+ApJJSQivXafuZklzH4ZEp0osUikGEHMECUyiqIgKCPC0OPW7avEocre3gGFQsLsXIkkFkiIkIRLAL8gSIQhCIKApl7OyPq+T5JESNJlp0DTdU5PT+mdWJTn6xSLOb7xi+uYXkjzokecQBQEOEGIWOrSWJjHUDXm5jPkcjX0XEwkWwzOdURZ4+HzfTT5GtVSicloB8/r0+o4pLUGitBgOJjQarY5PxmTyUlcu1Wm2TrBm6S4/9rb3Lm3RPtiRFpvsL7yJr3+BEkSMFIqlmVy6+4inV4XSUxRKqcYjfsszM1y3G5hFOqMPZOXrz7nxfNXpDWJk8NPOXz1KaIT4FpTMlWXRJOYW5hn0O9g2uecNy8wlYTeeEBFMjjYeUkul2MyMHGDFOvL7+D6fbK5FEoq5MGjx2xeu06j0UBE5vGDJ/z8L/wahp4jSQSm7pBXr85IZ0Qk0SCllwmTAdOpiaqk6bZtasW73Ll7nWazzdTMcf/tTVY3quwfbCMkWbKpq9RqW2zdnL/kMasZer0BTx40+c77v44fWSiaD4KHGBUJfZ/Osc15c4qmF1iby7G1WSRbzPLwgxdYZofR8HIO8YvPP6GcXWFj6W3Sepq5hQKEKe69q9JznzAaK8zV57CHNuWqxtFpTCeqst86xBrFlMs1lteuoUkyxYLN2D6ntlzGCVuMhy12nxyTkxXiyGPQT+hdHHK01+H+u28iixKnvTbnOwlZG84OTKyJg2xMaCxpjMYdPDfCDxS+9vVvYmTS6OkaSaSTykqks2kG/QBNT1GrLKCnbaZWwI2tZU6PjjFHA2ZnirjWmN7pFAmPnKHhBUMm7gHbO88QBYN0Ok2/P/irF5QBFq8OnvPjj/6ELx5/iOcNiWOL/ZN9IlmApEjg5KlXNkjrdTQ9gxclHLfOMAopVFXG8xVkNUNvYDIyxxRqIYqRQpAUeq1DIg/8qUAYxhRKDSIqTEMJkx6qofJq75y7d96k2zWZOgPGoxGVSoU4cckoacRIJQnT9Noh477MdKwzNRM+/3SbyahDabbMwAzotMectU55tPuU806L8eSUZueEMPGRVYVMusBk7OBYCYZq4PsmkQ8IMaKQ0Gt18SybWrnMoNdnbNqkU1lcx8d3PRQlIIwmqJrA1J3ihDYaeXQUREMmr5eYDkIq9XlwLXpun1IuixC55PNj7t+9Sac5AiHAyKmMIxNrGqIEZfS8wLPDHZzQxo7BN2SMQhbVyJHLp1GzLpbdojUY8Gr3DMuLOGs3iQWVSi2NYmRJlIhspsTN2yuMzIhCMctsQ0GVaqRSBXR9lvn5Bkl0aXbI5LKk9SVsW2QwblFr1JhYIzTDp1RX8SKLVNZhacPAyBSZJqAVY27e2+L621WsbJvKgoGQHFIvGdza2mLp1jp3Xn8NIc7i2QKSmkJSJLqdEe40YdKfMuyatFsjSqUKpbkZRLGA4OeZDj2KmRwJLkfnx5iewEXzhFymiIiGKIookkq/Y9JqjajV6oiKzGDYBSHhysYNfAesiY2qCLi2yVRwON0/JhWJ5DMZYllFMEJyBYGFhQUqjSyirlKsZFioL0GYQ0npJGLApNtGVkKOj08JfYXAh/nZZUQMRFEmTGzShoQgBownbUY9D1lQCDyXxCsiChJJLJDSi9huBKKAKApEUQgJeJaA0xcRkwIT10LQbSI3TRLEWJYFpFBU+at5RUVRLsHnUXSpSUkYiRZqMcPYceiOLJS0gZIGRVSQfIn5+UXG0zMCuYmulvCmIiQJ46HH05e7KFqObEGhNmcgKxG5vE4UOSwvrZGS5omi6CtsEPDVcxILCEjIikAYRGTSRWbnGpSKWTa2yviezfJqnatbFVbmrzGeDNm6nqZU8+j0OuTzDfzIQtVE7t15l3KpTmf4CMcOqdVnGVkdFCmHF56iygpSUuVv/s1fZ+mKSzqVYXYhhWt7LKwVyRg1pMjk3t27mBObmCylwiZ5bZEQk5n5CqvrK9Qq1xmML7DCc5Blxu4Fn37xAZ89eIjjK18tz49YXa6QV8uIsY+igmCoHD4+oLXT4/nH22TCJcpSlsnpEXtHh1RqG6TTDQbWGe98axM5bXDRdBmMbcb2GamCQrPTRrRl9CSglMlgeyl+8ic/5h//w/8Seyzz/NHHhE2Hm9cW+NrWL/La3dfIzS5y5+1v8MMH/zlW30aRakiSzPvX3yJ0bN5677uYnSZ5tcHN63+D2eoVssUCM42bnB4M0IsqU7fFuB/zve9+kzBqct7aQUuNmbovmZ2p8a2vv0emkpAxMqTyOmu1VTJWiuW5FSI5gy100WWPK1sb3Ln1C1zfnOf0+AuaZ4/oTENGvs/xwYiUXqRQKCDLGXrDfTq9fRIx5LzzHC/exw26yJJKHPuX73GYIMnRpeEMicnAZ7axgtnPkyBhpC9/yqqEoFroRhrPVqjVK1TrPtOpw8bGBk9ffsH8WgnXD9A0FZKQJIxIwghR0L6CmQdBcClKBQExiUmSiDgOEUWR7rDN/vk2rhxwcvyYqdVHNYokEQRegCjJnDeHPPj0JY4pMh76GKkCI3OEPfVZXFwllymTycGw32MymTA7s4JhGOipLIuL82ysr7O5/hb2NOa11+8Re1XODgU21m8wN59nf/eUw51T1pZXODt9xYsXL9jcuIPnTwmtGrlMjVanSaO+xniQMOq5xHHIyek+tiUQRzm+ePgpXjSlUd9AkRq4toQ7zvMnP/pjnu0eMLbyKIbA51/8EDyNfDaHkSqhKHXy+WVKuSXqjWXMqY+WyIy6Fo3iIo3qJopUJEpiBK3PBx/9iGdPOsiqz9071zk7P6Dbb5MuhFSqWaq1IpIsMDL3yBYEFhcXKZXmcRwHe+rhBmf87Gc/I/BjHr/4A8YdhWtXb7G2usXsosZF+xmi4tMbmty+e5M7d15HMWxEEZqtE7LZebauXSHyDVJGkUq2RKoik63KNFtPCbwitqkSuhAmHURSzNaWsC2Lxdl1Hj35CE2Hd9+7xWho4oTQb0Y090WOD15SqRnkKnU6PYvTsxeIQkK9UkbTDGRJpbN/jN0UWZu/SaM8i2clzBZUlFBhbf0aWiFie+cpL54dsnplAyke8ceffcDI74A1IsxOmX/7LqKUsLG1xPJGneF4jGHUcSyNdEbmxfZzptOE1mCXqX/B2dkZC3ObtDsXuE5EFAtUSqsUiyUmQ4tvvP2L3L7+Bo8/6bC28C56KiDwHSrVAp4boigR1eIK5jji2rUtJNH4SwvKv/wM5dEuipzBsh1su8P9N9/myaNXFKtldncuMM0eQiSh6SKCIDK+GGCHIaEIZr+JpinoWoap02MynqJrAtXKOtWiztHREdm0hjW2cKYaE8nDcoZ4UQpJTRCELMdHLXSlTH94gSB5CImONUmwzBZZo4QbKRiZPMQCo4lJGAdUKjXazR5T02Y4sGj2BCbuAAkPEwnEHPl8Ht9UcSITSVYREwV76hGHAdXiDOPhiEq5CEKEICZYloUg6iSBBDFMp1MEUWI0GiMmOkk8RZBcRDGmWJhnMPYYjSZk0elOTebqZVY3N3n84Dlmf4ggTclHOs5IQC57NBZVOq0OuUKRWI0RkyKaHOAah8yVa7THZ1RqCzijDKPRBZ22x+ryCD8SiVUB3wqo1ebIV+vsOLvM1WcwRwboCYOeh5iTEVJ19s62SVISki4xm1/i4HhEuVrm6LhNLmeQJC7ptE6/P8QehCj08R2PaWSRyCK+I9Pt9JHtiFojx6SV5qXd5cpNg9HpMRm5wttvf5cPHz5gEidcqRlMPIVb30/QqxP+5R/8JrOVHBNHIHIcYtvAnpq0pxZpPY+fCPiihB8IfP78CYqsYchlkiRhaI/QUyUmdgdVTLA9mThwEOIEcxKyOFfh4HiXKNBptg9IaeAmMr6k03E9xucXZLNF+oMOYhJTEXO4gku+lEGQZKrZLHvtbazIp66V0A3I6yk8e8hbb9xgb++cbEln7LWwoxhFk2lUcww7NqGd4DKkkJpFStdwHQs/HDOeDGi3ehSLVVJpBVnWscyQmUaObKZItze8vElHMpmMQkJCnICEhKb55FMKIQ6uJ1CrZ0hCBSM9YdQ1QFFIkhCIEQQJVTOwpkPCKEQOfcI4QIokAneCXonJ5vOMnQ6CGKCpOVKKwnAyBCmDZXrouZB/8Pf+Dj/+8e9weNKmVKtRyqexkjJjc4qiKly9uszkiwBVjciVfMLDy++WJIE4Dv9CXAqXxoY4SpAU6LRMCpUYy5owP1+iUKiiSzpx4lGaEdALCtmiwMHpFyh6iXQ+Rb46iyL6PHrykKU1gVpDwxoYdNptMlqdk5NnFPMFTo4eMnV6ZLW5S7OQoXF48YAkKDFxpsiqw9byG0RuRKGcoj34gvffeofuwKW8kqNWznBxfkIS+vzSL3+boTUgOK0jZLsogk+rtc2Hn3S+2hOtSZ/QqeNORGTZ5eOd51QWtpjL13jjjfcZ93VkvU2ltMLbb99hJIa0+mOQVCQBnKHOYuNd+toAeaJwetrl5PkF3/r5n6cs1ukPbIYHPu++821migLrG3e5vjCDMZshNxU4PnyBb+S4sliE8YBOaPCdr/88RjTH8d4RdvuA7ZHLsHPB6soG6dsGz7b3ubV2lV7rgmIS0Nv9gkY1xWKjwe9sb3Nn6RqPP3zIYfecxY0lfvrJTwmcOvdv30fyhuw+f8zG3F2y5RJ2NGVubo1XB89p5HQKGQN5onO685hT8zGSJJNEKTp9i7Lcp5yfI45tvDhC1QROz4asX5vhwYMjikWFRJjg+EMULUDVNCQZfD9GklQEMSAMJBTRwHV9Xm4/oXnmoGspglBCllIguIhf4n6I0ywtyxycfErrQuL6tQX2z55z7dYV/vhfHRGGPqIUw5eIS0lUsD0bQ9UQuEReBYEDgKZphKHJ4uIit9++gyJ6fP7kcxqZOTaWrtA6+hzHmZJO55FlCT3KkxIlnj55wY1bNxkOQwpFg3pRYe+kjxUeXtICRJGRNUCINbY2vkUml+WnP/kRuUwd251y9+499nYvmJtZQTFGDAcWoiDj2EOWlpbY2z2kWKqTzYFlmlhjh8pyQn0mjTXxKBSzjCeHvHX/HmkjRasVU84kdFo9Nq+sMbEtTi/OSEll0mmNfHEFJS/T6fQopausztVJxykkQWapMcPu3jbpQo3uYIieqpNPaeiBzExeZyaT4/nej9FVnUJeZzjsk88uY+WOEOWYhcUVnu/8hJnSPbKpr2OoAr6dQUEmkzJoVOf49KMPEZQiy8tLCGLCytoMZ6fHVMs1pqbIm2/f4fhwF/cwZqaxQBS42G6fTP4m9rjJBz/7CbP5a/zy3/gb/PGf/ujSKGVIDE2fTE7hpPmQhfocdnzGzouPePfWdzBjh4gI33FoLLxLPiuQK9o8fnzO4vwyt2/VkFWJzx8/QE3HrKwvYTbbWF2VuXqG3e3POD464/37/wabmwtYjkVoF2ksSLzYO0SILDA8Wq0TSoMOrryBsJ7FHcLwcBuhXGDr7utIss3T7UP0pMpS2SCTl1l68wbD/gWHT5+TSff5k5+eoBl1SlWFUBiTLxVxgj4X7RMGPZf55RqzsyV6bZ/nL3ZZ36phOyNiZD774hNef2uD5pHLRW+b/vgcX2xxcjbi2z//c/zT3/y/0LMSlmobNJtNjJSJrPXY2RtTLs3/pQXlX7pCedId8fDZPtvbF0wtlQdPDmmNxuwd73N0+Ir+yEHLpHB9gcF4yNQJ8BOTIJhiW0MESWQ06SORp1gsc2WzxHDQRpS6BNMYUa0hajFGLsb3Q+Q4hRyE5GQR1UvIZ2tousLO3itaF2POmy1UNYMzFRHVBDewcTyXILRRJRchsegPerixxcxKHSGdo9k+QvASwtC4hM6i0zrpYSU9ohDMyfRyI/ETKtU8mi4jixrORGRquaiqTCaXAUFgcWme6XRKKpWhWC6QzUq4tomh6QixhO/GmKMpmZROOqVQqGgkfky73efh44cYcowz9lheXufa2hqFmk6SO+FicoYvD7n21iy3r77BTEElhYmWmuF0ckEmraCFItnClGG7T1aMOT09Z+fohIHtkUoVmLg+p8dHeMmYQjGFmOSY2irW1GU8dXn6/AxUn08efsTTnZe8PHhKd+IjqQKZtESc+AwnU5T0ZfyjnAg0zwcoSoA59JCTPMXUOrogM2pOGA8cAnVAy7/gs5dPaMyWyGpZTg63sbojKkKZvL6EIM3y8mVA9zhmfm4Ox3XJlD0SLE6ODpBlsH0HT4wYOx6RCKKkkU9liJmSMUR0SUNAZTxxCH0JAQPXdZlaPnEUoKkiUyvAsXys6YDBuIc5sfEGU5xegCZWCDyVZrtDt9+hWMhiSAq3qhmmpkvXHdIeHJFKNApqEXs6xTZ94qKAkVc4OjohUST8cEpRqKNbNVJijqmvoeR1SAfkZnKMvSEnrQMSUcL10oSeiCylCIMY3/cZDDrUZ4rkCzpjs4We9pFUkOUcru8jiqDrlxGL1Zk8+bJOLhNTMCRSqsFFa4qd6JeGpLhHEASoqv5VBvGfVwxd1yUMQ/AMUH2WrhfJVDRs06WcnYVEZRJYIAoMxh00Q8XnlP/jP/oPsdwjZhYS2t1DxuYZlWqWSrHG6/feZ7a6ysriEu3eLoo+/BKkLiOIMoh/EbsYCzGiCFGoIIghR/st1tY2qNRS1MpXCKI++0f7nPdOmTgDGo2r2LaAFyasLF1F0yQ8t8dwNMEJhnS7FvY4j6w5FPMZlCTP/s4+oVvke9/5O1y7cRUnPkOS6tjeFIECWjrkqP2EiWUytccMrSaqJlPMrdBqj9lcn+Ps5JAPP/gZ+Bk0OaScbVDWlpgvrJKJK6RkiIIB2az31VparECosLzSQFWqrF25TjUS6HbbaLkVfv4XfpVSZQG5UuG874HdI52RURWBUmaW8+MTZHHC9c0tdMFgZa6K6Rzw4U//iMPjD9EzEv/qX33Ev/h//B5qInL9xg380GTnz/YZWmWavQtcu8PDn/6MV1884+yDT7AeTHn1eIeFxioLBYP97U9RCiVenuwz7Z6zVc5hT87RXR+r61PLVHh9cYVHHzzg25v3mVu6wsQ8ZSYvkPNyfH39F7iyUqZjPeBPP/sxc6UN3rj2GiVZ4PjsmEeHL0jLHvNGnkTKEKVtKmIBMYjIl2VS6TJ3X/8uWaPO9rPnzM2tIcsiw+GQVCqm2T4irVYopstMLQtDvMLifA0/MBFJgSgRJ+GleUaWSVAJI5unTx/T7tiksjlEERAjREkl4bKlHMcJN+9V0I2AmUaeXLbKlbU3mZ2roGgyICIJIr7nXMaEigmh733FT/V9/0uTm3g5h5wIlAsV0mKW3adD1nNX8OwCp71TxNhH0zUEMSYKA9JGmZnqIj//y9eozPeIhC7Zok+/3yaTLnPtxmtIFMhm61y9fgcvnNJq9nn+dA9RTNjYqqPpDlPLw9DTXLu5gqZpGHqWVveCza3r6FqR6cRjbWWZankG2/SZr80SBn0ENEoVA8dvYppjNDnN/uGzy5ANWUYWQRFn6LdtOs02ajbi5OyYidXGdiU6rYTZ+atYXkKuMUttbYUoHWMFLlEcEEkh08TE6l7AuM/GxiYxEt2RS0apoEoqSwurOJM0d259jcW1KqfnTXx7gZXFJYathObpmObFMVfWr1CvGxxs93jt3n0U1WY0GrMwt46s+niex9rqDF50zp/88ENG4wlKOuGiPWQ8nrCytMzDxz9GDGu8/+4vYLnH7B6ckMpkOTi84OnuLq8Omsiqh5oKOO22KMk1NK2PqX7Ks+cfc2W1TKGqMra67By+ZHdvm/WN66AMOWq+QNYlypUl8qV5MqpP+6xJz/sCN2ojWXOsz17l1e7ndLsJZidB1DTipMvinIET9JG0Ik/3TCpvLPOdX8qg9l+yvlzj+//aOm/Pj9kSX7JWPmBBOCddisnXVKK+i9m3MZQZsppBKBlMzIjiTEQiJsRxxPHpCaKYMFOb5crVOlJiYPaNy7hWMaBamWNuroFry6SzMBnbTKcme/svsUyXUlVCzuzz27/zX7PYeIN3779HIaeQUvPMzjRIpXRCt0S33/6rF5Tt3pTBwMVyRkxsh0fPXtAbd+n2LPL5PKLg4toOo55JSjOwJiH4OXBKZMVZdKWGomjkcjkW5mYIHIlcJkPgu1RqBpViijiEcrFANp0jm1VYWMqgyDKeHaDKEqN+D2IDWSliWxGOP0VRDXqDPqHn49gWoihiZLLk8mUUwyBVMnDFiASbSq2InoJiLoDERss6zMzp6GoKUfSpVquMRzZpI4skC3h2hO+H1OpFUkYOBAk78DCyCn7skM1mIZFJpVIUyyWqsxnSeYFiKU+1WmFpqUw2JUAAPbfHenUGJczj2waZnMTW7SVmaksMvBRELhATCipaBsRMhCjn8CObYi3LxDlHEh2EKMPMXEBWF1mcryKmbfxQ4Oi4ix8IKLKIOTWZDF2CUKJ93iZOPKbulDAqI0dZvGmb0C1iOy6oPk9OP+ViuscnD3eY2CadwZiu2WHvtMdwKjCzMIugZshqcxhRFUXMEgojwkDh7vX3yDDDxdkYLdH57I8mJJLG0O5gTltEQYgXTdg93iaKoSAs0nze52T7EN9z6DdjQjtNrpjDc11kWWYwGhBMI2InZNQfUEkXiDwZQ0njWCZxYGNaA1RFoVap4fsuqpJFjETyuTSeMyWTSdBVgVxhjoSIg945jhfRPxuTknNsNFb5+tU71ASdoqbRsUQu+hfIMbQtF1twyEoJaU1marWJzBDHFEkZBS7OT7FHIXo6hZJ1SBSfJBFotduIqszYdGj3Rti+h57W8aII13UhCQgjG0FQkBWwnTGjQUAQeIytMQkBMVOyORUQieMERZWxA5NiWUFPFZlbXMBx40sTgZswGE0R1BKypHzlTL0UlQEiCUl8iQ3quRblxQKSLjGxTK7euoHjRwRRRMooYk1d0kYB39Ww7C6pXIr9AwvbSlHMFdFSHrKUZmPlBqrk8+zpI2Q5pJxdY2FhAUGU4Mt2N0DMX8xSCoKALCukUzm6vSEvXz5FkmREMcXFxYiR1aTZ7qPpeUTZYH31PZbm7uD7PlGoEoUgijL337mBLOZACHDcC3qDl5QrMqtLK9Rm6uy8esb2zlNyhSJbW7fQMhHFmkKhVKHf76KlJMyJR6Veo1G7T5C02Dvdp9aoE9Hn9mub2EGf5fU19o5esrP/hHRaYNT36Jwk5NQSBS331eo1e/QHHaaOhZa6FPL5VIHXX7+FJJ6yvfuHNOZkCEyc6Yjm6YTdgxNW19aoL6k0+0c83X7OBx/+kJVGhXDUwYsn5KU8D3/2MU8fNnnte1vcubdA4sKHn/w+xdQsI/OI/dYfs/P8nA8+e4zlSqiKSbWu0UscBu0L/vlv/jYf/PSC9+98k8Ui+E6Tx3/2gPPzMYXUDJGaJqVlEBIJO5R5+877lKt1WgcvWJvL8c1v3aY3OqZUUbl3+woL5TXWK0vc3XqL0dkB5rnA1vIGfv+CjFKl43b57CcfcXXrl8hnG/iRwsneEXu72+w++5B8towrD2g1h5RLDcyxT7/fZ9A2MRQdopDEjTA0netXbyAIlwxVYgFVVYkFkTgOCXHI5kqk9U08P0CQAxATVFX+8v/q8h3M5ODnvvc9ArNGpVjh3XffvfwdMaZcThOGEUEIekrH912SJPpy5lj66jIGEATBZTKTIHL8/2Ltv55sy9PzTOxZ3mzv906fefKcPN6Ur+pqj24YckCQBDHkEMOYmNHoQhf6J3QhhSImZq50M4rgkBLJETAEicYQaHQ32nf5U3W8S++293t5q4ssNKUYRggRworYkZH3a32/9/d93/u8x6e8PLwPksvu2RPGVp/jkwFL9Q1UQSUNBRRVIwCqazpnnSf0x5/x3m/4NC/to2QkIn9OrzdnEXoEQsB4NKdZK2OYIoVqkc2tdZ4/mlKrriCIEaWKSbtzxrBvU6lLbF9ew/XmLKwR3/3u74Do0el0qFdzLDcb5MwSmlSk34lQhCaGmfDgwQdsLr+JLGWIwgxT+4yj0xesbxWollrkTQ3CmM11eP+9Bn/4T2q0cj9js3CI7HzI+ctdPv3kC5w0YNi1mA5sLGvKu2+/h5eEnM4shiMbPc4xHI/4xY8PmQxTVjdVXjx/xKun55TzLZrVNTrnRyBNqdbKbF9Z5eEXnzHouTRWMoRiH0OXUBUYz454+eox25tbBG5EIowplFT6owOEtMqlK2tMJmM67QGXtm7gBwtKxjq3rr/OT3/yS2RB595r64znZ/R6M4qFBlJaRlfynO8OKAt3mfcDvv3tb+POA46fBBjlfabuKZlsiaPjY2xLpdZo8uTpC+zgnKfPXjAaL9habXB9+y2IdS5dWuby1S2KuSKrjRobq1s4Yp/FPCKxUjJJgYxlkQmLnNnrnI4PWW3soR7+G05//hDduE48Uxl8csxk9BHb5i56LLN27U0sYc4kPuPTh58iqRXe+Du3GM3OeXX4KbIqsbFVYWrt0WkP6JydUikWyeZkclmDWrVC+6zLy+evsN0Okqjh2iKSbJMxDEhASgtYzlMkwaGcLzAdzDg7cmm0ijx8+gGBD/3BGdPZ8G8sKP/GI29ckVwmj6xJaJpGIioEgYfniyy8hEajzKg/J5vJk0opkWSTJAJJEpM1VRJsdNVkNp8gpDna7T5b2yZJlEVI8nQOFhhyATkGsyAxGMzZ359QrqySK+mcnR2jyFXENMGyh4xGIZlCQJJG1Aur+F6ELKugKiSihBssKBRbJPICUSqg2i7zcEwh04QgJo4cZF3BH4XIUYCW0XEci1ajThKLJPECTc1QqclY3gjdMEhEiSSKaA96ZE0NUVAwjAy9/jmN5hZOEBMmHqpkoioXwqhSbuI6EeNZRLwe8O5WnSe7I6R8hiTJ8vHjT0mSiM2Mie3rbF/dYdzt8OD5Y5rGCEmZ0R2mCLHJUnYTXavz6NFjQtdn7qb4WpdSpsGV7QLOBKZ2ApHMfOHiRz6B1MHQsiShR29qURKL5NQmk+kcqGJPYOFGrJVWcBcRu7s2ucoExdTJ6AZjK6I9HNBs1cmYGVCOyeVa7B48hziiVI45PPK4s9Hi7rsrfPW1lBev9vCnCW+/UUEWTfbbz/nd31tnuvDo9eeocURrskHoO4TiFEeRWWms8+jBc+I4QREFZiMLuZrFsj36gwWppROZEYYOQ99FlnPYi4i5PEaTAmRRoFBpISQClUqBhB6yXiGRdVJ8VtUaZlajVMmTrRpYroVnJ7RHM84HA9RSjlQ0WAyGBLqIHwngzfB8G1INx0txkpiR06dcFAi9iKO9l1RXSizmBsFwxGpxifFogZJLKehFgpnDqNNjqV6hPzpB+/KdaS2bSILJfO4QJy6l/CZq0eX09JRGrUF03ieOE2QZoiTBclNcW2LhjugdtMnny9y8sYws5pn3DJSMhJRohKGLJCmIXxpjfp2pHcdkRCgIIg0pB6ZKnARUmlkWixk50yCcRszHI4q5JUSxSblew/ImTMY2ohCjieusby1xftomsm3u3bnD8+Mn3Hv9Lb7/vc8RxS/H20Ly5WH8pcBMRWIERBySWGYxtzk+aVOtVnm+9wm6XiGbv1gfUcUMvj/AsRIW0xlX3rzGoK/i+w6t+jreXGF76yqWt08YrLKxsWAykPi9v/8HfPHkx8RiGyMrU62WGXZ76JksesZmNJhQry5RLBaxRY/2oMNaq4pjZ1BMi96sjxA1EVDw4xmjqY8TxiSCTn/ksLl1m1FvSq2q8aMf/vDXJfGNd97FKITMLYvh+BhV0Miu5CnqEww1JZPLkMsarC2JRFKWwSImnZ7xq5//gm9/411Wm5uEUZ7RcIp+vUqtcZmPf/YX7LxZ5dKtm+w+n3PtrsF8coQuyrSaa/z08z/n2sYGU1dh7co6S+UatbUmI9fi8Mkjvvr6bf75v/4BRU3E756ze1SgcmWVb3/n7/Os9AUL28fXBVazJcL5hGfPPmP3xObrf+9/T6Euk8k0SCZdzl/5mDTIChovnu6Sy2+R01UkMWDqqdTXthiOZvTPxsjyEZNQRJnN+fgn/57np59j6SaytISedxlMeohPT7h+6X32T/8K2bjB0lKVRlQlTRwOdk/Y3rjEzqVLTGd9DvdPyJkGUeB8uUt58U5JooSsBnR7C/7dHz1BN7MXaC1Rx49cDFUlQSYNRUjnfO97/5p7d27TGZ/w8uUuJ/sht99ooeohWKCqGmHooapZwtgDLi4FmqYRBNEFnUBVf53as7NzjUzzhAcf7bNz/Q3EcMFgOmbYmaGqGnESQRKwVFligkchew3HXuHhryxu33kLezbGsrqUCi30vIgfRySyxWzqUK2UMItQzb5JGP0V/eGQnUvfZTBsMxiecXQ0YD4rcWlHQZYDiOFg/yGvvX0X19J59OwRrXqEJOq8enbA0qbJy1ef4Fgxd27cIhVs7EWCb7WBLKpUJKNkKBZ6+N6C2JX4xfcfU5DrmNWYnXtVQitDokdYyS6GotLt9lnaqpMrlOhMhhz3J9iGhO86RMqUjLHgwYtzVlevomctuu2Yr7z9Ozx+9hPs+YylpTzjsxrra1Ucf0ql2mAynvHgizO+8Z17nLVD8vk8khIzHUeU8+vkM3UonuH615ktRDZWA14ePSMN1siYOv3BAFFoU2+ssLxa5sXLBfduv8nJ0S6hrZHP5yhnlpkMzun2XjAfRdQbFVaKTT794gTBecSs53Ble43e9BTfXiNfrHLSeUYQBATRgsQrEoY+smTihBa9gwNWtq6ytr6ONXfJmDnOz19SK1xja6XIf/gPj3nr3pv0pq8oL11mabnGvYLD/n6PT55laFXfpb4qMTk7IentMt7/jGx+h1LhNg8f/Qxp3uEo+wCp/hYbm0W8cMbh4RdUnW0iTwd5QEQT160RejnSGDaXLzPqP0TLVXCdFC+YEkUKubxOb/SE892Iq9cCxNQgX1I5Peuxs3qLg70E/Ab1NzV+9TOfm3cuM7KesLxRZjqWqDezHB2e/O0LyrVqjd60T5pIOJ6LvfCRRI1yxcALZUbTIYV6EcNQSMUcoTMHySdJQmK/QB6DJA3wvJAktpHkLCdnUwJ/hqEF1PMVEtFnMIrwF0Oq5SKqFBP5Np5nospVcnmVhT0HSUVQPHJFHXeeJUoDkGMWVoIkuITMqJbLCIl6wZcMLsbq9UaTNFEY+QtSPYM791kuNml3u5SyTcLwEEGY0et4aHpCpWRQzVewFh794TFGLo8fJNQay4S+TSZbxJovUNSI3viIJEpxfY9iVmUxnVKrZukPzoljD0NXOGnbTBY23jTAlwLSTAZDUDHMEkfnAxLTRC92yUo5hmcWs3CXd76yxGknQhLqnNlntJ/cR0FGMTIghEh2ES+CYkblrDNA13WatSyREJOLMvhJQBwE5ESDzKpBqAqkErixzdTuslnfIVe6wsHuIY1ahdaqhmPlyIh5wkWAKghIqkatpmENArwkIVtNMc4K3L2xxnvfKBNpDvfeqvHs2ZTEHPPGN6tE8zpZo8zJ+AVS1uVP/sNfMesllCtFao0Gl64sc3r4jMlpQLFe5vj4HE3SSUWRIPZJU5XuYIxRquOpGoE2QTNl3J6EZAlkyimCrjCe+SSKjOo53Li6yen5mChKsGYCoRiilyIW05TEnRMkCsFsztnQZjia4cQyoqSRMw2IfZSSiuP6qPMYSZSxDY282kARJRZBQC0rE9syczcmnxPRHZPZKETKh6heHtsPSGSZRRjgWwswJEJNItEVZMnANDIEgYe18JGkgDiJyRhlwsRlcDpBUQVG4y4tUUZVVSAhDEOSVOJsekoxU6ZSrqMoEYZpcnreRpWvoqQJXpQgSwqyLON7DiIpJAlREJOkEVEqo+YUXDkmFGNKeRNRSlHUKYk4Q49T8sYaqhjhOAHtzozJVGTtag0JgUHHwbZ3GbT73Lhyg/29AwSxyC9+9X3yxRaSnCIIKaLIf+T3pSKieCEIUtlBEzNYgY9jySRVl4U1pz/2WV2+QquWZa11hdl8iK5pZMwy3bbDYDBBVco0S5tE0ZzxdIIgmMSRws76ezyYfsr52ZRbO9/gp7/4M1Q9YjB8RbG8jOc7tOrXmA4eMx4O0Vpv07NfkioK7fHHpKJGZzjHUI/ZWi3w2Qc/plG/Tfv4lEgOWWmtIeFyenDAa6+9xmIy5b/8x//1r2vi/skJxXwWN9rj8qWbBG6f0eiAVNxASg0UIce//+M/4ltvfQu1MGTWmVKplpGNI57+6kPWaxXUXIFBOcNguoeSKXJn+00e7x5Qry3xD37nm5xOOqxf2aZ9+hnL1Ra5/GVOjg8Q3ZBEzRN4I8rFG2ilPAUZAsnkv/0//B852XvAxkqBsecjRQXC/oSMlKFSSWiWV9g7fgCei6pusLrh0zl8gjNdo1gpEHFO7Gg0CllMMeHO9St0J32ur3+FJ7NjlFqLtL1LMLR57+u/xXj6gpK8htmqcnr2nJEc85033uCsY5HPv8tnBz9g7+gRlVTHc1LsWUIQjHn/3W/Q67SZlaasNq4R+yYj1+D9d5b5n//lSwSxQCoKSLJA6CtEiYOu5fECl+FAxtCLkMokXEwj0kRAVSWm0ylvv3GZ3/mt19h7PuONO7/Ls92fMLeHaOoaX/vaW/zP//pHaKpBKkAUX7i5k0gkDMNfs1RF8eJ/XTcBkaPDY67nXZqrRSpKgTCREBs+4tDHDT1EUcAbRxiBTm/uMrWK3Lp9g48++jG7TxZk1QJSLebdN7/NSe8zHr0YsXapwovhMe+99w6ffvwZbkGlkK8zGcU8fvYLTDNLpdzk8lWRydhhMY4ZTw9oVW6hinmePGyTSCP8aMpgWKBgpmh6iG15rK9tMOgfEURzHjx6zsryNQqaiTSW8H0R19LRdZ3l5WVy2hQ1TPnv//v/EcXYZm2/xte/dYf+SUSt2iJTdQlo0p/OWWq2aOByeLxPoWhwenSKUZUIJZGvfvtd7FlAtzuGJM/esz6xK/Dq+AAhuEacDthceo3HT0Z8cf8IRVFY2pA57bRZeCOiucrW9iYza8BkMKE7eIWIz2wKVtjHsmL0zILpLOXy9k2SVERVVSb2ES/3RDJqlmLBxK+WqNUkXj7cY31JIa9tkXqHfOO33sEadTk+OuPu2zfIZDQymQa3v7nFn/7xHsVins8f/AJDz+MGPUwtz7e/89v84ucfkTOzKFpK49p32J0+IDoVUPE57abce6tGFE64vzvgzbdvoiuwkd2k02tz2pU4mUEpLXD96ianL0TMXI1MaZWZJxKVZ/zoyTG1Vkit9TY7bzQZDXscLQa8fHXGSmMDx36Is9gjlRNK+TX2907JZX00DQoFeHz/jLv3lvjV/U/IZjMUqjKuVWK5tcSjB7tcurxJ+2yEpizTdx6jyiWevjyjWMhTbhU46/VY3W4Q4lOqbLB/9CtyxhbZfEJGTf8TivD/T0E56nUJwoRYcRA0g1RWsJwZykLBUBISw8C2Z8ymEprmYcbFixxhUSeKAxIrZm6NL+byfkychCiGiqFKLKYLwqlFpZHBtRNEOUHJGGh+RKc7p1gL8RIBKY7woxBVSzEUEXeuk82bmNkKljtENoaoikFOrhEsPCLxBUGgkiQZBE2k3ihxdtolm1EplUza7S5JJSSfb7Kwhyw3V+n1p1zaqVMp5HEnEVHcZ319AzccUV/ZwHQs5pMJqiQR+0MKZkK9uMSjvT7r1RZhusAT5qAJdOwe2byCpuiUKlkQxmS8ZSaCR6WaYeK6jDpj6jmPpfUMqaLx6lUX06xijWxsy+WsExH4JqP5kHg6Z7W2hILJOJqQCiLuQmI47RKFLtliESkJmcwWCIZKLIhIrkyxrkEk0Zs4yMzZamQYjrLIxU3Wloo8f3nG22/dJRAlPvnsp2zr64QlkUVvglDzWNnc5OzRgHKrhCAGHB2eoZopWr7Iv/qjU7xkyGhmMHPbvPsbBSz7hIU8pWPFtOM2hWaWK/ItZtUZqqETxSmvdj/FsRVylRILe07ByJB4Ae2TBdlsE0EaEU0UwjRmEB1T0XN0RqOie7IAAQAASURBVANGsymCqpLEgOARBx55rUzV1Dk4fEaUiEyHIqqZxQqmDM89Lq/c4EFvHxUZUyygSSBnG6TjAdlMFstZEKZQKuYuVio0A8uyCN0FUrlAKiqUxIjYMbA8m7kXgBqSLZnMhileELC6UuXo0CZfF5iNfRRZx8ykREFKu3NCkkqMunMK5ZiYAGIRx5YI5R5GJoMpy6haBS/yMUSdOAmQJQlJklFIMEUd33Gpl1vY1oSMqVPNb3E4SJENGSmRSdIIUgUxkYnD6GKhJRUvXNaySKfTIZPVyBRAzxokwpjr69dot88o6RrVRoGDw2OypU3mkzG6otMZTNEME1URGQ4d4kTjlx9+jmFIbO+s4VpVbt28RTZzdJFuoskkSQiJBESIBEiCTBKWQLXxQ4vQ3iRn5hj3YrbX11BkA1WXOTh5zPrmCr7vU8zlkAKJTKrhOyOG3Q5aTkNKQUq3MDMuLz/7GDFIePLhL5hdv8LGyhLuIqK25uI4Mrt7HVJHo5U32Vj9FsOJx3zu8e1v3WLuOOxyTppEmKpKrnaVKy2PkTVjkUwpSCqFmsDpbE6hUeXVwSNuXH2Xdu/o1zVRySUs1Zu8fPkrevEpN6+/hWOfMHen+IsFr/Y+xU9tnuw9YKNW5ubVq0w7Q2Jxg82VJU5GLxj0T8gVG+iLEDU34Y3/7Ktsnm5xsv+cSJxwdnJIUdeomAUefv5XlOvr9NwzWuoyyyYcHPTJyr+kcf0q3c4hqpvhcPhjiuUNjIJBUZMYHDzBNlJI88h+yv2Xn/Dwx58wtwPuvXuT9XIFc2kFRQuIR33KBUiEEWe9CXLuKkgS4bjHB+cPGCwOeHi0ixlvcf3rV3j0wz9l8/Yax/0BBc1hdXsbfZLl4FGXQTon0Ze5snSFrhaztNRkqbLEpw8PSaYu9x8cEziH3GreYdpVWNiPqa6sc7b/iDRMCXIiiuuRIpMIEYKk48c+giqSiAmCBIIMaRx+uTspI8gQ+yHvvHGXrLlJsX5IGE1Zqd2kXuhy//F98nkJMdWIpRApFiAJSUWDRPJIkgTD0HDsi515AenLNZKUbM6gVI3YP+jilFMWvk9W2iDwPyOMAlRVQtRgbo8h1aguCTjzDqWSwmmvx9WV6yxmE37wk3/LlaurNIoJm6UbHGaf8cMf/YJcXkYyppyen6IqTaKox8nJC1Qpi25ECKlDqSLhBnUSOUDXEqzRAEWQMdUaiTgjTvIXCVST50hiDZIMJ2dnlGsFnr94crEbl81QLGfwvQDH1emejGlWK7TPOvzdf/IHBGmXYrHI/u4BD+8/4w//8A/Ze/UcSfPxPA9da1Gq5Jl15ww60GgUePWyTb6UZ3/3jMQVuH3vLs5CZzbco1zWWWle5uHHn5Er16hXIy5trdI+PqO+VGTvZMLB4Qtyao5LV1o49pTBiUsq+kiqz2Sy4LhzhGVPuXz5FoknMZzMEQSDjc0mz58/QEZlf97GD0FwbbRchoWfUi4u4/kz+rMOa8sNipLBxM5Sq5cgURj0fApZne//yc+YdOe899WvsXVpAynVGPUOCNyE/cMHrG+uMZ/3SdFR6i5hW2Xt8nUGg2Nwh4iKymKcJSMt2D+f8P43V/jkw/tMx2f0nrzkzhtfZ2AOMSyV7csypjrgw4+7bN29hMrr3LlWp59z6U9Vhs9jVDMllzdxZicEYYtqcYdP7n+PWq3KzWvvIy4PCeIp+XyJ+198gm6s0baGiGqJ1eVv8e57t/jen/wxzsznrbd3mIwTtq9VGfbbLOYZEk2gkC1Qb2j0OueEcQPdmHL+PGBn5zJXNi+jsEZ38oxizf0bC8q/8Q6lFXlIiUw21NAXATU5z2q5gZD4xEGKbQsgSqiGRxhZaFmP4dAhCCJSYYYfj6g3a4SRgO04aJqKqekIKeRzKpVmkSRN0XUdVVWZzWYE4ZylVhFNUdHkHImfUi83UeQ8aaIjCzCddjnYfUSv00WWAwTRYv/lGZ7jI2OQzYrIypRYDjjpnDKdjZFF6J53ieOYIJmQX3LI5cu4/hhBDMjmFKaTOUlsk4RwfPiKG1vXCSZz7O4AJRbwHYHpNGJ59Rr9kUXBEEnUMmkmIYgU4kigruSpxSWSQGM676EqBoXlBbIx4+D0GNs94crNDWpLVVqrG/iJznQmE8sLBDWlWGxyciRwcDBmOhmSJgKH3RGnkymLOchihBrGXMpd4ubNu/jBAkmOsCwX17nY1wtCB8+Nmcy7OPaMxDVolJcpZ/OUCzpJ6lEsF2iWl9i9/4SsvIxQSAkDGzGnIFpF9g+PsLUhs6TP5ds38GODtUtX2T8YE4QhoV3i6eMealTn8OMqx/ereKMFFd1lpVSlnFvn4cEBalHDUBVmg3P8qYU98Zl5A84nbSbjM5aaVXYub1MqZ5FEg2LJQJQs4i/3mIajPhtbaxQKBWaWjRfEpLJApmRwOuwTCjpumNAfdxEJSUOXZiXDfLpPIVMlo5fQNQldN7HsEcVCldncAnRyYpXpYMrCnhMpHoWWipYxmY59XGuEH4p40RxR9UmIyedbNFeaoHkUKibn7T65vIEQprx56zpZQSH1NVJCbH8BWAS+xXziYShFMnIFUzMIPZcotJkENug6iqljOxYiIkmUIosid68tUW000UyNuTPC8efMnT6dfgfD0EmRSPjrcTPESQhcmGEEMUVIUpI4ZHt7DU1TGXZ99nYPGfS/3OeaK4zHDoeHHSYjj9CXEUWRat0g5eIbBojjBEmVqC8XcCOH6cylXC1h+ecX3cn/j86OKIoXu2fSl6NvyUJICuSyZY5OjrCsKZouIQlZomSKa8VMxkMGnSmxF6KrHtWKwc7ODTY2dxguOhzuvsDIZWlkdWpCAT2psFLbwWgWyRdmhJ4Picvxs4iT5yPeuH6PG1duk6YCsetTLkIxs8wHPz9kMoaDwx4rl5bpTc45fXlCJldnffMy33jrGywmQ07a+5hynv75c66tbTM6eg7S9Ne/rv+Kl4dfsHX1Lmoxj++NaayuIntZFCHhtVvfZK1URjFFepLIwaiLV85SWtGItS6baxu8e/UKSuAg6wJx6PDnf/rHDCyf7nzOpz/+c+zRCV/sfcaDF88QjDUGnz0gdnTEKMGQJTQj4OXxKb96coLkFbHSkJs7l9AyNjkRBD+iNz9GkeqMUgGl0WD6/JhULLD+2vsk4RKDsYEXz4nmAw5OTul7RfS0jN5s8ccff8g//5cfMlcSvvlPVvnW3y3xT//O63z927fYWVrhzvtXUbN1kqnGg5cPefDsGT/4s18xmM+5e/c6gv+YyxtNbl/7Oleqr+GPQr71zXuUm3XwfJaLVXIZlYX7gIP9Pf74//n/wmzWyFUq4EYXCKA0RlUkSOJfxyL+OpEpuWBJ/jr2MzEw9CLXbizx8tUzrl29xMnxKYLkMBwsWF1a5723v4YfuMiSevHtCDEIEYKQkiQX77osiwhiehGPK6QkaYxuajx/FlOuLaGaFpYzYjJqY5oSiiwShRGyJjJLPZI4z7xncXzawVAzXF5eYutak43lu9QaIqeHBzRKa3zy4U8JXIFivoQsCcwXIUJawDBl8vks167dwPM8RsOASq0MiCSRiaYWGQ9d7t19B1GKUFWVOze/ThRN2N87olq4gzWfUK2a5Mw8GUO/aJi0fVaWG1jzkO5wH8edUiw36A5HREmWeq1J/1xhffUGlVqW1966i56JMDKwvvQes7HE9k6VySgmn21QrhqcnfYxMwq6qrJz6SaXbqzQ6R2RLToUajVcp8RSq8y3f/s3KZbLGDmH8dDj+rW7kIYIYY5maQnSAftHn/Pk+Rd4yRzdNLn/xXOOT+Y0WyuIss7nD34JqUihLPD05c/44vPnkMrk8i1WltcRhQnd9gv654dEjsdydRs/inGSHtW1Gidnp0ydEx4/OkaS89TrVSpNmUxO48rVq9jTCCVxOdrfQy+0qG83sN0BgjAiiiIMtYFvF7hx7TpGxmE6GhM6EtNxwHD+CQPnhFxR4fHnZ+RVmYJhsLnZwrP3cNo+prbCcn2F43OHIFXYf/GKRFyQqD6LbsTd2zfZuJZnNAtoD0esrd5ExKNez/Pbv/3bnPZm/PKLH3HamZGEZdy5wXe+/g+5fmON67du87Wvvc9XvnGZDz/9M5Y3Ssh6zHw+RRQUGrUSi8UEVVziq+9+B0me4i5Erl+7giBNGI+nRMKC4bTH04d9arU6zVodVTT/xoLyb9yhLBZKaFmD8SxAVVUW8z6aoBMmAm7ikHKBC9G0ApEf0u945AoKsuwiiiZyPsWPHSr1MmIqIisJvu8jRVAomdheTJoYpNEMVfMRhRKeNyGX17FHCpNRj2JZI01dPG9GmvrEkYAmGuSKBmOnT9bIMB65bGxUyRoXh+J84aDqEY7jsFhIXNq8yng0YHmpies7xIQcnLyiUV8lm1UZ9EPO2ruYisHS6gaTnsF4vs/xwTGLWcTa2ga9wRlp6IEocHx8TKmYIzOP8bU+DlNMQ+D6xptoqsu457G1kuNseoaQhARYKLJDqaxhOQHPnh1x/UqFyWSFq9duMZ6HWM45qiZTzhZod89RFFCDPKaYIy65pFKMFqjoQh69IKLJCc+efE4UCSSJSL6g4UsClWKNs1kH23eollqUigaz2Tll8x76+vwCp6BrLL0ms/vyFe+8f4XDvRNGgkdmliUVJyiiiBikSHKVolKie35OoWJwetJFU8akUUq9ViFXqTMZjpnMEmJVYXKQsNFa5ubVm5x3zijoARmjyu7BC7auvsvh0Qtyhk2tvEaOlNHwc8yMTyi6eCOLcrmGlEgkxPhiipAaCKnD06dP2bi0gShLLKwLgPPBQY/L9WXEUCN0fAzTJBESSBWKpTrds1OWai3CNCFMHdJQJgxjKistZtYe2UKWJHaRIx0hEel15mTyGs1GhfFoTuiplComI2fEYhSSyzfQlCIHh69IiClVVGI7RlEVBDmludbg2as9RMB3Lna7SrksGTVFSApoksOwPydKxmRzDRI/JnZtnIlLQoo1d4kTBUkUSdKEznCIrcdUalUEEhRRwXZc8vlVHOviQJRTkTQBUUoIQw+4iIwT0pQ4jSjm8/iOR7fbplJt4Xku54cSo86EbAGGvQgvsjEzEsPBYxQpQFPKCMiIsodrqyiKRilfJAg9Xn/7NWxnzNOXn/KVr3yFNE0vDuH0Iu5RVRTiOCGKIiRRQ8IkFTwQHRwrg0wLQThnMpqg6gmedUy5UCZ1A5RsQEHP4TsW02CEE1ncvHKXWj3L7ssnHI0HZLQ8jXWT5nKT8iwDXkqpmTCez1A0j5WtLLNxiKa2MfQmn++9ZGunilw02Fm7zuNnH3Lj6g6dV31G53Mymz0ePhuRMcr4tsPq8jUOD85p0aXfbfP9v/oJt+9eYdKf/7omlpQSaZxytLuPpkvYjk4rI7KYvyBfaeAJDolRotW4hBv1+GT/Q97a/hpDa4HpuGwslUANsGwXRIvOvM+bl9/CVHyO8TEKmzRllak3QQolWo0amXtfByWl0+nw+aPH6J5IXFmndHREst2kvtGkLOV5+pf3MdI81bUcyxvbPH/yknIZXozaGJUyd1fqGK0VrPEBo+4Jq8rbqJbI8o6L3TlkEBQxFY1maPOb//UaO2sxvaN/Q5GA9Y1NhsPPOHw15c3vlnh4f8o//PZtZt41lJUGtdJVLl1aJVo4yF6WYn4FIx1y+GzIlUubfPzTzymVTZorGc7P8ohrGwgHoOkTSksSb92+hxh/j0AScAMfRZZ/PZYWUpH0YjzxpfksQZDFizuLJOHaY27cWGM2cbh35xrPnj/nq2/9U3724b/Atnzev/pNjp500XSIohRRUElEiyT1kUSROE4IowDhSyetJKVfIoYE+oMuXnTKjZ2vsdRoMB0sGPaG6EoJUegRpyKiIDOcHyIFoCRFImTO9l+wvXoZx4rRNRNBMlhEId7C54tPHvLm+xu0Gi3G8z6ffNYmV5pSKTS5enWDT+//lNDN8dob9xDEiN0XPXIFCd93CKI55+fnvPP219nbf84nn/6Cy+trZM0m6+vrCJLNcNrHMAWeP33C+spXeP8rWxwcfUYQu4RRRCZfpT18hYRBvpJnOvb4zm+9yatXz7l14yuMBn3avXNmU4fNdZk337rLfNEjjkMULWBtbZMUD103GY0c9vaOSLWI0FtQLpZoD9sM+z1KBQkvdtnaWeajj55x/dYKh8ePOD89olAuUctfZaSPeXL8nFJhlSS1eXF0n6Xmm1QbK7iuQ6N+l+s7IrOpg+vayGpIEJh4voUim9RrOVQ0REVEUiQOdp+hFUOKzSJCWmH36IzLSyt0x6fcunsXN5hh5nPs7Z0ynfcxMiArOrqRpVYp4NodHj99SF7fZjw6xygmzHsDlppXkIQC5XyLrS2fiCGfff4J5dI6aBLFkstsMMXUJYbTkNJSkUhZoKpZ1GzCQXtCVDK5sblKcBqwvlrluKOzkunx/OED1reu8Q/+we/x4x99H3/uISYCg84Ey4Xvvv97eG6fRx/32bq2ytQ/4NHjU7LZFcRUwHMdhPiEJPVxfYmd6zWOjqZEvsF8JPGf/eZ/y1H75xwe7OLYNoo4x5rFLOZnZIwmUZxyev6KSj3LydkD5lOfer3+ty8og+kcL54zsjXUOCYJHTKyAW6EICZksiZpqhE4oGkKplpFUUEkIYo9Aj/C0HUSYYqsaVhTCVlWUJSIKExxQp80nGEqBtZMRtMdlpZW8f0UQfTYvJxDlQu0220kxadcyRP6IrqmIYgRRqwzGfmoYoFCTseaO0xHCxQ1Q15TqeV1MnoOa7EgiWJEBMQENKHAWq3JcNJm2ksJ3JjmqkbnNOAgGBIGNp4vY2R0QmHM4f4zUjGlUq4QpzHLS1UkQWBEgUxmwPREQ9Q1+r0zuiOJ+pLJtr6Mni5AmmPNfZqVVSaWAL7F+rUaKiqkGj//ya+QdRlDKpBvmnROJlzebnLtxjqdY4lPPvqIajWLJjRI7SGB5SLnYjKlHK+99U0++ewxcWixs7PD2FpgWyF379wiFV2yhsrCCqiUVxktFhTKEa1sgaPDDs8PnrJz9XXK9RrVQp2fPvuE5UKOYn4byxlQLph0bAeJlEo9gxdGhMIALaNz89prHB58jjW3Uc2ErCqxsALMOM+0bTGvD5HJksvrzN0FmVyOmTWgUV9m7+UjcvqMeuUq/fMOg64EqonrD9GkiDC0MXNZPM9iNJ+xtNqkN0w5ONyjXKnhexESMouFTV/qYRgmppFFFGU6nRFrW5cZ9ifMFyHjwQmtlWVs24FUJZurMFvMUEwdQROQBY0kAEMVkWSfOA7onk5IRQEkD2t+0V0zdUhEDzfYR816mLkSKRG6mWJbIYW6zsv9PUqNAqPhOc1mkzh1CN0FGb1M4MXMRymGqhKGBUb9DqppIoQKc3+GoisYap4kSbngSorMPJV5MGA8XFAu5MmYZQxFw1qoiKJMGPjIikgUCiDEBKHzZexiiMQFNN1zZ7i2iqEUaR8NEdWYSqXEeDSmnL9CoThFtC3iIMSQq0iShCJrX5oeYBHMCO0pc88hjRNiMSQMh5g5A9t2yWczTMcuSMKXGcMxSZKgqjKBHwMiohSiyCqu6/L8SQ9RANs5o1pVMA2dcOHgmjMS16d9MEBSRAQjIcDBoM3JmcCwOyDTTNHLPiezId35hEyaYTaUmS3GDBfPabVWiZwMBaPFk89fsLa+xftXX6fdP8VZDNkfW9SrGYpFk3Rao3EJZnHI5pU6S+UbPHvxmKk9I1qkzHo+2XyeS5stHj3dY/vStV/XxDS1OT1rs7pcx5qO8GcOT60jIlxkM0c0l7i+tEPbnVKtbHE7r+InHn3LYadU4PB4TLGq02jUmPe6NLK3OZ1bfP3ma/xWbY3TvVfErs/cs7h97QrmrMerucPZYsjlG5fI2g2qqzkaV1o09RznJ33EaUS4IvLWb/wukX3O6e4Bd975KqlcYLH3iDgSqV7aRg+6YD+kXqiyeOkjOR4Td8jlrWv84sVPyUgOGxspr12fMer8kv2fpqy9cZXSxip+PMZsGtzdyVFoiNxQl4gyHZLpkN65R9EwyQoFAnVGIJyjiTqLcYBeE/EzKZnCCkEyoH805cala+w+fUXoaRRWayRji0mnjaHIyDHIX6baCIJEGMcokvhrcsBfO7FlQYT04q+3ULhyM8HMj3CDFN9xkRSPq9c2+eKT5xwe7WLmZLY2qxyd+5iqAZjIooKQSqSiBamAosj4/gVGLooiQKTX63HrDZHtS2vMhiPEqEhGV/BHE5I0QVZUVMXg3Xu3+ZdPf8qVrduUSutcfe8SL1894Qc//XOurN2itaTguwvchcA3vnmHmd3n2vUd/uRPH7GxusO3v3MV12/zs5/8mHs3v8V8YTNb9CkXV2m2TKbWAUenF2lbj58+RNU0ztt9ihWTTsflG99cYjQ/JRF9blz7Gnt7eyAeUaoJWHbK7at/QG/6BftHx+RLWRbzBSQVojjm8Owj9PwN5osxu/v32Vy9SbFYpVgccNZ9TpwEpMOQZn2ZxWLMz36+y6VLW/i+D4nEpctN7j94zOXtTR58ccjmlSb2fMzTZ32u3CozmL5C0WIMPQd6zFK6zmQ6xA16RJFKwWiiSln64x5Xr3yT23fv8PEHn9NcKiBiEHoLjk4fMJ3Y3Lh+B1FLiUOTcqlFu9chISbxmqxebhGEj2luNDg6OyenypiqyMef/4AgsfHjmEubN5k7x4hiAUOr4tlzbHFCf+oha3nKYoa16hb15R3mbhfLPkA1MhSKKi+fHnJlu0GxWMSxc6yuWHh+yMKa4O47rK0sMZ3GtDYayHJEFOSRFQMvVBGiId5in/bLA3a21vjRB6fo2QgphFK1wpNnnxKFCtmsyun5nJW1dWy3zWH3kFeP8/z+P/w2tdyQSn2d6aHKtVsJ5+dPGYzOee3em8ymC/LZAsP+AmumUi41GPUXaLLBo0cfoRsXqWvXdrY4P29jL7Jsb3yF8XyXmR0w6E/ZfuM19o8+pFHbYLGw//YFpV5RGI1nSE6MHIVEHvRnA8yMii5niRKFOPbIGDKKkiFJbSb9CFmWWVhTWrUlhHQGSYq9iAjj+EtGncLccnCDAF1KCQOPSrHFcDyiG1jk8hqiEOP6EYpcI1+ooeo+uikgSSHFXJbRuI9uxpiCjiDajCcWRbNBmtdYWjcQlJBkqtHtniKoIog6SSwRuBFZVcdaBOTNOoPpiGvXskzmCltbJmZmxmzYRPLGHOzvo0gRjWqBYrHJcOLhxjbtUQ97EqCWdJRhwEq+SKTm8JmQz8Oon/CDwa8o6hq5akwiNjjrOLhpRCFbZTR0yGoSHfsMpJAogjSqQqwiyTb2zOXZFz0CcixtVVnaVBidTqmaClLS4Gx8TujPON4/4+rOJsQOnhsy7k8xDZmj/Se0Vtd4cXSGpDpsbd7lYPhLtGkTKZGR1ZDN9WX8cMbgUEAp6txsXUOKeoimxOBkiqIFWNNzupaEJGXIZy3U1CJMEwbuEwx1hlhTaC6tcPoyIvJC6lWd5dUW48UI306Yy2eE7ZRLK7exFxYz22Z1eYViNaLT+RTZCPGjEIUK1uyMGR10USaTqxL4c/IljcF4hO+q2DMNz7K5c+sGD794QLlcJVAvcAh+4nN00mWptY4YCyyGQ/KGiYtMv9/HjRwKeZ00lbC8KWEAk+kcUYpIBAfL0QhckUpZxygYDEYTZCXk7KBNPiejGzJhPMXM5lEzRYYDi2lP4cbtVZ4+6jA6n1DezjCfTlgqFy+cnEmKKuRJ4pBMRsWxfJIQkjChUqwxmgxpri6RSgELa0ocChedk/hibO1ZLmRUdFGgkM1wcnxIrlhFooUkyqSiQZQEgEiapgSBhyjKxFGEJCn4UUiuZiKkKjIGa6s5To7PUQkoZmQOX7xCzAqYGR3fD8mXdVxvRiqEBIEAqEwcC002SAIPx57h+RY5wyQNXR5bDzCMFq4eEafpRVdSUi5EpKIQBD6SlCIIKqQiphnx6uUhkiCQy8rIks+8r3AaHlLIVam8exNr0WM0n6KaFey4i04NQzG4/cYbLMZHDI8HSHoeBIOZluII+1y+dY3kSRs1CkkXMZmsyXqrhWkkVBs6Zz2P5lID34s4Pjtn6EhkjTJTO6JQaCJpIp8/+YDjg2Nu3LvDXBzS64+IVYnPHx6RkWWs8dGva2LOqFOIShy+fMzJ+Jzf+uZ/xdnU5rwbsr3sM57P0JbX2KhtMe4fsLJ8lZ989BM0QWAUl6jXHeJwQeLXyS1t0Grd4S9/+H/jT2yP3/07/4y80sS1TtmZV/HjhMJylXp9ijQXqKsKN37/79GZDJkfPGW0bFJau0wcegz3j9nYucz5WOKtN7/CcJZQyAdsfOc36Z31cMdd2lMfFJGrNZOrb1yl032JF/jYn4tkyis0VQiVkN7sKtPJgPW7q7w8nbP44QtWlst8dj5h76RLyZ1xvJhz7JZ45/U3aRkem9tZqkseclAgEW9jlnQGSZ/Ql3i1f0Ls+Ny7+1VODz7jxYMBSiakVZRpXbnK+UGb8+EpsgSy7ZPqGr4AmiihSRJxnHyJ97kQlH896k7SiDRNyeZC7lx/GzGOGXZtFC1i9+gnvHq1z5Urt7DdgDAcsLqyxN7BPpIZf9mVtJHFC/QWXHQ7k+Siw66qKknsYeg5yqUSH37yl6RMkdMsiiSzs3MFgVckUUIozrGjz3j7GymVwphXT09BXCKOZFTq3L7zBifnH2Jka9SWU3o9Cd0ocf/hR0RYmKLEs4dddNXgdH+ByiGlWoGZ3WE8tHnz9bewXpZZahXxHIGlpZhev03iLaFoM7SCx8n5Pr22z+qlGj//1c/5xje+Qaa64IvPnrFz9Tof3z/Dtm1yZYPZbIFtj4hshc2NDTQ94eS4SxAEjCZnZIzCBRovq1EurhImA6aTBaKkQ+qxslbGc2SWV+o87LxC03XWlspoWsDW1RKj3jGBn7B9fY3R4AAzZ1KtRfzi55/wd373XVy/x9Xmazx58oQbN97mqqLwx3/6P3J1512Wlrb5kz/+U7Z3mpwfDVlZbTKfXuD96rVlMkaDQlmg310wG6eEYR5VjlEqFo8PP6CsL/GjH3zBt77zBs8+/RWF5iqZfJaSdAkt67J79EuSRMeehjRaOrXGCt7E4fSox+076wiEiOMIRJtyrsyiN2Tz2jrj2RHD2Qm//NhGpQRCAKlCs74CaYgqgmIaTNwOt9/IIM5cZgObSIrIyh6JlvLW7e+S0VPawz6ZaszR8Yip7fP+m99l4Xg8ef4Jy5sNipsjToaPUeM8lazJJAr5+MVDLt9Y53/603/Ha7ffpKJomEoJMRTpn0U0G6vIJPjzLN3OCN+LMbMxi2mP1ZVNDvf7tKp1Ik9nfaVMp9Ph7KxNFBcxNIVbt1tYM5ml2h0Ojh5TKP7NO5TCX8e1/f96tr6WSZvNZQJHZjGeE7oOqQCFahFBUfHnLqqeYpgikqgzG1m4rku1WmY8ctB0CVUP8V0IfZNY8NAN8J2QZmOFiTXFGnsYuk8+X0BRSrT7HbK5gHJ+hZE1Io1EGs0KYgqWM0KRU8r5FQb9Kak8x3NEVC2hWMhgTUCRDSJhQRCmNI0sc29GbbnJZBbhBw5BOERONGQpix3NyGtlJMXG8hRkwyafVzk9mqFqDTaWCgz7LuPhgJWtOsPpCDWjcdLuUMg1kFQBez7DSPKkos/CdckoMkksMQw9ioZApiCjay2kSMEKO0jyxQqBIlkoShE/mKPIJggxqqQzn9qkiUQ+L0AYUqpWCFKJKPS4d/USYqJxcjJHzznIscb65U3Oz/d5/vgVRqGA50yol5qIqoJPj1w2gzPLYIkDTMUkJ2ep1FaYDxbEnKPKFarNdYanxyRShKjnGTszYndGIqVkJIdsYQnLc4nHKZeuGAymc+xJiFmTGdgDotDk1pXbPHv0kEbjMqOBhW/NqC4LDAcejcYW08WMfCZHLifSPjoh9HwCQSR2oVbbYu/sKbmChCZmUaQSc7dPnLiYRokgVHDsgIyhUchlabf7+E5ILAlE7oLllQZz20GRDeIoYjQaUS6XmYxnaDmZKBaQ5IicWUORYvxQwI0c8qaBHyToWg7kENcZ4s8F/IVGrVwhlucUiyUyGUgijycvztFLCoI8o1m8TEGTOWv30TSN5aUKziLk+OAILSNQqlTwfZ35YoBAEUm2MNQsqTBjPJKQZB254jDszclli+zMEv5PB+qXu4gS/+dbIh/afTbWliEK6fZ6VOuXEdgkiiGJQ5zgwp1K4nN+so8sykRBgCrJTKcLbt5q0VjK0TvvoWamNKvX8cMxqqagaRpzt4OAiu9pjGdDYgQyZpHFzEFRPRLZxLYdCrkihqySJhFpGJFGEaahMRtn6PcmiLL8JbIIUkLyWYP2aRdBEr+EnwOxim5INJcThFghl1MvdpcklcCPkRSLVqPIymaZ+VxheavMxsoqughyGiP4MvN5hJYTkA0NJ7bpHn/I1776nxOkHifnR7heSq3awHcDshmZxy/2KRRV4lCgkCtSrRbp9DqEnoUWZOi7Cywp5MblSxw9e8Hx+YSl1RZnR8+xpwKtyhL3Xn+Lk/YXv66Jqq6jannms0OWSlUMqUiYxuilBqOTzwmrZYzKJSrzOaFoMyQmbjvEYsq7r11n7+iEJIlZ2l7CmaqcDI9ZXX6do72HXKluUlxfYTo5Zz1fQ282mAcDnONDjjp7SL7BzZs3GY5tdFFlMu2iGXkKWYlcsUWIxHzWQ9eb+PEEMbUx1QxyweDhR5+ytbPFYbuHIpTZubqFEDmMjuZcvraEGxzz/GCIrJepGBksUtqHZ4SRw73r17HmMXbSw5FXOH7+Maaaki99hZVmjiQaoZYyhM4Ca3COlFtFqeX403/7L9jZuYeqCDDxUdWNi8lA0eLq1etEkwmjfsC11zY4OnnB//Df/ZRfPD4hn8kSKSJpGP8aHwQxsqwhyxnSFETp4gyLwpSMFvP/+J/+O4aDZ8znA9qdAYXSCtPFCaQBlp/y2tUG/+Zf/Zx/+S8/oN4oE0cSqRBBkhIlKZIkoGkG8/kYWZZRVZ3RwOGf/Te/wd336ohJiiyAkEik6YL1kcC7//wFkphiORO+91/AB36PKA5Y37zBaqXC8FXKzpWvoNQkVNlgPD5B13WeP33AztU3EZQ5k7HHbBJgmlDUbnN0vMezl7/ijTfepFDV8D2VWq1CMb/Mp599QLlcRVTmqKpGwbxKu/+YcXeKkZOxLId6q8R4NKXXnfLaG1c4PxuzsrKEmRN59riNpmm899bvEvICa6ISRwKSatPuv8S2AoqFFoocM5sPcBwAk3JNRkgKyIqIpNi4bkirUefosEMUe4SBwRt3d3CSIY+ePqKeX2NhOUhqSmSHvPf+36XT+Zxe30LWLkgW1cIqulpAUkUG50M2r6zw9MUXJLFBxADHG3B8cMbl7VsEfkKtVuGT+z+mkNni9Td3GI46jPoOmxtXkBWF9uhTothgu7HKWWdIpbRCNlrQnsVUV8sEnoYbdPCDOa6TEMRt/MAho9zg7Xdep3PcRVIUnux+hBDXUcWUmzfepNIo8mL/l4iSShC67O095trmb+G7U4iLVBsqmiryF997wPq1Mju3L3N28BTr0OP5wzaVNYV337tBd9Kh2lwmGyloikcoeYRBE6FSQ0h8HGvBy/0HDEYBl163qCrvk4lCrOOU2jWZR/s9PKFBq9XgylbM3uM9Mtoy1ZUyK40NomjAqxcvqTUrHJ93ODvvsbq0SbVY4MXLU+q1JmdnZ1y/dgcvHHJ0dIAXWiSJyb07t9k/eM5oOGD70jWi2MMwVP7p7/9fhf+tKvzfPn9jU86VlR3SABzHYrGw8byAWqmCnCgkgUOpkEMRdJIwxvO6ZIwq9doqcRyTLVy4lpKkQCooKEZEVi9gqBqFgkLnbEBeN9labfDarbeY9D3cYES9VSaTK6DnfFQTUGxmiw6T0ZTQ1fBs2N3dwzAVWpVlAjtEE/JUKxnyBYFiSWDcXzAdORTKBXLZFoeHIxw7YrGYEcYB88AjFME0K5iZHKXCCoZh0mjUCJ0iGys3+ObXv8J8PEQRdbav7BBGDoWihD0fUi1l0IwY3ARRMzmdHBO4EfVCFUWuEkshWU3CMDL4rsywc87ZaYfQ0kmkCIE5xCqzoYcqGuiSQOr7SKmIEAcQ2KShR0OvsVork1FclpdM9s4OebD78cUIOo7odAfs7p3iOAmCpBGGIbl8idF4iuvbCGoBUciRyynISRNDNtlaucHJsU8qqaw0LpMvV3AmPeqtIqJWI7HBmrh4logXZun1fI7PhyRak9KSxrSv8vCziEkUcPByzOQ4oIDC3v1naI7OrLOLkLaxoyGLUUw2F9M9P0WMItyFy3zosbDnpKKIMzfZ2riJJIAQ6RSzTQizuIuQ6chCl8sEPnTPz2i0imQKOvuHByiqzsJxmfY93FmKPfUII4+5M2ZhORhmjt5kxPJqBVWSsRcWgQXt4/ZFfqtnkzNUxEjBUGTm8y79syHOTMNzY2Q1pNfvMBt4HO31mY0tXNvj6rUGK+s1VjYKLK1myWo5ivksiALt3ghBlGg2Gyw1KmREE81IME2TJA0o5rdxfOsiH7ukICoJsW2TUwzwBGQUFFlGliUEAXrDAY1mncj3mE5sTD1DGusIooIoQspFso4qyQS+jST9x0vihUEH6rUCS60cb717g0KujuNaVBs1wiQkiEVy0jIKMrWyQTavUCjmEKWYOE4hUsAVSR0Rq+dy9KLHou9CIBAHCufHHr7vI0gKoih+uU8pk0YxhqZSKheQRQlFEpHQEcQIz3PY3r7C1atXIdFI0pDJJCBMRfxAJoiLRLHM8VGfn/3Vp9y//zMOdtus5je5//EjJN0jCCwODw8JuhYV8Sb3f/KEvUcDBuM53emUhy9OOT47YO/JgOVmhelkQJIkHOyf8uDJIc+Purw8GyOVVhhNxoSRRXdgYbkJa2trRKlBfXmdN95+g6VLLX755McMuuNf/4aHbcaTPaJgyGLU4XxwQD5Xo9t5wtbOb7Oavcmyn2JkDaJigclkQq1i4iUxf/WzXzCzPc6nXR4/3iWnqGRMkbrZZ22txvPuC05PHzI5OyCb11m4Y7748DNMJSHfvMaZ53P/2SFeXoCixPX33iMvy/SGC3YP+1j2GLWWp7mqUItjFA8G/SmHTzrceet3qItZ3ltq0Mq5TM8PMWSDnbsVJCdhMgypFTM0Wk3IWWSTBe/caFGoNTlfJJw5C0rFOjvrRao3Kty89zWu3NYYLp4glWRyRZNLSzWErI6xmicfCTTSFmYxwfIt2vMx+WbI3ddKlCQdZzCn0x0y8Uc8PTyili8TBTGxIiHIMkQxKaAo8q/frzT9a3F5wVyVJAnHcbh25Rb1psh8PiBOHCaTCfmizHh6jmVHaIbM0dGE27c2MUwZUvnCvBgHiFKMokik6YURR5QEREkgii6iRH0/wLO7jEYvOTx4iuvMsG2Xx08f49ges5lFFGoY0hJ/7x/8HXZubzNfTFh4HfYOXzKb9JjOBpycv6LeWOYnP/05iRwyXXQ4OhgTRym6nrCyvIkVP2FpI8vv/6P/ikLFpF5rkslIVMotxtNTGq0irRWTTz/9lO//xY+ZzPcRZZd8weDGlTcxTZONjQ1ypZBGK48ilahWauhiHmsSs7qWZ2OziiAu+OUv7qPpWc67z1EU9QICLqmEoQ+CjK7XWV5do9SMGI+HrK9vUqtVODk9xLFDBMXF8VyMjEFz+YIusrd7SuoXKRWuXnTMJINSdZ1PP/8l9z97iKHn0JQi21t3mM9cXr78lIUzQtAnLBYT7ty5heePMOUVGpUtWs1NioUqKysXrN9Llzb55m/co9s7Q1YjYlyGwx7zxYjZUKaSX8cbhzgjEV3JM5u6aJk1dK2AGx4xGk4Y9zwCb4Y1jckoK9SqWc5Px0SBgyKkvHPvXTZXl1hdb3LSfcJPP/pLVteWOT7qU28uo+ktgsjDMDVUNeHz+5/iDCr8g396lyTMMu9FNJstwkLEW7/7Bs0rq/QSiciM8eIcxpLJ/eMX/OSjXSYjnWJGw7XPCF2HkplnpdJkvrvJg5/vosRlCuUQ1c/y+uXrnL78OVra4Rd/+QGxUyeTbXByuuAvf/RjfvHLz1nMJQZDl/2DI0rlJRa2BbJCuVrGdgNWN0qMJm3a7SmXruxQrq3QHZ/wcvcDnJnC6voaU+uUhdvl8bMv/pOa8D/1/I1H3ofdDuFcollWWb60zsyGWHAuog5FEdeGOEnxLZdSqYagWoiChucGTEYRlXqOhdWnkC2Ty+WYTjpY8xRdybPcNJAlg8V8jrassbG5zDjaJ4hCgiCikK/iuTFJdBHNmC0ZjPoLKsUCkqiQCjYKJQp5g0w2wV8IZHUNw8zy3e++z2CwYDKaki3U0T2b+XyOrCjYloSXiCRagBI5WNOAWiVDJiuRYBELC/SMBekyhllFyYYUqy5xX8QayWwurRMkNlPHo2/ZaEZERi8RKimjyRBBMkCVyMgCUiIShjFmmkHM6YhCxHzmoiYSUhpSz+eZjV1EXUYRMuQKEtlKnTDwKZVKxL7Iy1cTNBM0JWQymbHSWKF3fsbE6eIvcky9Q7J5BVnNsbAXmIbM6loL1czQdSzCSCYjasjCnNDymQ77LLwzCuUcL48dqqUMni9w++5XiU/2OD96wVuXmsRGwq9+touWV/EiE6t3ji4mqMWQ2mUBMQwo11t0jkekkcb6zhpnhx00WWE2t1lbvkYSzPHdFCXNkvoCxWqB0bRH1lhBkiNGE4fHj59yZesqd6+/z2C8jzUf4fkLCoUsaZpiW1NW1mpAxHA8RlBFxlYPvQDFVhY1UggDn4VrIyCSlUtEkUuzJtM+7aFoOkvNKs4iopLPkAYpjUqd826H0aRPsagjpjoFQwdpyvbdNc7PhsiKQjln0O1ZZLMml7eWOVt8hhXmEBSNIJoy9xVKrSW8s3OkVGE0GF84m7UCC8tGFjMkpFTqKqNhm3yhwHTikcmrxKJPTr3M6eyAVjODOYU0iUiSGIC8WeBk4LLWqlAq5siXZYZDhQSRVNARxRRJCiBN8H0PWRGJgwtRmaYxsizjuRb1eotHTz5HViUqlRpJkmFu9bi0Y7A46aKrGmmkEXo+gqrQ7S3Iq+t41gx7MgUEwjQkiQLmYxj3bPwwIE1jGkvZL53dFwc+QBj6mKaJoug49jmINr4jo+ops6mPNZOZTo7QlSJxNKXazNNYKiErkMvk6Q9f0mt7VJZTrKmOVlX46Rc/YHm1ij8fY/kR2+vX6I18LPuItdo21SqcneVoGHmMnMZseIwkhjz56R6t1RXsqU/kSHx+/3OOh32arWs8f/iXtFopulOmc/wAWfJR5xZ37tzj9HSAnlVIvYRvvP4Ov/eb/+zXNfGXv/gLvvfD/zv1xhpapcrJ0UMWrxKWKsv0Zs8pRRW+ePUFq1/ZwR1PuVxfxRSq1JwpBydzXr+3g5FxeHJwyo+ffA+zeAV3d4wjTVE0EUOr4DY1vvfBj2gWGiiqysPdNvkr29SyCvWchnvykvvdgLXNU1bLyxTyNTQhYW/3Fc1inW52QaFuUEzzFMUSs+kAe3LMk1dPeffOPWrZFrm8yYc/+nMqO7eJ7TFGAkKYkq3GfPL0FL1cYhI7rGYb7PsBStKjKypc8xXu1u7y6vghil/n4e4zFl/8kq9/879AsQcU85fIs8z+2a+oNJpEZwFr9S3iVY/Z8QkPj8/ZuHqD1c030AqPefTZR0gTjZ/sP0PUDdRYIYnBSyJMUYUwRlIVwvAigFsQU9L4gi5wkb0dsrKS5/R4Qil7k0D8ght3trFsj3LxMi+O/oQ7m9+lpayiblgYugixiiyFCIJKSkySRCRfCkpJukjNURWZOI5oNZqs5KroVYUg8vjo/sfkisv4E5sokVEUGUEWaO8t+PH/5a/Y3F7inVvf4IMHf8lbv/WPaZ93iA8cZDUiij1qzRKGKeIsFHJ5gdAX2Nra4snDAZvbq4SRw2i+T6OV5cHnzygVmzx/9QnNZo1w0aPdGfEH/+i/5C+//yOePHnG5tYSpWKGTD7kzs1bZPRl+t0fc+XydWTVu4C26w5xLNIZnJNbfZ0oHRL4MU+e/5KzszaXLl2i2ztDUQziOGQ2v+j87R/uoWRtctk6UTLj8eNnhG4RVxxwsC9DKiIJOcRUoDfpcfX6Ok8f7XLUe0CzlSNjyARxShTbLK9uoCgS4+mCwegXOBMPQxc5OT5ANyX84IS5K+F5Fp3Op+RzNd5+8zeZzE5xXZelpSWqtVvsHzwh9AwypoqiWFy/uU67M6JSbFErZkn9hNCZMRp0wFXYerNC9+QVtr3Adx2uX32TZ88/Ynv9dYplmacPjnn7tSwn83OmPRtVMSjXs4xmUwKGbCxv8OrFGRmjzM9//gn7Lzxq33ao1Ro49iHXrq4xnH7B4X2oV1rcumTws58fUqqVGPRGXF7L88XTJ9SLSxjVkM4oIpZWKbQcDqcfoA/forWyxfPHZ+QyW+xcFXn48DHKsszuYESpLtHtjrlz8y1+451v0apnefxgyv/yR/+Kf/qHf0gm73D/4RNuXlvj9LRDw5fwfAk/UkjjmPsPnrG6VCcVHHZfLRClGMsOQJZQ9TzrG1dpFusYK1my+RJ2cMLPfv5jDK30ty8oy0qDRXaEXJRBUhGdEWkQYpo+w5FCueCgyg56KpGKUxaLlGxBRdIS9FxIHMzI50LmgxB3NqW5lGdghcSCg+uPSOICVhzw6OBTKtkKi9ECRdSp1LKEgk2tnCNyS+hiwng4QVE0zrs9lqorVAp5TvtzyvUaKys5MkaW/b0zOsdtFl5KmNj4soDvHJEKIMopohTSaORwHPBjFwmdUrVALLpMxjGKJaFndFzP5MnjUyK7TcHM0j4fEwQylVaRS9t1Hj96ijOfkCnk8L08ojgnCSOENEaMXXxXJsknqAvv4qMrSyxGcyIhRhNCZDFDVs8gSAlGXkeMA7zQw3EU8rqJL0vEgoCoxkS2hWsFzGZjVlabOBOPKA2QlXVSbcZg3EZTlnFsi8iPOT7oY97ScacQzRb0BQtNyKMKRRzBx/UC0n6CvmUyCnJ4lsfSpTz94bMvxeAJT6Uh1bLEa+/mOTxI0UQJRZc56swopQnFXIKXFtBLWW6taBy/HOI5CqoG1mRBqVDG96aYpkAS5JFNh+nE58ie4CxU8hXQBA0pjhFUhf58guo67Fy+gWV9jBtMEKWU8XDKbBZTa9Q4PxvgeQ6yGgIi2WwOP5jSmVrkM2XyZgFFFRmPuhSKFUZjD03OU6gJILpcX7mFvYgZTtoc918gpiUUSSWJYlzfQynOEaSYUO1y7fUijz454+mZTGupTCAO+GJ3SLncIFosIDWYa2NEXeO8neL7AnkzRJI0hFjAHtr4loKftSiU8oxmXSpLJqHnUGuVmU0dolBAMFzy5RJ2lBAmIaIoICCQkiIlIa1GBlE2L5iAMSRJipbKCEqIm6YIsYKQpkRuhCqpBEJCIkGSQJymCGKZDz55zGw+oVCqsv/gAxSxRpz4RE8EBKaoaY58yaOgFxjPQpI5BOoA152h6Sb2PCRwfSrlOgvHZm6PUZUMoijgWsGFsUFQCKMI3ZBxXZ+3336bH/7g+ziOhaKCqgcIqMSxxWQy4dLWNtPpCKPYoFosYc9tspkG457F2uob3Pj9hJOTJ1xubjId2fiJzPqb63RPOyQInLUH9Icx9arOZD6gsXaL3HqN9vNHuH2bqSDQP9mn1NwhlUL0okx/OqDV3GJt+zWEyOblq0847+bQpR6iCLKhYprw4U8/RDZGqFqWREg5OHvJX/7oP97WK40WhdZllHwetWpwO/MbGNkc5YLO0dmYJHC5/e5dQtdFkBXy5RJnu4f4cspv//53ebl/QP/xiPWNIt1RnZ31FV48PyRJFiytXscJLaJkwe7eS7Kv5ckWFZ5Peqx0iqDIdK0uk+GCzWaTYWeXp5+9RC+XMdWIq6ubPHr2CkP3KBSbFCs1RPWQwIpRK1nyazfYHzsU8lM6Y1jauY7TnxAVTUJNYtI7ZXb+hHkwIuonCHEWveBy78Yb/Pv/9QUt/5TnJZliRmOrucbTgzO+/eZN2kOLYH7I6e4p1U2LwYMfIufK1G5c5vjBC5Tzc0ZArbLJRq1Ef3jAiz8dcPVWgWqlxeH+PpmVKhHnpKlAKkcoLkQypHGCJgjIiokoBSiCQkxKlApIsY0Sybx2c4Pz058znYp87Wu/xaMHP+X46S7X712mvPobCPMCCDKnwwQhTUilCYmgIQQ2qXKxmymLGeIoQpYU3MAnlSVEIcWeieh6ne/98Ucs73TpTbvUK1WWt66iPjknl80znQ1ZvbzNb1bf4uTsJS/3Ixw7w1ZjnXlnhmgo2O4+oS+y0lzno09+yXKjynpBo3e24NbtEpd3dHr9E5ZWTTxX5tMPDy6IKYKGoQecHEY0LuU5OfuMw4MK/+j3/oDPH/+C/riDKBd59cpHMAvIA5t37n2Lzug5hw8jtKyGkTFo1E2ms5RFzyIj67z1xmUevXhAQszH9z9mbW2NIHB4+eKQ97/6Fu3eEUY2ZD4qU6/XePWqh5LxmDkzNpdvMZ8OEAWB5eUS7dkDSspVuqce1WqVhXeCmC4TJG1CV2SltYK3WHD86oDrN7b54kmPRFQJPYWyniGTTQiCCUd7KZY9wyz4SEKKbvjIU5/FZHiBRxJsFgubyWxIvrJKrZnjwcOnpLHAW+++QxCJnMwiRpkZcuqTLeaY9vuU1CaeMyebq3F69JK717/GckXnL370EZfvvYlqhHhOhlSOMXWdxXDMrL9Ay5mcd/p43hRnkXBr/TY31hcMR1MOPYc4CLm9c4Odmwk//cEjQndBZ+izsVWjP98la5SpLG1Qn3YwVJ2UAGsREscSfhRQqtY5POiy0VhCUBzUYo6xNUUSVVbWaziLOY3cNTxzwAefPObm6w0evnrFm+/8Dtdu9ykYOc6OOtTKGVQ5JZv10DMbXL+SRZOzPD/ponsKgZ+iKia6llCqB3Q/nyKpNo8ePqZcqBHmHcRI5ujBPrfvrfE73/k92t32376gnPampEJKbiXLq8dTKlWRXLaCFYxZ3zJIfZX5wkVRVKRUI6soJFaAEHmUMyaiopDEWZpLGrqhEgcyZmFIJpujdwZaViCTSZhPFgiBgCFWkTUFhATHntEob6GURHxvwXg2od+XaLaqtHtT/Hh6USTOe+Sy1+l2zgnimGIpy2zRJowTSEWCUMTMpkhqQqtVo99xEQQPQyqgaSHjSR/TNBEEWCymyEoOy3VRFRFdaTA4nSMZAVNrhKL7HJ3N6PUnZI0CWSPDaOAhKmBmG8w8hyCZkvoxkaUhRBGSkKfXHVMtZZlaPrlMhnyuwuDExkscKhsxzrhMPiPgOh4xM/wkAFvGizOU8zX8dIaiF+j2RlRqS6RpjGTN8XWHS7fXiFOZrGDS7x5T1gsMRkNWl5tUqqvs7Z2gZnTSJMZxAo57JzS2m8yDkPVtAzkUmI1UQjPFUHTWLq3jxCKu5VBvNFhd0ajWV3l58BCzNGNmB8SRiabDaGxxuDdBEmA+nxL6KsXqFoO+z2n/kOVGgQCXydSnnKtQq5bpq6cEooUpbyDKFomQ4ngLgsDj2bNnVCtLnJ70QBCIooRKPcd5p810OqVUNom+NLzM53PyhkrJqDKfWtSqeYhBFQtEPmRMsOczjl+K6JKOWOtyPuyhFSMMqU7BTClkM4QkaIqDZApU6znG0zmf/WpAMM3SaOWwrQBQSUOTiQ/ZgsF8MUMKZTIZlWoxpHSpyNHBCamoYzsOV65c4vS0TX80oVhRESQPP0qRZI3pYo4XepSqGYQopVjQSASRomQgyxeQZQHY3NgkLV0gTtxgjuP6IDZBFPCChCCKkESVJAoRJTAMgzBwgAtzgaYpDK1jwjAkU6gQRFVE3UXWIVzIoKoktkmpXidfspjPQhS9wKBjUK1XSJM8h0cjpnOX1ZUWi4V1EStYVBGRqVdLdIZzPNdGFU0U5aKzlCtU+PnPPub8bMi9124ym805PjpFURJUVUVA5g/+8T/mh3/1PSwrgTDFzJhoGRUtirCCAVGsI4h1RvNDyq1Ntpe3OT05Ymd5jdhzsNwxgeEQWR5XX7/Hn/67j7nxxm3UUo7picvjZ4f8xm9+k4kzIHFV9nef0GsbWI7P2s4YPB9dMQnFIvmshDUeYzseaRyTpBqDszEiczJFk8PjB+Rzm7+uiTPXolBoIMYSkWPSGbzgVuldnj3dRc7KjMQCd5eu8/zkAeNhQKNaRDIG2LMJn376lGxRQTUVjg8WhL7CFx98gS+G2M4Ue5GiyRqt1jKaWsULXe7/6meoZgE57lMr6tRrK0iixdHpMdlikULNZtQ/wxVVHrgj4lTDHY/J2hZG/5grW7cY9M9oyGU0WUfOZnn+ZBffG5DLqhi6zPf+7S/JKlmWmlUql5Y46s/4va++h9+esXf+Areusby8iq4keOGc01OPNE0ZhD6HHx4RulNalWUwfGJZ48bNr/D9v/oTmqUGd9/cxI8j7pVuMrOOqdQL9J76VMsuzx91qawWMYyUtdYy9cqQOD5FUUXiAOIEFPkCmh+EAbIsEMc+oiSRigAJigp6LuT47JwklfnhT/6cN+5dYzrtMh63WXhZ1GCIJ01Z2dygVqzRGS2QdIlElEhTGVFKSMP0S0rChan0r93k8/mM1e0NNq7vUaoVKK5sYaY5xEkXSRaZW3PSVCAIh2xtv8nGxhbPXnzKb9z6Kl88+gnjic1X3v8u/aGEbQ84Pj3g3r179CcvOT4t01qpM5+47L58RaliMuja3L5zGVFI8SOb2eyUREnJlWRSR8c+b3LjToODV09RhVWW1006h2csL1VR5TyVSonF1EKOW8jGcxrLWQaHx5T16zRbVV4+b+MrM4rFIuVMi3tfW+Pf/sUP2dm5RLW6AkmGbntEvVHh+HCIkYlx3Dkbl0r0ej718jUK2RxR4GLNe5yedLj3+nc4O+khJDFx6iInLSbDCctL1xjE50SxwHg2Jl+oMZ07ZIsae3ttNKVEmpwhKHly2SyXtpaIWeODT/+MjGrw6OETlmotNi+vcdbbYzIc48591lt3MLSQVy+HKEqBm1eX+Kv/9WPqzW2W1vPIbgTWGKEwIw1UptGMyk4d3ylRNiKiyObjxw8oljKkMwcxV6HT6VBpLOH4C3JakbXmFez4mMurJcb2Gc8enhOkMeNZn+5wyNraMtl8lZP2LmPLoFIroSgRrhfhLnRkKYNjjVEVESGWkNIMGWWd4qpAqdzh40/6bG82WL7WoD3q4i1CVpo57PGCfNak07b4ra9+hfZBm3koUyhHbF9Z4uXBTzg/hLffvY3tuLihQL6UpdUsMuiGJI5AJCo0tnSclxO++o1vc7D/grMzh7fe3eHh/VPyuRq1hon43Meypsw8Bbs/oNVYYWF5lCoGcaj+7QtKJI9i9cIpmykYVGsFzk47FKsZFvMBjqMhIZKSoqsiaeJhLzyyGQVFlHE8F8e2WVu9jCB38FyRjUtFOqcRhXIeWUtZqq8wzrh0+scUChpRmOLZAaqWYTGZsLG5Rntu01puUi6VSGSLKPZZ+DM0Rcc0Cpy2zylkyiRxihu5eK6CZc1ZWmlSKImkgoQo+/SHA+rNFtpER5RSzIzOsJviBTMUSUZIdMbDGaIkE4cq4+mCcskg9nWCYMzJ6YjTcwVTlckYCqETYeo5gkhgOuuRyBHOwiKyIZ+LsTwfOZUQdQ1JAEPTUaUsopCwul5hIqRIioBSdtle2SQJE7qjU0qUWNh9Spk8TugQezZqVMTM1JkNZ+RNMIoms77AdOGQKi6akcWNQ1YaUDQzXN5Y53x/TjGjoesyk5mFaQg0aiU0VWYSTjlvD9lYuoShJYTCgDQ6Q9YyVHUTU28RLwTytQnHx59RLTQorwmM+0NyURXN1Hh1fMbG5dtMxnMG42OKGZNCtgaxw/b6PeI0xPc8xHKHNOwzdRaM5jOKuTrT7h5GvsV8PidNEyQxZe64LBYzllZWGQwmGNkMlUaRB49f0VxqIAguYioDCaqqIJNB1F00LUPopywsD90EQfYZTW3KZhU9F2IaOSbTNitLFdzIAnx8FIIwusDUBB6lsoIWTXn2WYwk1aitX1x4PCvBGYtUK3lm7hH2KITERFEFZFLm8wGpL/L27XeZDR3axzO6BwMKpQAjUyVKI/KFCrY/Q1FCFF0kVyzh+XOUNIumifiRjef4xLGCKIgkaUoYhpyfXvAPcxWJODEJ4yxpmhAn0cXulaFgO/b/N/RZEEhTUFSJrF5ALRt0p/sInkAqCQiSwsJygCnRNKJ7+oJWY4nT00MKpRyt1jKWPWFhjUjkiOZaFS+yKJQyiBJkzByyoFOvN5hYAbblkBIjSglxFKFq8OLlQ+Zzn9lsxs6V68SRwHjSR09E9vYP+fiTL1heWWM2W3B+OiYVDXoDmzBO2Dt8QSlfQJU11pvrVPJVDp484bu/9VUePvicrY0Vbm69yfHeOZN5QFYP+W/+d99g/dol/sMfjTibhVy/c40ffP8X3Lr7LolzTnvPp1A1+c3vvsv+qzPUzBxrNuaHHx2ytmogJC7NZpkogNHYI59fpn26D/0CgrjB+uY7vy6JugbuaIA9mmJqZUJfZfesh+tZeNMR5eIVugePeXH0EVeW36F7foYdhji+xbQ3QrEFZgsfRc6iZbIEiUaagGMVsa1TfC+iOxpQLucZDBwKxTp+PGdm9QicMpPZlEajxtAe4skWo4lFzWyw8GYc96Zcvb2Je15lZM8IhnNE9ZRsNs8vP/+COH7KpfV3qdayyJ7Lo1dPubL1Fb79rb/PKABr2iFDlbrpcf9wl+VCCzNvkMQi73z7Flb/mGp5FTfIMJ0f8HYlQeZ99NIWJanIZ89/yL/793/Gne2b3HnnKxycDJh2umhBDu2KRqRY/A//4s+4ff0aa+UKb6ys8PnHP0QvFrGmHWynf2HgSmVSIkQRkjhBUSRkWQaEiw6mEJPGKbbjsbLU5MrmKtPhMZY9oppvMmiPMdWI+dzD9/bYuHSPWU8ilytQa0qc92VENGQlxI8gjWNEJERBRhAkgiDEMC5SqMqVIh9/+Ct0PaFWWqczO6WYqXPp6hrh/7KHJAroWZ1C0eTg4IBCLk+1rjIa26ysbjKzP+DVi13eeeer7B9+QjYjMJy9pFLcRBQVKuVleoMDNrfWUbSQQnaVg/0T1tauMpq9ojfYZbhnsrpSQfAV7ly7gzWbE6YLFl4HUytTq5UxcwX+/D/8ETvXrhO48PX3XiNmQsbMcfndJVw7x1mnTiY7ZGNrHVFMWYwjNM1gdbmKoVU52DshkxMR0Oj1eoiCgSYUKTZlet1jVGGV126+w4cf/wjXuwgzKVe26XXGDHp9trZbHO6VuPfGBo8f79LuvCTwU4bjDjuXryLEeaJgQUCd119fxvU9XHdEmqiIUkyS6EynC964+w2ev/oQSalh+V3SIAOxTiHXoJht0lrJsHdwSBplqa+afPLoCaurb6AXfRbOPvLCo7k85uvfvs3Dn8+JvZD9kyGVlRa9QYfpNGFuO5hKnnffW+f48BGqXKFR+3+z9l8xku15fif2Od7ECW8zMtJn+apbdf1tPz0909MzJIcE1pDiiisJwhIUuRKgB2mxb3oRZCisoDXQasUVV7tYYElxqCGJ4UxzTLvp29dUXVM+Kyt9ZmR4e7zVQ142BUgPDSwTCORLRprIOP/z/Znv57tFqZbj0Rc/YTp4yLWbNwliAZkG9brI5k6b/sMjqrUWlrVCs7RDvSHx5dMnrFRvEKQHDCcx9tzjrXfvU2/0eflsj9XGCr7v8tknj3jvaw/wbZn19QL28oxDd06WeZjVTUyxQlYY8vCzMd/81vv89NOn3GtcpyR0CQWfP/6jTylXNphmIVN7wHn3klJLRUtXKFsiZ9EJjVKZOPR4fdxj0rukezHCdRKyDAaDORkypQokYcadW9dYTjUOTvZQsjVKpZRw1iXMFLr9w19ZJv7Kppzt++toVQ3JUim2AmzX5c6DMo1mhho3WFtvoBtVPE8jS4sgSJRKFfwwJEozTKnGxmobz7lkcO7T7mT0LqfY7hhJTtBUgdNXl6RZSK29gpLzUXWJKBaRUwtRiDg9HqLnPWQNkAVePZuSLyWQFImjHJpeJI7yLJwUL8zoD+ZIaoJV1Lk4m9MfnzJ3ulTqBfKFGla+QqnYxLU9FssZnm8jEFEq1q+SWkoFQj9gOBxilMCOZ/TGIVmax9SrmGoDXSnQ64VIoUe1YDDqTwj9CBVoFjpEqcr67XW2NraoN1RKJZ18waBSzTFf2FhFDVGbE8wjvL6PyJJ+f8iLV3sEiY2mm2CUkA2JMAlZsORgfIqYhhiahqiVWbgikiTjLn0iW0aNq2yvtalXZdorTS66l8y9CXfubdLqKOzeqKNrMhISnjPlwZ0NECLccIKsLhg5F3TWyyyWCTlLZjg+Igx7eP7VaFFVh9j+HDOvsnOjTBgv2ajX8dwRWzcNdu6sM5wsmMynzIc9FhcPsQ9/TjrYoyrNWMvbXK8vWMubjIZj5oLJ0rax8nmmiymKJoEYI2kpSeZSbeSo1lqMx30q5Rr20mU8niAKGrpiXGWopx6CJhHEAaKWIZsZQRqQiRJ5K49csijVq5iWQK1Wxo1DkqzJZOJzvj/ASGtYBZHKaolMqvHsyzx37l+ntp6Qyj5RliDqKlohZbQ4JYoEFk5GEEe4nkSSNdjcfpMw1Hjx5IRSvkS1blGt15HkMmbBIIwSfC+jUmqjKiaSJDEdjzA0k5XVIlHsIQs6lfLKFaNVVZFlEUXxuHGzzo1bHTRdYTz2ETITWbmCl+eMAhkJUeyjaSqyIKIbKoqsfcXQyzA0kyDqIWUyAgFypnD0dIyULUkDl8lAoFTOEQURWWhyuufy5OFLJv0RG+1NPvj17/DO195lZ20NS5QoKQaVfBlZMxm5HuOhzWzqM595+G6CpuZZzhzee/+b/Pv//t9C14r86Ec/YzIZoWgK65vrbGxt8pOffMrHH+4ReBKtlQYbOzXqKya6qVOprKNpq4zGIseTCPQc733tbSq1KqX6FnsHHj/+kxcMT1Pu3Pu1q7loJPPyo9e4lxO2O23eeusdzJzEzptbOOKCtTvX6C01/ov/9oeoFYUsl/DTh1221yoYRYVl7GHbIe7Q4+TZJb0zH0UpM53PULI69tz75eP3//GfspwniEKR5WxO5rskrk691kFTS7Q6G6gVnU8PnrMQZS6CgL2jF8yXDsVaCUUV8fw5g9EB3rLH5GIIaQ5R0lC1VVJJwI/nXA6GDMYjFssUAR2JMrGQsgw8Xh31UXJFxssAUTXYu3jO2LvAsmrMxjBfLvHjFCdz+fDxE77cf8mF7XFuL3m09wtOBzZC8RrFtbdwRJ/cVhM9Z5KzNI6Glwx9lzD06Y3GlG9t8fSLL/n40UMePf6Yv/f3/y5/+vE/47J3xNmTp7z85GMOfv4z/uAf/d/4h//V36dTLlIrNtH0Iu+8t0lR0bh2fYOlCHphl9/93feRUChUtsgUA2+p49gxk8Eh8/kRii6TpTqCJCNKGaIokSQgSwZRJEEmkSUgijL2POSdd95gvBgwHPe5fvsWqd7lYvqM8XKJnteQ5BYvj7vYqcPYmREKEgkuiDZZll2xicWrW2KSpSiKgiAIRF8lde3v71EoSfjBnI9+8afkTIlCKaA/eI0iy2RZguv4TCchOzcKvHj1OWEE+aLI8dEZ9doWSm7BbBYhCHm6lz1u7fw2nqOwvXafg1enKFKZOMhxejLlzz/8GacXz/nzj/8prhfguhIPbn/AxrbG6eUeiaBSX20SZiGtZpH++RmGLtAdHvLNb34TP57SaOW4HLxg7i4w1TUGjsb56IBru3W2m1UWFyO6hw5mXcHWlvzmd3+LWrnNB+99/SppiyK5XJ1UuiAVbRYTGyEzyRKZx198zkqrwXQyRFEz/CBkb/8hjrdgNk24d/82T558yXw+JYh81nfyxILNdD7j9dFzpm4fy2pQq2wym4xRFIXGio7jzTntPkLShgyGPXJGg8UsJYgSTEtC06qIaspwccFPf/FzPvn0CR987T3yuSJxkJElXUbjc54/PeD9D9Zx7YQ/+SFs3bkNRZPWtQ6xJHNwdsDC8dm8toUT73F6ts/cc1jZSTi6/JzBZES1tsG3f+ProIx4ufccbxHQ6TQ4vzhCUSS6vVeE4YxHn/+c1wd7qKrC2D7ncjAniHJs3DL4oz/+IfncCrdu3WJrZ4VxX2J7e52p8yWPPn2IqZYZDHoYap52a4ecvuT45Bco5SI7Dzoc7h0jSCbqVsrlaIqbSSS5hPe/91t4aYw991GihHA4o3d2wuGrPrEQ8fJgj0xJSBC4sdWkYeWu7vO+zXImgTxg6nzJ4esuk1GAqNukSYSMQmetztlpD1HK88E33/mVBeWv3KGMfAMxrmAoyyvcDgrDy4zAllhbl7gc+oz7EwpFHUGcQDJEEIq8ce86o7HNcrLEngt01kpMpZicIeHOxmRpRq2SRzSXaJmIpIe8PLRZ7ZRorRQZL2Yk4ZhSUSROI0ShjBdkDPpj3vtgi4PXz9na2KY78K4ydIMEf+6Ttwpouk6SRSi6wNa1OlNHor1WJUNnNDmgUW0gaw5+ECJIKrlCxPAyIgmnjCZHdPQOaSJTreuIokkkJFQ2FKajgPkowLeXNJt1wjjlcOSQjl6TKQI5rYY7mZG3lnztvVUUA4qVEpXrec76fQLXJohgfauMHcwo6CtMR8d864N1upMAnyVrW21EwaN3ds4UG8fOMLQmAtpVCkkq4HsOgq7ghxGKImM7EYYhkfg2iR0zlXTGwxGKEVPdKHIxudpHLFcslMkMq6QQ+Smj84x6uUOtJnNxcYlVthgPQnIFAWIFS91Ek5fY85hGdY2QCUcXPa5vPsCdS6x2GkiLPI9PPufw1SUla4W1zV3m8xm+kBIFIbNFnpWVNpHrsVJoEjpDbq+bWDmBs6FHrVEkIcOyi8ymS9IkIZ9TyRdzJElCvz8gzHwyFAIvJJczqFdaTKdz5EwFNWG+8FAFDde1kVUJU80TuhmWXGThTxkNUsQoopxvEKUu/fElmyub/PX/6Q/Ye/I5z18nFLWY4UCmUwsQfRMpUMgZPpqhMp9M8Vydzuoqjj0gSvPklCpWQWS6mJOOUiRV4ax3QtZXMAwDJ43IMoPQniNr8pWbeTamWW1BJFBtl/H9CYE3R1MVNCUPAVecPUAQRHwvZTmBIF3ghxmaVIUEECI0XSQO52iqRRr7FAsWaRQiR1fJIrKkIkopU2dIGCcIooDvp4gEmLqApZQZnS9QFZVhf4pnjzCkPJsbKvl8jmZ7hf2XfS7PpnS7fayCjh8nbO+sI+oqJV1DJmN9s445klBkFcfxiIKrbqlrB+TMMtW6gplvEcYhWQaFQh5VVQn8iDgL2N87ZzTuo+jQaDQw9TLbW6vkLJnOVg5Lb/Hx4+dsb3b48PETGpU1VCtHraFxcXbOh4/+lGs3rnN0ds6yN2D1eoMsU5GjlL/+P/63+fGf/RQr0+hsWHzjvRv84Q9/xLJ/SKtc53d/6y32X53iegJVa5PlaMly2aOztkOxbvFyb59ETPn227u44eyXZ+Ibd++RyDGFahtdyjHr97E0j9PZiCgN+PSTH3N4cYGo1+ifvKZcamDqMt7Cwkkjxssz8lYJRUtRZBFPjOn29giCBF1pUalsMJoeEfoplWoTSUk5Pxlz59Z1ypUcH338Y1ZXOyRCiiCqtFqbZKJEv39ErtRlPq3RO1+iGjaOK7Gx0eH41Ws2b9zEtHRkKePpi4ccnx5y7dpNJtMxsw+f0d69g1as0Gx0uBh1saMlw8GAP/qD38co5ljb+KtYhQ06uyrzZZcnlzPCxQAxDtFuvMf6WpU//88Pefebf4W04JCgMOpLbBZv4y269NMRXxw8Y73SZjbtMxqNGIcz5HKGYubY3ihjWUfEUYwoyKiqihu4KIIKWYYgpF91JyFNYjRJJggCtrZL1EspX3t3gyiJOT/skokOCiKz6RWwXFMF8pZIzszz9oO3efjJ72FVc4TLGEG6Ql6RCaRpepXMIwgIQoaiXBl0UGWKLYnW1jrV5iYnh+fcq66TpB+REmGaZVSpyvn5ObV6Gc9WmC9PMc01zJzI0vE4On/B9WsrPH3xGfO5w81rt1nMLjFFHdOsMhges3t9lfnSxPZOGR1MmU9z/MZv/Jt48yX7e0fs3Kjx8PE/48GtX8fIWRwfD1hdvcZocEKmy1QrLYbTBa3mdRbzj9E0k1RaIvsSvhPzdP8QjxF6QWQ2FdkodNj7+Cn76Rl3793jtHfM7Tv3ePL8OZ32BlYx5uTA4/r1NTRVpdfrXk26rFXWN5tc9k+ZjVIatR38eEQUi5xeHCNQIpdzSYWEF3sHuK7PzB5y894288WA8WBI4Lq8/eZ3cZz5FR8zt0oU7TEZeeT0HIVymyAdMfuqoDo8+jHvffA+k8Me7dVVvv1r32T/xQX5okg+b7B/+IxM0/nN7/4OknNKb3JBTsp4cd7nYnFBp3CXxcXh1f5ve4Ob17d5b3sLMQlIqWCWK8TZCSfnH7Lauos7kannryNF53izjHzJxZmndFY22Vy9yXx2yf07N7G9HuOpTaVuct4bsblRYryYkS/Ds+fHtBpFTk7mfPM77/Py4Cc4sxxvvL2J7wWUCxv0LxcoWYImWpQ2K/QPL1gOVM6WI/6tX3uTV6fnvD45o7GjMZ/KvHxxQqlicH7Ro55vEyb7XL+5gaYv6T7xeOfNbxJmAy5f2bS0W6iKRKlQxF4kDMfHNNvr2LZLGgtEoYJu6tzYukMx38KezSnmVV48vUQ3BH7ra/+aBeXsfIBuhlQrLRzfIl9IiCIPxTTZf95FFg3euL7LdDG5QgUZq4iZihCLGLqAVIX5OGA2X5Ir5Ik9g/X1AlnSpNkO0Rs55qrDZDDl5k4Fq6QQRwtkMUUpKkz6MlP3hHq7iqSI1Jt5FG2MLjVYzPoUajXms4hW3eDyLCJNIQ4Cokim3ijQGxwhyjnErMR0MqdWr3B+cUQYeCAvcRyLUkXDcyWi6JL1zRqDSwdBjkgSkSwJOOl2aa6t4gUJBc3AEEUWiwWCapIv1MiZEknoUsjlidIKiuVTbqns7y351vsC9kTlwYMH+M6E4SJEDBWmC4fN3VuUG+uUciGfv/yE5rZMGE+JlzLL2YJqp4kSw2DkUS6aoEkIWoKYCiwnE2SthKwL1Gpt5vMeMi6SoLNwYrx4QtVq4Lkyhwf7vP/eG7zaf83bd7/Fy8ePeefddxj3RkymQ6b9jJxR4eT4iNK1NoG/IClmNJt1AlthZ6PB9voGxyd7dCoCX3v7O/y3f+//ztrmBtVync5Wi1f7EfOJQ5pOmc7GCEKKoaygV1IifcR4YGCnXYaXM+Yzn9WNdeJE4Oz0CFFWCOMMy7BYOEs0RWLRnZGSIcsK+XyLOJbQFRVJihj2hui6geu5hF5EyazRu+jSbFUoFA0OD85oVlbRJIOUALO6IHY1sjggcuDf+xt/na2dAsifE1+csXG7Trli0VwZQLTC2fmYpeuiiEWqDY/YK+DOIxazHrohMR3NWSynNMUmhXzCcDJBLSiYNYOD8y55tYCQeohZQrVR4KLbY24vaa82iTyP0A1Yu77DeBIQhRm1cgV76eG5S0RRJIpjkiTFcwxcQyQWHTwvJomv2HtZqIGoIUrK1U5ZGmIZedwkQchSFFEiVhTCyEZUCmhikeF8H1kok88bFHSdcddn2PfI5VzyBZPOSgtdgfZKHUlMyNIAw3QpNDtUa0VKloll5UmyBEkXKZbqzO0FTSVhc7OFJKtomsZyucS1Xfr9Pn/wh/8dN27uYuaqBFFM4IHvxqRJjK4bWLkyznJCo1lE1XU8N8CzQzw3YD6XkRSJyF3gORIff3RISowgLtBzBkkyQUKnZjZ4sv+QYjnk+koHTykwm43Yaetc9I+5e3uH+XCMqK9wcHTKe+/fJ1oaPProM0Qrx5sPrnF+fkEcwYWzRK3VefO9B5xenqIoGpZkcfRyzJd7D395Jt7/xiaCpvDx0w+pqLf56I9f82/8D29xcrJPksUUJZXuy5C/+Tf/JqJ3wo//+R8R5lSENEIvq0znEVklIxR8ltMRldoGRpzgGzNmsx6zpYTnz3jzje8gZAWOjr8kb20wGL0mziwq1RKz+ZhSfp0suWS+7CKSUSluEroxYTyn1myy0r7F2eEAS62xsRuyVm/RG53hCQaFqoo7OeT8dEkqNQnnA04vRtRKeZxCjYWfIIg6d3Zu8LV3HvDi9QEff/oRparFanOd5SSltdrBzy3wM4+SWqbgrfJXvv6ApukgxBUm0wXOsM/GzRsYYodk4aAmoOopc6HHq7M9vn77NvWayOODLrpSQxCLZMIUUbpKUhNTCSQRSYIki6/e/4SkWUgYJWhKRpye8cXLV5jSDnZwimC4OHaMs7wgTTRK1Sbp6AIjzBgpAg/ubiFEMmJqoioBUQZJJiCIAlmWQnqV7S2KOlGUUKs3Uc05k4MDivEGhYKEbfc4XV6wm/pf7emHbG43eSIsGA1mbG9tIOsSvhczm8wxrBKGISGLeba2tqjVahTyCWczkdcHnyMVwDRLtJu3OTz8M/YPT7CKBVbXKvT7XVIEVN0gCTTu3nifYr6IG9vMbQfROGMwPKdQKeDFl8RxzNnxnJiEerVNmM45PjslX6whiRIaOmQBa1smXz78gnpTpiTLJInHdDrm0aMDVtZqHJ08R1GUK8/CxQmr7V2CaMpseUlw5LJ0BgiiweWkR5hIiJKEqoRsbBQR+ktODn2MvIxrZ2yu3qbRLvP46VMevHGLxaiPldOYjMbUW0UW3ilpYLK1+h4nRwdcv97icH9ElhUpFQzCwEdRNHzfprliUS6X+fGPfk6n00ZTGyRiwtbNdUaTCD/ymFz6GNU6tz/I8+jzH1Mpb7KYe2SCg2m1wRT56OnvYY822NzZJOdHnB37rK3dR6vkII0JmCKkFdRcGVnL0WzmePj5j/C8Xf7iX/yAP/0XU0JrxPHJHqJWB7FAriBj+32EMKG50uLZl48Io9uoOYsnL59x/fYOH3/4kmB8ZV66eXOV7lmXMOghk6MkvUF/OUISNb7xvTf48ulT8kaRW29sM19e0FAb7D38Q4aTOeXyNoVOBaG2SvdsTD6vUTE6FHSBycJgrVVhd22VuTfAUGRqNZOp95InXwy4fesW88WIpT3idvMvkC+d8Qf/5CO2dptX6VSByNzZ+1Vl4q8uKIe9I+q1FUbBFElOSJSrxV8n6bO7cR9FFxgvL5HUBFPTMKQCqqwwnbn4YcjCXlIs5PjWd99idC7x+vAZt9+o47tFBMlBSHQKeZ2N5iaXywNSbUFObOI/H9Mf+mys6pSiN+jbn2MVqkyGE2qVNqXqnIr1BofjHrOph5TmUUQN11lcRS06U2SpiUKN6eyYvecBguST003u3rrPh3/+JVZJp9aokGUCuhGx0qkjCAKNpslgdIzrQt406bRWUGSN0kqbgl4idmE4O0VQA0yzwrB3Sa97ybvvXuPNt7e4HJ4yvDjDEgs4joOk5QlCF1UuI2VzonTCtes7JLHISqXIR599QrtTZDadoao6QSywefd9uuc91FJMErpEaY1UUojsFDcCOZ9gaBH2UqZYychTZDm36XQqiLKEPbRJgozu5Zi6VWY+9Flv32Rw2eXu3TcY9Jacn5yQyZCgEiYevufy6vVzckaO14sjttdCLLVA9+IIEcgyibu7u+w9+ZJCuYKspZxNXlIo6OTyNrq0wqAb0SwWiREQFYNO1eLZ8Qus1SaBP6TWrCFIS05OlpQqBSqlKmEckzoeiqKQy+WYTmcUimUyUUCRIPISlssxvhtSLpfI6RKiJIOQUrByVPJ1vJmPZZgkSUK91kZVFdJwBpJKQV0hdCPKKzF33/8WL85e8CefPObWusneU4utazbd05BSucDCtllbb+Aslhy9GBD7eXZu5EkijyiQEUWD9c0VJCXGdR3G/RhJUfHtCfm8haYbmLKEEGkISQ6ZkGKugqmVaJWr+O4cWddIEsiZNS56XSwrRVV1ZDEkjgPiOEEAVC0jTGLMvIwkl4hDFYQlXuwgKxq6KeDbMaQJopChSBKqJl/deANI0pAkiDDNCCUuYBoWx6/OsFQLzXC5f/8ardpVByZFpVjSkMIqtaZBe3WDzvYpVqnMbGwThz5WrkScpTiBg+/72PMFvpMyGY2QZIEg8EjTFE3TKJfLFAoFgiCgUqmwUqogiSrL5ZLxZIggCCwXMyRJAlEgiCJyRRNFlBARWCxdJMWg6w7JwhRLy4EQIOkqnmdDEqNKMpfTPiureV4c2Ry8dDGMiEiKOKouEVKPi0GXUq6NK+4xn40pmFfittCpI+ZEDK3OrdstFMXma19/l/kiQMwkNlav8Tvff5OD5y8JwogbOeuXZ+Kl+xmCrZClIjNnilT2OLkco2o5jo5PORrNEI0cO7d32d26zU8/+QlqWGY0PyeRG6zXrpM4PmkooGcaF88vKDfKzJwZhUKbOHQoF/J0L6Yo8oTr124SJwK+t+T48CV5/TqaNCXy51SKVS4uLqg2LRaLGUVrm2q+wWK25OCZRGfbZHuzynlPZzqboJkJR+cPWW/dRdMMXu/1uPaggZ24xO4Se+oyqxZQ9ByBnVHQcwRCheGZQxiFhFnA2cVLarUOmaCzvlFnFPpMLqe8f+868/eGBFqdTCwRGkvWNtaR8zKf7z3GSDRKUh0hmfPu7m/y6Rf/hE+eDLh/97vMvX0GwznzuY+umZBlV+geVb/aCU5SEDOyOCSKAwA8z6NcyuOFFwz7KUJo8+b9r7P/+vcRNZ/NdofXr4+Rs5RCZY3R+JBYW/LFoxGZmBHGAaQZmSghSzKkElkWIogikiQQhiGSLPPq1WuG3Sp3r3/AeDjh8vIL3nqww+WPPidNQFVMgsDB92KMWoV88RRRueDZl5ds7WwS+Cmra9CoVhEzC01sosgw7sl0Oi3u33uPiT0j9mVOjveRUPmNb/87vPPON6nXc/yjf/yPODo8Y3Utod87oprfRmmWCKd91jod7t39Oqe1x1z2B+y/7tNqtdAEicm8QC7vcXRyjKavoOslgnhJUf0O3cmnJEFCPleipJcRkjwvXpyAchVE0usNrsw98ylGIcWzRSbDKYVCET+cUsgVaDRbnF8coVcyNE2ms9Hg2YtneLHGiyevWVtd5fhoSKGwymQ0pdGoUbKaXJxOEFC4fXMLxICj0yNMtY4gz5nPF2ysdTg9OkYUwRkLVIo72OzTWS8yuOyx0dlFNyGwVdZWbxLENre3bzOfH1ArqLx48hSJOqX2Gj/5+IcIsoXpqgiija4b2N6EIJColu9y9nqfyWeH9PZek4RFvvb+X8Os59DNHMVyk1cnn5OJC2rGNfaPR+xee5/peMDDhy/Z2K3y5PE+RqGGohcYjsfMF2NEuYMk2VTr17l9T8Z3XTIlwU9cXjyboRsS7fY2k9klk+EIVV9SKe1gxfDyo5+Rtq6zttvg8OFDgjClfrfJZ4/30cQ5b+7c4v7te/z5hz9DEUKCqU2xoOLKIxQtxzfe+4DZdEQQmBiqghstKJSKXJzs0xsvUfMmt+60EBHZ6ryBaXYZDC948eoZb7y5i+N41BoGgtbDiq3/v5rwv5egbK/dJUmSqwouDZkvPEpFk2BpcDJ8hlTMMXNHmGaJmm4xnWc4zgVnlwNWOx3yuU18b0FO3Waq9Bn1Q/rDIe7S4datGxweHpLTM85mCyItd2VeiRtcXxW5s3uT6dJFNETWct9jfbfFj/75AY8++YL2qsbh5AV9N8C3Y8R4gkyGJimE4QJVNZiOF7jOEk1rEbkJvhtjrUocH73i2vU6o0FAEMVkCTRWTKIQBDFBVFxyZpU48SgWJGYzhd7pKbmcRZCLmEwmLBc+OzcKFOUlcTGh0Xgf1ZS5HB7jzCdomoGiC3iugmcnhAzY3mhRKMPlacpsnFIszHj86jVpSebydMLaShVV1BiEE87PxtzeNAmWEWvvv4MTRHz6yZesrq4RLQNULWGa2liyQP8sZqW9w1J4QRjPCZcasacwv1BQFJc0sRgMJihBxGK8xI4c1je20RYFIu+MXK5C4BusN2+xt9+jXlsQUeDk8Ij1tS3qtRU8f0GxZPLqyXOmyxFb1zfwgxRlMcPv+dSNOqphoaQ5+r2XlBoijh9zfDqn3a4wnRzgjywyI0OuZpiCihhD3ixw1j0jjkM8F3w/QJSyq4JA0/GjK9yMIotYlSoSEqqcEWcZogjd8zHV2yX0fMqrg0tUXcUsJiRxgq7okNokaYNIm3FwlvDF/p8gFX3aHYEnx0t8OWT/vEe7soa9CIiCGGchoWpVqjUZXItwAhWrgFIyMI08aQpeMKZZbTBIBgQOCJGGEqbIQoQu66SaSqvT5PK4j4JAoagTuAG6UsDQLXrdOTEBtfI2g16XnGUiyQphuESSZGRZJMk88pUKo9kAIcyjylUSHFwnvGKdouD7yysRmSRXO18IiFxF05ULRYQg5Px8Qpy4LPpTimYekYDVehNFthADEVHNKNQlClYdz9YoVKpEWUxrdZu950dEIbi+z97BK1TjanleShQ0VUI3QFGvCAL5XJUsy/CCkDgCWTYI/IDx0GY4nLOx1cLxJmiaRhTEKAq4TkiQhqi6QrDwvjLGCWiGyWgyYKWYRyiqeFFAGoX4ywhDNclkCVQDNQy5GEyQMwuDgNlyhqTnce0Fct4ixmLmh6SGhOtaNOs1SjmR2ImwgxEXjk1npUK5nuPybEBZLfDwRz/hjXdukmkWb9y8zf5pl15a++WZODqeUbQUzo/nOMsLokXCzvUBYhTQXG+j7ayxmMR4yZLf+70/4/ypy87NNrd2O5xPIw73X9NpNAh8nVhQ0MsGj784xLUTNq8tcTyH7d0VaismR6+6tBsdpvYxupLjzu17rLfe4dX+z9D1HE50SbO2SpDMKOTzOMELgkmDnCWj7YyQxRWOLvZxZynr623m3pSV0g7r9Zs8637BcDxCevEKSYvoDaYUzApiUaddreNFXfqLL3BncySpTklK0KwKW+sbyPMENBnb7TO79MhXRF7uf8T55IRbssnJ+SPktSqrq1/j2egpqg9qwSfMTZHVAko0xEoTRLHE3PXZuv4GrZxEnISkqU4aKyhyQhzHxJmALGjoOtjziCTNEAQJzw2pNeqkqUuWreLHxzx79SmNepte/4Tz8x6qlvGLj/8FnfU3MGWL0O3T2JaQdREkmThykIU8aRqiSBJJlCHLV0WWLKvIsoqiqLzx1ps8+uQzCkUV15tyeH6OCkiyTpyEKKrMcuESlTJWV1dZ2H0661X88JJf+/b3efr8jxFCg2vXG+QKEct5zMaWxcvPZzQq7/PujTxZlrG6tk2p0EEQQ3zHZDE/R8hExv1D7t76OoXrKj//889BKyJJGaErsdGqc9bvYygVVjd36Z5ccOftHR59uYTIYHfr22TilPOTEdVKDbIpK40ig/6Uv/KX/k3293/G/v4hxUaNNFsjV5DJ5Yr0e+cUc20axQ6ES4LQJZ/Pc9lX8PwYyUxpNvNEyYLZxGb/9ZBcQaZ72eP29fdQ1ZjFwuH69RX2nh/x9MlLypUiGxtryGkOsoQ4Dhn1UmTFx4/HpPEFc11l0r9aPdi8WeGs+wRBztNopKSOgD108SSRX/vWe6h6wni84OXeMzZWd2jUTPzgKUkyJ44Uqtoas+WC7qyPJKYUihlBPEcMYsbugspqi/XmLbL3ZwwGrzGMIb4nsRxJOP4R7mKO7y1p30/ww5SZ9wKzUqA3+YL+JOWNt97irLfPYDBEyHJ897t/mSid8tHPf0a/N6RWzfO894xMNFDVFGc2pVYtMVv0UMQcYQC7W2/Q7/vI+SJb791n4k7onuyRq4coQUb/8oDdXQv7UsJUJPqvI9r1W2C49LrP2BHe4tq6jCsoDEevCROZQAiRU4PD4zPSIMcPvv99/sVPfohRqhGFIrGXY+n0QPBY3RB5493v8OzxBbPFCEWzICmgSr+y1eZXF5R5w+DV3skVEkQM0HQBL7QQJYXK2haqLBOEItPJBYZhMJ7OiGObSq3MeL5ExGY+U/i//Gf/FeWWhlaO+bOfBuQMg97EJvRjZCHg9q1rZF7CqrDKxeUl3/y1W5ydH1A3BVY6TR598SUXB0sk7RJNqZBFCYV8goHFmIjLbImmC5iOgSCrRJmLMtXQjSJx7KHoLpKmMpi7WKFP7JlYsoUmy5yOT9hYbWO7AcvMu9ptKJZwXI3W2gZn3c9ob7XRzIBw6JFFU6qVNgVdY3VDRR5eY5o61HI6o2mG50ckrk5zXaNY01npdDg+OWA4jvHDJdN5SK2p4UsOF/0xXuxTyK1y1j8nlSLiADq1NiNnQrUgUepMufh4xPfef4ee0kM0Z7QKuzw/75EkBm9f18nCGYK2zfHRMcu5jKaEtFoylZrJqDtjfaPJk5fHSLpIcqJCcEClUqLY+IDZbI9KWWZmw82dVfqjS1xvTl4ocX66ZGc3z8HhC9bW26BGdDorBI5Lr3fMxsYGr/YOiBMZea5QqGjEisjleEmjVSCv6XjOgmKuRl4zWC6XRGMdOclIRQ9Jr2KpbUTFY7bwCJIRslLF1A1ce4CkaZhaGVIFGYvh4BRZjSjkVqjkVMQiePMllp7j7QdFBuMuhUoZN3BQjRL+8ATXO6DWLLJzt40TgqGVWGuuIichhqFRKphImUgmCLh+wP7hS9pGQprmcdwIy9y44j7GC8yygannGfQjBMllo7OB7wbkrQaLxYycWcRexLhBj4vjV0yXNoZSQApCRCmlO1rQagiAS0EtkvhjdrZ28YMl6szHNHSSNCFNQZBNzHwZc77DXFTQlYzY0RDEGCGLiEMVdz6iVMgRhj6qIRMEGWqWEocJoZvi6S6iHFDKlREFjcHlCDNvcXg8JQwvEWUJwzDYYIMXLw8ol8vMXBvbWSDLV8I2DGOi8CoCr1Fpsrm1Sq2ep14rUS2VrzqhqUccxwR+wnLh0+2NSFN4fXCK5wY4Xsijz15iGhaaHpGmKVmWUczrmKjEcYzjh7ihh4BItlgiyypulF2lGuUtZN2g7w5IgqvYR88eI0kyYiqSy4kIksFmvkGzVSWMAyRJZGW1wHRy5WwkCugOLlhMlpQrJTY3VlnYXcbTJbKUp1VrsZhMefN77xNEKotQJVHzyGZKJ7F/eSbq61tEixUevFOiuSLRu7ygUKpzftylbFXJNfJ4zjkXH71mcPwzFG2BmCsgCXl26x7H8ynnB+cU6+vMoxBJkEgjk1JZwPVHvPP+Gwx7Lv3ThEXfZmLNyCTtivEazHk6fUhzZZWGZtHtuhibFVy7Sn++YLHwabZLnJ73KFoFRsNXVNpFKs0afioQem3ajYDYtdnduEGtnGc5k8jEgJXrKzjuAmdqk3UuCYUly0yivbJGQV8DMcCZXyC5JjE6kirgxyKt9SqX5/tEnst6vUB3/IpCp4TjXvCTL/4B1ZZFrIWUrDtM+3u4TNCKJVrtG2RCA0WJWPY09K9VETUDQYlAihESmSQTUKSEJIM0FJGEkCTUkNWI2FuysbbN1vpNDs/OEEKLzD7FVwWEVCFYOpz7UzZWvkPv5BntnRoFNhhNLlEkFTkTSKUMMU5BEr5K4rna0bxCYEVIIoyHLs7CYdJ/xNEzld/5K9/nYO8VhbBJEp0jiCKaInN58YLW3e8Tk2P/9QsaDRldyXNw+JI0yhMILj/+2R9x7843KLY9zqaP2Lr3ATfv3We5VBAkl1jTeHHxBCEp0bvcY62zghuAXsk4Orng/be+w1sPMgaDQ0TRwDJ2uRhd8uJzm80dneUowdLKDHp9CloBy9pida3I0eFrEPs82/uEnc23KRQlCnmd10cvKZTuU65KRKFEIS8hiwuOno9BFGnUBD794jnluomiaCzcEEU2MbQ6ubzEs+evyIghKrO+1WYyEnnnzft88ujPKJkdbt94QOItiNMZpVoVSYw4ObrACxJu3bzO6dGYXEmlWpKolN9nOlkiyAKfP/4Tfvu3fwvV8PEPuqyt11nOumhlFSHXIsoW+GmKMxvy9LNn1Gt5Xjx9xu0bH7DW2WY8OyOKDSS1wGhyyb1rt2m1RT78xR6i7BHMn6GZZeRIY5kPKUkGo37ISD6lVV0nXw55tf8UVa0SqR7/+Pf+gN/+nV/Dno3pLg9ZaexwcTIlyz5HFH0qZg1BVJkPPUbTc+7e/Bq98zFCEtJobvPJLz5CFKHdqeKHHqdnh2SpRKuxy9wJaDRX8aUTTs4cXDvDj6ds7W7RvXhBisbJ6T6xazF7/BmF+hor5QJnF8e4C5FDfY9bpTbD82NsJyCIEtZXbhJnI5yFy63ba3z05acsXYVWvcY4mBELl1RqBR598gtG0ylWrsRqo8p7P/iAR4+/wHZFbt7d+NcvKOe+S3FFJcsiFMUiTmDhO0hywrI/oWiqZEKGqlZIYgVByCDNoYhlAu+EpSwjaWDmZAajPpKqIeoqmRJxennGRn0HSTb48MMvaDfqSHJGIgl8/PAps6nN2rbM0UlA92KIMFiC5nH//RbLQY35bILjDXnr/g5PDwYsbJ/E9TA0lTBasnF9lyePj6itKphFDd+XEEUZBJuZPSSvVUhkE13PsXREvEAilVzIFgzGY7Y2Njk/2scUNCxkhv0FudIqOUVGyiqc9w+59DIKZZ/D8zkrd2TaWx2ePx7R6Vwnk10+/KRHzpqjayme26dV3sKfxaTBlP2nPVrtPL5tYicBg3lIOR9SrdS5HDjc3GgSiGMef37E+vZdQj9ms1LkSHBZLGy+tZ7x3nsK7d0I2ZD59KdTHlzIfO3rDcx6lT/7+IxlfxW1MWU6DzFNCccOwDjn6IVG+2tvM0/HnE2P0dQGnheyttphZpssAw+1uEkSdLnoSaRiwnDao1xcZT53EBX7K3NFShIWyVkahUKFwfQQSRKYTmM0dU67apEJKtOxSxgtqZbbJOkSXcuzvlXhw58cYWgKge3TXKlwetEgSx0qpTVmPR9LkygVNNJUx1m6aJqCKGp4/gI/dpFlC9cNWVsvEIQeiiIhyiGJm7BYdFlda+PPQXChouqsFAosJ3ns0xAhs3lyeoxhKRTLOue9feYLn0xQaa5UEQyRzfYqpZLJZDJhOnLwFue8cfcmqqCQU9dIUpdMkfDdGRtrLc7OT5gtxghyhoxOu1lB1/KEYUi5XCbLLmi1C4iY+G7E9uYbzO0xkFFv6SB8ldYhi5Al9M9PEGOVVm0Fx4HYj9FFhZSYNPGIAhtFKiLLKlHgXe1+ZaCqKsvFjLCnoKgKXhKzvlZlrVMjEwXiCKbTOYqsUiyXCIIrlqWZ01ksZ1iWxXw5w5RMdMNA1cBzfMajKb7vUu4XaLcb5MwhtXqBcqWAkcuh6QK6mZIJCkvHptVqsL9/gGHoIMrEcYrv++RyOZbLOcMwRBTFK2ctKop8ZbTQVJM0TZktbSDF8V1EUcQyclcQallCNyyyLCLLROLEQZENlvac5f6SlCu80XQ6RVElkkxgOJzjBwGGrtLqVJENaBVLGDkR3zYJMoH21g6O7aJmM+a2w/B8gGnqlEq7vzwT220d2x1iGTrzuUOxbKFpLrdurnN03OPo+QWtAjRMhXSzzX7P5+DwM9588xvkytvocxVfHzB2B9SqLaJQ5N4bm6iKydHhJUdPU+azOXFgkxNlBmcj7q2tYOY7rO40Odl/xfjZGYtcmUKtQWKHHB/uEaQioZ2yWCzonneRVsrEkcDlRZ95X2DqvqbdbiKKbXQlwQnmlBoallWi3qziej0Mc4XzC4fAGaEpMiVth+MXfW7fl5l2J9Sq1/jy9RN2t9dxzwN0TSQxFTKzQX9yiaDmCcQ8f/RHP6Kxmqect1gcZaw22mTRFEMqYy+GPDnbo7GyjpceMzsxSV2VFy/OCf0IMgVR1oljD1GUrpBikkgUBaQIxNiQCmSJwWp7g4XTY6PZIQpi/PMRBmM+vnyFld+hvWpRqV1j69ca7H/yGmcxZT72yeVlsjRFxCCVJciuCiZRlL8qdgRERCAhyzLWWrs0mlvcXtd49fAnuEmLstijkCuTZgGeF/Hg3jt8cvxzkEzu3/0L2O4MRZ0ixAECOoPLCbKUcnb+lKE9JUl9altj/sP/4H/GxvbbFPMtCiUNCQ3DlDHkGqenpyznCXfv3ObyfI+9/c9BUMgXBcrlKsWcxkd//pibt+9c8SKjA05PvqS/12FjfYuPHv4TOt1b1Bp1PMelZK3SbDZxwhFO6KLbKb3eF4RZwM61ayyWMyqVOvtHj1lt7vLi1RFJZNFRW+zv73Hn9nW22uugXHBy+pztjU0qpTUWS59ay8D1n/LJRyestN4mS0IK1gYL+xA7XLBZMYmDGcQpQRLx5PFLNlfXqK1HfP7RKxRdwksHDA9z/OAH38cPErKsSrkkESdLavUyF+cDZKnH9vU8f/JHH1Es5nnw1g6nJ13qtQ5hvGS1nOfsTEU1NKLsHKsYY+oaaRDxzoO3SMIZk9EMPR8wOZtRK2QgZ3Tam0iyiqwkRKHD7Zu36E2/wDBatKs7PHv6mhu3V3DsBEMv8uCNWzjeIa5tQeIwn0WI+iGp6BClLjdvbXO43wV5yDvv3mcydtCMgCgK0I08rjdDtxJcd8TCdpktUj77+Ig337lHu3OL7smQQnEdz+9RzzXJNXIkmUm5XuH4+JRO4wHlWwaff/GEX4xdNjY7BPYe5eoNzEKF/mBBtW1w3utTK1Yo5WL65z2qqxXsxRIta7BRvQWxSKtmUDBVHj/6ElkweOt+jZcvj+DX/zULSsfvkSQJYZRR1ixce0K+pOE4Hu4iQpZVhKxAFMa43oKLiz7EOvWWhGXmcOwMVQ+xDJlIz5GmOn7gsvQSIl/CsSbEUYqq55g4DrMXX7Le2cGOAtJYYz4u0WyVuH2zzGn3gu7wkiAY4doBqpLgi/CzD5+hyBAHKe12njhLCGYlbM4xyjPSTMOeZ6SxiKrLKFoBZBnH9xkvxqzUOwiCgL2YoygqkS+yu72NklgMsi6pZXI6OaVUKeJFA5qNCs8+PKKSF2jfqtDtRhQMlYvzSw5P5oiyymA2I1UinMBBMRRSXydLRAaTU1LR5fBoxmV/SJjl0HIWSZpiqSmq4THon9LM7zKaDNi53iD2y0xHU0IzJQpSsOFWoc/NVYmD8Qmf7XukvsjudpW/9NfWWNkw+C/+6z0+3tN4cz2jdzQglzO4t7XKp39+hrkS0773JocHNt3FUxTLYaNTII5FXp938RybqlUmi4cgKLzaP0TVMjprLZ49f0mppNJqF+id2sznMYpaJV8UWS6m9M6H5Ap5VNEiiVIm4wW6ISFIJlJq8vqgT6FocLD/Ct+vYBoq7cYGr/ZfICqQ01dxkz18v8fdWzc5OT9hPh+gysWrQ1ats1yEX3WeBghyRqVcpz/oUSo3MEwVU9Xo2Qd4vsNICtGyNvbE5eRogKyphGlCGCU0VjSkuMTR4JLmShPfFTHVJoWChe8OWSyWxEHIcHKBaerUa2WIIPYSYj8iij2QVYQ0g8zn7PSUxWJKo5lHFAzSzEASfYajOaVilcXco91usZxPadZbqHLKs+dfEGUzGq01JFkhSVKyTCRNE1wnxE8zskSk2SoxD6YkcUaagmyaCElEHC8IwyqBK6CoGYokEiUQhxGtVpN602Q5X1CtVkmiCHvpM5pOMIwclUqFwA8Zj8dkWUapVLpyucoycRqRy+XQVIMgCJAkBSNnAjCbe9hOwOlpj1qtQXuljpkb02g0ABgNJ7iuS2/QJ4l9ZEXBMHKUqznm8zlJkuH7LpIkEcfpFchdEIiiAEmSkCSJNIuJs4QkzJAkEVGSiKMUhwhVkpEl8SoirVYlTWMEYpIwwvM8dN1CQGYwuCQhY9ifM5nOURSdJBURBJlnzw6QZZG8VaHWlFnOHXJWQn8ywF4kJOKC+USAWMIPT1HU4v+XoOzgB0NkWabd2CWILhFUi3qjwN23rvP/+gc/5Nvfepsf/fkXhLpBc/0HPLgnc3o6IkgcZnaGIOS5vrOOIYrMJlMyEXTN44MHOwiKiqgZXJz3uThYcP3uHT4/3OfmSpu6rhNKJpOZynZlg2m3i6GkKJnF+ckxYxtmMxEtazO8nKDqEbGdIOVnrDXuk8upPHn2kO//5l9ksnhGEGvkizqSYtI7CTENhcnUpb26ShqPcRYR5UINp98lsmMW6gX90ZRW813aK0Vc54T+xYRUyrj87ISNbxg0Cpv87d/9O/zs858R+DL1jTJe4NO9uCQNBDTBoljYJM0iyvkCSSYi6iKVkoW99BElizTNkGT1avQs6CRCgKwGxJFCmrqkiYEgCJSbI0qFPJq5RrgMsLUptcIGdzdNnh9e0ijWOXrxZ9g/j9la3yT2Yq7favHjf3FIEl29r0IhRUEiSwUyQSC7SnhEkET4apXk6dMjWit38OwxATNUBoTLMkE4IU1jyDIkUbqa1Mg6Z5fPKZerTBd9cAvcu3ef/ZMjsuyM4+OHNFfX+fDnDxG+/tu8d//X+f0/+GP+9v/8r/G//l/+x9SqDf7q33if58+ekDd26DR30OUck+mI27dUvvxiH01dIUkGnJ8dUWusYwfH9AYKb739deaTR5jNAlYhggsfSQvwOUdSZFZbNQbDHt3pEVauynB6Tqd0nUQJkYg53DvgMFrhG9/+Op9//oSN9jbPn+1he21u3d8hDEIiuhwfnFBttEBLeH3+AikqEachlXIV33dZzGZsbm5i5lU+/eI5kiyzt/+Cir6FqcvIkoplVCiWLPpnE95969/g6OLnXByL7F7bodt/juc5NGpbbF/b4PGTn5GSp7VSx/X7fP5wQnPFoN875eQ1vP3BA7787IBarUYSCXQ6HUJcvMsG+dwCVx6ztnqTvYPX6LJEobpJqgzJ1QRGiwWKlCKLeQwD4nTGoOuxuqaT+Su0Vt4kThyWkwQBhXanzKPPPmW1uU4W57lxY4PL4wtKFYkoiijlO5ye7zG46LOxegdZE3j27JBiyQRBIw5jdF3i9u2vs/fykHxR5IsvHvOtb/yAr329xsnZC+bzMRvrDyhVEp4/u6BYUYmjBUgakT2jWs5RyuXRDJH6Spvz7gWj8YT1tXcol6pMF2dUqmVKhU0uL06oVIusb1dZzgIyMaA/XHBx9oKbm/eQpATNiJmPZlSLLcJAZ97zuLm1+/+jB/97C8r1NYNycYXL7ogkS0kjFX92lYHaWaledSKWDpKcEoQetVoFSdRIsjmiqFOuWphWQhhAJV/j8mKKpqaUyjWCICATUpIsQ5HyeJ6LbuQ4Ph5QKZsspzajYY/xuEqhUKJYsJgtKpAlGFaE6/gkqU+9UyNyPG5sFklVn7OJS6lRw/VnVzs2kYgiKnT7F6imfvX7iib5XA2sjKk7Qq+IbN1vcPj8kubaJnqlzIuHp2zv3uW8f8p6u0ngO/RGI/JKldVWE82A41d9JF3BmwQMEal1VAJfYhJekkh5sjjEt32ieI4sGRQs0IsWhwdLOjtlnEVEJMX0j+dkUUROscgckPIRYi7is89esrplUsw2ePz8HDufcaPepSJLfPI6RdNlqoU8xXWDzq0G/+U/fMHxa4Hp1KfWrvHsyQuePR/zze/fZf9yzMqOya3rNfoXArHRJbwMMXNlxGBGp1SjOzC4cW0NRco4uugxncbImku+XODsvIeu63iewLPHYwRRI3ITlvMuti2TJhKiIFCwNAInRYnLJIJL73KIkStxcmwTRi7LRcj25i6mnmOrU2Y2WyKJKkJmkisucAYKcRihVl1yhoFlKQiZjue5qJrMzZs3cZwhhXyD4fCSYdLFzOksnUv8MCHL6tSrNaZThSy2UPIyaqCwvb5OEAdMlkvW6kWW84RWu8JarsTh8QmaqbDSqOLbS5ZLh5WVLYp5nf5gwnzmU7Jgfb3F5eiQRq1JtVCl0biOH8wZz06YzXx0XSVX0JCEHLOFDalGo75ClAbIssRoOCPwHEhUwjAmTiOCSGc6c3j1ekIUZYhiRpYl1Kol/uzFEds798kZeWTVQ4lDYickjFOy5ZI7tzfpd33ETCfwfDRTQlGUq3Gx65ImIqVS6atOsowfXJmWcpbKfDlBlnKUSqV/1ZkBdF0nE6/YfF4YEUYxK9Um0+kUACOXI44Tkjjm+KTPq9cn5E2DJElQFAVVU65QK0CSBNi2ja67SOqMOI6JoghRFBFF8er1yuVYLGbIsoj8L2PwJBnXDYmAOIrRZRXTsHAcBzd2cV0XZ6mzmPtUa3nq9RKlhoFhygSBB4g0V674pk0/h+c1ydIrxJZtO4RBjO0scNyI6VxhYfuomkgShQiIxHGMJBqkSULOUhCjfzXyfvrsGZquoKsir199SaWWo1xTmc67fPn8hEJzm9djkZ++PmIymVCsD3lw6xaCoDLqHuM5LrVKk73nF9x/8yZaPiZ0FZqNPIu5z97zPUoNHUWuIesyHz76Eqms88WnnyMuHTqbDXZvrfL//H/8N6xUGmx2OsyHGcmyyWww4uhwxM76NpP5gFqzQk4NOTkeYC90imWJYX/Epx+/wLGXXGRLWi2fzW2Rbu+MJMnY2FnF9kIMs0ahGHJ+tODBtXuMgwvcWKWSXyONNczSJq9Pn1I0VvjgvfdYLTVQdIFHX+zxxps5FEtlNp1QqdwjDiAJRmy8kSPyVKbjHvUVk5/96AUPHryJ5y5R5BxZJqKqKmmWkiQJoigSBC6yJhNHIkkSkSYSfjJH02XuXP8etdKSxXzE5WRIsaXh2AKSFrC2dZUr31qtYe1sYJgp7XIH2z7FUMtMIgVBSZCjGEGUQRDIEMkyEATpl2EBnucRBAGBO0LRZFJJ58HNHYRMQJBOCHwXwzRYujHru29xfPqUJIl5/WrCzu4Gp6NDTk766GbCfKby9Q9+i95wwb/71/8O/+gf/Md859t/g7/8l77J6/2P+e0fvMn9+/f54z/7PSTRZKUmkK94eAuDrbW3OT66oFW/gyinHB/vUSrlieOMJFHJUpXReMn61jb2MuDsfMIbd99lPB6zXEjIag5NaxMkYz7/4hG1aoNafou723Wm9jnTkcN793+X/aNfsBgr5I0GOVOnVDKpFCTkTCYMfQQxplmts1xOEaIGRDaaGVGrVOn2FmRpgO/5jEd9SBNu33iPP/3JT/ng6+/yjXe/R6u2ixfOMAyBZ188ZX1ni0p+i9Udl3/U+xGWZfHu9i0effIKqwSD2UPa7TZWXmPv5SGdzjqOsodAlc7qLYrFIk+fHJMmMk+efcbalkkUZpye9di+0URalNEljePXe8QhzKY2nXYRRS9TlCr48ZKDVxc0Gg2EMMBxHFY6BexlcOXg9z1OzrqUyjmW7hn2XECTOmxsb1AuFljMl1RqVSqtAk9ffEboWuxuXGM4uIouTDORTLKJEomt9Rt0L/rkckV6FxPSJMBS1/nmt9aZXCpcv1ugNzwCZcnF5SvcQCYKVDSjTcQYSXVwvYDL3ozSts74PCGOMrY2GgShy3D0OZNxkXK5DFLCyb6LbIz50Z+f8s4H95hNZkiSSLN1jchb0r8ccPv2XbIkQijkKOQqaHmH58+7TAbBrywohSzLfqUv/B/9ra9lp8djNKNELmeQZQnD4RhBkK7SC8wSC+ecNIYoCoEUSdSJ0yW1ep69Q4eVFZ3B5YxiLoeV01BlgSdfnFMsFtCKInGUUSrWCIMA04TIS9Hkq8o0FXzEzMR2xsiKgZq3vkpDUZjMbMRoiZ+CnIKl5HATj9J6HSlTSNwFjiNgmhCGIZqaQ1Jkpos+hpVDU4qkmYKQevi4OJ6NperUG0VOLy946/Z94sSgd3pMyygxnFzi5zQWtkBZU5gMp2Q1hTQxcfoOuXxMJtgQNTHyLr44Q40tojhDNSUKJYXe2RJdqWFZFmZBJ/Iynn/+FM3QuXGnjCHohIFGLILnLwjdCMtUqHZWmY9mXF9tUTPOOOpdoOVzGGbMqCdw3o3ozedUGmUmlxGKYnP7/jZz32Q5HJI3WmgWbNxQGJ7bCKFAHPpcniRs3apwun/K2kqD1e1dFDEH6YJAkPizn37KSsdiPg1xvYByPWM8WqIIJeqNApZawrIKTKcXDPpzyqU65UqeR5//OWudLSBFlAVsN+bifEGtWuCyO0Mio1opoRsylmXQ7/n44RyrnJK6LSpVicBJyRLIlww8N2ZjYw3XmzPoTzA0g8vLy694oSaj+RRZzSgWywiCRJw4VMpFMkUiS1Rse0QQBKysrOK6C+IgJZ8XUNQ8w+mEQqXOZNrHlDQss8l0OWYR2liZhGzkiKIIQxbxvCmN1Tq6bNKurkJ61VHzoxmaanF+fk4qxKhakTD0MU2L6XxCkI6RFRBii7JV5fmLLzHMEppa4PzykFLNYHuq8h++EAABUVT4e9+ocdmukrfaLKYhE89nPpnjuzGoEuF0hKmNGXQ9SoUVJCVC1WUkUcdeOMwXY0pFC0lUrvJ1lzaKol0Jzq/gzYjBFbNSFInTjDRNCb4COsuyjO8HRFF4tVOWpui6ThzHZFmGrht4gYcsSgRBcOXKleWvDA3ylYM7k64OHCEjjmMSMsIwIMsyJFlA4CreS5ZFIEOSrqp8VVWRZRnXn5ImX3VlZflqFJldXc9xHGOZBVZWWuQtHTOnIUkZQRB8hRgJCUMXx/EQBQVF01ku5xRKRaIoJAgC5vaSycghny8gySaO46BqIGQiWeIjyFd/w78U0wCGpqJrBmma4nseAgph4KHJGsgCmmViuwtqZRMhkRElFS+JUCSJyHNRZBFJUtB1HUGCOI1o1tcZDo4YD8a06h3Oz7toqolnO6xtlpGkgDRRyIkSBUvCT1V2736N0LMJ54c4QczLFwfMIpfeaIkYZwiSSBBLrFQrmFWR5WICUcakq1BqgpkXmYwdrLzGvTfW+fTTJ1RaFvWmhe9lZGGN7//OXY4PxrSbFT5//jmev8CxJ6w3vkar1WDufkFFukmuqjCZBnzv175Ot7tPIiT8+Ec/5/aNm9RW8uy/uEDXdTa2m+wf7tNcKTFd9CmXWnhuQF5sUakZ/B//zz9ikpgYqgFZQhD4KIp09f8XUybjAXGUEfgOjWqO/+g/+jtMeifMByPq797AsAOOX++TlSNWikWmwwGjQYZMkbXNJnN7D103+b/+J59y2k8RBRkplcikGAGZOAnQFAHfi0nSCFGEOBD4P/zdv8ks+QJvBAXFRDZ1qoMhv/XfjYjjEFEW+Of/1gbF791iMJqzufImon7Ow4f7VCstxoN9KrU1lvMZlm4wmPZIY1it7XA5WvAX/sJf4E9/9E8pFzew8goZMnfu3OHxs59CJiHFVcoNldlizHh6wtnpKXeu/wYbWzUODvYplDR6/THrmzvMZjOOjg4oFWs4C59vf+s3ePjlT9jbO+HmzdsUauDaU0b9GWlU4nu//gMG41c8f3rAtetVJFHjyZdHfP8Hv3P1fUo1Zpc+d+7u8uT5z4nTJctFSBQUKFYSzi66dFpt5tOEW3c2yLKM8SBg7r7isjvinbe+QxT1GU4yCoUYXS6SCQ1qNYMkcBG0jLzRpD/sASIHR695651rHB+O8YM5qlxhczfPs8cXNBstVjoGJ6fH7D2/4NqNdYrFIilL5PgOqxsxP/nZH5K36qxv7OAFfcZDA0kaMu4NkI0i09GAQr7OtZubZIHPeDEjV8gzmZzS7y0wcwKaWqBQyBPHCZ5vE3gm9VWfx5/53H9wnWazTZj02X9+QalYY7o4wXUSgiDge9/9dS5OhyyXNqVa/oqM4b7GszU6q6ss5zHj4QjXGXJt9y6eYzP0hmjpLdZ2fT5/tIekxWhKnU6rRffyCQIV1jdWUPWEV0czbm7sIIlznGGInwj0JueUKiqKFuLaGZ4jUSmt0qgXcdwR5eYKzjJB1UOSALR0je2dHEkU84d/8s+5+8Y98jmL13vHWCWN2SzAypX4d/+d/73wq+jEX7lDGSQp5UYJM1cnCBKieMnNNzaZjRecnUyoluZEExvfi6hUaownc3RDoLnSYr68JF83GE5n2HZC3jAQBQXXC9jc2cL1JqiKgSxHKEpEoWjg2FNyVoEwWDKajCnVK4ShS7FcwXYDMkLCMEazQpoNmfFQIxlHNDfKV90UV8B2phiKRc4yyIQARZFAUCjXqvRHQwTJJEuvKg+FOSBQL1doldpkcYSeCZTkMkcvulhtiVpb5vRFjyBY0CitU6rlmY5OadWqzJIIvSSQT5YY5QpC2ianG/R6XexpiVzeJRVkqvk6vjeikC+T+B6R7+AnbbY3LLizSyREhOkcQZBZaZf42c++pNNZpZhXWAwj9KaHkjNZZA5ffjbHE0B9KbK+dRNVinnnAeztD7hxe4tPlicUm1soVoyqznl/4x4l2SIQfF6ffUkWybjTDAmd3R2LwE/ZvL7L2uYKcaLw/PlTVldMAk9B1ibMpj6eG5OzLMZ9l/lUoFDIkCQBzZoxHC3w3QBNk9nYXOH1/ilWroAsGQwHUxTVYDbJkCWByXjJ2so6vpdwebmHYRiEvoaspUhyxvgyw8yNmc6uLghTvwq2bzbLJEmM484olFRUOUcpVjCUMrP5iFpNBElEUQIuLy/xHAj9iNVmA9efYGomceJwfLLP9a0HpKqHvVzgq6BrFr6dIEYGUSwz91zc0CMTPdK0Qmp4lGs5/GlKpbiOomr0Bj1ySg3Pn2EaBURR5vj4iJiAYsnE8WyyVGZ0fkS1WbmC0schiS0zHs1orrQxc2WG4xlbu1v0+mf4nkyWyaRxSkKGMxfpiwPSsspsHiEZFoZm4jkzpBRCz8dbLlDVHK63QEfC9V00NcZ1PYr5Ajmjhu+7hGGCqioIcoxqKMRRAIKAJEj4YYRlWaRegB9FyLJ8ZbAJQiRJwjBMkiQmTVMcx0HXdaIkxvVdoigilZWrrk4KjuNgWRYCIp4b/nKcbZrmFYZFktBVFSOXw/M8VFlGFGG5nFOtVkmzmNraKrIs4fs+ilzDdX3mS/eK/ZpmhElGmkpkscDStolOz/B9H0kSKRbzmIaC7SwRRZBlFV3LMZ2NcZ2ritt+efqVQAZBTknTDEOHOJkiSDFRJBNHAbIgkyYpWXb1+vzLD1VVCcOAwbBHo9q4Si2SDTJURDMhDEIs1UCIDTIpJcxkiK44epJhEcQRXhARCxJJFFIuF4lFn0xRuPfWGywWC66VtrFtB99XyFVKiKKIm4hMnZhu95LIjVnGD8lEgYMXn1MtFQlTHyvXpEUL33HRjBTbHzNbTBnMBGQh5J23ruM2YvaPXnN8MiX0ZTqdCo8/uyD0dPqXDo8fd9ncLqKJ8OGHjzg+OaRc3CZLXAxTI/BKvDo84uTkBRkuhpUSvJzjhzmevXhKXjNobtfwU5fzvs3p3Ca2fbKlR28+o9HKcXzWZzhYcuf6Lk+fXPDuO5s8f3GGHwsoqoIogu+FCJJIFEUIqGRiSBT6AKSBTrlSYOT8KZfDhHXrLZLxjL3PuqzuXkMoj/FGAVKiEc+esL36PbLZEiE1uPRm+GlG4i7Q8zkiUYY0A666kinSVYGVXKX0REmEVSiTCCrN1jrL6YROe43V8gph/ENkRSEjRjVSXr9+wsnxHNNqEE1i6s0GoTPjN7/3V/n557/PwdFLGrUVyg2L7vkFaXaL1bUS/+Af/jf85vd/g0HPJgjnpInATz/8PYazL3AXeW7s3sPvFpE1kVevn9BpPEAzUj59+CUffONdFMlFVkUG/SmjkYNuFNEMk0p5hYv+ay57Pb713fdwwy6v91+ThAqarCKp86sYy1BFVhacdbvMRzoP7r3Ho89+iD0XWP1ak4NFlx/+6AkIIdXyOkZhQOKfoeVucTn9MxIvwDCaSLKB79uouoGWmhTLEq57zsXpkJ2db9BaURj0L5E1jb39L7DtHmfnDuudVUyjglVOiIUTnj2LKZerlKpFapUqj58/xcjnuZw8B3GHjIBavUgQX3J2NuH0eM7WZobtqRwevaJVzXFjt8rjxx+ytnYNVc+TJEv8IM97771LlPXoTo4JlxH5Uo35vI/jBKyvblGpVPny+ceUy6vEiY1ra9y7v4PnObz7PsxmCx4//QiiFR7cf4+MkHpS4vPPntPuqDx6+AWLkc47X1/n5HxImE5pVXfJki6nhy6lisbqWo3j/Yizi1cEboonZAjiI0afJZQqq+iGzMHrc5qVGtVqnaPjM1xPZzFJKebzBN6EakPCm/rY84ztnWsslmNEIUehFJDJS6b+GZKbkjPrHO4fkzfrzGY+0+6EaxsGQmLx7PN93n7zAVatyBePXlMsCpycTTBNk0Rc/qoy8VfvUP7Vv/VWZjtjkiyFNE8cqhRyZXrdU+qNGq9PvkTBImfquF7CzAnQLYFyxWI0nJMqIMSQN3S8ZYBl6KRpykq7yXi6JIpVsjQmCDx0TSZKbFS5QEZIEsvoOZGcoTMdh1iWhaSIjKczdD3CtcELl7TqNaZuQKq6ZEGGJBXxE4f12i7+YkFKguO5KJaPoKSEjkjespAziTD0qa+ucnFxQZakXL9+ndCbkAkhaaoSjFLu3G/w0aNXlGsG7kRn0JsTiHNcd8GdrbfRyh7xJMNVRdxghiwqGHrK2XDGSmuDxWKG5wXUqkUUVNLQRsx0siRAFAQqnVWOz16znE6otZpoQoYpVkkkg7kTcOtaGXsR0+l0ODk7otHsMHPmjLoX5AWJ0A0IeM36ztfwg4RGrUV3+BDb91ktrjKcJZg1kzQbkgVzhvsalfIKxYaGURaQ8AiilOnM5u237zEZB7x82aPfO6XdznN8NL/qCsQBmprHdV0UVUBRNBx7yEpzi866xspKiyhQmA5dUiFlPvPZe3HC6mqHk6MBhimzdGw8z6NaaZAKCwxNJWc08KIx9jLCMAxW13VevezRXmteRSxGKo1qmdm8exUJOZoiSRIb1woMu2NyZokbt27y6OET7KWDrEhY+SqiBMFsiaxriAosHJ8sy9i9tsXDR88BkSTRKFclzk+7fPDeu6RJQBSC73usra6wf3BEqPrk83m8QUyjWeN19wWlfIXV4i6REOIHS4qFGnGUohkwGJ5h2w6Vyg5ROML2bIplFd/xMbLmVWGGz2g+RRRBpIiQ+dRO5vzv+iZfpcDxn71Zod+qsNXeZhEInF5OSVJYeEs0SeToxSusXIZpWohCxubmBkEUslzYjEcjbt24Qa1hYNs2IOJ6AZ7nE0YJoij/Mm5OEK6SQWzXw7Iswq+MMoIgoCgKcMX8k2WZMLy6yctfmWdcx0cUBBRZo1YpYeUNAs9nsVgiICHIVztoi9n8ahyuqoiyguu6CIiIoorv++iqgqyIqKpCEFz9LMMw8AMXxw4QxKvnQorneVdGHknF9T1s26VYLGKaOlHocPP2Bm+9cwtJTvEcePH8Fa/3z3C9CN8PQbzao1QVHWQfEYk0Nsjw8MMAWSkiCSFpnCGrGpomkZH8q8NTEChYRXzfJ8sS4iRAJE8Q+QSJDUmGlIrIUo5YdIlTUOOvUle4Qt78S4GtiGDldURJw1k6NJsrTKZzpvMJ5VqZ+XyOKApYSh6rmKKaEnHiE7sqWeCRs2oMlwtEP0DL6VdxhM6YZqtKloi4ywWZpHF0cECzssN/+p//bzjq/5SjkwV7z7t89sk+BdUijmO8ZMHCdZnPIwwzRJMTmm2D1697mFYOS8vjLFM0S8UPEtJkiWZEaMo1NmsWlZU6B6cvaXQsRpMFo96CMFzQ7hRoNreYjDxm8wHrnQ0OXr/CXbr0Lxzu3LpGpVPk2ZOnBPENEt0ii0LENCURIfACFNVESJeMe+eokslw1ON/9R/8bUrVATldwE5ecfxkjhiFmPkO966/QeYGTGcjNqp1ehWN5fCQopGjsSXwd/+3D3n5bIaes0gSCUHwIFN+mR8eBzFhdGUCXC4C/gf/k9/g2oMRh705TjDhve13eBCuc+Pv/pQ0i1BknT/4twt06xprq2/ipGfsvRzywbtv8OLLJ0hijXxV4fTkc4S4xNe/+evYCxd7NkHWCkyXB/QubO7eu8HZ+SErrW10rc7Y/QkrzbcIlgZ7r75g92YJd2YROhphNqZQaLN7ex1dDnj48Dnnl0Pe++ZbTKaDqx3lMME08ji2RLkKT774klwuYzxy+dZ37tPrOZydXpCXi6yu1fCCIdd2b/PF58/I5xqsrOkodNg/+jmiqGIv0qtrzYpRdIdeN8MNLolck2vX3qTbO0VQYorWBnfvbnB8+AJ36VKrlNnavcfTJx+jyVX8LKbb3Wd74xa1FZPXr59SK2/jxQP2n3m8+V6bzz77jK+/930S4ZzPHnZZW29RKJgMR33m8zm1agPHG9DvzjAMA1NvUCgp9IeHGPIuS/eEne0b+MGS6QhWV6s82f+UWn4DUdCotk3kzKM/6bIYSjSbConTJpVH9CeXbG/fY//gc67tvImVVzg7O2NuO5CUaK+WCcME306xCglfPDqn0ZHwwwB7GbC7/k0evFXm4cMvCJOQa+sPkEQFx+/T7Xa5c/sBl71javUiX3z5CavrN5Blm0cfn9LptKk1THq9LrpsIiYm1ZZF6E0pFSxmzhRD1pmOHGq1DmaxQ8yA5cLhtPsKWdGp1KscnH2IItzgzbfv0jvq4TkLSuV1dlfW0NQBti3Qru8yDSb0xmPGg5BSLWU6m9FqtRn0+vwv/uZ//a+3QznqpkhSCS9yEcWElVqT/uUYQysQODNWG7v0Lk/ptLc4v/BYBues71Tontl4jki1LCHKOs2SSSgJ6EqRMPYQUg9ZEElEH9Oy0AMJx3GQNYk48whTh3K1he/F9AZzclbM1J2iig1EZUYU5hj1bdrNKvbcJZNUommOvKTQut7mZHDO5KKPqEgsvQvUnI4ga4ShhJEr4DljNFHHdgXcywOiKEYKFTzfQSHPbLAgUxPEMGB0PiULNSTDZOz1MDsSjXyb4bGA2Vqw9Ep4wQByFVxvTuYrSHIeMVNYzi6wigaQEWcjBFTiCHbWbxPGYy77M54/+5hitUU+v0NqB0x9F30notVYwRif8OrVSxqlNicnZxSVIt/87jf5+//l38e3x1jtGnq+QE69hahJRLMhWtzAG/u8+e5vkjdXef7D/zfbTQ89VShU3mDzWzusbehMLmJOuy9RdYfRNCLIVIZjnzQqYZouUhIxHkYYms4iWKCKedw5KJqOu7CZT0LWNlZJwyqnR10uz1IEJGpNlfl8ymi4QNVkEF1Uw0ekhakn6JpIranTvVhQXS2QERG7IittDStXxLLyNFdDZrOIUklCESSW9pyVdoU0c8mEMpKWMPeOqdRqKDL4rsNbb96ie3mBquks7RmqlifOqSwX7lVGu6lj5iRO9/f41vt3ODsboxkxSQilrRvkdREnctGMIn604Lx3wOb2DufTIxxvQrPV5Oa1LVIlote/YDDaQ81vMhidEmchObNI73yAYUKMzenZazqrZVRNueJO2ilZeEGhUGThLGmsNXj16iWdjsFimCGlOrquEMcxhq6xsZlnlggcvnxMcXWLernAeDonEXxAZDQZAA1u39plPO4iyzLFcolbt27xz/7J73PZu0CUG1iWCaKAZujItoQkKdi2Q7FQJkxSwjAkChPK5RJBEP6yQ6lpGpJ01bUpFovEcYosq3i+T+AnCIJCpXq1ZG4ZJq2VKqoiUNpqc+/eA549fYEgRyzmcybjGUEQkKYCju0RhiGu45HPp+zsNNF1DU1XrtZq0hjbvqII5K0tpjOb6XRGlmXIqky+sIKp59B1nUJZZbnwefTpS+YzD03N8+TxOZ4j4vlLTk8HyLJKGCS4gY+ZUzGsq5jOWIQoSNBllThMEaSrkbsgpghCSopPlAQESwFVVX55JooSzGYZQeAhCKCqBok0RRBSClqdKJwRZSKJ4CEhYsoiTpoSfoVeEsSrNA9NV1BkEy+A2WyILKXQ72KaOSplC3dpIycyChlRErMYp6Rj0Ismvp3QKNXodNZpKxGPH32El8WUtSLbpTukUkaajKmstKmvrnJjJwdpkf/kP/0/cTGeUqheoZ/+vb/zm7xz9x1SZvzhDz/i5cs5C/8FF8fnBMs8saOQBQYD2+E48FC0mFKosra6yWwiEczh9t0GejXj4We/QM5FhGcOmiBRq+oYuTbTwTk9d4Gs6qRRjGlJvPHmdYajAY1WytraGmOnx8bWJk9fZIiZRJpdwfpBRJZ0ZFkkCgKyTCAKRTICFC2mXGrws5/+MY4d8uvf/hYxl/ziZ/scHJ5RVEtIosnopotl5xnPfdKWjTkrELsuWSojSjKZMINUI00zREkErpBWwC8/XxU5Fi8/fsbN+9vMs0u+fHLJVmDj+SE5Q77qpifHdEcvOD8I2NlZ4/lnPUbOPplvs3PjB3Qa2zz6/A/54R//U77+zq9zfnrOyto2a+1b3Li1YP+5gyKbXJz32N7Js1J9m+PDHrdv3MHzRXxHZ7J8TMW6SeQt6A8V9Iscd67tcO32mBfHj5nMZtjh1b7yauMag16PmTti2G9SVFdRlD7f/s77nHXHhF7MrZs3IZZQFJnB7BU//vkP2d54i4OjLxnPdvkrv/tbjN3HLMY5vvXdXf7FD39GIU44Pz+FtMPOjXXUYpssqZBI+xwf9fmdH3wTFJsstdjdvY2qurx6NWZ15T3KlYjhNGbY9dF1BcuyqBXeIwwcwsAiTq4McbNxysHrU2bTAZVyg8ALuLTHeP6cnFEibxkIok5YrhD4GflcCxiiSmvcvrvBH/7hM+o1m2arxnz5msAvUas2sadTtnfW0fWY2WWGvzB5+8Fdfv7zP6aaN1lbvwmSThBcufMPjj7GsWVu3r6PZuRBSJktz0ljBdf2aLSu8+1vvsn54BXdwYTmmsHx5T7Kqzz5go6QlDg7nrB7q8jl0SWiEvPTj/6ED977HhP7kFAIsOc213fu8hf/0gYPP3lC5Oe5f/ddhCQmTV0sWeLxZxccvBqysbnKXBbJ6x3m0wxVjYmzBe5URcUiTefMJz5F9R1qTYuT4wvyVp1StUxv6DCczjDklNHYZ7b4nEguU7ZWqd7t8vzpEdPZiDBw2Fi//qvKxF+9Q/mdv3wnC/w5vpdRyBVI46sKu9MxEaUI3czz6Yd77Fyvoqh5jEKFg7OHrNZ3KRoq/X4f10vJ4iKlkkSWxLhehGguGc+vdq5c32Pp+BiqRrXS4vziGY3GLo7jsJjMabRMUiFCM3TixGN0IWJaAuAz6susr9aQlQzPCzg8WPLGOy1s54KV8k26vUu6/RG1LY2lEyOLJVTBhGiGYy8o5ppIFUiDjFwsMFnMUUsaqhRR0bfJr+Q4OX7O5uYm034fP5HJFauI3hIn8XE9Cd8LyOVlwjCAWKBYLDJfTJjObRq5Mvn1EqPJnMh2WC3ojALIZTkqDYWpN8BLLAg1cjkdMQr4/9D2X0HSLYh9H/Y7OXfununJ4cvhxr13L3YXwCKQiARBVVEWbZNlkWWVrbIt26Uqu2S/2VWWy+UHlSzrwbQFSrJBmhEkAhGJBbHp7t745Th5ejqH0ycnP5z5ZoHSCx7geZqv6puec7p7Tv/PP4qZTCTBZDng+uYqCgYFJse959y4fYt4oWNVl5z0BkiCQCpELEYB9253SYnwx6WfbOdehbNXKYvwjEKcMBvKvPP2e4ynxzRqTQ6fLKnXA9odlSBT+fLROav1der1Cq+PnzE5Cak1LOb+gCgQGfRnVCstGi2Vly/P8TyFa7trzF0XNbURnTG20kJrJthqm5fPx7z7YRc5tzgdvEJRq/RexwTJlA9/epPp5Ig8qDMeuQhiQqWqsb7eJS6WLOYustRmejFla89GVjRcd04Y+Riqjenk5Uyh12V7V6Z3MmDYG6OIBopqkUsB5AZR6lFvFTjmTeYzD83yqDfKVZHRhcT9d2qI2Q6D4QnD8YhWt467zEmSBISM9cYO3/72t7l+YwdVdpi6A2bevLwlkzWySCIXFqRJgWFYhMGULKpSa+Wsr9xlMewTL3WKZI5o6dRMk4W/pNs2GY0zRosBrXYVsoL3iwr/7h+8pshL1vDXvtLhwLGo1CtIkoEfyQyGk9LHFgS8enWEocBb77yLZVkEgYfnuSSJR71e4dr+NnEiMhqNkCWVLCuIopgCSPLS7yMrGqIolms5eY4kKaWMfbl4UxQlyHrz5XleuVxSFGiaRsV2UFQBw5RRZYE0zXAXAb/0i7/KfOLzW7/z/8UwDFS1nGZUdQXD1FBVhSSJqNVUTNPEsixq1SpZltDptEjjmNlsSobGcrlkufSJwhgvCEmiuEygaxp5nrKxscOf/tvv8ulnj6hWasRZTprkKIqCYelX55NRekSTJKFIM1ZWVkizGAGJIChB7p8FEXmRUuTCFZP7ZwFllpQ+UlUtOzTDKEG+fJ7SNMVQFRRdu/Keyqp8CVZLL6kgCBS5wHy+QBIVbL1MLWuaRpIVl57WjDhOS/+3opapdMtCpNyZztIYXRYpkpDj4YhOexVZVlA0hVa7Rq1qM5+PURSFaa/PRX+IXTU57x8ThiFbWzv8xE9+gBde8IOPvyArFrRaBs1ai/kwAV/n/PAMGQh0lVqjS2/k4schcTjn5vY1LEEmDce4UpWD0wsa9TobGzovDp6ysbrPsH9BpbvCeqfCbOoiiCmRlyOrJpVmgawJ9C5mRPOcSDQ4vzCwzSpF5pPmEgkxmiKS5ynJcsn4vEchJWSpxEcfbeHOj9jY3uTpp6fU6xX29tbRrRBJ82m2dlAMi2XYJ/Ua3N++zdNPP+WDb9zl//Sf/SaH/QWqBmKskcuXRExRADlpmhIHIaomM5/P+eZffZv9uzLraztYZo3jwe/j/+GM/8PRFnHkk+XwL/9mnce6j+XA229/nU8/fc3p6TF7ezvs7zU4O12yv/MOL5+/Zjp7ycb6Crd23yFKTnn45ALT0smUKUxMKisdBqNHeLHJj33lHV4cntC/mLFzXUKTmowG59hmF1kTmbtTutV7dNdXefb8e4xGPmu3LV4fSXSqCm3DYTaXqNRzNKvgB5/8MdXaDW7duM4f/d63eO+tt6g1a/RfHrJzt8bpxTG60aaWm8xcnZ1b1/ns8UPW1uu8fPCMd9/e4+Xrzwgjhf1bbyGKNidHh3S6b+FIAaG/IJ65LOIL+q5LxW7y1fu3efSyR7t7DctQcN0+B69PePf9r3By9phxT+CDj24zGJ2zWCzY3Fon8RxWV2t8+fkPiIs5vqvy3lf3efT4UwTJY+lqFDmc9V5x/9ovkMsvUaUOkhYx6dX48Bvr/MEf/TMk0cCuamXdmb1HGPkEUZ9mq8JgOKPVukHiZex2b7CyXsWPE2YDn/PpU6bLBaYas3Qj9nbfpbMm8fu/92+4eeMtyDVmsxmtZpNuc5OT8YiL6Wvc5Yy5t2C9I1BRNkjzFuvrDU6OnxAEAYLqY5v7pQfcqDGbHZIJIo68y/a2RRgmxJnL4fEr7u5+nUZNRbfXaVgKzx495GJ8zuvh5+zv/ByW5XF+dEaztcpgckEcy6xt2AyHJ7z/zq8Q5nMePHzE1sYuktnn4PVZOXFKjqVVkZQFj764YG//OrLqcno85tbNfaIoRJUU/s7f+s//chnK2XCO7Ui0aibj0ZJwmbC1tcV4PKLZUVh4Ca2VFt7S5/qtLUaLOU61YDQIye0KXphjmg6DoYueaVQrGna9xWSqMR+eUW9LFFmBJmvkREzGp1jaGsHSZT6d4lgtJAUiP8ZzfQwLnIpOHICsZbTbNvPFBMdWMAy4caPDT/xsh9/9jSWWscnF6TGVlRQxraJLApZlcfLqnJojYxsNLAfE3GAxn5BZKqLVYBl6KGKIogeIcwkDh+PDMZqqIiIymy7IQg9VM7A1BcMw6I1OsGwNWVZZ+HM0XaJa6CTAcDxAlQw8QeRivqTS7CIUKn13TrW6jr9YEEQukgSGrHHe62OsOFhVm1wqCLMFuqKzfX2D894AqbAIioypN6SIJDJi9ta38N2ExkqdjXadk4MBh4/HVJp1WDTZ3FnlkT/l5OwUOc95dTRgZbVGc73F8XkPhJy8SMkVj88/P6LTqNFqO/QmEWksUoQGUbBArCsEocXufhchFXl9OuH6jQbD3pKbd2ucHqgknoHeiDDMnIveGEkJaHeqpJFArVOwVWvy9IszOt0VEOaoqspKp0NrRcYwTI6ORwhiTr2VYKgVrFpMlibEwxzTMGi2LBYjSJKMyeQF81GLYX/MxnqNW7duc3o+Is8dwhBMVWE28nE2U3b3bZZeQuAFZFnB7XtdohAcx2NlrcGd+x/ghws+/vS71OoGwULjrNdHNVRCT8KN5vjxEknSWbgj2q0uUr0gjDNit4KtxWioWG2VYCHxW//4e0hKyPaWTTCN2Nm+jlnxsZoW3lIlEabc37jGfOkiKBqnR2OSpEBVVERJwqi10DWRMMxJ8wBR0hmPB+RCznA45NatG6XvdDam1+8jqwqNepV6q4EkFJxfTJFlHVWz0FSdLMsokAjiCE1T0XUTUZKuQFaeQxiGJfjKMjzPI0mSsiPyEkSWwZgSmPm+TxLFIOQURYYkiciiyNL1+LVf+38xmy1wFzNWOt0SoOU5qq6RpimKIuH7PkUmXgFaTVPwPQ9ZLnsp4zhC1TWCICAvSj9k6ZUsWdayiDohyQss06HZWiOngDDEcnTiJLwK02imga6XrIMsy0gIzOfzy3PL/tx5pWl6Ba4RS8+kIPzoupplGcLl/bgsl+n4arWKJIrouo6mKcRxjOu6pGlCURQkXoJh6Fi2RqNuY1kGIBKFTYIgIAkTkiTDC5dkWX4ZfBJQVRVdlxGFDE0VSdIFAGEYU63WKSQF1/Xprq+WwNSu4HkuJ8dnTCyLNM6Yzxfsbm2zud1mY6fL29KHfPbZF5AK/P3/8p8ynQ9YMWxWN7tkcodZ6FCrrmCtGIhGk63tBmLS4A//6HfY394qb+SSjI8+fBt3fsZv/osn1N5Z4fZKl4ZWZb7IibImw3mGG6bEkx7+3GMxD0gjWF2zIfI4PY3Zu9Zhrd3lyH1GmqdYdhXy0tcqijJiFpchLd0hJSif9FTE0myyMMOdRTybn1Jd04ijjIvRklZmsJybhPMl7a7LZJAyz3pc9M84PzrlLB4iGxpJnKEaFqGUIOel7YOiKPe84cr2kec5muTw13/5FwniJS+O/pDnXxb83P3bBE+HKKpEkaaQVRGECMe4y3e/8wyrHrK6ZTOcuGiyhigJPH76CWkS8PUf+2nOT6aMpwvcRUiRSZiyw96dG/x/fv2f8UHLIV4ugIjvf/GCtabD6koTf6Jx6/7XMYUDFt4Zg94AzTDBSXk9eEl9xWG6GOIvZWZnfd7b+imeH75GN23koornjln0E1baLRaDPjdubBHHLh1rD3VX5Hxwzm73Z3n+6l/Q0Dzudr5KRUu491d/kovDV9z5hZ/hnTt3+P3f1vBVnfnsJWkY8P7uLpmZcnryiN7Q5ue/+dc4638b8WjM+s4N/vTzh3z0wX2WyxFpXEFUDWI5Io48bq/v8PtHPyQIdsiSjA8/+IAvvvgCwxjw8ImPbjVo6gY3fuwuZ/2HaBj0+wPWV66jGbDWWkXQpjx/eYChZNh2jUR8RpHfQNbAX8DG5jqPet+iyGSmY58PPnyf8XjMtd1rDEenDAYelqIyma/hNAPCJKVa6SBpOokX8N47u4iizPe/+2+IfJ00lpjMTtjfvgNixMveAYIgge8jRR6OAe50hVRPWNv3OO/NmXsD4jhHy3WW4iGSqKCbDsVERNV0luEhj5+KXNv/AC+YYzotvv35v8IWb9FYOSeN5whZyunJBU7tBkenn7HWWaPWNPHCgHbXpFbZxXWH1OtdfuNf/TqbO9exrCqqZnByMSH067TWmjSaCp9/8jmrKzXaKzZROiQXMla7dYZDl3fevc7p8eu/KEz8iwPKze4qgZdD7vHuu/uMBkssSyDLLTSxxsHTIz788C1CN2B0NgIlxS72MFoV3nvvFj/4Ycjx+TOaTZtOawt3NuXV8RdEoUjDMQiDEEkW0C0L3w/xpgIwRZEtGhUTp6mzWCyxLIMPvnKbLz57iumAR4ogaNh1MMQtRj2fXv+c3Wsdnn1RHnt1rc/WfosUmckootpUcCenNBs5UQCR7yOjEsUTNM1h6C2J8wJbdMixOJges5d2qOoqwTJgGsyRpYKoENC0knmIvBBRV4mimFzMEYoYS9MgTKk7dRZBGXJwBA1fzskzgThO8cIISc4pximKriNoKYv5HLuxze7OdU4Wp2iWhjtziYIFu9tN2p1V4uU5pmJzcTHAklUyyUYSTZazBWurXeYTAcfSSGJYbV5neNJjfa1Cs71GffYxfhgQu212d9qM3SGD0xMGkylZZhN48ODZEzqVXYbzgAt3jKVrRFHA+toKXpCzvbfPYh7w5acv6Da6CGLK4HxKLgn0ThJ8sQ9eHWUe0OgGmKZDzAh/ucLw9ALJlBEkmU5dJU4nOFZBISSc9YbY9jZ+WE7zFYnE+eAZdWcDd9xGM3JkeYplqKiCxubqdXICbEvh8PCUrb0Od+7eYj6fEiUhumFj6hPOjpY4doX5JEDIJSynjlERYBqiylVG3iGKmVFEDqNRnxeHP0RWEwI/4PQ0YP/WBtcr+6T5lEEvZ317l8m0RxDVmU/7xFmOpoNQjBHSGnWrgyaIDIcHfPjuLvvbGwTJhIppcPHoAiO6SZb6fHC9Qtfp8o/+1Q+xah0Uw6ZSEZDkJVkhUOQ5gqxRb7VZLJcYikaSZ1QaNvOZhyQpuMspjWYTTTfprNWvQF+cZBQFqLJClJQS9ngyAsC2bVRFJ80z0iQlF8oEN5JIFEdIqoKEgqZpV2nuPC9lcUEQLgMpMYpeyuFZnKCrGsulj4CEKElsbDTRdJm19RWEAqIoKutfkkuQlWUkSY4syyhGmcZWFY28KNB0/QrcaYaIqkqousblJzx2YaMoyqV/sSDLqhiX16plmFCQURQ5eRKQ5xmGbeF53tU0ZVmKLpNmJRAOwxBZli/Zzvyyyki/AhRpml7J/m++BEFAN6wSWAoCjmNQUJ6n7y8JglIi73TaGIZBlMQUSUwYxkReznw8I4nHSJKEokpomoLuCNQqFapFeX0o+0ZLYJ+kEWkIi7kHiKRJjiArJPmSLA2p1Z0yKBUXnJ0PcewqkiyzXAbouoZTtQiz0ibx5OlLVFWm015ntd3h2rUNcjwWwxnj6Yz+aIK77JNmL7l/7x6SIGFabS56U1SjhiSkpQUmSPjux58wGE5o7+7QtCweH8/5uf/xL/Cv/8m/Jk8TRDXm7p2v8NmnP2Dg97HtDmtbErWawuMvh2xvb0Oa8OJhn06niyZU6PeXVBwLJBGEBFkUiKOINNXIWJDlARQaui7S7ohocot242229hVev36FpTc4OTtgMY+YLB0OBlNWuqvEeY6XDtHaBY+PnzAfNzE0CwBJUhHzS4kbEPjRzUNRlGyxO/X4kz/4Y07PLtDMgnff2SQ7iSnIiaMUCol37t1HUidM51NUNUAQRGTBRsDj4OiCIFhSq9VxbIl//i/+kH//7/xtnrz4PfJ0lSQJ6Y0nyAdL3rp/g+EgZBzY1GyHneZ17r29wT/4r/4JX//mNT755PcpEpG9Gw5puEZntU0QnbGcvOBipqNWYvbXP6CedgnHLmvVNoeLM7IsQ5IqrG69ha6cM774krWdKo1Nm2qrRj3IEc5W8cIxS3dM90aT42d/gnP2OR9nOZXU5sOf+/dYuGNOzp7S2X6HOFcI0iXfe35ItbGO4ic4tRmfPP993r7xLsniNd/46nU2axqjl19giSlLM2foe1QrGp43J8pVGvUWXtCDXOfg4AHBMmM6KFjfrvDyxWveurvHweFT+sNzxqM5RVJmMUAkjkOWrk+rdp80EXj/K3f4/NG3eH38QxrNCpPhhK3NXcaD8ga5WoHRYEyzY3F0+BTbMTAcyClorCSc9aboSp3B7AFxbHJtZ5skFFAUmYrdplbNGU+GIBSEccir1y/Y7t5Ca6XMvIC63mB1fY/+6Jg8EJi9DMGGTqdDnLoIeRXZCDg7niMWp/jBEj112N3bZL6YMvMHHJ6ecfvOTTa31hn3ZjQ6Cp9/fsrdeyvk6iq729epN0yeP3tFs73J6ckYGbO8ZgtgaE2++nWNi4szHGeX8fIFUTAnDTUMzWTYG/Leu+/T75/QbOggaizcHqpssrbR4Ft//B1arfpfPqBcziPizKVumUiCRV5MyXKJd97+Sb7z/d9mddthtvRwZ2PaayGwgW7nFKnBJw8+xc9DaqtVVFXlqHeGqpgUmkmzVSVPUoQoAEFgOhtgah0MvcCplDLTzB2QzhbousjO1j7+IsQxHXb2WxwcDMlzkZVVG6nQ2dvb4dNPClQtp95UePsnqvzx735BhImmFkRByNbqW5wcD5DUjFrVYzxw2du9zdMXxxRKzsqqRRaLmHGVIHNx0ymBPyfKMwTVwLIV0iymU23gLRf47gxNsHEXU4RcQhFUQi8lzcCpVAmXWZkwT2PqVRtvnqIaZlmeq8pookHoR4xHU6qmzEq9wd7mNvPJAm+2ZG21iSasUO9WqdXAX0Z0V5tcHLnsrN9k6jWYeGOicI6m5NRqNU6HYxZeRLu7gesPseoKXgF/+L2PSYuI7Y1VUq3Ay4cM3ENMw6HWNZhPZeSwQuEvSKIlk1HK/s19VNnh5PUZcZqwtacT+UMuTsf86l/7ab7xtfv8J//J38ewDdqbDi+eTFBXF7TXTQrFBEFFUguIRUbjMzqdLqJoo0gjamvbKJbISe8luhWyItWYzyWidIkoKRh2kywyKXKHi8GYlS6sra/y6skRcRM2u0tWVx3cpcJP/MTXmE7nzBdLXh32EYSC0eIc27apVGucn42oVDPOejGN6g6yliNrKU69jTttMB27WGbBcPQYQU7YXL+B6w24dU8GMaW/eEq1XqOyYrAIZiw9ia31FtPRBMFU8ZYJQTBE9B086RQhLXj/3ZtlwKjikCYNamtbNPQDvv7eR0jzp3z70z/lHz8+5sb7P8Hm9h3mQYAhnmEag/JGJQ6Zjxe8CgJ0s0aeewhiQrPVot1ZYy/dQ9VEwrQMi+h6CRJlsZRdwzBGESV8v3w+Wx0HQRCI45giTZHF8kNTkMvKH4BWq0WSJFcdkmmaI0lcMZSCILBYLK4qfZIkgaJgMpuXoCsTyfMEX44RFZH19Q3aLZvJZEIch2iaxunpOZ7nIwoyQVAWpgtiXnYPZgWybJV1QYFHURRXknGa50iSTCGAHwaXBeY6gpqUx5wWGIZBlkFRiCiKRkGGtwyoVqvEcUxepOU1zfNR1VJqz9Liqu9Q0zQEQSAMwytA6ThlCO2NHP4GZLxhLdM0JbpMrwuCgGlZlwA8Yjqf0R+Wr6epm1QqDrKWUm3ql12hZTBqPncpUOhdTEv2L4rL0u+6w+paE0kuyFMNkRIAe8uAZZjgLnxkWWU29XCn/qWUXuCFC1RZpCgEZKWJZVWp1essPRfh0r0QJxlPn7/i9u3bmLpCUhzSshsI4zE7NypkSYJIzvlRj4MXB9RXdvjgG+9Qs2o8ffyERqdGrb7KrXsSx6fHDIeHeMsev/nr32LwbETX0ZicD9h6611W2l/n0x9+xnTik/gGgwCazSabWzpnp6eoWGyv3eGHT47QNIO8iBFFAUFUiGMBRWogigoCb0B/2WN6eFBuqju1jBevjyAPqbQU3u7e4vDojELMWF27wdKVmC6ec2v3mzx58oT37t/gi3jIZOJhKgpFmpPLIuLlTcObTtY3izmapnF+dM7TL0K6G3Uuzs/Rihq1EOIoRJYligwuzg6ZaiphnjHt5bTXWsznYw5fL6jWW7TXHAI3pWkYDOcf81v/+nep1nTi+AX+QmH7zl2enXyM58e01S6NRoNua5OHjz8hy8OyUN9TseycaBmissX+vsfDp59xNpixW1tFThZIYZv5ywJFF3l48QVpfoHDh6SKQCZM0Fc7rCqv6Q97mBOHhZQieh4HR0OipM37777Fy/4G/XmVxrrI4cEZilKlsOG3//S77FcLPv3WH6LeTmg1bORMRas42C2DW81dHr04YDZNiH2Xm7sGf/KvfxPVyfnhdz7h1t13CIsIyyiY9Ef0xgXbb91me3eVxWJGnqZEvsbZ0SF333qLyMvRDJWnr15gWysoagPBmJIVApVOwWJaMJz38JcF9+6+jyAUHB+f0u8NkVc7nJ17rK6uMRqN6KxU0fRyxc13M+ZDBTFbJ0l6tFYdlosxJ2cFnz/6NvfuvUO7s8V575DpdMrOjkMQzMkzhYV/xnKu8RM/8eM8fvEZS9/nbDDjdtdi0UtpdyROXg2pt1os4yO8VCB1U/K0gapVSROByXyOZbSwrSqkEo2qw/e/9ylr69e5dkfj5dmYV68e0LT2sA0FP1giawUPnw4xNZ1e/5zFrM3mbofjkx5IGZrhkOQTBsMlppagqAWC6HByNMK05yiFjsiI0SCnXusQRz61mkPgpywXOdf373F8fMxwOKRW1xiPvL98QCmqPmISEmYxg+kxyBGNTpPPH36CYeiMhxPShchmd426nZLnGuP5IZPpc5JIR9FE5KRC73hBnC1prypoUoXlNCGNQSJDVBPq9gpRmmBWdARFZLYYUamv4nouQqHz7Mkxhp6TxSoHL3qoukYUWRRFRparDKdzdm9UMOyE05M+3/vBK5Z+SOjJZV3GB9ewnTmb+wnHhx5e4HHn3j556mLXCtLUQ0pFFNnk8OCEWrWFmVss4ojOWpPlzKUICyRD5/xwBEWMbVepVaoEcUTDtBGE8kMtjmNso0JuSCz9Gf4y5ii4oK21mblzxKpWgtL5knpDQdJUwvmYKPK4GJ6xmIZUnAbeMmaS98nYJRVFZvOId969hygMOT07YR4sCIIM3fTZ236Pi4slvdkxuSKxuHDZv97hZf+Ujt5G0RYs3Ixvff8JextVbu+9z9Sf8+rllJt3W8xmIzTNRogM5smYlfVbqGKN737rY8yKRJZbbG9bHD3q81Nf+5Bv/szb/IP/5h9y736bxkqVVy9f8/43bqBpb/HFs4dsr5c72OdnHraqce+tTTx3QTguEIWcKJ/gXmQY1QDfjTBrCokfMZ8lVBsy/VEPVd1iscgQlZxBL0XN6lQrBYriMxpPmU4SLgZjvMCnVl3n2fNjkrSgs9ImzpaIYoXAM6k3BbzlgFu3bjGdLPB9kbrZ5PGTF7Q6HcJpwqg/Q5AFFD3h/OIF7eY+QlEQZwGGrBOMcwxFRjMd2lqd3vARqmQhFBKThYdjt1nvrpOmHtWKxGTis9qp0xsHqEqEnnUwnCr/5nsfMz58jty8xu2f/hm2tqvEkYApibRbTSRRhKKUvR27Sq1mosgmWZ6SFSFZVhbZzmYLFLWgEBVkWcT3fcRLeVYUZcIwxM9zhFxAVSWky1CJqamXgDOkYVlMp1MsqwRxYRxhWCZL1y/9fHoJkizLYjKZUJBdreV4gY+iKBQC2FUb2zAv64lilr5HGIaEQcr5iYRTsTEMDVmyWOtuE8cxk9mCIBozd12SJCGISqZUSJMfXaRkmTQXkCQVipSlF5HkJdiVVZ0gShCKFFGRyfKEhfujTk0AifLYdb0EcEtvgWEYVKtVQs8vk+uSWi7z/BkWtpS7f9St+cbb+OYrSZKyr1JVURTlkvmRrjo0i6K8DjiOcwXGfd8nClNcNyLPwLINNC3DsMoglSDmNJoWgqCwdAPyTMRfelycLymKgjAaYxplh69hmJiOSKtZRZZF8jQhTJcUuYI7j8vgk+chigIXFxdYlst4XDYK1GqVkvnMQyq1CqPZmMloTJIG5BnU62soikIULdA1hf27b0GWk7MkzwTaq1Xs6n1ev3yFKPv0zydI4hmGusKNukGyOGZlv8bezbs09ILv/eG36A1DghR0zeL0+JwPv3ab509GnLxyODodkAdVhoNvEwg1zHaXOIspipw0chFFGUEQKTIIvSXi5aKNXbHorMnIUsJgdsrW1haKIlOIC16+PqFIy+3pyTij1pCoz+s8evB9umsNFEnEnXvoig55gCiqxFlGaQgp13KAsgeLso91Ok/w44RlNMBdGAjygPzMI4kBMgQEnj59xbe9c3SrTS6HjJ8FrG2usoy+QEwWRCcm0+kMzdzDsW9wdjFiPF/SbKwSajM++eLbLBc9ElXE2XAoopz7N27z4Nkn9CfPyeU509E7NJoWjw4/Rpe22b9dZTpccn7cY1V+m/e/eZu//1/9Q37lF1pERkw29anneyyCKVVhlXtbe1wMXpGcP2DF6PD1b36D/+J3vqDThoPHY1Z2dD578hTHuo7SUYnEhOR8AonNeWqhC2P+1e//W/zEZstOMSo6upezLlUJsjp+Ds9ePadTv86ffutPaDoafiwwjX32fvyXeXL8gGg05t6dO3ihw2pznRt7Kzx7dsJoOGU5T/nG1/4K/vWQ2aLH2sYqorpOmhf0zmbce2uTKBtTrRoYtkLvNOba9bucH/eI81dowk2u773HweunVJ016rdXGU4OuOj1mIyX5fu/EEDIefDkYyr2Gnc2d/j82XcQcpt2U2Rra4+Ff8HR8SmdWhNRhNFkzDI4ZuEvqThNNFXn8PQF7sKju7ZJlPsU83XuXHuLcTzn9ls7TM5C9le3ePT8KQ1HpHc6RFEURFFGV1epWI2yhSHPmM4mbG1sY7WOOT0Z07a3yUKZ1eoOteaS3/idf8za+luI6pAij/CDOY26zXxu4iUu/izDdkoM5fs+zXaNpTtjPnfZ377OfKJjqw6bNyNevzpmPDrHcMB1l8hiFcduMBpJSHKBLKmsr3WIu+n/HwClFjGdeKxVa1jVnOkk4uTsOZZZxZsLqEaV7f0qmxsr9I5iXh9/imE0UQ3orm1wen6C68Us3Ig4D8kGU2azGZWahuGY1KjQ6jYYTX1m/QcEocqNG3uEUZXD40Na9VUk1cdfFFi6hah7xLFCrdZlPu0RBU1OTo65dWcTRVVYzGLu3nyHzY0bHPe+YOFNMFSbjb2Us5OUk5MxtbYFWY3z0zHJNMHZyKhUWpy9cJH0mOpKhmkmKEUVYpFFFKDKKuPxGCUVycIC27ZQFZ0480HICIOAmlMhSTJ0WcZxLMI4Y7ZICeOCXMlg3kcxVSazOWGcoSCT5SmK0cDUVRKWuJFHJonEWYobFMjGkidPP+ed++9g2zYff/cRd263kdWIvc1VXj7qs71znRevTumfj9l/a4M4XCIoAS9e9/HCJZu31pgMfFRJIF3kONo+5xdjLs77CJnJqvMWJ/F3CIIlsadw+537TOOQ3uiC3esSpr2GUJgcHT+m097l5fMz/vhb36MQJVa3GyzTOXu3OiTFHDHUMQ0Bf9Fn0mvTWNXZ6OqMZ0Peeusei/MQL8wIs4yVboWzuYtl1PDccodb1QzmC59CkPFmLpJUruXIospZ/xV5lGM7ErIyploRqNYajKdHxImFogpUGiqz2QjDtgiCGUYlJk1ybEVBtQWKRYFdFwnSC1TVZDweIxYCnW6Dx0++ZHW1zWTmEixPuXfvBs+e9zh5OePm3hrZXGQW9omK11TqqyimzvlJn+3tJp1Oi/kiwl0UBMGSZn2NJAtprq5h1QxMu8pabQN3KyH9sTsIkkI+ixmOFmV6WhLKPAAFgigg5AKSqqJIAmHkXgIWEVXVyTXQDZU8T8kF8XLCML4ENBGCIFCrVa7+1LMsu/I/vgFGQRDgui6tVqdcnwnKsvEsja/YtzfeyuFweLW//ebn37B5y4VbAqbIR1IExFQsF29klfFkge+5KIqEJAsEgXcF7sKwZPx0U0URBfK8IC8Kokt28M3xvvlekiRkTcNQDHzfR5IFVM1ARsf3fRzbIU3TEoRGIZIgkmcZcZSSpmnph0PCWwZEYYKulrVHsqmWHs28ZEnfnPsbxjFJyg7KP+uhfMOclkAUQLysXioIwxhRFVkGAaPpFE3TrthI0zSxJb0cgBBTLoZjdNcsvVeUCXoASZLoNBuYHR1VK72mcRYTBzFCLnDRmyLKKkHgkUQe1YpFpVlOQ5bduzmqVlx1cIZhjDspU79HRyeYRrV8fpULas0akqiQxR6iKBNFfURBRtVkwjAhyxakaYxQSJz2Jnz740/RVIc8EXn3nRUkXeLz77nc++Aulqnj9h063Rbnwx5aZRPVbnDn2ho//82vM3V7vHp9QZwk/PV/5zavXlzQWfkGViXF1Gy+/dk5oRAhCSJZrKKpKmmakyYpll2QZS4gQm7S6TRY24o5fj3AVnao19qcnr9ivDjFMbdLe4AYoMk12rUNPvuTl9x//yPiwmc8nRInHuQ6GTIRARIqgiCSU/AmgvXG6pDnOZmWIRkKk5mIVvG46MdYoY8sV65Y7cE8prK+xcuXz2muWUhyjW996wVb11oImYq7WFJkOU++PEUUNDqrCae9Pi+fR2zcThmcj1GKCqKWMV4s0OQGzw6OqGot5q5PpdLg+YuX3BR2ubn3Tbywz/FBwe72HdYqq0i1hKPzE1Z3DL5z8M8psoIN4xbblS0e+T/g+NkZK84qtqPhJR+xu1fhN/7Ypd3eIlcDxEaCIZsUy4B+kBOmCw4ePmantcPGtdtkfYlb7+zRdzPe33gLy/F5+sND4obK5PlrpMRnsWjjRzkTt2TbLatgkSdY0iqVik63uUutadAfXeC0Wki2yCcfv8IPfarOGo06XAzP8TyBdmuXLA+ZzhcspiH1WhOBhA/e/kVevjggWCxp1E1Cb0m91qbVkPj428+xLIOtrT1OTh6jqA7Vuk6Rq1QrbU7OnqJrDlFU8PO/8Issl0uS0KHT2kZSDJKgBFF5usLPffPniIOHPHnyjJwmo3GAU5WoNaokkYJpCTSab/PwwTPW1+sczF+jWlW6q1uMXo8Y98eY5grRNObmOx8hCN/n4efP6KzssL16jbyIkQSBLE+gEDB0Cc/Tyvoyt8L13T3qtZTQz7h//edJyfEShRQPUZaZzKa8ePWY9z/8Kp7Sp9e7wKkKGE7BeOqCWLYgaJpOq9LGWwSMzhfsb75LwDnPDr6kUq0jCwbdbpvFFNor65ycvoDUob1i/eUDStPocP16kzQXePHigorlQC6w0mhjdCLcqUfd3KZ/3mMZzLEMm2rDJ441eqcHHL4coCk6tlEg0UJWLG6/v810dk6zYbLVvsXSj+mNj/C8FNPQCH0f05TZ2KjQbtQZDAY0WrB/rcPnn35BtbLK9k6LwHc5P+5z/XYXTQd/adFuWTx79qB8g8QrVE0RTY05eD4jSiUqzjpnRwN291RqFZEglpFEhTiLEOTyqVlf6/Dq9QmS1sJQEwxVRi4ULMem0azjLTyGiyGCKKKKDppqkIfxpXQm4zhVxuMhQZTheQGO3cBLYxbplI5dwQxgsphTtatImcDSdwnmEwolodtpsIwWyJqAJCq4IwcxjxgP5ii6wGLu0TuXeProgvo4JwzGPH3mUjEtbt7dQFY1ijzBki2Go4w7azf53g8e0uyaNOWc3VWZ8+EjLGeTWqWL0ZB5cdSnYteZjqfs31jj5csnyFqBpdeJkkYZFvDOiUONSE1ZuDNaa2v0R+ekUkQUVpjPxuh6A9d7iiDljJcBmmlydhyTRDmSnPHpD44RxQmb2x2KRfnH7QZL6vU6sqwQJwVIBZpqEwY5Wb6kQMTQuwTLKf3TM1r1dbwwwlI0njw7Yf96BdPYQlNtcjwqVYNmq8L54AhNq6BrCl4xIQzmHB9rTCcLNjY2MJUOcTpmPJ/TrDUwdQMKEYEOgSvQ3enyxZeP6Da6/Pzf/Wu44z5FYiBKPsNpn1xQmLshaxt1cgQQBRTFYf96gzgKWF/vsrrSoep0QEoQRYFZWpDlAf4oInbnxGmCrpvUmiZxktHINRRVAQQKtWDv2gZqTUdWFBaLJVGYEyWlpy5JU/JcJb0MEEhSGQiTZZVCKBPOaZoS+hGmaZJmGWmekfiln1BVy8nD5dIvAzGqynQ2u5Jh34CpN75JXdfL7ke11EyzLCuZeKeUk2VRLL1kokSz1SD0QqIootEqvZ1ZlmBaFopaAiS7YiGKInFarugYqkyWFUiigaZpV7K7JBVkWUGcpZc+RhlRVC9ZXB1J1FCyAkWRr8CgWJQ+xyQrl7uCoJS935yTLJTgst3sEF+eoyAIV+etqupVOKdc3fnRlOQboCEIwmUNUo6q6peF72rpQZXLx6s4NWS5PK8kTplFy9I3qWgoskKlmiEIYBgqcSRdrQulacpkPiurjdIU3VDRNRtVUrBrGvV2hU5n9eo4z45PGI6nhGFEll+GnPIM2ypASLh1exfyBISUWq3CRW/CxUW/DF1FEV4QEcYpipQiK1BkZcJflmUW8wnVmoMgS4S+hGE3EASZNM34/qefk6cZ3a2bzHrPOZcsJrMhg6WHgMxodIBmdRj0h/zX/+/fwXJErt9cZ7qI+N3f/YRGtYZt6bTNVtmDGUJhCCRhgqboJLkHFKiaSJ7kRCFoskWcJowGY/IkRxXaREXGlw9f4NgN5tNDZjxGzqucHngoist08ArZlulPnhMmMyyzQaPV5HCeI4o5qqyTpyAiXAVyiqIgKwokQBQhiVLSpGShqw3Q1S7XJQX1ZEieF6i6xHB2Sr5xg+v3NlksckRpQaUakAdrOJbIzO+hyDoVy6HdKcNYDWedSTrAO5Mhr5IbNSqqyNPPDtjZKfj1p/+IPNH5+kfvcn52yupGHeQhjn2D1EiJ4yXd7j65WuUHL7/FcDgkXmq889Ff47MffEra2WYsBmytfQVTe8Xx5HN+9vYv8CR/zPcfFdS2TFIvY73jMD2eMxMilm6f2XSE1rnH2sYdZpOUeljw7OQ533nwkL/9d36e4cURD5+fceujXcJJykkuY9c14nmOU9d59uK7bG++yySAwWjM/es7qELExeFL0mKVzuoKzx68YqVWK9mxqUB/cMH2+nvs36ijOxoPPz1ma19hMctQFZ0bN/c5PnlJkU6o1Rp8609+SLtVsvs5BQvX4md+7iMmoxmuN6NSqxGGBYu5wI3r28TZkodPZ5hGhyyf0B8uWFtb4cEXB3TW2ghSDRKBXu8zdK1F297k6cFr7lzf5HsPnpCnJp1Vg2Fvxv7ufabzM1zvmM3NJrJiIQRzpMBn+eoZO7WMb/ziNr/+mw9Z7V7j9OQZ7UaL3W0BQZDRzQKn4vD0wZilG7G9UydOBDS5QyF4tLY32dp36J8v8MOI1W2b4fScRU/mxp0bnJ+4nPafkucap0fH+J6LZYuIYhVvGREXp6RpjpI1II9wTB1BFbHVOo8ef8bdd29z6/qHLL0pSZRwfnaBNzdZXSuHTmRmLOc/GnP4SwOUYeQhSimKWkFTCpJsytbGDrIcI4s6o2TMxXCCopgo5pxskSMrJmEYIsgRm3vbRMuA1dUajeoehmVy3jvAGxmkywg16xHmYyRBoN2s06zt41RFNCNnvvQR8gwpryCIE4ajC+7cfQddqbC3t8fzpyesbxpM+hn94pxWR2c+A13ROTvo0263KeQWC/cZSagSZVOc+gLbDxmex6w2a2QrAqEv4U9m1OsSelvk7NRj0lPZ3IoxHIVJf8ZW+yaFolCkCaap0q1vsDhfsEh8cgSSVMSPU4q0QBB9nIpJkSY4hslsEWDZBlnVQrdVaprFbNQjISV0CyRBoWJbeLHPdOximyrLzEOVdRTLQjIkjgcHtDs7eLnHi8MLclXgbHRKQ7fo9QZ4tkFoB0TxgmubNygSmyR+Rc/t8dbbbU4PfQJRQdFtRuMzjnrP2Flrs7LRoHc84sfe2+MH3444GX9Jfb1DEed4oYhhGihqRJUGs+FrpPUFDXMVVTJZeimt2gqhGzELZyS+h1pvEnh9gkJAkjMcR2KySNhe3yCKXbxsSSddwQ+HoNRgJiBkNZazEYKso4odKg2dpVeGSCS1SiYEeFGfzb0VFAyiSCaMJFJxRhg2WF9vI8kZ3/3ea1zPpN6sU6vVMC2JJ58f0mne4PYHb/Hy5VM2dxxW21UkmpwNelgm1Bs2z54c026tlezQqsXZ8AFVq40mGNTqFsOxQCSGdLsd1hsdPvvkO7RrW5i1fabTCbZt02w5DC+mVKttZtOA6fCMentBlkpoQkSuKliKQiooiLKIZtVQ0hRByLArDqk7oqBAFEASRZpVjdq92wShi+/V8byc+dzF9RdXBbpJHCIIEllaEMU+kqSUbF+WEMcxpm4hiBJpFKFpGkuvBCm1arVMYbtu+e9arQSeaXrFqP3ZPkr5clIxL9JSdr9k/SgAROIkR1NMRFFgOp3jeS6Vio2mGWRxQlGoV9cUQ5fIsgzDMBASjywtSNIEWRSRpIIwXJYgLk3JBZkoji59m9rl74U0LWuPJClGVgQWyynqJSjN0wxLtzBtC43SH/kmla7JpfQfRxHCJTh8E8Ypa5VCoJS8TdPEMk2iKPpzoRxVVfE870oa9zwP0zRZLn0UXUHOZdI8wzBKhq2UbbNL6XuJpml4Xoaum4BEFMhkeUAQJLRaLQDm82XpJ8wDslRgHk7RFRV3sSBNcp4+OUdUZOpVh1qtUi4d5QVLLwAkdM0kS2Ey9jh8/f2SrTRE1lZXUBSFnZ0dmk0TwyiXqERB57x3ymg4YdCfEgZF6bdFYjSdUbUcJLEgD2UqVZm9rSbLpUWj3iYIPHJ2qAsaG+01bMMizFOWgYcmalQzGz8ob2J++MNj8iLh3t23yfOUpTtmNlVQNANRMEgvPYtJlIAEimKQJgWFuEQQCuIkQJIEFFlAl0wqRpPrN2/w4Aff48X8CZJWYRoekykDpHyPKNG5e00hOh0h5Ta7q22QfQ6kMXEhYkkSqZeDJpIVOUJRIFxaJvK8DI6pqkYyX1BTt/nZX9rnn/7j36TW8ihSDUGg7NFNCtr1Xc7TpLxxC5cIKGytW8Thgsgv+Mq7dxgN5xiaiTs/ZzhyyXKVRmMN01IQxkNEcYgsNEFpI0s2WSoBIl989pThcMjN6zIzAyo3Zd5//yO++8Pf49XBkP2NPT546x0+f/KA6VRgMrzgV//q/5Ab9yX+9be+y/DgANk0ESX44sEDTFuiXteYD4/5wZ/O+bt/71cYDh6yVtFxXZeV9ZsURcZsvuTGrXXOz3rc6lzD2+nz8Z98hw/u3OfWL7/P9z/+PbKkwQf3PqQ/eY6fxnz9x3+OTqOOo3YYDHvsrHaoVTQODl3uf3Cfz7845P67H/DJF9/jdNBh18w5OjgnK2KevvqU7Ws/x3gy4+s/s83rl302t6ukUZU0yykKC8VcMJ+7OHUFLyoXye7cuUd/cIzvKWzurDBbTkmTgu3dLQ6OXzGfl4SHZdZZhlPMioamVBlNz5nOx3Tam3RWmnzxg0cEfsHP//xt/rP/83/E/+gX/y+011/xu996yY//+FscHH+CzAruYoxtaQyHPUgV1tcttjjl/Okrfuyn73PrG0t+418+oqrcJU4vmPULGmGH7uoqS3+K687o9yM6nQ6L2Zw8txgvnuPYDfwlaLrLdKZyeDpCN1Qm3pjz3jmWscp07CPoE9I0ZXOryXBwTrO+juNozOYX5U2zUWAYDramkYYpTq3BxfExOxub7CgyT54/pt7osraxQhLCRX/E/o02i8US3/dZ6bTQrNlfGFD+hXsoP/jZm4UmTPERaDWa6BKg5DScDZbeBc1GG0XLOX69pCAmiCZYlk2t2uH1ixGba02iKCCMMkQV3MBHFjPWO5tcu7bBwycv2VqpkOVrPO89ZmPV4uJ4hmDl3H/rDi+fL5CSGQ+OX7DVENlu36C3jFmp1kjjhFiNWM4lVNWgXjWZzWZEoU+cuKiqjunU0LWCwIO+O2DkXxAELnZeZb1hcdD32N26yY3dfc56r8jVHtVKnf6rClHymvNwjpJp3Fzb5fj1mKJI6IcHJHaFalbF649pdqsIosxoWt45BW7IxsYGCBZzd0EYhkTJjHp1DbNScHYyR9YkwniMqrgYlX1iV8IxcqIiQzIr+OGYNJMIllMETUJHI1rMKAwTN0gw1II4C+lUqtTtDQ7PjpDEENuuMQlCwjRip7aCYRX0R6dYlRYSEbamcnQypdK22eg2CP0mvh8ShDPixKXT7BJ4IUkYsL2xz3A+xI9UUGOG5yP213ZYLoZUnAZL3yOKZ7QabeZuzMgfIcoiqqQgiwmCoqNqBS1plcPnR+ze2+D45IKmUwLogTvBkBxs22Y4HJImEoJcIGhjjIrG1vq7vH56wDwOCJKAdtUgXZiomkEsnKAJG8hKRt02UIoaXjpEtQqKJKJWt3GDC+7feAsvELDrVR49+JT7d+/x8ukRng+mUadIXfz5iNbaDY4nE0Qh4vDgKU6jxkrrJtvNLeaTc5azCR99+DWm0yknvXM6K+u02hv0Z4cYhoEsmDh6GwqRIFzS6/Vw7BqiqDIaTiky0DQFWZVIiwxV10mygjwqME0dSRBYH7r86u98DwQQBJH/9iffpdepIikiSVL6B4uilKJlZPxgiWaZV4GRKIpwHOcKAFUqlfLGTihXjeIwKi82skwQBCiKciVhZ1mGZpRAqQRDRgkuJZkgKmXoOI7JkvQKhAGol7vgaVr2JcqqgqHpZdhGEAij5RUjWhTFlS8xy/NS6smFqw5MVf7zTGBRFFc1QYryo6T1m8d+43N8wxTmeVqu64SlH1MRJaK0DO1ADnkpiQtIqKpadltG0VWXpCBLiAVXQFqRZLIiJUkSKo5zdVx5npdg9hJ0e2EAiGhaefxvWE1RFK8WibI4uTz2SwB7yai+OV8hL0q7jFVWOcVxjGmaVByHxWJBliVXifOytichixMksey9LBCRJIG8SJGEUoLP8xzHsUoWVNNYzF3SFIpcwvNKv2uWpVSrDo16lc2tDp1ViyCYU6vViJOCOMqYz1165yPCIKV/MSGOyjnNggzLMpCkkpEtE+sqlq2hqjLa5SqaKMhIokiWxXh+uaC0urLJcukzmYwoxJTYFzkbZqRyQSbmZGGOJOYUuUBOgZAHnLx6AImAZav8b/7jf5+jg1c4Sg058al1ZHoXS/7gD7/k/r33+J3f/Wesrq+w9DOCJKbTMqk1ZCJPwjJ1BrOQg36IYSoUoUYqR2WgLStvEuI8JvI9yAt0WSPyBD782g7ucsLmjoNiJDSOUv6jL8t50bzI+c2/ucGnxZzeaVkAv7XdJc3LKdiW08aowXwmIhZLjp8nbGysoVoRw9mcNBKJYg/TNDD1FpIY4ocpk8kEVZNptzexjJzRRUqraxEuM/57f+Nv8O3v/Q6ZqGOJDrpVcD56QRwZBJHL13/sJ1kGz3CsFY5Ol8TZgIKM+UDkva/cJY6WiFJG4Odl6XueI+khg0HI/o1NHn1+ympH5+5bTU5fL1nrbrK55fDll1/izxvcfecemTDh449f8s5P3uLkxSta1Q6vDo/54Kv3UNIKjx5+zty7wDHWKPIehrTDnTtf4dNHn7G2vcLB4acspwL7OzfZ3OoyGo2o1qvIqkEUZjSbTZL0jP6Byva+zunJAWHksrLW5MWTGd31CstFwObuHmdnZ6hCBb0Sc3LmEkQu3c4O7aaMOw358Z/6Br/12/+EMFqiKBrtxh6VesHzpycEfo5uKNiGyXj8AjFX+M53vsff+zv/a+7d/og/+M4/p1JZ8ujhl6xt3GC+VGm0bT75+Et+6a/8HL40YfTkmFsNl7E/5xs/UePjPxrh1T/E7sDZ2ZDEr/DOu3fIOOEH37mg1tTZ2NrEXY6IAonprFw30g2J+XyKobWwKhmTYUGShOi6j6F2cdo65xfHZJFOo67xxYOnrKzVSd093vrA4uzkNf3BCe3GdWrVFotlRBotsK0mpglFvsRdRGiGjWFbJDFYWsYsnrIMJoz7IZWqxnwR8X/93//xX24P5dId0ei0SNIJi9mSCBGzLhDrYyQknr94wtrqNZJsQSEmZKnJRb/PaFxQqdcYBGP2NhqkS4mLs4hOd0mYu7jxS+JEw9IUXh0eIGsXbG9L6AoEsU8SKvSPbUwrYH4x4cb+Jv5kzoujZzQ6m2hFwnQ+ZpSMqNVqCEWLo6MDdFNgMYvodptUnQ6vjgfEl/uvy3SKqjqkQkYcJhSZialI9E6OmPRGVOqAHOJ7PVbXG2jK24w++zaWlVGIMZKlk+cpqlh+mMeJiLXiUO9UmY9DVut1ZFGm0ukwmJ5iVjr0Ly6oNco3ieufEBcatZaAH+QsRj71bQXkJcPTPs1r11m/ts93PvsBll7gWHXmk5i1xgZxlBObCePRiLbTQcozKlUHGZuxf45mZNham9FshqgKVHWZIJ5jGQ7LoCBJF5iCRGWzgulINIwavYsZEh4UEu5yhiYrTMYzHMtBMSV6F2dIioSQjJGUhHZHZegd0+g0ifMUScsxPQsvnDJcnKNX60ynEd22AyJM5xFOLCJWdbZ22lScNkJxgaapuPMYTatz/GqEbJ7T3aiTxBpFZmI5dYrc4+zVBaIhI2SgFCZ+DChzdFXi4iBk72ZpKo4jF1MJiZigZjKWapOFMlLRYjQWuJi8hjOdNDAZj30Mu4mlW6zUmpwPjnClhOOjU4IkQzQEHGMNaakTGiOOXI9Wt4Mo1vhHf/JHGJaJmKa889EHfPyn36G1IVJXb9Nt7iGIKb3zU2yzxQfvfxVNkxmM+iDEmFoVbxmWMupySZgEKIqGICSEfkohSAzHE/Iip1TdMsJ0TpIZzH3/CvS9AWepkKJbOlzOI75h1MIwpFKpkCTJJWAo+yQVpexLNS29lGOr9p8rLI+iCFGSrqRt3/fLyiHTQhJEwrCcWNQt6yp5bRgGURjiBwGqql6B0DfF3Z4XlKxhmhKkUVnQrWlIooiiQJrJRJeAUZblsuYoihCE8t8IAqquXXkp31QFKZcAMAhj0iS++r2ybBHH8eVEIwRxdCXhF0WBaljEQYgglFaUN6BPEARERb5Kbydh+XOarpMk5ffZJYCGN7VBRin7ZyWIVRSNNI0pioJGo4HneVcAWFEUsji5AppvzkcUxavQjqqJJetZRMRRSrPZZLFYMhx75GnJZhZFcSXbR1GEkBfIsnJZgp5gmDaybKKrOmlappSXyyVB4BG4Cc1WFUFMsasaa5sNLMsiu9yrvugN+OyzL4jCnMnYw7IsVFWivWKysdVmZbXB+voKmi4zHA4ZjxYMBwsG/SmuG5LlMVkKgqDQbq0AApLiX4JOjTwtWHpznIqBIms8ePCYJC6wbZOsyPDjEEE2yYhI0wxBLF9/xPLGIfY8yETyTEAQDD7++DGapvH45CWZO0M2VHRD5pt/9Wusrzf56tf+Fxy9PuLLBy+4efMOop7w+ugFD14f07x5p/y7kUXidImqgCKZpHGEKstkeUyepIiifGkViImFgqBY4rQKJE1i7o7oKDqQoSgygiAzn06YxDG1iolddYhTj5evehSFwEpd5ORFRpoGmLbCnXctJqMxi0FBwgDb3mLVqjIeLdFU0DSLKFzSaufoeotOq4MsT8higUYHPnn5nIcPH5NmBl4w4XD0DKcqs762R7O+xVn/Jd///HskechKt3y873zrC+rVOpba4uOPn7PRXaOxkjGZDQh8nWp7zuCsyu7uBhenMxaLc+7c/Bqjc4X5NMddPGW5uMlHH/403//kW5ydvybJXVbXHLQ0xhANZAS0wufjP3pGta7TaNQJ/AmS0mBzfYff/ud/SLW6iyhILJc+oaejaQLj2SFhGLKYRzSWIY0VjePXPjd+ucvnn5/x4nDAtVs/z+npOZ32GoEnoNsSk3EZRqw4myzsJf3zC9KpglNr0GzrzMZnVJ1NzvrH/Lf/za9z89Ye0/k5aWQgywqz6Yzu2gqhP2c2FtFUmTxZ5cOP7qFXYgazUwrnId/59vf56KOv8c57f51UnCEMdMRgwk/d3eH+6ojY3uO3DlxO8gZuNuT//g9mbG7vMHFPqC6ugXiNIvVoNKv86Z98SWeli24W9M4H1OoWVtPANBpMp1M8/xDPd8kSnSxp0+4UeJ4EmYEgxqhKE1msoagxk9Ec27YR8wo3b9sImYSpddneaLHW3eDl4fcgN8gKkQyPhw/7qGrM9Rt7FIWBLBkkgkDIMQcHA/b2VpDrPu3WFiNl8heFiX9xhvKn/t31whQbnAwWGFaGXMTkRZVWJ0PMK4x6E4oiI04ETNMEUaLWqNFs1uldHNPYaRAO5hixRqvRZBZPGPvnpLHH9e09yBTcKCYOBfxlj+39JotZWXFyfHZIIafc7L4Hho+pKhy8PqFRrdFUwS8qvDx/SRKp2IaJqi/wlilioaJpAZq0xnCZEMYTciFFNURkqc5sMscxTAxdotVeZWN1hccPXmJWY7wgJ04jEuY06m1so8JgMGA8mBImIqKgE+Zz4sJnRd9EraoMh0M6LZv5IEdXVMxqzsyLCdIY/IJcSKlUO2RFRlYsyVON6XSIIpuIFZEsndNQGyiKgy+luK5Lu+KQxCIePlIuQl5BUBbYikruqQTFAlSVyXDBPFpSMxpsrqzz9Ohh2bWHhu3oEPmEWYaoiMiJTL1uYNU15m5B6M3xfRenZpImoCoWmqIzn8xp1JsEvkeYF8hyhKWJTCcesqFgGlVMpYmmZZwNzjFUCz9akko5cRpTtQ0kUcf1UrrNFoVbUGtkFHKN8eICIQnxlikLv2B9c4MkDcqd0qpDxW6T5zAan5cycA5pHLD0cgpJJg17rFRWkaUmseoynI6paQ6bnesMpkc06l0GZz1ajSozd4Gf+ei6QpH6tBorzGYhjXYDTZEYHA2RahrzcEERp8h5jUqjwnB8QNVqsHB9LFGj1mhxeHYEEsiaShzEtKt12tUqrbUGYRhiqAaG6rC/v0uSxgzOF1ycBci6gue51Ot1VEViOh2zsbbJqD9i2B8jSSYpMZ99+Rl/6977/A/++DGCKEAB/4+vX+e5Vfr13gC9MnXtEPplkERR9avt7TesYZYVl0Xj0hWr9cYbKCtiyezH8aU3sLgsFS8XaWy77HkMgjeAqAz9vOlxrFarV5UqSZKQXjJsb8BuUZRSaVn4DaoqIwjSFTP6Bkj5/vISSMlX7KYoiohIpJfJ6jchojSLrxZ63lQfaapR1rpQViX9WabxTXdgkiRIkoKqypfsaHC1TR7H6RXT+Ob838j6mqahyQpBEJAWb+TwHzGnpQz6o3R4kqWQv2FPfwT4NM24Sn0Lb3oOiwJBlq6qiizLIgxDFFUvXx+13DnXNO3qMaMwJE3jshoqTalUKjiOQ56nBEsPz/MIA4Ew9KjWLBRFola16ay0aLWr+P4SzwuYjOeMhnO8ZUKRi4RRgKqK1OoVKrZBrWaj6TJpnLBcenjLiMlkjrcMGI1mJHGGbqh0u232rnVxKuaVv1RRFKaTOYPBGG8Z4y48wjAmTku2XJVsVE3CNMvnrdlsl8X2RcbO3jafPn3G0leIiwRZUiEXSLOYIpdQNJlwesHpqyfIgkaz3eDr37iHH3rs79+gbuuMJiGCmCHJOTXb4vyoT9Vy6KzUeP7iCWnsU4gFWaJj2Arf+u4XnAxyjIpKGiaIkoJADlmOIECchORpglAUlzdUMdu7De6902U0mlBrCHQvQv7nX+RQSOS5wH/+tk5/RUeQYgpiJsMY22kwW1yQRAnN+ga1hkoYB+TiElmpk6chzXqNs94ZO5vvsL5eJ4o9Xr04p9aEp49nrHabLLzX1MwtKnWT8zOX9ZV9losLksIjyCNa7RVUWcFdhEzGHpKsYrcynr8+YHVjBy1dcHBwwLt3PoJcYm11ndnIo70qcfRyRihE5MKEJGpTb+qEUUqv/xpFtgiiiK9+eI+j133e/8o9iiyh013l+PQJYRQRRQYfvLfJ937wEMswSb2CIFpQ664y6rnsbbU56A/wpgn3b93B1m5wNvmCs9FTbH2f3RsaB89OCL2clZUukpzh+kM6K02iQGE4fM2773yEtwigkOj1+jQ6DqqecHww49qNdeJszvGLDKuSYNgZF+cJq509BqPHLBYzbt66SxAukUSVIJpRZDr3797j4YPnNBs67tyjVb9GlEyxaz6mss/F6DFZ6vCrf/N9/sk//Ge4E4edvV26Wxq9sz5763v89Ic/Thj8Jgcv+hS1f48Xr5/w+MW3cIx1pCKnP51SEUMq3W001UBSQhy7wWR+QL8XUCBQqddQhLLmrFKxePr0BSudLmEywLG6bG1tMZocshxCZ9Xh6OIQWRNRpJz+2RKntk6r5qDqMY8fvaLWFAn8FENdoZDPmEyXVKpdlosMQzWRlDEnR1NWO3eprwg8f3GE4xhUKg1UQULIZyiSShD5/C//p//kL5ehbK6a1OQax6MhmtqiYiqgSLiLIVHQQxRVkjjl1t1NsrTBZHFCHCccHfeRNYvxcMT8fMF6ZZWT8wOCVKTQYnRTJYptWk2B4aiHbplY0m1keUKr4zA4P+be7T3cRYggppydXqCrARvb14lmCr3zp6zuNohSG7tecHw8QtM0TDtEU00k6xZB0qMQXQTVQxFrFDksvSFV2yKNJaZugKguGA6HFLFA3UwQZYGFl7O5scfgIqewNGqNNtNBSsWEJM1RhDXizOewd8y6sEJ3bYXe8IJcNUAryOPSoxadX6DpFeI8KmU7WaHfC6m3gEKj0bCZJhGmbaCoCjXbZnZ4yG6zy+p2l289+pS6UX5YuolLt9EiCz0iY06j0+DV58ckhUijYVC3TabzERW7gh8tsXQVPwoRIwlFE8hIGc+XOFaF09MZhazSsCokkc2oP6DZbKLIFr2Lc3RV56w3xrE0JvGAqmWji5cAvNEk9jPESGZwNkdv1MkjhVl/SqFEOFWFmrzBoOdjN8eMZ2McTWbiiYyX5d2YKJpk4QyrVkXIUjxvTCZGTCOFwfyYiqWxvbOBKig8ODxgf3eD5esLQGI4iZCFBbWazGLhU2tUmZwNMVSHLI/p9/tIUkF/3EPXa/gTj+mRwMaag1jkiJrB8+MjdLugudZlPJ6QZzKirFGt6CwGLqJcYZEvqFfWMDQFKVe4s3UTy5S4GPZRqgaT6Zxe0qc3HLHabeOJEZYeM/lkynp3lUdPH1OvNdGdNqP5jGgYUK020TWbj7/7PUxVoV6r4rke9YbA3/k7v4T8+BxBLMizDEEUyyCL6VyCnVKOlmWFxWx+WcQN0+n0Sk4uCqH0JQoFaQqGYSFJwlX6OYoifK8cEhCkctqPokBRJKIooFarlP61MKJZq18BNEUpJxgXiwUUEKcpmmqUIZxLkJTn+RXge5PkfgMUBaFcfSkBXplofsOOluckX/2sqmqkvn+1VPNGvjcN+wocG7p1eb4lGFZUCUVTr2p7giBAkERMzSJLC0TxMowjq5imxWw2w7Ksq8L1N4xrKa2XKzfzpXvFnP7ZhDeUBchZkUMhgFAyp7qqlZ7DosA0TVzXQ9O4Kkwv0uwSeGmIinz5/JRAWdM0JFkgigPiJCxT8EHZAScJIrZtkudlmt0wDFzXRRRFDEOj2+2iaRpzd8BsVnZ7+n7AcuHx+tUZiqKh6yqapmBZFpJo0Gg4TCYTdF0mL2IGgx7HhzKSUCBJIgIplq3hOCY3b+1gOwaWZZHnKbPZhLPTIa9ennB60iNNU1RNBjJ29zZxKib719vY9i62XWHQn6CqGoNhj+FgwkVviO9lvHxxgmHoiHLMcHLGUijZVTErynUmWS7fl3GOIGcEwRRF5srK0V3dIswSRlMf3y0BsqorqLrAk+eH/Na//COWrs/1GzvcuHETW5LoDc65ceMGab6k3jQ4HswQCxtREMjyAEUWEQURSVSJ4oAsT1AllTQWECWJ2Szmiy9eUOQx58cajt5AJCNOM2RZYDGfs7Rj5jMXQUrxfdi7uUtzXeL0lYuqSST5iKcvDtjZv8Vp/4S19hq9swsWiyXpSs5yLvLZF09Q9Yhme49re3XibEJLW6GIy97iQgjY273FaOxzeh5iaQ1ERJ49e053rYOiKHztaz/Fx1/8FptrLcLApT9dUG926I9cSG0azZip1+PoiyGLWcb6ZgeSVTS9vMkaDYdkeYiiKliGysMvj8lyn35/wcnpaxpnz5hMJshKlUZtnT/+t88pxIDXB49oVW+ws1/l1cEJm933aHctHr14cjnlGrJcBLjuBWLRodqUOTg6wrZWsKwpg8EFViVnvlgwGve5vvs2N/ZvMp3PkKUUd5LxwUfvcXp2TBRnnJ6OgAKrliFrAoKg8+LJhJ/56V/AD6YUrCKmdYZ9H91WaK1UWJycQSZwcnLCee8YKb9Lpanghkc0mjVqtTXa7QqDGQz6Q06OA67f2OJ3/+Vr3n3/Fg+/eElGQBpr/NRPNtGNX6Ky+dv81//s/8je2q/wjZ/+Rf74u3+IGjs0VjRmswK/N8KuhZBrmEaVV68Osc1d3vvKJuc9l5rd5OTsOQUqneY+ghBg6haaFvPdb/8plbrN+so6n3/5kLW9On/y3W+zs36TNItxx6e063dIY51Gs0Je+Ig0qdRsBiOd0PeIYg9YUHFWyBIdUZARlSVhHGHaGdP5GYPhGXVzg81Ng9g3UDSJv+jXX5ih/A/+V3+jIPFI9Rm+N0Wjil6tcfyqT6Ui8+5X3uYPfvdj7JpGFOY02ibHxxMypuhmldj1aTUqKLJMHLqsbdoMhi52TUYSNRxT4stXp1zbbyAuWsh6QhRI1CoGx68CNler1Lod0iLEm86ZuSH98Wta2jpZZmI055weTxn0I6r1GpqdYNnVkqJ3XYK0z/pui5qzReglTIdjbNvioj/Hi4Z0ujV0TSP1K6y2HSpVm+PTJ9RbFnFSRZZUXj17yr27H/D8xUP8KCNLDHQd4tRHkzOWqUBveoFe0VEFhf2NPRb9BUIeESYi1bqC76c0uim9XkC9ukqW+qRpiCGLZEbG3MtY29jBny243tnn80ffZyyOWTG2mPQzFMckVVJmkx7drTX8aUZ0MUBqSKhClSQtsCsW7myCqon4aUSeKYSjELWWYygO6TKl0m4ycxdEgYsoSdStDpLhM5mMoVDorjTJstI3lcQhZqVNEESlrBj6NCpVqhWT494hQZYTLkVsPaJq1wl9C0XJcEyB05Mxdl1nFiyo1xzyVCfI5tRqDQzJYjwZUW02yBiTBRbjgUe9qrG20eTkbMhKe4vpfIIXjLhxb4+zkxmJHxEHGYqmUm/mzEYe/dmEW1s3EFOL4eicSrVLq65jWDlT1+fFUY97O9fY3dzns1cPSaQQS9TQZY1lMGE2W9Bo1IgzHyWXuLn1PtNozuH5l1xbu0OvN8DRm+QpLNwphmkSZymmJRMlM0TahJHP9s4qhlUwny/Y27pH73xIkgZoWplyHowHWLpBnmWcvD7kzrUbrLTajE7n3Hpnk29//zs0LzL+o0dzFE0mzzN+7RvXOG7Xy7vVMP2Rt0+RSNNyoaMQy73pN8BOFN9MCKZXHZKu6wJcBWyumLUkKVkZStbNtm00Wbmq7JEkicViQVEUeF65zpNlGYIsXXkvl5dy+htQqSjKJfNUyrOSXKah0yS/+j8F2VUQppSLS9YwiiJMw74slC6IogRZFv87KeyyePzNik3JZIaRf8UivildV1UVSZAvOyPLbs43u96CUILvQoDQDzBN88rjmSTJVZ+sIIn/HYbyjaT/JlgkyzKhX84CvpmqfGMbkCSpXOMRfsQuI4m4rlsuaDnWZbdoWeX0BoDGcYwmK9iOiVBAXCRXqfs3K0aqqiJeppLTNEfTlMtjyIjDS1/ppQc0SSIURSPPRIpcJI5jKpUKcRIhyyKiHFHkAq1GA8cqfZuTcdljVxQlK9xqdqg3qqxvtJHklDBy2d7eJkky+r0FJycn9PuDElAJAk7FwnFsVlc7bO926HZXEKWyk3M6WXB+NsRdRCxcn0EkEUcCilKC+DgsGcUil9BtOH/xCG8yI03g/Q8/4P6t/dLSoFsoAmRBgijIWA782z/9I2qNLisrK8xmExyjhiCE2JUKF70BlYrM6+MeT19OysWnIiOTBMhFVLnAD+ZUnCbTiU8Uuth2RuQJNBtdVro2UrHk/GDGe1LOfxo0SNKMPMv4f75v83nuc3o2pN3tYtY9xpMloW/Qbrb58Md2+PyzR6xtbqFUYx49fMbbd95GzCR0Q+To+DWWcpdcOGU6K1dd1ro7pfQv5GiSxGByysXwmPv379O0rjGYviCXTIo8ZDieUK236fcSCmJWV2tMR2MUJUKRmkgSNJoOwVIjTqcgJAiCSLPZwJ2PmUwj9ra+SpCclmX54TFumNFs3UAVlhjKJitdm+9/8jvlso2fYegOzVaVi3GfdrVCs17l8PiUGzd3ODt+Ts26x1vvXOf04AjDEjl49Rpd7ZLLGSgmWiXG8yMajsB8FJOnEGcztrb2iMKcPPMRyTk6HfHLv/jLTIYzUgIWrsdi2SeJodNpEQUhih6hqTbLRUGjskK1pvLo8QO2Nvf47g+/R3dDQ9dXmM9ionDA5uYm88UpYq5gOQ06nVWWY42NHYXe+YSlPyJPbL7ywX0++fS7rK2tsbG1x5MnR1RrEqNBj73NFf7aX/nbmFaF+eKI3/mt/4LPHp+TKBX0wkTTFVZ31sl9AS/sE/oS2zsdLgZDxKLGxo7GfC6QhgX717b48ssHjAYR12+uYNsKYZCSpWClEv2Lx5jqBo31DicX5zQqLS6mT3FqbY6OL6jXtti/afPgwQMm05TOmoC/KDtr43SOUIAsaBSpwkpnnSCec3R6QK3apSjGLN0EVW2WVjmtQ3elxf/s7/6XfyGG8i8MKP/7/+Hdwh9lLPMMTfNYqe+QiDAdLlnMlxiVgmZ9E3dSYXXdRtccDg+GWNWQk/MD/OGU9ppddsbpC/Z3uvSOcw5OXvL2V7awzJgffNLj+q0VitRl1M9xTANShSgQuXf9Osu04GJwzuZqmycvn+LFHsVyDRGXvRuN0pyf6GiGxPNX58wXGXY1pXcxoNGysKw2h69G7O9tMp2M8IIBttUAQWJ1q8mw36dIbaqOjKFpLBdzqvUaiCrjwbCs55BlposFfigixDGFP2dz/zYPnz4ok512Rq2ySRxG2CZ4s5z5dM76+iqVOqxvbjIcHyFKBs+fjNi7qREGBYvpgkySqVh1ZCllrbvN6asZZxcH1HZMhhcxa+YKp6+HZMaC5lodb1kWkLrzJYIeMHotsL13E5Q508Epm1sbLOIQXTdYjpfYVZWTgxm7G1vMgykT16NetVlMPAI/5dqNtVI280VAxF9OUVQB09SR84xMhNL2n9Ot79Dv91GclEUYIysFRAWOVeH8bEi9XqVZlymKjFwyWE7HIEjEqUSjaSFLGpHnAzp2RWQxjWh2HOI0ZzQNqdebkOXoqsJ0csHNW7v84MEPMLQ27969jlgofP+HTzDNEF0UGS5T9tY7uBOf1bUqimJhGyrj6ZTT3hR0hZotYluB8UOxAAEAAElEQVQr9GYBSAvW7DqVokaW+jw4PsCp1Wk3qrjDBbKtsba2xsPPn1FrWdiaQxylLOcuy8ArPZRi2QOZpQGt1TVUVWY4mCKKKlmWYDuXHj43IY5TDEdhPLlAEArEXIHMRC0qkIj48x43762RiRm7rsbf+jcHKJpMUeT8396tcrG+iarYpEkZ2jBN/VJyLaf2ROUyuCKWIOZN0EOUyvDIdDK/9MplV0zcGzApSRKSCIZRVvXIQgk0giDAMswyLEPJAi5mc2S1LOXNBcguQV+Wlx/kbwCUZVk/AjuiSKVqXx1TOU3ol9JtXAK34vJ3akpZyJ5lGWEUYVnWZbAkvXq85XJ5GXQpt6TfBHHeMKJxEl6Bvzd+xTiM0HX9z+2Sw4+qf3JKplOR5CtZ/w0oLVPkEmEc/TmWUlGUK1YYKIHjpc9UltU/B6rfLPBIlK8TknwVohLFcjbOsiwqtoZt21dg2LIsotC/2iKPUulKIn/TjyjLMqG3xDAMfK9kgz3fLe0RCEBOGC1J04SK0ybLErI8Is8zVE2+eg8EQUASlCxypVIjS0NarTrrGx1MS+XZsyfcvPEO/X6PIAg4Ox3iLVNkyUBVVZrNOitdC0kWWF9fpdGsMRyMieOEi96IyWTO0WHv8qZEolp12NvfYnW1zWq3hSiL/No//Q6ioJUdnVlOXoRIqkyaCIiKz9mTl0gFuJ7P/bfusLu1iigKBElCEecUSoaQiShS6eWt1+uIUoYiw3w+LxP8afl67Wyu8u3v/JDPHryi0ajj+1MKyabIEyRkZEkH0cOupnhewKgXYRY6rZUK/8F/+DeIwmOihUXt6JBf/cP+pZc3439rz3mi5YzHCe12m5WNHIQAcge7bpCmOXYV0qRGJJygGiAXbebTEc1GhSwVqdWqV6MCw9GovA7rKxiWRbh0ieKCiXuKIui899ZX8OILHjw55M7dDxmMXAQlZjw/RVEyJucRTafLzsY6k9EJWWbx3le3efjwKbVGg+HFgHplhSxPOTp+wcpahX5fQXfGNGpNznsXLEMNw6lii6AZFQQUposegpwjFCaiKoAQMJv7JO6SH/vwbWZuimnaZP4EIdO4dnuX0eCUJKpw8/Y6aSbihQK5OuJieMLrwxnXNjYZjk6pmutle4QYIIgp4aTC+x/s8K9+7x+xt/l1PvjKHb748lOsShtBnnJ4eEyrsU21qnB+8QzHqdJq7PDyyRmtZoUgCqnUK+iGwKdffMz29g5hqGA7OnGQousJ56dL7Bp0V9vkGVQdiziMCIKA1bUqvZOUamOdlCeIyTrXbqxydHBCs7HG559/wtlRyt/7n/w73L55h46zxQ+f/Rp/9N1PUVQHXZWZ9XX2d1d49eIcy6yQCReEEUiijm2bGHqd4+Njdrb3qdQkLi4uWC4y7r3b5tGD1yznMoYg8uWXv8d0lPHehz/L7vUNZDXkxauH7Oy+xfHRa9Y3V5nPPNrrGifnr/H8BQgWlqVh6RYkDcbDCzbWW8iSxdPnj2iuOsymc9ZW2xwfDbGqKnZlhSBcErkF/+n/7jf+cgHlL/2tDwshXjKJLlCkCkUsYDZzHFtjOD5FE3dYLmcook61WiWJRExLIQhVwuScIHSJQxex0FldK8gTEVEUaLSr9M8D7t/pEgQB5+MebjJnMZD5xjd2eP0s5uaNu4wGAYp9xHxSUKutkqURzdo+L18dI8sziBaIQh1J7mDW4eDoFbrpkBMQeiqDxZQo9LCNCkkaoqoVoniJZVgoKggSWNYKVWeNh09+j72tTRaTFASVHJEodalUNPqzBSI29U6def+cGlUWRcRgOKG7o5PGCudnQ27fuInvT7AqEpOJiFORaTQkVK0g9CVG4xirEuKFc5zKCiYmg4VLtWYzmr6m2domXFSQopgsCjjwfKqKz37rPlE84+DikFRUQPWYeBbNikzil963otA4PzzB0gWa3SpBkeHNlwgpaHqTNE3JYx8/K0gKASmKaLW3QQzKlRNPgkJDUXNyfIRcZr29TpjM6V0M6K52MCQbCgWnIROEMb25h6WoaFLO2eCcjfVr9HsHiEaEl2TUFIPGapXxxMMyDNI4Io1SKk4dRYMk00hin9l4xs7WNpJcIIgqx2fnODUNSVMZjS+4festxqMzHMfh/NSl3awyny4o5JymqdCt7jKZTFhZWWM0PGc6nfNLv/I3+PLj76PYChfTIdNFQBDFGKZKq1Zl0p+yCCLqHZOm06J/VqZFTSNhfBGTizartTaOYzIaXyCKIidnp6xvrTOfzKnXmgThnMFgxM7eNRaLBXESEoZ+WVlVCGiywnQ6ZjKfXTJEMvVKDfICIc+J/IIwnlNtmGyNA/7jZ2UwJAxD/v7X1vlCTdBUi3plBcOwiOOQKIlLW4dUgsgShGRXrCBAHIdXFUBvAFiZgC4nEEWhBEy6plz5F9uNJkmSoOv6pQdQK8M4okgUlRfYNE3JKJBVhSzPiaMISVQIfJ/5fE6r1UK5BJCGYZQeR1mmXq9flXzPJtNSlhYK4ji/Yh7fMISSJJEXBcvlspxHvPRUvjnXKIou/Zgi8CNwnOXJ5cpLdMVyCpep7SAIKIoCy7Lwff/qZ6IkRhb/TI3R5ZziG/DzJuCE+KPrap6XE4iGYSCKYrm4I0pEUYKmaQSXIaXSixqU/k5BQNUN0uJHizwAaVoCqEqlclkmX8r6lmVRcZyrHfLZvDyetMgvJXUXWfjR8UhSCZSLojyPXq+PbRrICti2xXy+xPdCgiBBuPTFWrZOpapjWhqGqtBodJiNF2VTRpTQOx8ShuX7b2OrSqNZ4/79O6haCXgUReLly0MuekNOjvu4C58wTFFknVrdodWusLG5wvpGB0kuLQJhmLJ0Aw4PThkMxkzGCxAkUqtDlhUEgYdhWOVmNyCgURRTjh8/RcpF/DThzv27dBsN0jQFWUVBJxBnVByLPC57Qbm8obJ0C1NVWC6XOI0K88UEU9F48OVjnjw/xiwRHoKikic5klxe+6IwY7GM6KxUsayE41culuXwK7/yDU4OHlJT1vn5lRbf/I1PEGUB07T55z9zg+/nR2zt3WQwGvLxdz9FlgLee+cDDg4fU61uoOo2k/k5ui4Shw7d9VVG8yP84ARdadGod4miAEk0cH0Xz58hiAVrazscvOzRXWswX0TcurnD84dPaa2aeEGF2+9s8J1vf0LFaSBIHsPBgmrFZnujTRQFTCcuga/TXrNB1MiyhIplUSQxk3GPwPXYvbHJxx8/p9FsAyGrnW2GiwGxdIGd3cAPE3THx3VzJA2i/JzQ0+msNAnDGFWQWS5cavYGu9eaHLx8xfbmFnFUICszKDoIUs7qyjaL8JjBwGXsPsXSdqlYbXQ94eTQZW/fJovaKLrL8atz3rn3Nab+CSevJ7zz3nWm/z/W/ixIsgU/78N+Z99zX2rvrt5u9+2+69w7mA0zBAiQEEiaZIgUadNWkGGHHdaDHJbDL3L4TX6zQ46ww/ISMh1+0GKFKdEiJYgECICYwWBm7r71Wl1de+WeZ9/P8cOpriHf5gH51FFV2ZmVJ+vkd/7/7/t960skVcINJ8hKjaEO8f0D8rKkyEzevPcOcXqCu6zp927y/PAbbt24g6RKXM5OqWqFR+/c5Oz4CF0dcDl5jiCUlInBeNxQTjo9mSxLUKURg5HBeu0xHg85OUjZ3BOZnKX0xxZxNeWrp085enLCX/qtv8wPf+s+d269y9Kb8Id/8gmTy3NW5z7vPfohab7mYnrCYNBBpI0XnLG10yNPNdbrNbNJxDvvPCQrfC7PXYL4FN/3iQKRm3sPGHQrXr08xG5v8c2Tx+zv9dnYHFJWOoIksrlt88WnU1SrQrNygsAHNSSJS0adb1EWAfPJkjfu3uWzz3+K0aoRlZgaGTdYQtJBEjPeff+H+G5O5Lr8r/6df/jnKyj/8r95u86DJaK2jSjobG1VuHGIpuWYpokgbnB89gXdroypbPLi6Zz+qElbrpcpgq0TLyMsuabTTjBsC91oUUuXpIFNSzaRyponZ6/Yun8XEgNVPyJwbSxL4oPfuMvjn/hs7pmkUkYdpYxbDzmduGTVc4RaZDr18YMEZAG73WE2XSLLjf9rNvPp9kTy+ApWK1x13OKgKgVRFPPg4VtkmcrJ+ed0jE2KTGCyeIGi6QimQOJV2B2TLChZBR77u1sYpcHL1QSx8uj3hhwdX9BpN9w6WZaRtZIg8nD6FpZhsZqvEGSXwaDHdGKyfVtEU4ds97t89vQllxdn3Lk5YO2L+GuNlqTRM9rMiwUzr6BKQzZ6NmfrKV6YIZYymiOy8jw2xx28WUjH3uDw6Ji+biNLAomWoEkFomIQeT59Z0hS10zXLnUqMhprdOy7+P45rndJHFVsDG7R7euE0ZLVIqHrjFDUFEPpsvImaKbA5dRFERU6rS6CnLO/tcezJ4fIVoph6XjrHD9zCasSQ7Bpj2rSQqFMKjQtp2VuY9owPfPI85zRWOOtNx/y5WdPSdME227Tao1w3RWX/jkbvQGKnHOxnKBpJnWS8cPv/Tad9j7/73/0H/LBW3e4t/ktAj/DXfmkoc94tMXDd97ln//zP+R0OUfRCjYdm4Onp9S2SX/DRq5zVlFJtl4jKzo7t7dZTs5oGZ0rLp5Gnte4/pICAd3qECUxslzQdxy4qkA0LB1RlFm5Lo7TRtVETs+eYzkiWj2i2+3i+gFBlGEaFmEUoEg5nr9CMxVEwcJdunzP7PDvfZGRpjkg8H/9/oCPK5dBf4ted5O6hsVijqqqdLoD4ijHNltXXdzitchZr5cNokiWrydyjUjNrgRXfj3po6qvJ4iWblyHdYBrcVVV1XXoxo9CnHYL/WolnL7GEknqlUAq0K7CI5ZlkeXNc6rKBjukyDJVVRBd+SRlWb1eC78GmhdFQXAVwnnta3wdsnkdPnotokVRbEQZoOnK9c+8/jlDa56Lruvouk6SJMRxfO2dDKKQPM2uPZmvfZqv7QXN1FH+11beFc3q/jWWqS6rq9ewafSxbbtpH4qiay+pv1yi6gaVAFXZXMgqstys7AURWdOJ/OBfg6b/q6ikoqiuLzSqqhGVjS9UvAoqJddr+LK8avIRRMoqZ3NzjNPS2djYIIoCViuXwE+YTOYEfkYcZVgmtJ0W3V6Ljc0et27vIooi8/kSdx0wm7qEYcR85hFHGbbdotdr0+vbdHst9m6M0fTGEhEEASfH55ydzlkvE1bLhivqtCxG4y5b2wP6gxatlsVgMOQXH33CH/ziFbbVarridYOyisnSAlnUKesVp199iSprrCOPv/l3/jrDdpvJ5IIoKyEHXW3je5fomkZZVaR1imF2UCWdNAmwDRuzbRAmAZZq8fVXz3nx6ghJNpAEqIUMWRApixpZ1giTgKJMiKKIltNjc6/i/KXI/+zv/z2Onv5TutKU7ykOv/mHTW97jcQ/+1GfgzZoPYta7uHHKaBzejKnqFZUSsj2+Ies/VeIVUWeiWSlijUoyfJLDGWH84sTXNel3VfZGLyNKAu40TNct8Q0JAxtgOv7dFsak6OYR+/c5HR6ydbGbex2wKtnS3Z2N1hPJfZu9jl49YI48bBNBzeeISgiWdpcVCXrEkUsGQ9sygwqfU0tyZSFTJoskRSDgpq8lMnDBYP+PsPtLk+fP0OUO3THElnhcvwyYnM8RLMVFBSODw7Z37/LzL1AV0zaeg+xFknKKba9iaRltJ0WcViRRBLtfo6ubbJYHGFqHabTKS1nhKFJPLj9Fkl2Sl50kWUP29KZLo45nx3Q791BVhOWswC5VlE1ixu3R8wvfSQpZnYh8hu/9V1OJy+YnE7QtD6qlaEZMmeXFxwcPOfO9m/x4O0ejz//lPHoBp9/8Qvu379Ht2+SxSYFq+ZvsoiYzEPuPHjI4ekxrVaL/VtdLl75qHLJrVv3ePrl1yRpHz9/xv2d9zB7OR999gWddkaVb7C51We9bM4jqmLQaltIasjRQcDGtsqLw0/Y6L9HHIqgXFCWNfPlEacnl9x68y0e7j9icjblcnaMqiqMujY7W0OywkBUS376Jwds7vUoxDWSAvPZmuFgB6uVM1nOUUSNxNP58IN3efbsGVG8ojPQuDyX0bsF7uSSvjNmuVyiSDKSEPO/+9/+6Z9vKEcVNOzxEM+PCeM1L88C9vZ3aXU1Tl8FbI4DdsbbJNGcw+Mn3Lz5ASenp+imgG13qOqQ1rbAbJKiFT3WhzG6mnLj7g381ANLZrQzxtjPMHQRMb8JQg9PrOl1O0xevkRUHURVI5g3ANJJeMjNu2NOjhXytIvdLVmn5xSpRu0pjDZaPHtyzI0b+8RZTi2ZRGWMLJZUeYW3DhkMZbygwk9TLhYzhCrGkBTiYEWUpQx3hnihRxlDp7VJuLpEH5houUKwXHMcXtDq9UgqgyCt0R0Lq6VhKQpHR0s64x5pKGGKGadpjFLKfPvdHxGtF/TuZqyFmOAy4I+eHtDq2PRHbVrmkIOzCU5HZFNvs65SvEmIpfZxo4jL5TGmvEGSLwiTANsYIhUGh08nvLm9T+jVDLoFq1mMYJYomYwgKuhO88Gd1TlVUdA1dNzUx3GGXE5PaXcM4khFFVXKKiLxa8K05M7WDc7dCetzH9kJyIoEqTTpSBpunvFqfcJ7e3dIJ0uKOCUTmulLWcU4qkNaBGiWSuzmFNUaUVHJ0y6HF094+8E+ZSiwuacS+yUrP+DNh2/x6ugAUexRlY0X9MO7+xwdHeFFAo61QV3DKp/z868+YWtzSnc8JI8SvvjmcypF4cZ2C0lo0p5ff/UZvbFEEJrERUyuS/RuG4SzgsRdMty6ibeesAyX3Nn/Fv2OROFvYg8MijJk1Bpw+uoQZ2AzP5oSuiVLxWfvxm0MXeXo/Dmj4V1KVqRRQRmp1KqEF86QKNHZpKw0jk/PyNICp60iywJZEmAN2my1b5DGl9y+tY+uDol+8jFFkSMKAqIkk0ULhjsjJFHi6ZMvsFpN8MOxRqRxQhJHaKpMTYlQS8i1ROS6GLqEKNckSYwXrmm1TMqkQNdVQt9HU1uIYkUQpshyIyTFuiLLIzTNIoo9EEpUuWnFapLeIkUOltlGrGXSMEeSZAzdupoANlNDWZDIsgJNa6ZzNSXuak1dlBiaTppG10EZUWiEXxg2AZTX6WZV09jp9lksFrTsFooi02t3MA2t4fKpMuv1+iqV3roWvnmekpVNVoYakiLHzxovZZ7nBEFwLRSDKLwWYLquYxlOIyrzBthe0rTMWEpzqny9Cn/9b1EUCf1mfS9rMrLUVDhalkmcJM3a3jFxHKcJ01zhmsq8uOZuuq6PLKloLQvXc5tUu9l4MlvdDp7nNcicLEMVKzRFpsgEirymiNOGDyoKV4n6RvgLdbNKF2oaBBUiF5dznj7xabVOaDsGlm2yu73J/o1t+v0+IDKfLnn69Clnp1OePD7kn//ez2m1WoxGI/rDHttbG3S7bTpdh/V6SV0LLOZrTk8uOT8744//4CMUTaY/6KDrMju7m3z47beRJBHLNnC9kNNX56QxPH1ygO9GyGKF3tKp1AGm2aGsJGRZQ65UiiqjFAtURSLxQqpSohTBkE0uX00IWx62bbEx0MiyhJoERXeoa4EkLtBlkzTNCeM1kiThhzHeVUtU3ZYRJBDqHO3q/UCtk9UFmixT1SG66FAK4PSagJU76bJ1K2PtHyOELrK+5GJRE0QqVd1cvPzBHx/wpRjz6Fu7tHoySZ1y826XepYyHuxwebFgMn8OQkngwu6NEVHk0bVtzmY2olWysWVh2iWBJ7BYHfPwrTukxx1qZ81otIHvwnhocWv/Li3noPFT3vl1lu4rUk/F7HZYeSve//Xf5vOf/wlB7DPsDFhHEZKoE60jLE3Eddfc3HqDMPKZTjLqouTNdx/gx+eswgBTGxOmLmFYcmPvFtNSwC1dlk9c3n3jfV6dPyWew3SyZtTtN3WrfsEkX1H0SuaLc7aHe+j9DklwglyNMcUG2VPGAUkFZeEhZWOqSGUdvaQIOvzmb/8lXr54xdePf0Z/s8erVy9pdWTW8QmamHF50uKd99/FDxYMOg6a7hAtEtqdNvfufIssLznNPsePfN5976+RVzGff/Q5qq1zb3ufk9OXKCqUdcDOxl3uvTWkkmPSwGQiPmZzY5fpdIpm9CmzCEMXkQqb0/mCVttEKDJGVp+L81N6LRlRyilKg+nMR3da/NoP3uTwwEIQaqS6xbi9w8ZWl6++esbsvM1v/Ob7/Le/94dsbOjoisr+zbtQ/pwKA10d4nke402b5aIpaLh7831+4/sb/PRPf59gWWK2EwbViHu3NvGWEZ1WnzQrOZnM+eFvvs/x2Su2+w+YzpdsDlUkdUG8GJCGKf2tPhv9DT766BPG3V1KIeDyeEWaifhezmw2Z/DmI779/W2effOSXufuryoTf/UJ5d/+u2/VcQiKLiGpApLegIo3N4fIioBmDDg4eEyRlMhKG9f36PTbqKpG4AZEZYwfhSiSjTcLKd0SR2+xe6PFYNyiIOXpsyNagwgBhUF/m+HQYdge4y4qTt1PuX//HnEoMh7ucz75isQf0nIMXHdFhcCro8c47T6L9ZTuoMd6meAHa6IoQlRkoligrlQQfYpIpi7sK39MSFkUmJbI5miT9dLHUDV0zaHTG/PVN18gmjW6ZFFUJVUlQ11gqgqq3Afd4Gx2hJQn9MZd5t4aiNCkPexewmrismH3uCwuubd9B1nxSQKPljVmLXp05A5lleAuQ27c3OPV6ZKL+Rlb4y0cwySKAmbTNZ32mCKHy8kJde3QH3WYzpcs1wuMusbq2CTrmtpIKbBQhZz1KmRkdXHMPstgTs6a7mhAFLtYTofjVyGWbZDFC9r9PlHo0RW7rOUUoSxo2xqj1jY7t2Wef3HGyltTaQ5VLWBZPrOThELUuWk7THyJjY1GLLzz6H0mBxO+Pv2avCUj5im2bZLka2S5w+WJy/ff/5AwWLNz2+Lw1QxRNNjod7ich/S3BPxlhiBL5F5IyxkQJJcg57RbI6pKv+qXPkEQM/obXazSoCr7tIYW0fKURzfeZzK7xC/OMJVdvOiSP/34a/Yf7eDYbQ6/XPDWW7d5+vwY20npjttEgcjbD/ZRBJunL55RV41XVu8MqfMcqzciWh2SpDmZaIBbMdiyuby8pKZgc+MmYRBRVyWe52HZCstFwObWkOlkiWls4rQMTJurCbqIpssoosNkMmFjtM/WdMH/9OfngERZVPyfP7B55mj0un2C0CXOvAYo7zeCpiaja42bwIUqEMQ+puHgtAbEacLl5JiWbaNrHVTJpiqbFbEo1eha41HMi5g6byZalm2wdhfIUrMSj6IIhIper3cduKkEEAX5WpzppkGZV9fNOUVRABWmpQOQpRWqJNOyHbiaLGZpQRBH5EXj8TOM13gdgawsrpp8elcXJymqKmMZGqNxn62NMWkWkWUJw+HwyktasF57TCdL/CDGdX2i5Co4Q0ZxxYAURfnadyldcScNw6DIm6rBLMtQFYUsTojSBN0wKMrsqoXmlxzKLMvIs2Zia9tNSri48l8KcP3/vvZRpmmKZjQTXEVR8Lym7MBxHOI4pixqBKDdbpMkyfVK/DW0PcsyJKVZU7dsh8gPUFUV13WvA1HA9UT2X232MU0T0zSJkxBF1ojDkDhKASjKDMsyaLVadNomnW6LXq/DYDCgqioOX54wmy04OHhJkSkEQeORHQwG9PoON/c3uX1nh07XJEkyDg+PWC581quQ2XxNlhYkSYM70s2S3e0dhsMxaR6RZxmGpuPGIZ89niDILapaIEkiqATyKqQUK3TJYHr6FeuzBbKmUtclf+tv/S2CwCMIPUQRkiS+ukgSqeuSdschy2IsW7t6vwu4q5QgiK6OtcGL54cslh6SpCKJCnkdURUZTstElgrm8xXdlkMT/qrQFYuld8qv/+AdWtYEUpHOocO/+zIkL3KqWuAffucm4b0B/YFIVYnM55cEQUKW5gxHA1TZIYynFKXMcuHRbvXY3Brw+JuvkOwY3ytomR0kucT1p5hGB0U2cdpt1u6yqReOV7irmvtvvIUiN/aU2JNJ8yVJUTOfrdjc3ePi/CVVBRujDRRRodUd4i1zsvwSSayJEw9Z0K86zyXGwxFRWCAqOeeXF+imTZJkbO1sIqkhTw6/RrJUNlv3Gfd1fvxnP+P27Qe8PHzOg3fu4V5m2EqbJ6dPGOwMEfKQbnuLpChZzebIlYyiJlSFhm455EJInRtkyZzt4Zid/S0kSWM9S/jN3/ouv//P/hmJa9HpS7zx4AYyFi8Ov6Gr72K1l+SChrsqeevBLh99+hE//OFv4oWX/Mt/8ZTv/cY+P/3pl7z5xg85Ozvi3gOVk/MzqtIiz9v88Dfv8o/+q/+clnKfrT0FL3KRFZV4LXDn1ojjgxmDbotF+DWGOcbzZHQhpTPYRzR8JFVhNV+h1Db9oUYY0WwKKpFer3tVC2vR7XZ5+uwrJFFFUHQsrUWrrfHV4yc8emcHfxVQlzaSVjO9PKKsCpJIxOmULFcT6kIiCgvefechg+6In/zpp2i6SV5EjIYd8nyKLt3mw+/e4p/8t/8Ng8EOutZm7V8yvUjoD9o8e/oNN25usPZ8ijojjSwe3N+hKhubU5CtWawS9nfv8+1v/RajscV/+p/9P9FVEcMS+Ad/6//+5zuhLGqZMIauZYG0oKgLbKeNl1xSeBLucoKix+iOTpqGxLmHnhnM5i5bWx1IM7ywpMLH6lTsP7hDuPK5XB0jOy3anRG7twx0U+XoeM7Z7Gtcb8jw17ao5IibNx8xXR4ilV2WyyWSoHLjZocwjMCr8f0FdldGFjU2tlocvlxy41aH8JVAfzBmvnApa5eiEFCkHpJc8dY79zk4fIlmhdjCFkg5ValgmDZ1HeE4EpE7Y29zyDQMCVcxo5td8qgiXCvEdcrupsHhhYuipWz0unihz/bmHnHiMuoYzCdzvvPOLZ4+PeHNu3c4fXLM7bdMFKVELmocQ6Pfs/n6F2vuvrnH8fkxNTq9ocoyPKdggFy1aA1MNE1mcTJnZ/s2i9mCKolpqQrOqE+2znFXAXkmUAogygm5kCEWFe8+2qcW2hQvS+ZuxOnxipu3xli2iioX+K6L1dfJ/IoalWno0e22SaqQCpGjx8fcvv1dTk6fMehvECsujthnFUSMh7dRjIowChD1ZrokU+AvFqRVTSrkqKkIeUGcB2zs7uMnl9zcsci8Ne+9+wYfffk5K3/G1sYuHUfl+NTj9GyGZSpEPmi5Tcuwefr0gk7fomeLiNSs3TWaYLCzfZesTJAoSCufWqgRhQRRiEkKn/7GiGQd8O47H/Dond/mk2cfcXbxgv5YZTZzQS5pdRSKRKJIEwKvZtTrsFoGBMEaXVX5Tldh2FP56fqQMCzI5jHmhsbm0OZy4WK1LObzNWfnc8abrQb3U+tousLujR5VAg/euMvaX3M5eUU33yLPUzY2huRxm9VqheNYVKJHgU9ZlY3gEWUEOaeqS6JYR9csJLkGEuo6hFKl3x2RRjOyoiJOC4q6IsliNMsgyjxKMUQRTWIvQDBBEkwUVcIwJfK0pKgSgjik3+kS+DGKBtPZCbf3H1GVEmk+pe0M8NymLjDwI4qioN3uYmpNQ09VCERhIxqkWqDdaaMoEmHos16vURWFipooCJBFiaSKSdOcvGjEZbvV9MVKQoMxKsuSUhDw/GaylGYFvhdzkc+YXK54/vSUfr9Lq9UiTXyS9OI6FS0KMqZpougGk8ns6msieVEhiDLyVShJUcxmjV/l+F7YpM0tE0My0K68j1phoGoaVd2svcvilxfgVQmtVut6uvo6oFOW5TWzM4oi6qLxftqGSZRkaLpCFMQokkp72CEMfURBRrdUVEVhsVig6/o1wui1R1PTNES5vmJeZlRSTZzFtFo2xZUX1LJbzWNe+THjOEaRJMIgIEsS7I6GWKf0BybjjRs4joMoyCwWK05Pzzk7nfL5Z09QFA3LstB1lY3NAYNhhxs3v4/ttKnqgvV6zcuDV1zOjjl4+ZI/+kMTSVTp9R1297bZ3Nql3y949LbMaDzA8zwWiwXnFy5nZ5d8/PEzoiTBNG0sQ6VSa3R7hyCIqGrxKij1Sz+wLNXkSYogC4giCJLS+FlFifHGDooiXQn8Bv6/Xq9ZLBZXUPcY+SpAZTsKo/EmkqTQ7QxZu1POL44YDscEgYsgXXFFixpRtnDMBEnS0NQGYC/bIbvdfZLcIs3atNotdh5tIhx/jm2YSFJNu1dy7l7greqG9UmJphqMhl0uJydYdpe153Pr5kPiWOBiMkWQEkbjXdbxKZaVMZ1Mubm/g1FoKJKNJndQ5JwkDLD0Pi1bZz3Nef74kNt3dnj54uuGOWhYbNwYMhioHJ0esDVyCBIRz1shKzVz95hw1We0UaIoIrOLgk5HZ7RpsV75nJ3P2dy2cd2K/riD54cUJZh2xZMnp/SMTcY7XdrtEYvgFdaogyC2sJQh0TqlKAUm2RlFGXJn7y1+/vOfs55DXUVotUJcG+zdHzO5XFOkCWkZIlCShBWuktD2EnwvYGPU5xc//5T9m/fp9fqslmcs5yElAZE0oa1afPr5S5zNXXZvWMi6wY9+9y/yT/+zfwFygBeuWUy3+LUPv0Uc5zx6b8g333zFcpIz3Ozxznt3efr0gq5zm6pwEcUtVBmOj6bsbPY4Ojjl4Rt3KQuPotinRqTd8ZGSDqcnz0mFiLu330YqS8SyYr1IkHUFhBJJMjg7O6coCu7dfsjJyQmiIGHoDmXlghQxmypsjlVePvuGxVSnN1gjizq9/ojj42M67SGud4Ii2ZR1iqLWPH9+Rjhochib220Wc4mqMhBkm4v5CR9/ERHHcHZ+RF3q6JbMyn9Br/+QO3d3CX2JN24+IEo9gqixZeUZdPoOs/VL0nST9978K+ztbPDxZ39Alqj0ugrr8OtfVSb+6oJyvpoiyAkLz8GydboDmSjOEWWFXnuDIlpT1CaqMEC3dHzvmPUqwrY0krjAnQaQ12i6gZ+smLinaKrJzUcNQ+nsNABB5NXRjI3dFnHioVoaTw+fIdFj2JUolTXuSmP/tsV8PqOqRRarGVanQ5QJRK5MZ6iR5ipv3NujFny6douz4xCUCNswmbkpZqc56H/64y/oDxVEWScqA4Rcww+XyErNzsaYxWKG47SxzS6zeMJ4a0Bdijh2m4vTb+gOWyzdgCiKcZWKfg2S4JBWHqpVcXJ+BFmGVA9JkgllKPLo7rdJinNs02Bk7XG8nPPk2QGPHj1AsW1e/PwTfuf7v87JzGCVHROuUqyuh1I5rFcLhv0WWRZSiykVElFW0O32GA9thknGdHJGHEtUgksaF9y4scvHX32KZjtIqo3aMujofRazObrap2frVHZJKUvo9iZRsiKqPAq5mV4dHp9C3eV0PmHzvk6wiGibIEUe6Swj3/RR9DbH31zwV3/0fc5mMfP1KZ8cPgVDYmvUpyO0SM2I0+MZoqChSCMqeY7Vz3nx6oDL2ZQP3xsze5Uh7xj0LAGjNWLhu9wamHQ6HY4Pl9y5+ZAky6hLneX6AtuWkQUHEYlRbw9vecLl0RmKqmPQ5fD4gs54gOsuoSzpOZtU5HT0OfQsbm2/w2Tic/dtifBcYTC4Q5p5XJzNef/t79LvjXDaIu46pJvMyD2Bfq9PUqXUnZBa2SRTRdSeRhJUlJLA0fkLnN6b1KVEEJWkWYSixGhVD002KLIzen2BPFljaC0oNC6OL7hz/wEFU7I8pN/toWsJeV6AINBut9keOBwcHNDrd4iiCFMzEWuBQb9PXRfUVdqsNwuRGh1JhsvLlwBcHs/xlZTBYEBWrFEUHaNyiBIoa5kgXHHzxm3m0ymddo8gnGHZKknqEfg5na6FLDRezNdtNJqq4q3dxl8sCPjeohE/qoaiy0wuLmm1bCRZ4NHDh2yMTba3tlivvAZ3k+e4VxO6ly9fMp+HVyveHgIShV1SFBVhEl9VNDYsx77av2r9iQhOLlGV9TU0nKqi02mhahJIV0EXscKwZMIgvw72FEXTavO6i9vQLXTNbBLpskRdQhJn1JSIkkRZV1RFjaroFEJxfU58zbt83azzGmtkXvV+G4aGu1w1r5uuUlUZsgi6omLpzTRWlmQKzUAQsgaVVGSMx2MkSSIIAoSrAFK318N1m8arqqooqgK1r1LmBbIgMp1MUGSFxXpFp9NB0wySMGIwahF4LtoV1uh1ittbx1ycvUJRFExToz9oc/fuXWyraVI6OjrBXYcUec3pyZQvP3+OKEogSPQHHfZubLG3d4t337ORZQlZblqITo7nnBw/5+nTb0iSnLKokWSBXq/LeDykN9C4++vvoWoG62jNYrpivQrw04Spm1zXaEINQkktVAiVQJUl5FGCqsoUVYVQ1pydnf1rMPyqqjD1xn9rGi1kVbr+XhP0SUgjlePwkiSN2doKoFavOa6KopCXAWINYVRQ5grvvfcOUVjw0cc/R9dVDCVj0JZZuz5KZbJez9iqU8oiJ6vL5iLKmxJpKZ2+w6jT5eT0kqqCJ4cX6EqLiiWuJzCdr9ndeYf58o/Jiozz02OMDiiiyc1bFrPZKV1nB1PvM1+esfZCxqMhljXg5NTDMDTeeOMel5cXKLKO1tbwA4/F1G9A93FOZOlYdpfJct1QBpQSs70iTWxkWrz17oCL80tmywWem5KlLUbVHmF6Rpn7mJZJGCUcHfkYZo+CCLnscvz1OXFR0dVbqCVsb2xjjXvUYcCf/OIbek6X4ycvqWuDzpbJ8jzFsh2E2ufVwTnTy5C79x5w884uiiZwebFGVw1ePD+m3TFYuy12ttsomkheurjeJYfPI27c6CMWI26/+QhLU6kth739Hn/2h79gd/8mdx9u8OLwJa32bfz0hOB4yJ3bDxgM25T3LM7NM8bbFj//2VNsu8cHH36H+epLTg5PCIKQrXEfqRjT20w4mx3RNsfcf3iLFy9PmJ4UbO4o3B3v8rMfP0XdstjcvMWz54e0jBvcurvFYjkhCROytOLGjbuYjsZ0ntDpbDCZXKDLV3W0UoFMG6qM3rBC1SMujwNGW7vEIWxv9Gj3an78L3+B4zg8uP8Gk8s1Z7OndNsbCIJImsZsbQ/46BcTBLHGWtns7OxQZBJL/yt05Rbj3hucnj9nf+82pSZitXIKV+TW5pDFNAVBohYSygIM2aLbz5hOn+HYKr3uiDiO2Nt991cWlL/yyvt3/sZ+bZgiAjKzScT+nT6CUEOtUBZKMy1o9/jy66cIkkiS+riui6obqJpCEbnYbQM/EoiygLJKMe0OAJPLhI4o8OCdTeqqxen5ObIRUqPSHekU9ZzxlsHpixRFFhm0xnR7DoPOA2Q148tvvqa/IVJnLQbdAVmSMZlc8vTpM9ariN3dbXSjw2efHjPe6pMkabN2UyriNGG4abJepbiTiJZhM94eQFmQhSta/TbPTya0OwZ+WDDqt6/YjBXD4RAvKUiSgEiOsUODreGYI/eMUmrSUu2uynqZ8lu/+RBTGiNVEmglQXSJUK0xnCFnlxV1mlLUU56eeNzaHpOXBk8Ov2Gvp5KLPYI4IvYr9va2WC4vCYMYXeuSZhk1OZolI+YGi8mE+3du8/zpBWZHwDJ7HBy+ZLw1RNEF/DCmqgRMTSOLKxTFYO1OaW8N0OQRs8kBYp0w6nbx1iH7uzv4ocmzg0/YvCuy9tuUcc6Dh33SecX9N3b5yR8cM9xs8ca9Wxy+OuP46CWVriDoKWJSUWoGipZTJSJ2u01aZBRViKEKREHAaHOILhZ4bkp7o0uSeOiSjmPtkQQBktoimi8RlSYIIUt6w1QUUhAKRATyOmfcaeO6Bf3xBrook0YpdkfB0ESyXEURajTZwksm3Lj5JppaMJmeE5c1bXXM0eErbu7vcPvOfdbeCY+/OUTSMqIo4fnjVzy88yanQcginKHKGbnn8Nbd22SEfP7qczbHt7g4mVPXCb3WCEWQiaMFm6ObaIKI1VI4P19hdwzCaIlYdNgc3eDi8hRd1zk+e8aHH36I+Pkx/84nLkVZoCgy/4eHCl9pTX2h4zj4vo8gSFhG4w+zDaepIExTHKdNmhXU5AhiRRJWhH6F3daQlRpJUVi7c7qdISAjSgVr16dttmjZG0SxT5ItEAQJxxogCBKaLlHmDY9SkVTCMKbVaiGJSvM3rmq0u53rIMtyvkBVFcbDIePxmG63fQ2jVlUF27axbOOqnSa/Cs3FeK7P2g8a5iUiq9UKSW0mS1UtXae2BaGmqpuObWjwR/bVZK5ZP5uIooium/hh0HgiqyYdDlw32NR1fb3iBq5DSXVdIyKRF2lTzaepUNVXQueXN11vIOOvQ1ANpzO+SmZbJElE27bQNQXDaFb/hqYhiiKL1ZokSQjDGE018IKG2eh5zRo7yTMM07xGO71mTuZZRn4FqJdVBdM0cV2XIstxHOc6NZ6mKWEYYhoGUGFqOnEYURUJnW77GhNU5CVJUuCufaIogVrCaZk4LQNNkxiOOrTaFrIsEoQeUSAQhSmz6QrXDa6Zmf1+j1bLpjcwaLfb9PvdBs5fVfhezOXllMnllOU8J44CdMOgM3BwnDZbm9vMXJcnhz6alrP2g6vJZHYt3KvI5fCbLxFV7SoIZfPtb3+bWrwS8VehqzLnmktaVDlZ9stKT0EQsGwDXW8ugvKssaUEQcB6vW5A8XrEapGThjnUOU5HQCj7/O2//W/S6oK7WPLNk5/R7W4xD2dQLdg7q/j3noiUdQ61yP/+zYrZjkGrKxIHCtQaZ6dTNB2G/QHdQZujowU1DWpp6X1GmYpU8Qb9DZW93Zt89PHPmzCaILN/8xai2ITtNjaHuOuAMMiR9YTRaIvVaoHnBeSJSpJPSKIcs9UmSzxUs00WubR7fVYzF6fXgUIn8mOSdMbmVofZpYrREpC0OYtVRdsZoeg18+WiEfBlRp6AYVXoeo9hZ4toOScMK8b7bTRJ5vB4glsnvDq4QDIldocmaqGTVjZhcsnYHLK3a4BYsrvxNkUZ8+z5K+IiJI4NHr67xfODZ4yHLebzCULtYFhQpUojBnOBIl+yd3PMV5/O+dF3/iZ+/hVaS6ZMO9y4OWDQ7fDNi6948dzlr/z1b3Pw/JLV7BxNUfgLv/E7xNmaFwfP+fybP6NlP2S0o1LkK1pmj8vTGdQKqqVyeX5Jy7awrWbepsgmvUGfk5MzuiOVyWGAmGncu72FYklIss1wtMVsfsbx6QnttkNZKNQVtDtmQ/3IKlarGbKm0h5sEMTn9Byt0RKVxco9p20ZFFFCVqR0W/sMhm1eHb1kNO6SRDF1DYqmcHF2ynDUZX6ZQqXzxsMu7kLj1s09SjHDD2d8881XDHp71FRU+Bjibd542OeTz75CV03SzEOWLIp6htPqc3aU8Zs/+hu8+bbD4ctTNEPnX/zBF6zCz/FWFf/H/+C/+fNdebd6fU6OFnR7KoqZc3A4Q1El2l2RWzvvs7t7g1989C9R9AxRKtHNmroyCAOZsChRNZHZeo4sjgm9lO6gjVAJpOkKW7UwnZLJZYyqNH84mpyzXPpEuc56mXFyGHPv7i7vPvo+X332giyt+OzLTymqkFu39zCMPnq34vNfHJJlaxxb5IP3PuDLz19xcTKhM8z5zb/4Nqt1wosXr7AsC++qNcCbZ3jegp2NIbrkQCExn59yZ2+Pg5Ozpt+3ahHlHmnpIusZZd7i8OUltAoGHQVNVajigsXqnI5to8gWit0COebXf+ct/uynB0ipy5vvGIjCgrJyiFYqqmrhrV4QVgEbpoVSiTy+OOPW6Dabxl1yTlEVhdNXc7aGd1i6GUUhN1Mm3USoE/IyIo40tBq6wyFutEZUBBRNRhDLxvda6qxnl/RHQxRNxXMDXC/izbfu4qcxYZCSeBfk7hp9oJOEAX6cYHR7tDsdjs9kygBabQtlc06iL3n03l1+9nufYWUqf+e/92/x//pP/ilvPOxg9e7z1TcvcDCoOyJe7NK2ehQ1nDx7xf33bjJbJNh6F13KMFUJfBOn61JECTI9enYfMS5QDZXFfEJSJFiGjr/2CMMaRTYRBEiSgF6vi1hbrFcRw8EWp8eXtGwb09SZLgKCScqjt35Ey9ZJuSTNKi7WEzp2C9PcYPrqGE95xtbNHfIy5ecffcT21oAb23tM5k9xLIM7H27jxyl3Orv435wSxiW6CIfnS5bBE5y+xenJHBkdTa9ZLqe0rR623YQ8kiIjnM8xrTYCCmlYcf/ePlVRMhy1idJLvve9d/G9mE2nhSC4SCIUec7meJe4K10BwhUMvUVZhiyWl1SliCTJhLGPIutkRYpmSNSlzHIRMOwO6Tg5brQGSWE2C+j3e5RkqIrI2psgKxplHRCna6o6I048hoMtqjpFFOH4eE630+Bc2k4HS9fI4oThsEHIxHGMqqvIYoPb0VWVKIqYzGZUVcU333xDmcsNLzFJEBWZVssGocJQNXq9Dn4U4Hk+YRhTVtDr9ZBU5Ro+XuZZs+oUBCRJJklKDMduerSzjCAKUWXlquGmwLZbSKKGKGTESUJZptfcxrIscZxGhDe4IxFFkahqrvmZAlAnJeUVr7NpqrGvvKHNLUmS64rGPM8py5J2u40kCdep8Ha7WREPBgNs20ZVqiu+pEaUZmRpQbvdxvP8BvIuaYRhzHQxJ/tXethf1zyGQkUlVKim0qzFKbG7DqqiNyIsrXB9D103GY02qOuSxXx69RxT1m7FejVBURS2trYYDR1sW0dRRRAKJpcrVss1J2czJFHlchKyXi8xLYN222HQs9nZ3WY0bjciXJNZLV1mswVnFy85PpNRJfUaT9Xvd2l3LCzb4te+8z6qXpIlOZ9//hRBlYlCn88+/5SwyEEZN1Phq871uqyoBQFJVIjSGGoRRZLxk6au0/d94jhE13VKy2oEu66iqOLVe61CkewmlBMUTSCxyFBVkCQR29HRTIPBRp+aLS4uLpBqBel22YQ2RANFFVEVg48//ZfopsN2/yaK3Kc96DFzp9S5iKw2qCtJUimLpLF9CC3Oj6Ykgc54PODmVpdOXyauTpjOV0hqjFTbvP3uLrpt8+M//IRQKDh6NaHd6lJXIien5zx8eB9JFvG9gF53xIsXL9neHnP4asbmVp+L05Ag9VG1ikIIKSqFbq+DnyWkcUa7P6YMKwTBoKjXdEYVqSciyCVWMUbXOujWhLzIMawtwugZojzDFC100yHLU5Ik4e7t23z55Vfc2L2N4sjooo5XzFiHOocHX6LKbcRWzmhQIKg6SSDQardZuxfoSpvRdp/PHh+Qui4/+F/8XbZuFjx4a8DXX53zxZdnnJ8ucBwbSRFJEhHqDM1UiGODB/ffZ+0ecHw6QXMcfuevvUm4ntPptbm4fEkSRzg9jYOnL7lxZ8jdBzUnZ8f0x20kYYhjbBHEF/z8Z08Zbevcuvkea3+OqjgcvXDZ+eAR6p5Flq05nh1jGm22NzeQ6pz5YkpehhyfZjg9lSDMafW6KFLCx88/Z3t/jK4NMToq66VPEhfkWYAoVhy+OuDRw/cxTZ28dukNB8znU/xwiiDIPP76M1AuEJUdsqjFyGzTHqhML3JCP0JSffJUpspt0nRNr9dnsLmDokskrsCbbylMTsGd6JgmXJ5PyISInZ0dHj74NQw758d/8jPee/dDprMTfvwnJ+zsO7jrBbKkslqs2b99m/Xa43vfu4+mH/P7//yU588ucNo255NLTKfm9q03f1WZ+KtPKP/i375dC4KAt66oyegNNPJEp6o93rz/LTq2xXIZMZnMkOWU6fQA024hqW1eHB6i6zo3bu6wnickUYx0BU9rdSRCNybLFaLYw9a7qKpMe2Agyw7z+Yy1P2V7Y5OdvS5pFENlc3l5hiC7vPngHQK/ZOUeMRp1uDheUJUyti3Tam2iaAaff/Y19x/tcHas0B6mZJnH/BKC0KPT6RCEMVG8Ync4ZLX0kGyLNAsgqtHMLmq/jWXKPDl4jGMKbPe2iPyUpRsS5zW+d0lrvM3OqMP56RLDqehZfbJIZrihYYgiiqnyxhsWh8dHBJ6BIJbIoohYmaR5iTbUefrx18j9hEreovISqDIu5yVCLiLIIWlYce/uG6xWPmG6ZNi/yXw1Ia8DHL1DESYYHYfYC6mJKQWFlqlSaRrJPEFWRSzHhlrCdVes/YDR5k3C1CNXauRKp1VVtEddposLOt0Ntlotvnp5QGGGaLnMzb0Oqn1JnY+QQ5HnX9SMd3Ru3bvB+ZnH+eQb7tx7ixdfvaCSKpa1j5jV9Fs6iqghKSZrLwUxQFcrDL3LxfIJhrKF3K7pOi0UzSBaRmiVQSmICFXMbN0gRzw/RhJ1JKnhNsoSaKpBkYpoWoao2OQZtHSDTl9nsV6g1n0ePfgO7z16nx//4v/D2XKG1XEY93tkYUZ7IHBw+JI39j7k+OSA/miXKCgR64S8XHM+X6IILeyhTbxYMF+EGB0Lxyo4OVqxNWwT5yrHRxd0+zq14LO98ZCXz8/Z3+/grX2oFSyrmfyIQoxQtVHFHoghiipyevaSJM7Y2trj7dLmr/7jbxAEyIucf/zX7vLSsTg8eUKv3+LyYs5iscIwJYbDIXleUsoFWQxhkHBzdw/HarFauQ2M3F+i6TqmY+O5TWuLrDQ92lkes7e3x8XpCe464saNG4RRgKZoBGGIadq4rsv29i43d2+zmnm46whdN7FtG93SOT8/J8sSxuNN+v1+0zrUHVBVNfP5shFwdrsRfaqKqqpkWdaEROKQ9XpNu+M07TUlFHUz/ZNlmTBuep5tywCaytOqAst0mhOY0PSUu+uAVtumyLJmUlXl2LZNdsXlFMVm0qXrTef5Ndg9y66A701Vo6abVFdrcWhg7xUlaZyhadp12xCArChXNXL1dRBJv2oOms1mzbSwLNja2rrmURZ5fA1bb7U6hGGI7/uYpnn9+yn61eTiqo98uVxe1SWKiJKKrusEnneFXKqRFJk8K5FlFUMR6I+G+GFwxfQssUwdRRQQAPlqahiGPl4Y4Lkh7jpGlTSoZay2jmFomKbOaGPIZDK5goGXLBdr0ihB15tpaVFmGIaGJDW2DFmWqWvhWuQXWY7neVf8UKk5bo5Ju2XS6/VoDUyGgzF5kvN7f/THFNIYoUpIs6oJ5OQuVSmgqRZnR18QTpeoRrOd2Nvb4Y0H93FdF2gmsoIgIIlcsUdLWq1WA2l3OgiChCjKSEaJIIjkWUmel7jL5v6v60NHYx1FsHny9FNa/RTfLej3b9Pv94nSczwvY+GdIKkVVSrQ7414VMj8z//MQ9N0oOQ/fEvg07JENSV6/RFpUnJ5scJpi+h6hSJ18NwL6rLpQH/z4R08f4Ftm3jrkGdPz3njwR5Z4TOfZGxtbVGWLopsc3F5TLvd5fDVCXfvvUWWiCTZGqdXUdUZRVURuhn7t+/z7PETNjY28PyG92oaDqg5RSpRlxl5rLO3tcdsNaMkxrBVXP8c35PJSg9ZLSnLnDyRaFubxNmEdZywf+cm8UJkd6PP88OX9EZjnr94it016XcVwihBk7uM+h3Oz84o/BbbuwYL1+fy/JA3bn+b3/6t71FUp5haj4fv3uP/+4/+iNPJMZZtkIdjbt83+dmffUrXvsnf/js/4PmLj5EkAX+1oMz7/Pp3PuDLx59SSy3eefsWh08O8dyIu2+8yc03uxycnHJx5qFi8/0fPOSnv/h9Br2HfP34E7JMYLRpIUopRSSiSiaWVXJxGrO9t4lAhibDeh6hSA4HJ89QTBXVqAlDhbt3R5imwacfHXL7UYfz4zmxl7Ez3iPNKu7c2uf88pA8T5lMfHZ3d0mKZuq+Wj5mGYh86/3vcfp4imF5rEKBzf4DDPWCINHQzBBVHnB+fkxeZBi6TVnWiHJAoYjsDB+gKQHffPmSH/7wR8hKzX/5X/xjykzgwx98iO+HvPFgm9k0wFun3Lq9Rylc8otffM325h3SKOTtt97gT378cyLPYbzRYnNLo9M2eXLwGET4+qtTbt99hKxUbIz2+Xt/49//8+VQ/sW/tVdTqzhOm/OzCe2uhL8OMA0NWVLp9Nq0Wjf5+pvPuHt3jFQqHB4fIRoiktpiOV1x++Y+F2fHCIJAFJZoeockDVC0HIQGeSFXOrIWEa5Vuj2H/mCDUnQhEXn2/Ev2bmyxs32Tolrx1ZcHqFqNrg65cdtgcp4yn6wYbysEK4fBlo0omVhtGaHqMJ9HGC0XUSrxFgJVnbNcTyhKCZEWZ4eHDLfb5EaBuw7Z7W2AJLGIchSaVpK0nGNlInJlILUl6rwmDApUqVn3xcSoWpt+SyUNK1Q9Z9geYtgdXPeYrb1d5vM5UTHFaqXUSUWVbJGlAUEaMs1LpKCmbXaItZLwUiGNE0TDQBBjdja2ef7kDEld07I3CIIIP1gy2uwg5RpJHVMmMVVVEaNhWilR6dDRZEyjTbetc3G6wDRF3GCBqe+SVD66DmUIhigxvrPP4uiA3ZtvcHD0hJPVCW/cf4fEDzHbEzZvfsD55wvS9YyW0eIiyNhzetz+7i3+9OMDhqZBFhW8mh2y0W4hlB3QIalPKQWRy9kaXS0ZtbsEcUVU1cRxSs9pM9rTKJKI06MFd/b3iWYBrrvAsMcsl3Nsp4u7jlAUCctWCYOAupLomCZJUdEdbXN5cUDhr3nj3n38PMZfJQz6Y25t3mE8sjk4fUkhRtjWTSS55nL9Obe3HrGeBeTlguHOHb56+oQoWCErY/wywELDzwrS9IKetU1ZuHR7FtPLmI4pMur3KCqRrIw5vzgEwaDbblOWOcHK4fbdLbxggrf2aVtdHLPLZHpGf9BCEmz8MGBnd4wfLDCeTPh3P29wPGWZ8x//wORV20KW4WI6w7a6uP6KIFyzvbVLmogsgkskQUZXTQzVoN2xOTk5pNPbJIs1NLWp5PPiCYbeIgqbdWK31SdOVlR1CkJJVSqslh6WqVELoKk2pqmiqlDkMOxuYBpd5rNlg96hIAga0dDp9EjijOFwTFXWtNt9qEXiOKGsm27grKhI05xOr4tlmNfYn07XJvAjsiyj3x+S5tl1y04ch2iy3TTKFA20XJbl63VmWZZIcjMVTKIQVWsaezTVQJDEK4EhICBdMy3TNIW64UVKNOfA18lyXTcRqoZFKSoiZV1Rl7+srLw+eQrClQjVr1PcpmFc+zQFQaCuBCI/wLIsVFUju+JI1nVNegV1byDtzdo8yyPiLEVRm7Vsg0SycWybuhLQlRLL0Llz5w5lXrBarLi4nJJlBWdnZ2RlE2rKqxLLaoQbVXYltAqiwKfb7bJ/Z/+XgaIgoa4FDl8ccj45JwxiEKVGtEsivX6H1WpBmmfYpkMcpddw+SzLyLKMJElodxySJABAFOWmLegKFv+aaeoGPoooIYkqpVhT5M0kslIVgtxCwiXNaqqiRqh86kpG1W0Onv4ZlZ9TUiKKTRhqf3+fIAopyxpZlK7S+wWGoV3VehZXGKkcVdWxrRaKaqLrTViuQSwVOC0T6hLXXUFRcn5+yjvv3ufk/Gu++fKcBw/vE0ZrrI6ILDr4YYQownxxhuLI3ItM/v3PBeqqQJYV/i8fSHxapyBVjMZtgiAhWmlkeYigZmhKiyLW0a0FuuZw/GpBJaRohsOb9+6SFwmDwRbHJy9548ENlrOMlwcntDoFSQRZomE4BSvPQxBFqFV2dkes3EsERSaMQ0btIWGqotQxUS6QBI3VRlF1qDWkuiJNIqIwQ0DF6YpEkYysZiiySZZDVk4wTZMXj5fsbo9w10s6nQp95EAxYG+nw1fPPkOq9hDKFXVW0xkMWc9DnJ7Kxk6fw+enSEUBkoXUjTh6OqEqCnr2Bg/uD7B1hXZvSGeocXJxyU9/9oTb+2/z4bcf8eOf/CGSkrPZf5eNTYgjl2FvRFCmpMsYx1AwN7YZDlPOn0zwY4Gh3UVtaZzNfAyzi2EUlEVMErW482aXV0cvUBSJ8WaPF09eMOpuoEotguicurTw3BTdqtH1LSQtZTo7QZOHdLsW08kCL/DQ9YKOtcfNvX3i7ICiLHnxfMH3f/RrLM5D5qtXIGRUpYggynhujNVyiJIFdWmyd3+Dw+cTdtt3OD79lIm7ZHtzi/t7e7w8mlKyxml1iJIpgV9we/9NfvHRn9Ef6lyuC966f4s8ThkNhsh6ytOvl3z44UNOjp8ShCqT6SWddpf+oMOrl3Pefu8+a3dCGsv0ByLRUmFnt8WLw2/Q9S1aHQt3UfKXf/vbvDh+zj//7z6hMywRZYv+sE/gBfwv/yf/tz/flbfoqYiGCGVBt22zXkxo6QPqOkaWZTx3ycZ4H1VVieIFbXtIUVS0GSAL0N7aJHDnBEGIakgUUk0STanqFMPaIIl9KkFssBiZQ7IOuKxm1FZNnUtEgc/g1jZBofDx10/Y6MmMey1MbYQX+szORdq2yU5/k8PjGV6cEF6sqHOfJJpw+/4tRKVmOReoSon2MGNyEQAmugo7Ywtd2yYXBfS6xuk6LFOfWpVRRQ011kFaIUcSrp8w3nAQcomSFKkN62UMQoWuSgT+EZawTxIm6KrKZLHEfXbK+99+wPmFy9JdYTsO6bqDJomspxmttsBi4pG6Cq1xj9UkRHMcHNnHaXVYJj6iZjJZe5RSTFmrrHKf7qhDUSYs5xG6UWKbBvN1zng0IL5co7b7LFaX2KNtSilmNQ8hKxBbXQQ5Yz4/wXYcnF6LWbDAcUZ89ckXqGZKevqEQsrY2LDJpQWyI5IFOv/df/0HdLoWv/ntN3GXAn/pw9/i93//D/jsZ5/T0greuHeLL758ye7GBm8+vEtdqTz++AmW3mFVnbC5Y9B1NhCrGmGd88N3P+Tzj/6UeTglOelgDtsMe6Ajs9IjpFQhSUDVDMq8RpRqRNWilktWXkzLaLMuEzYH28RLF13QSC2LS89HUTQUW6KoPD59+TPsiU4Ze3ScEXH4ivsP7rC6HHN+dkaZ5aiWxsHRCUERoTg2py/Oube/jYhIkM5pCR1C/xi7N0DMe9zZV5gvLshik6BOmXseQt2lrTq0HJvp8pRaWXFyPkMWrCZQUesswxrR6hHXJrYpo1YmWVlycHDI99vbCMKcqi5BqHFXCeehy83dW5ShSSUJ9OwNWtoYsSzo2TqisEctBKiaQBgkXJ6X7N94RKttMp1eglCSxjViKVOkHo7ZAkFhHUxJ0xTbMdDEHkEyp2vrhJ6AKAmUdcQySqnr5oOcykCTYyQZZss5nU6LKJwyXV5ydCJx/967vHz5ovHVjfpUZY2KzHw1p9frcfDisEEMVRGz5Qrfd4njmPnyhOFwSLtr4/lLDL1NWQjUYv1L+LpQURUVBSm6XpOlNXleoagldVZBXqIrOpZhUdTFFQxdI89L0iKHqkSRZcqi4Ro2XlQBVZIbW4sAmqGT58190yKnzpvAja6Z1GqNKP3ynNh4G5Wrtp4GBZRn2fVq/XUy22ldHfeiQFcURKluGm40uQm6UGOoTUhHkCUMybpOonc7HXRdxzZ0TNPE0GVatkWaxty5fYNbdzYZD3+I1TKYzWZcnM/Js5LVMuD8bMZi6bFYpHhxSlVBUaokWcTp+WcURc7G5ghJEnAch9HWmNHeGO/KGzlbLBEFmTBKQFBQZJksz5FV6eqYCMhIyKqBrDTCvdsbXfekl2WJZjfC2HVdJEnCabUI4wRZrVELibT2yFUDXR1j4JEHClQFVCWq0ibKA/J0gRBVFHWBqmuEYchut4fT7lyHcYqiEY+CWJLlIZqu0B/0rv2zVVWBUCFKBVGyIE5FTN3i4vSCjdEYQaqoBJGjg+c8efqMJMno9jsMxh0u5ycYlsnRxYRRz2C+9HAch3Z3hCyoOHJIVSUIgnyVTpdp93SWKxfXS5AkAbsr0u3cps5AEGp6wy4ffbRC12x++KO3efr0KZIokuYZgVegSGu8Vc1nnzwmK3wk0UCut5CIcRyD4YaDH31Nu71Nks2ZzzyiuEI2Suq64vhs3nhqteYY9bpdTE0jSH3SJOPGzR0OXi1J65Juz0Qz26y8V3hJSa/T2DvyfEBv0MHfE5ku1ty/PyDYiPCOFXakFraxR1x8ztiMSVwDu9sIUzepGOwOOFydsMDjB2/f5PCrKaLgMB5ltFo2L58d8PRxjGaB2Q3Z8oaUkcfI6XFysmC0PeGdR9/mpz/+Q8TiglpUEROLb737FrP1V3x69AUnZwN+MLyBEoxRK4UHux3GmyNW04jups67338LQaqpiogyM8nlY15+c4wx0PjxH3xFHBV03zNZLc9RjYSzizPEWuHgIOZ739G4uKh5+913OTuaYXZ1tiQbZyYTyQqtocLLZ1/T3tb44ukBw/6A81OXzFtAreBGC9IoYXtjn2+995AnB7/AUEdsb++QRgHzoxXOzQt+8MMf8cU3f8CLZ5/jTiI2tgeEcYyY6qz9M+qsxWq5QKPNzvg+trEgXXdoD2C2XCFJNYYFn31ywb17H2C2D1ksI+bLOYulx71796AWEao2dT3l5KXIh996G7FeMhgPuLw8xey8yZuPdviTH/8C07H5G3/zr/BnH/9TzqcnGG2ZZXj+q8rEX31C+Vt//a1alXNMvcur0+fcfbtL6DnkIfjuOe2+jqJaiFJOVRdMThP6Q43VMiLwFbY3dbKywE1cBNXg9HJOyxnQ0nXSaEFRlWhKD9vKiBYlVaogSDpGRyTN1tTqJkWyRCwTsrBkc7yB4xgkUcRiFnDr4QZZWtPqGaRpzsnJEYJgIcgCYRzRa280PiQjZDab0e3ukCY1dV0SRj6q3GVjr0eUiuSBQJ4uSIQ1cZ6x2R8RTiNSCrrtDooAJ6cvkWuRspAotZo8zSgLEVUZkGQXKEKFrpg4joUoGNQkiFJOuzXg5PQV2ztDqjpnZ3vAn/30c956/xEzd8HkLKY9uEkRLVFMSIMKWelSVjH+2qVt2URRRCYLyKZO7HlsGW3cJCVJQzRdoMqbAMLW9i1WwQI/DMjEhDzV6OkdIjdiHnqMtiwUKSXLBcpUpNd3yLI1deGQVysksUdW+dSUJJVHx27WkEZ7TOUFOFKOZFqMulsIlkiZ6bSdPpqacXr+HEW0mFzO2Ls54HIR0O01v/vmxojjV4dsDvcwrC437pq8ejFlEcyJVxmV3HzY66JMpVV0jC6zSYGqiEwu1zhtE0kxOL14yqC9SVVIiEKOIlvoFiBnlNlViCH36bbGmC0NBRt3fsTGxgZpLrAK1uiyRce2yJSINMnRDIO8kAkTl7LMEUoLQwexEpnGHnLeNNhs39hHqXUW8zmGLrK9OeCjLz9hvp6xORjQMQdMV1OSJGFzdIs4W+FYLcRS5vxsysbWJsPxmCiJmc1dLBUcyyZJQ95MFf7HP1k0huqy5P/x3T7nY4vYz9A0AS9wMU2T0XCTp4+PaTk2vXGXk9OX+MGK7Z0dwjBFElUkSSBMPcRaRJFtyjJF0wQU2SROMjSzgX0bqs5quUSTDFZLH99vaiO7gw4lCfPpiuGwz8sXz3jzzTcIogVlUVEWKns7d5ktjsnKFE0zEdAo8pqNzV1WKxfdEKkFgSjKSJOC4XBMGAes1lNEqcT3V6jSBvfuPiBPCyzLYbWeYZgmqqo33rckRRALBFQU2aAmp6qgrkRkRWguBlSVLC2QZIE4Dq9bZHTdbJo+sgyh/OWFdl6VUDbsyaRorABV1VT2vW7MaTySCnUtXE8dX98Mw0BRJMqK6+9VV01CwPVq+/U6/F/FCv2rq/I0TVHVBlcTRVEDiE+abu/6CoVkqM1q2TadRsxREccxtmkQJyGOY7OxscFo3IisjY0NVFUFaipKjo4PObu84KOfPWtCSUlOnCbXVZ3DwRjXdcmLGKfTbnrgZQnTaHBOWZZR183r+jol/1rIyVehlyzLEK4E3utJsCAI17Wdr0NQaZ6RRDG6rCCoIn4uoSpDsizAnS8RNJUyy6DKQKnIvDXHz58102RFxHVd3nnnHe7du3eNWIIGWSaKMqap43nN5Lsum+MkyzKqJmMYKjVl0/FdV4RBQFFkVFWJF644v5wiqznLRYgpDxj0O1RihCBDmAVkkYDTBs+LME2TbmfI9mzF//rrkpoaURD4jz5o82dpgNVSKWufVquFKrZJ05LdrTajjTGPHz9lPltRVxKqqtLuOKRpzMbGFmHoE/klab4myfxmGixqJIHExtYGs8UrBsMtLmeXdLpdJFmkygx6Aw03nmMYBpcXk6tj2+Cq0jhjMBjw6ugF/W6DWZosXtGyR2iWTJqUKIpK4BeUdcHOTgddgZl7grUR0271efFJylqIGTgD2mYbSzF4fPgC2awQ1zqdoYzvRQi2iSRJhMGUMJTpaBbDzoBCSykDmVQrkBxYTl5y+LnMux/s0R1pTC5nbHX63Nh8i1JcU1QKL1++wlJ7vHn/Lk8Pvua7H3yHg1cfI1Vt/vv/1v+QftumTBP2d4dE4QpMC8e8C2JNkZ3w7MsF56dHTM4PePz4T1hJKjduvYPRzfj686fs797EsWVOLs+olAi5dMhrjf3NMV8efURf3KJScnZ/4z7TP76gkiJESeXtb32Xjz7+AxavXO7du09YrpFEnbPljEG3x/GrE27eskg8me3xm9y7u8UvfnGAamZUZcJoNKLOZT748Db/5J/8HqPhJmHkYRgKi/WETz7/jNs3b3NxsqTb2qTTkwlckU5fYbghEq52KaUJ7irnu9/7DoevPkIWRuiOxnzmI0g+jmUxn7kISHzne+/w0598xc7OFjU5dr/Ni28+wwsj1Fpja/suYRhzcvI5urTLd390lxcHTwGB2XTFf/C/+f/9+U4oLVVGkFTCMKJIRIqgQMpz/EikIGQ1N3jr/Q556XFxXCMUJvPJHFVrkeU+YVhSZX1UucJbhvTUTVQlIasWKIaDJsgUWcJ6VSOrFXmZkgYeVmdMFOmUxSs67U3G4/uIikgYBzw+PqJlarRvdTi5mKGbJuuLGf46QtF9hDpFKMcMRx3qXGYyOyQJcuyWRJq6RJFImha0WhZ5JXB0+Yo4WNI3t0ijGssx2NkcM7lYY1kSl0cLTL1mNg2oUo1b91X6vQpB0ZmvIuxezKsXC05fmiiGR5UbSJKAKLvUpY0gyyxWPopmslwHjIcdvCBk7+Y2Tz87wWjJdHQbschYrZaUbo4m2WDqxFFAGoa0hgOkuubSW5MUJaNWhzSIqGURVTLQLYiDFFGoWKynZFVGVhbEYdFMMRbHyLKC2VFIspJaVqiEiBqZIFyzmK8Ybqi02lt4/hxdNalzGVQLlISWqCGm8HTh89ab91GknHPfpW91cdNXaC2Ljz8+Qbcq7E5CYuR88uoLbt18m3Ugoek9XHfFoDPm8mKOIK84nZRIWKiGhiDWjT9Qd0jzkiKqKFxQVAPXnWNaKkHooxsibatPp90m9H0UUSTJY6JIJIgCBn0dTRMQRYMsr4kufER5ieO0OJ+fItYKAiqSkZGJXc6PXqCbNnkhUZVN48x6vaQUY0xJZywPqSoZP/EYD/fww5wsWVAWULgxF1LTdXz35pA8LfFjEcPpY5gZQl2gaW1Mq42j9Vh7SxbuKYJa0+v1ubG7R+jNUDSdMA44Oj0lzUQUVUYQBeIkRVJ1FL3By+hGjWXLRFFCu9Wn09GJAo8iqem3Rw1earRBHBUkeeMXtFstBFTiIKWsIPZ92u02QdxUA8ZSiCrrqLrKrdt7bGztc3FxRlZGiKJFvxeSBCV3bt8HocSxhmR5zNbWBrPLiLqSaNttalEkCjPyqqCsEmpSJpMVgpqxWnoYehtpnZPEKaJQUlc5l5NzPnj3EecXJ/S6Q+I0QDMaBEq/1zTPUNXIYoVhKqRxiSiJFEWE64Z0u79MFOdFSk0jKkxDodbVJk2elxhKM50ryxJBUjAN/SqAUzW1h4oCiNeiT1VVDEO7bsRpX1kYXt8EQSDJUsqivgaI11fg9yRJrkWmJEmYV4lt4LrC8XVi/fWULc8b3+f1yhyorx4nLXL6dhfd1Gg7LaARsVkSoeoaruvy8qc/py7kaxD67t52YyEY9tA0BVWV+c53PqTT6XB+fkGWFQR+1NgOophev0OWNQl5LwkwFe26ajMMQ5QrlqOmNY9nmiaObXN8fAxAp9NBuvIqQnWdqhcEoVkzaxq+H5LnKbpuUlcZRVEhyyaidNW5LkvUdXV9f0NTca+abURZIktSNKVJ43ueR13XxHF8VcPZUEhApN8fMxxuXve/l2VOGPqQm6iqyPnxBWnqI4kl440+SRogywmdDYcgumBjx8AQdXa2NpjNJlxOJxgtkyBZI7Q1xsMBaRazXk3YLEEQJYS6hlrAcwP0joFpGrheROjXZHLObH6GjMJqdUpZC/RGzvXzL4jRDYnz83NM06TfGxFnNUfHPqqsYtpjDFulJqTdbeF5K3b3hpyfLRiPxzg9nbqoyXyJKskoUqA0MMwaTTMw1E1UrWQ0brOcuXTaGl27iyjm+IscsyXiLSu+851vc+keEq7g3Tce0F7XLOvnVJJPXSu8sT3g8eNDOnd+hCDndActLi7W5Ospwxv3sYuarw4nvPP2A9TSZ7e/gZckFFpKy7Lx4iUHz+fceHOL/VuPaBuH6KpHUdc4PQ3FAktR2Nq5w589+1Mmq5Df/cF3SIuXSKqCaakkrsm//W//D9je1cljj9nFKeOuhWMJfP3Fl5yu/hGff/GExWWOossUsoCmw598+jGPT0J+9zf2+Av/hkVZHeGGPXbubOKFG3Q6Ei9Pz1gvzvB6HW5svMXqdIqi2AizkDN/Sb+n01Z0PvnsJ0zDiIicl6fHJGJGKsZYRZeClIFW8vzLZ+zufQvXP+Ef/VcfY7YctMyg15M5OXvF3dv3+fFPPiLJYiaTkHe/dZvHXx5TZV3euPsu8+kpAiaDsUpVFWSliyR0uDwWUKSUwY7Gep6Tpwobwzc5OX1OlOsMBgMuJz69fQc/TJAllS++OOXuG/dBjplOF1R+j0rUMOyYi4MJg97bfP/7b1IEI4RyzMnFY3K/4nR6Qrt171eVib+6oByPdjh4/Ji3Hjl4FzLrmUh3XOEdrHjn4bc4vJzhuxGnJwuqvEWWCFSCzvbOiEqMUEydLJqyNWzRTXQODmYUigiShpdkCMIaqTQoCplczNEUlc1bPSTBZJVM6HfayGaHySJhOT9jo6fwcO8mC2/JOjyjUgLaHQvv0qLf7yJr+8RJhu9PCFcOUXGBYcs4lkarPWY6W6G3c/wwRbNVhDTm7CRgsz+iyObotsb0MmB5vmDm+dzYHrO9tUFb0+hvWdy916aszqnSnDBfY2ghN/YMjp5lgE9VKchSwmxa0xvrpEGGoArMvSV2x0RTZE7nK2xdI3JDjqcJvdoiSZbs3qhJspSus8n2tsV07lKkGZpjMEtW5FcfRrog4cUhra5JC5XLy0tanRaRkJMWKVZLwV24CGKNKpUoYo6mC3S6GvNogedV9Kwt/ChH1wtUWWW0vYm3zgmSFwgoxFHKqD9AyTWmi0s8RWWrLfNr/TtkLzyWik9u6ixP57zxYIvV8QVSEqBqJYvjJaY5IC9yFkcLfO8MWYKySum0DMoyRCxblIXEKvWIkwBVdzDVLYRaIs1D+t02i5VPGq3JixpRELGtLmWVUJQRk9mEPClR9RLDsPC9kMDN0eqY8War4cKlKR3HYbaeIKtQZUqzvlUjlnEOYY6qmNSVjL8OkfXGy1YjIgrQ0mzEWmymSIJNlPksFgvGwz5JkqBQIUgt2rbGYjbHbjlEZYRtG+iSQ7KOCTOPXmeIn4bkaoWiamRVhbcKoQiQNJkwzgmSlI4qggBFUcKVuFnMPfo9h9DN8Dx47513CYKAqjwhTwveeet95HdV/tP//D+hP7BJ4hhF1VANh9WqYLlyydOSTqdDVjTNGlnZdFv7foBl2c1ER0xxgxm9YQ8/mqGoMrpkUUkWpZwjVm0uj+doVsRoNOLsZE2crHEcB1238IKA4ajDZDbl+cFnIMoYeou6qOl3O/S6fRRF4en8BNtpM53O0A2LJ8+/YHdvGy9MOT+bcu/uW4hCszrOsog8r3Fkg6KskWQRz1tfhUEssixqWnaqFMexMAwNxzawHRNFbKZnvh82pvjFEoA4SUnyhLqqEGQZTW46yUuaIExTY1iR5w14PQgC6rps6hyvboqiIIkKolJT5HkjnEyTJEmum2le94K/9hG+9loWRYHjWI1QEuVr0fa6YvH1Y67Xa/r9PpqmsXTXRPGaKPauhZ0oCWiaRktsIamvuZgCWZYxXywQKgE/CK470AWxptPp0O61sW2bdsdCN+TrlbHnJizXKxRRuu4ML8sSy2zwJ1VVYRgGnuuSZxll0aTUX8PXs7yZziZJhmFI1xPiNG0mnGmaIogCNc3EVzVsJNkiyZoudM1oAltIoAgiVZKQBk0XulBXKIpGEAQYmn6FhmqmyZXA1YVAQZQWWLKBt2qCSYaqMRo3ZRZSndPtWQiKjCi0uTw/I88TZvM5sqowW583bM+6Qu+ILKYrAi9he2uXy8UJsl4giS18P6bdtpBFEakIqaryyqNb0+/3OJUzTo9P6PQcBJTm64MOk8UlGxsjVuslqqZddbzL5BlEeYyuq1SlSFllRGFO2x7i+yEvnl1y59YNBMFElkDXTNxFyN72DgcHr7CdBqcUuQI7O3vcvz3m5dFLHLvDdHZBt1Mjih0URaPVKbi8CGnZOqYpU2cKo5GE513w9fPPURSFX//OjyhjAUnsISdbDFp3uPtv2MTxlLqEnXFOrkJpyNwdvk8lJ0RigNXaJigEonCCLbUI3AjZgCePD9ndtvngvR+g2Qe0rD16Vs1sOad7z2G91kl9AWVY8vTpEX/hh3+fP/3qp/QHNu2+yXzSAk44enXC3/67v8touyCMp2iiimIk/Ef/8f+J2M9x8xVnkyUffPA9CucZoiVQCSb2Rpf3f+cH3Dj/iqz6CV8++Ra/9Rf/Pt7qiNW5C3WJot6la0T0b5u8fPon3L75F3j3Ww8ovD/i6OOfY0t9ZvNN2vsgeAmLswWiBXLX4dlPD/DrNXdu3aOKB7QMAUXdIEpLbt+1mF6IIMYIkoCqbUMt8fTZIUkyQxQV1otTzs/7dHoqrpfSsYcs5sdsbQ/Z3XnIdDrl0ZvvcmO7y0/+9Gf87l/9kP/iv/yHjPsPWbovOH2V0R+qpCUgVARejTsVKTKf0eC7DPYC1qs5jjFgNltw8dE3aB2F+/s3WHY8/OA5n/w0pDXs0bE0tm98j4fvqEiaz+FB8OcvKBerp1CVOJbOhx8+YuIG5LnH/TfeYD65YLgT4gcVptVBlRUu4jkbY5vRWMKwN/n0szWjjR5HlxFuuEbuaHS7Q+oqJwkjvECl3W7TGei8OknRNZH5RURZ+gw6JlIqUoVz0jyGLGDYfsDp4Sm6bSHTxRo2FPqsykESSeKMJG6SpFmSEOUpuiVw69Z7TBenZGLMep6g6ypBFCGmBT27wFAFFrMCkogsK0CSkI2Stefynbc20aoYnYzl/EtMs0ISmzq0bK3wR/805avPCwbjFqYyQlZEBDXGMAzc1QzLbGNZFoZuM5lMyJPmqs1UFd64dYOsDIjjC2RZpBQgKioGezf4+NnvMzCGxEJGIlSkVbPeWy+WOIMWsViQzF10SSVcB3jziE7Xob6CHrveklZLZjld0XFsdE1hp3WbtJWTRTFhLFDWOWmiorShv6lxelKjiAJVGXExu6AyoJAyFl5GxxlxGC7QDIXuZpfJcoa+oXG08FnPY8Zjh0LwyeuMi8klna5DVuRIikwQxFSlhmI4dLZHnB7P0GSNpCgw7S6CVLHyJjiOg2LXpHWMYcu4S5HBeEQQeLh+k16OgwpFiTFMlaKQWQcxspzT65scPD0jSVJkK8Jpt4lyCUNz0EqF1dqjNAUUx8bRbWYvX9Hqj1FVFc+bYEola3+NWAmoJaTpGmkooMkSx6crnK6CJOR4ywzdtMjDlGeHT3h07z5V5rAOXXS7qdwz7C6bm5usIoUwWhNmAbJkYtltyqTA8yLqPMHpDDEsldV6wUYlIismdVVDXSEpMoPeCEMTEU0Hx9hAkVrMpkeMx0NkweTFywO+9cE7bG2PiKIA13WpKNjc3qHX6xPHMb1ej+lkyebGBi8Pn7G1tYnretS1wGhzh/WzT/ATmRs7d/nym0/RFBVdHWMZBtPTkuGoTxBESFrO9s4mcVQiyhq1WBMFNYpgkaUxp2dTDLODqgnEic+9O+8hVBmXl+cookEU+U1QQpNIixBF19AkqKsYq92l1VJ58eILNjf2CKKS9XqFZdnESY2sNFMqSYYsLdF181os6bpOXpXIeQ6YrJYuG6MRum5imjZUNYNel7oWWK5XzJcrfN+nLJvKw9cr6yzLmgpKyyKKGjGmKAplWZKXv8QGpckvA0KNMBSvW3JeTymzrEmHN8KhuF79vp58RlGEZTnXPMw0ybEd63od/noVLooivu+TZTKIOrPFgrqu0RX5CkNkkSQJUZg14Z4sQ9UsRInrxzZlizQuSLKCaukzny1RVAnT1JGk5rkbpkZP7NDptJivlhiG0Ty3tGA4MAijZgXbJI/L67X+69R8UTSgd8exrnmQWdYErOK4EUy1UFHkYFg6kqThJjlF3rCByyJDqiuoK8q8wFtOSQIfRRIoy2bC29gFBPK0YDKbEgQBt+7e+mUtaFVcJ+PLsiZMQl68WGDZBsEy4KvHM2SlptexKWqB2dxlsQwJk5jOhomgqpBDGiecHB7R63dw/QhJEdFEg3XgY5o2y/WK0XAXJcupqxBBFKnrirW/QOi12L95l9nqhKKMmS8Ser1BUz+bxbTabeI4Jo5TQMT3VwAE4RKhsOk9cFAVkzhdYtoGmlpTSitevlyjKibdnkUtiBy/WvLOo+9x604fb9W8ZpezZ7w6OGLvxh5L7xxFUtF1G8/zuDyvsW2Hdj9j/+YmZCrj9wyePvUZb0ecTHLeu/cm1DnHl68Yb+3hHxVMX/l8tfgKxJQbN+8zO1tSSC3y0kDKYro3Ks7nU5zWkKGjU4oyaWTx6NFbIAWMe9sE6xV/+C9+zt/7H/0uX3/8FWcvM7a2+7x4eYKhtvlLP/wLHAdf0tqTSMopN7d22N6S2d5rU2QWSGOSNMSLz/nm8RGBnzOfrvj5Tz/G0Cu67V3e/OAux//1HxJHProFj7884zu/9gghWpOuYiT9HSzdo/ArDr8JiThH09rUVUC5rtnavME3X36J2Wnhro/Q97p0hrt8cFvj9z865uenHc4mC5K5Qh26DPs3SJcBN3tjDk9iDp8ds/VgiDcPqCQZTRX59OcLsjRBkQQuZ+dUhYFttbh5s8tXj4+IXYN2x8D116RhgBuf4cQ32bmxR8fY5+Gjuyz+eMbmTp/5bMru3ohXR4/Z3OpSJTZ/+Xe+y89/8ROefjNjZ2eLshS4c2dAXWZsjvYRyZlfLlgvRIQ6o9Ma03pocTK94HRyToXCeHPEl58/42H7HX720e9hWQLDfouOvc377337VxaUv7KH8h/8g+/Xi2VIUc+4/9Y+CDHzaYwotqlKGl9MGVMhUtYeslDQsjXSBEAEQ+TiOOLkaEWnbyBIEu3WEIQ1klgzuYS7b3S5OF/irQukSsRSTUYjmyQtKcqMNAPZrDC6Knml4PouYThHrhWkvEOt5MhahFDK3L+zQeDmnF666J0E3y/Z2diiSFQOX85IxYTRloEqaEi4RBOFpJzhdHSyXCVJJSgrLMsgEyryIEcXR+xv2gwG5036NJQo84DzZchiYrD2BHpDkygO0G0Jx9zCjV/S6Y6b6UCpIojQ7wxZLteEcYRmGOhm3UzNpBIviWjZDm6yIvBC3KjAtm12+gN0Tebs4pRWt0PgBqzna7ZvbHO5nrDldFkuc5yuxGpR0OtayKpIGCesgzkmJoIio6kytqGz9OOm/k2p0W0FxVSZTyOKIqc9KolWBlke029vksQllbcm10qyOqXXNrn54A6ybrA8ueB8OufeG/t4nsd0siLParo9m4qQNI1RRBVHbdO1bzI9PyMvKky7TS0XJEXOer1m7+YYVTdZzRfEKUCF0xbxQxdDV+nZG4RhzYvnL9FUmXZLZnIxQ9N1qgq8VcJos0WaBdjtHnna9BdXeUyRRFhOB9O08RcRg6FIKkTkUo/N0ZgsCMnrBo7seWtyIWXp+ZBD7UdUHtz/3gfIssxyueLCP0CqQK2G6I7BwGn9/1n7kx9p8jzND/vYvpvvS+wR75pvZlZmVXVVr9PDaQ7ZJMgZASL1Bwg66SJAc9FBgggKkgBtkC46CRIggAAhQTMnCpoRRHG6e6arq7q6MiuzMt99id33zfbddLB4o3tuJWAc8ENEuIebm5vb77Hn+yy8m79j6HTZGxxxdXtFreTstlu6TpuqTpENhzDYkGcBrnlMrzfEcgRubxYItUjL7aJpGq/e/JonccH/9JVOkZeoqsL/6fddzv7b/wH/7P/2z9nfGyKIIpqu4Ec+YiWjqDmaZRInPoaqMJ+vOTzZZ72ZgSDx8MGn/Nlf/CWPH54SBhnbTcDefo+b22tUxSArKrrDLs9ffM2Dk0/IM5m6ygl3S05PH6FJOlEUYlt9VCNBUmK+/vodrVYP1QDP8+i1B/h+jN2SCVMPKpM8TymrBF1zGLT28f0AEDl7eMpf/MWfc3R6wNZf4QVbiqzCNFyqQmDQ7fDh/B1f/OgLdl7AbudjGAaPHvwAz/NYrqacHj0hjUVUVSeOY1odl6qoydKCLE6QZIGWYzdxO5KEbdsYmoKmqI12sGoyJn2/ya9Mq4Llco2u6//GqLpxCMvEcYwo/a0BB/5WNykIAsXdWJw7YPp3GUlBEO4BVQMSZTzPwzRNwjAkDGMGg0Hj9k6aqr6PIKyJ/qlxnLuYJGR837vXZCJUfwtkEahFAUkSQBQIwxChru763pvRr1j9LRtpGQ2D2us1rHEcxxRlhqKq2LbLxtsRBFHDnIrNe6/q4t5w8xHIKopCmqbEcUySpvf6yo/60I+AOk1T8iwhTHIURULXJKII4koFoUAoII5DREEgTmNkanaTG1bLKYgCQg01InEc8+WPf0S73UYQamqBexZYt2zabRff94mSkJOjYwSxJotjNFXmw4cPRJFHLaWIQkFZ5li2QRiGbHc7XGeEJIG3W3F69IA8TchLjzBLyEuLPBew3JJ2q0eeZcRRyidRwT/5Jrtz9cP/4lHG7nTMoH/AenuLojRNPUEQUpYqslJRlClJEtDudZEFGcdpsdvt0NUWy/mKp4+eEEURF1cvefjoByTJhOVyzcH4E65uX9JpD7GMLlG8YL0oaLldFstbHjzYR1EhL2pabZP1ZsftzYrTh8fMph6trsHav0aoDUb9AaPOEVVZECcppXZNRJfPTocoqLy5foMgOFiomHZMnOpotsZiMcPSDQ76Xb755h29ziNOH2j82a/+nHANP/7yU169ueZP/sP/DkK14Jd/9TOKTOHzpz9lMnnH6ScDfvHzrzgaPyIj4GqywNQtDvoW44dPKdJruvYhrtPjdv6OR48/5/L8nCRe0+7qBFsF2+pxcfkWq5Nx/mGKZbZotwa0Rx3iTcH7D6+RDZNuz0UoZLz0HbPtLUo5IggTTk+HnB3tsbjx2ay2PHnyiJX/msPjP0XwVnwz+TPU1KKuRD47OmSv/Jr/z0uRuG1QVwq64RL5G2yxxS4MWXshlqxSFAlyWRB7MfuPDsjFhGADz7444cWraz797DEXF1fkqczRYZe89BBp1i3T1ImClO1ujmJkpL7No4f79Acd1suY8d5DdrMbWv1mKjKf5pw+7LNabpupRO6zXcHZ6RO6fShrj4t3EZ2+wGIi0B3KOO6IdxcX+CRoYkmSz4gmBnklMdjTGbVkBFRuZ9eIgoSm9FmtVvyv//N/9ltpKMXfFnnGSUG3BycH+4ilyWrro7gZB2cpmra5S4NPiIs5BRFe7LHb5Lz5RkItDzElk9ODLv/Jf/QnjK0ej4aPWVxs0Kou496YByc208kNkqhAqZCmOV/+0QGz4JZImFCPdojdjIPjEevJksWHBVqq4Yhdsp2ILGeEXoi3TcmSHH9XIksmSb4kCBN+8OQL9PKQ6dUNf/qP9zk8SdksCpa3a5RSpRTg4cHfoy7HaK6O5ajsDQ4R0LEUm67dRzMKbqYrLs67vHxr89Wv4Wc/q7i91giqiFrbUEopiqYBFVbXx3ZdZLOkKAXKWiKJEqqi4Oz4BFWW8DyPq8kl76M5N4lPqKhMvS1eEiErIv1eB9syUBwTzwsYtAYoyIShT3/UIU9ihm6PjRfRHQwRNQVJk7ldTUjKFMUwcDsDJFGnFiCNYbkIGQ5dHFdkb7hHnkpML+c4hsL+YB9/oqCrObamEfkpWVwQVTFP9h/Rax2gMkLe2Zy/vEWTTB4NB4ixzuZ2h64qaLrIZrcjTVTa7SOCJEfSXaIyYJftEIyCTF7gpxtkSefTz46RK53NOqKIUvKsIokL1rMNhtHD2+YIlcBiNgcxQ9MUqgJcc4hluJiGhiSrjfGk7kBlcPbZAKMrEqUCg9EeVV6wmM6QnZxULDkc9vkPfvgJqze33C5mrGYT5rMJIJKmOUlckhUiw6NDOifHhGnCi9+8bOJjKh1JsWm1DMLUI8g9FL2H3Rpit1qMj/voZsnBXhe31WIXh9xOl1AriLWNoevc3l5xcXFBFHkomswuWJAkCV/+4IfYhk1d1ciSiCSJd6PImO7Q4eWbb5ivVsR5gWFLaI7Mq7dXXM+uQKyRVIVarGnZLVyzR5YUfP3VX3O0N+LD+RtEscl4LKuC8bjfZL46Fpv1kqODY2azGb7vM+jvUwuQZCGKZoFc4HZ11tsAzy8ZjHtoukmNzmDcI0kywmRDXpXUtcJ6u2wW+lwAoSSpQtIqRlbB9z36/QGKqOGvIjTRptNqoykqRZqRRCkPHz7E3244//CGMNyxWk558+Y3vHv3G4o8YDK7AjkiSpfopsRutyJJAgSxpj/uU9QViBJeEDJbLLmZTHl/fs2Hi0sW6xWbu7iioiiohY8tPjKWbSNKEtmdU1mSpLvxrXUPGD/eVLXRBQpCM3b+yOZ9BJ2e5xGGTUd4kxlZNSC7LLHtxuxiGAbdbpcwbHSCmt4Eln/UXApCja7rhGFIXhQkaYgogaY3i0otVBRFhiQJSKqEJNVARRQ0I35JUSkKsMwOkmggKiK1KKDoGoLUdJev11uWyzXr9ZaqhN3O5/b2tql+FOqGVaSkqgvqIieNQpKkyQdN05g0bVqcdF3FMnUkEfIsIQp9pLtd9lHXqesqjuWga9KdjhQ0VabMK0ShQtUUBAEqQNLke3a2FhqjoSDWaLqEpgiUVUpRZqiKRLvl0L9zdW82K2pyotjnN999xeX5O5LU4+3bl6y3VyhayWDoEkUBsiwznazJc5GskPG8HcvNGkSJ69sFUaJQ1CayYmCaJm67Afa3t7csF2uESiZLgFqkqqCqgUpiuVxzfXPJdLImiSTSuHH8J/kMQUrY7TYkWY0sOuSlyPmHC+bzKVFY8smzJ5RlwWazZjg+5PLmA0mZEyUZYbzEddqUpYfvzxgNx0DBZruk3eoThDE7/66OtJIRqHBbJhfvbjA08Lwpe8MRilpRVSKGJfL4wWdkaYAsmJiSyPXFFAGVm4uc92+WzJYhX/3yluOjByjGjmhXIUkKt5M1UR7ipXOCdUW2zckY48cBi+0NP/vX/5KTkxaj0QhRMvjRT084PTyg1+7xyadHnJ08oGscst/p88mjDr/7o58gVwE9+wGynPDy1S/I0zUXb5+j6hVmWyIONFptG1HZ0e5YGEqLp0+eUdUShTDH2/o8evQF44Mue/0DNqstkmyjaibeqkIxbUb7NePOkOVkjR+VPPjsEbPNiiSq+f/++f+Z94sPnFiPCSKJ88WWm/kN//RfJWwKh93OZ3rtk5Qbrm7nfP3ye95OFkzCLbexh32wT9lTqMcy02DLelfgtG18P0BTK968ecfhwRmmlfDm1Q3eLma19JHkgulkjev0sO0hLbeDIFcgWATpnKvbG2zXIKdit8vRTIHVImMXrNls1sSBjGV2cXo1XniB76dQWRwdnhB5Ju1Oh8HQ5Xb6nvl6yS5YMxgN0OUTiiKjP9RJE4hrmdvNmvHpHlGuUykSJb8d6fj/F6Dc69S4tojdVZmtrqnjnGQecnvxAcWK+eHv2Xzy7CmjcZvBqGlH2D91GY0N3HZIulIJVjI389eMHti8n71meFqhaAlvv/MYuC1kDHRD5sGzir//33L49uJv2EhrMiukLJb0TjZspeeoPRFJEZBq2EwCEl9CkGp+50dPaBsdUg9qdDLRp9XucXp4iCooXF99zf6ozc/+7BUPHvb58e+0OdkfIhYWBw86mC2f3/3DLwjTJug2DALEWkalqfErq4RaTdmlJWENZk/HTwp2XoJiSkiahqwaWHaXNBWYTbYUuQWChWn0iJMSxdIJypDr9RWKrSIoEv1+H2mToYYyHamFq9SokoylD9HsGkHNma+WLJYrdluPreejGjpQoevNSGzQ6rGYLmiZFnG4oy5T8jRis9oiVAJ5lZLnKd2+xv5eF3+bU2UCvr/BNG06rS7d1pg0TpClBCHt0+20aHcqFCXnyQ++ZJHFaFbN0dBluboly3OSXOU6TLmdfUDWBfwwoixEdFVDEWQ6Woe+3mU73xJsE/qjPqIpktdNdeDnD48YKS5x5LHZrOjYbfYGRyRxybA9JslKTF0nTxPiIObh6RPypAmFdywTS9OpSpFWv0VdKSilzPFgTOSnrHYe3f0e23SHbOr84LNjbMWhP3iGv5WYvLzm0X6POoeUFL9IeHt9y2rj0XeGmIKJZVkgiXjpnOHYYbO5oqxEZNlivZkiCALT9Ry9lfPh9jlff/8LPly9Iy3jRrQfbDDaJm3L4fT4GWGosfBmSGZKlOQg1AhVflfRtaOuJEzDRBIFEBqTB7XA33z1SyQl4+jkDMPS2AU7dmHMq/cv6A4HBFHIZhtTVjLUIt9+84I8lfnxF39AmTVApd2ykUSBlusShj5BuEOUBM7P36OJOqKYIUvw7OknbHaXDPa6jPdPmc4nhJHH119/jYDKu/dXZGmOrGrc3q4pChU/DBgfjBGxSKKmCacxHAjIosVktmGz85rxeLzFtkUEMk6O9xn1+gRBgCxWyFIzWrf0FleXS0bDI/r9PqfHT5tRsSnhtmzCaMvt9DVROqWsI4oqZDK9JMt91pspCBnT5TVh4mO5FrqlU1Q5QRxQljnzxYTldkEtlcwXtwRRiChL98BOkiT0O6btI6CrawFRlO/vRVVSlo0hR1EUqjy7b8iRZZlOp0Ov17t3OxuGQbvdBrg33nwcgXc6HQSke5NGo+Ns/qbrKggCURTdj9iVuyijJM4QRBlJ1EiTkiItKIrGZd1pNfIj27bxfR9Jkpr4H1VvaiY/5m9mCUkaoBuN0UsQmh7xJEnugHBBVWXM57f3zOioP6DValHXNXmes1wu70LfK7IsaVqBVJWqqsjSmDSJqKuCNA4p8sZ0o0gyUZiQZY3mVJXFplazqhqgXBeESUhW5FSUTb5m3mR37nYbNssFaRqz3a7xvC23N1dkeUAQbvDCJXkRUAsJy9UN0+klcbql3WpaoXxvharXd7KMHQgF3Z6N2bJArOmMLKJyhhfNWG88prMd2+2WzXZOEtfouo1uyGiajKzkSBLId9mcg8ERaaoQBhmdTovLy6Y0II5KtuscSbDptIeNTjbckmUJ+wd7DAYDLEckilfcLl+hOzUVNTv/ht0mQTNMduuYYfcheQYCGucfbgGZLC2ZTdfUhUMWdlgtEv7qX3/Hu9czNksPVTaJwwx/k7CYhezvPURVRa6vzonTJX/4Rz/A0CS6rkJZCPzm+xcoasrjxwccPzjj3/n3/gFBcMnzXwecHH2Ot5hRqhYT38dVRmi6grfOkVoT/vXPvqc7MBkOFJKNjGka/MGffMLrDy958OURstEw5YbU4sGjNk7bpjPoM/cmdOwevV6BrkmY2j4PHxzT7Zq8ffuG1Vbm6OCQ+eyKYCuxNxpTl2BpbQa9AYoM0TzC99b0uz287QWj/oDCuCZX4Qc//AOGvTZ13mK+XIFgcTN5zWZVYrVU9LZNW3+At5zw/PUbDsdn/NEPfodvrq+Ydws6I4M47dLd61D6JUUUkCsipVjT0hWO9/fYrT4wnU5JcxXb6aDKbcpSxdtq9HsHpPmal8+vcGwXQYrx1gWKJOK6bXzfZzKdNnm8yYDj42Pev7vh6mKF2Up59faXqEZFJXr87C9fI1tb3r+dkhb5XTLBlk57yHKzZrGckRY7fvXVb7i8ec+b829ZrmLifMnJ6T4OGkWZcNg/4bMfPqNjqnz+4AGaBLYxZrqYsvAWOEOL4aPebw0of2sNZVHItGwXzQpxn7gstinbZU64cnAEg5UkMl1ekBYbBo5Bv95n/X2CXC9xWl8QpRtamsbaW9GxDvnRpxJJAmkWMz4yuFmu2DvrkmszSjHk29c5olDz7MFjtsuQrBJJQpGqkEjzBGUvZv+wyy6o0X0RSXdZ70LGwx5CXVLkPppcMXANbBxuJu9JipyVtyWJRFaXJnuHFuJ+jK4axCuRVRAyf/sbisDFW64ZD2LqQibLN7Q6NmXYdOX294csdh5aC8aCj1iLCE6NnIpIukTo+8S1gKn2mc42uKnE05NH+GsfUShYr7fYto0slZztHbJafWDQ6jSZdlXINhPYOzxlttxRRTJ5saT2LARBI6tLTEUjCQuivCCnpKpFRpbD0X6HcFNxeNDlZpqx3YqIcs12u8VxDcpMJchFtvEUVRORbIvVIsSQQkS1YRHW2x2jvUcoqsAu2iDrCtahwXJ9haGreMuIrWhSItCWbBJ/hSVp7AQfJREYDo9J/Jw82aH3nMbZa7o4EpSqSSaIFGFBUWwQ2gKT5YQP35/TP+mAWHKerOkoCg9OhuRhilKlSLZBZpr0T0Z4SsHxjx5R387QFImL9Y4wTbAUDckVyUWTKhfpGR3m+YooWdB3jhGVLUZ7DMsPTK5eMj7Y52odYpk6dq+Hq8DtboaSatT5FudIpEhj3NYxeenge++pRAXH7fJ41OX2fEOkakiqgC5KpNsat20xvX5Lq3XKaimhKRV6BabjsE48ZuvXGKMay9aoao1oE3DcsZhsU6wsohDXnF/4fFIoVJUACNSCRF5V1KJAKVRk5AilTBTvUHMZQZAp5bypqasiNqsN4/4Jmg7ffv8dsqFQyzXT+TWGbbHwFoR+xKDjokoqOz/g08+ecTtdUAYlZ8cPiLwt24XPo8fHvH7+DVku4LoinW4L3ZCQpQpFlVFliV7bxt8ssVsGSayx3i6Jsi09y2W7WSIJLbwgJIkjXKtLmaf4cUqvd0xZxFQsEXKZljpgPFSxlTa7lYefLJoMNcngevY9kqnR3mtRZQqb5ZaqqCmKGuqcJFthmS0EtaISIpI4RFYTFLVPp/WEUjxnE0zQpSfEfoqoJdzO53R7HVRNJMlTVAnyrLqPDRIFlcBvWlSqQqcGZKW4+1yamyJrQEZdi5RVwxC2TOvOcSxjajpJEqGqMmbLZrtbEwcV7XabMIyRaEbjcRw3DUKaShQFSJKAINRIUuNY3mx2hGHIeDxuYnpkuekNDwI0VUZVJKLIuxuLN7WXhtFoeD+OnA1Da1zQdyP0PM8b9jXPqCQBTdNJqgooEWsBWVNRNJXdbkeWNSalveFewxIKAuv1+t61rhr6fbNRU325I/T8e3OTYRj3+6xEQJQFttuYNC0RZAld0amoSKko6oqiyjFkiTyOqSWZVq9LGHiIsgilhlipGLqGLCtoYsHt5AaBNk67ZLa8JE8Kel2HJJ6DJKIbJrPlFNvqstnlGKJBZpT4SUGW5tj2gFIs0SoFy5RJYpEkknCcEXHis3cwJo4NhLqgpAQqDKUp+dhtN5R542qvAVEokZSAqtqC5LLZ+SAVKKqAY7hkbQVVMQijLYaqsVlNKasmGstxLW6nbxn19nGMPvP5BN2y+PzTn7BcXpOFEg/Ojlhtz6krF9MV0dM+lhGxWKyRJJm9fZO/+eUbCiGnVioeff5jdtspJydD3rx5Sae7h+GuoZ5i6hbDkz6/+u5X/M5PviBMIJ0viWodQRUZtjrs0jXr67cs65yR2OM//OP/mJW34pvnOaduwVFrv7l4WyR8+eXvY/UCvt6959PHT+l3j7hdzjkcP+L0WOX6fcxmGlDXFaY6YvigJBcreqtDnj0a89VffcePPhvz53/5X+M6J5T4hOE+SOA6HQxR5nb6hl53D5CIo4rRocL7d1NkyUKWR9Rtma1/ycXlJU57xMqfkC98vG3O8ExkNlvQdS1+/le/4uR4xGeffcZ6fcF8WaOrLQZHBYXfQddykmzNr755y4NHJ1xNZ3z1YoNmr6luHTquwYNPTwgikbLIiKMFUZSQhzWW1KeliUTbLWFVMUtaDAcQr2tkacByfUF4rmJjMHBabJY5K2lNLtTstluoFbLyNdtQZ/TgmChMsMwxIgWrVUEQ+bTtHsN+h6+//Ya20+X0QOPq/B2CHCJWNdv1Dsvuc/yZxrs3M073nyBXJh31kLoWGe5bvPh2yp/+e0f8xb98R5wG7Oole+4XDPcSZq8rzh73Wfqv2Vy7vzWg/K01lP+r/+Xv1poxIipvEUST5TymZbscjJ7y629+wfn8mloUMFxQ6w69ds33X18iKoeMTh1Ojlq8fzuh2xkhAHnhs1zsyFKLwxOT68maRz8qeflmznYX0HaPkKWKtnWIqffw82sCLyJNJWoF5psVaeax1+/gTQxkVcDbzHl8coKhqWRlRL/fJQkrykRndNpjs1kxnV3x5GxM2+5Q1xW36w/cLDec7D+iO5L4zfNvuDm3EQQLt1Xz7sU1B8N9em6Xq6s3RL6K5moYHYFO7wRvt8PUIS1KFgsPxYK8yHAcA1WUCFcFmqKyP+yR5SKKLpKmJVUd4YcLOvYeSRSjiim5oFGoJbImEu6KBnilAbaps00zZKXGMCyKrCTPAtK0xLI6JHnEuNslSwXCJMR2NPwgQNMd4jRntbll2OsjyApVKVBnFbVQkNYprZZFsN3iqjZBEmK0B9SCw2q1wmkLxLFPtzskWxUUhY9Q1WR5TKvTI8sr8tqjEioM2UCIZfwyw3HVJj7FabNdbCjLHMft4rg6NSlVVbFabLGtFhU1q+0aSYFuv8duHdNruciVxs31Of3hiLTKCGSJtmJgazKVULJeThHkxsF50BuyC1ZklUiVVZiyiW46xFJIWecEy4z+kcl6uaNlNAze9DakzivOHp7h50Aagi2ynuyQ5Iqykpv8SVUhDmpkEcSyRDb7VHioZRPqEkY5g1aHRMgoooRNrjAaWQTLGbnQHJeGZCOYCc9/8xq3ZdCWD8nlkNZ+RXIOe90R8ywlr9YcdtscL13++z+/paxyagH+j39gsT0Y8eHyEj/26HRaGLKOt96hKRqO3aISFRaLtzjWmLY7wgsvyfMaSTZR9JgkrgET260oKp9R9xlf//Ibzh6O2Rs94G9+9XOefvIARTZ5/eIDCAXDURdZ0onCjMPDw/tIk9V6gSzLDIdD8qykFiqCKGXnbbFbIlvPp+V2kWWVPKnY3+9RxE1UV8tucXkx4ehszM30gkF3j7wIkKUSseohyQmKqiLLGqttQG/Y4+XrN/QHLpPbps6wqqDTcbiZXrG3t8discDVx7S6JVlsouoFZVnhuG2G/TPeX3zFoH9I2zojjf1GW4qL2xqQZwGaZhEnGQjZXXezhSSYhPEaUSiQBBvTcimqEOq/q6EsUdQKVTEpC5Ek8RpGm6bn29AtxBqyPCHPc7rdbmO4u4viibOULCtI07/tGa/rRpv4UYf4cYj00XCiKEqTI5mm6LqOINRkd2Hqqqre1x7eu5/v4og+VjsC978H7rWeoigSpwl11rCmoiKz3W6bYHbHaUbudzrRPM+bmCVZvgt314izRkMpCAJ1Ud7HIn00S91rQGWFJImQFJntLmS7jnE7HdI8RRBKojCnyBNUGdLU5/ryAk3WEGrxri89p8xzjo6HqHLD1hqWiO+FtLsdJrsPCFWNbVqISkVeRMRRgesMWC8XONYYS9PZBQsKIUVW3TuHvc+gPaAQI5I0JApFdM0BEhTZbADJwCKpYwzNJM93RH7K3viAo+WOf/JthiBCVdb8757KBM/G7HZr1iv/DlCLWEafxXJGf+CwCxZkaY1lqwjoaIbBZjun1+6SJTnT21v63QNkXWBvf8Bq6mNZIutNCBL0+mOub75j2BtTpBXtdhffEynENUUhIWsdBCVCNyTypKbXV8kSkaenD1hsXpJ4Np//4Cnffv8VhiNRJC6qIbPNNhRlhRDHrAOb3//pI/7sz/4FVW2T5iV/8sd/Sh1NWK12+FVAmmcMlDMO9nSu5jfsjY9Jk5JWRyaIMn7y4y84e3DMcr5ClFPKpMvNzTuGhyK30ytm100+6B/+wWespktUS2O1WeFtcxS15utvfs2nT3+P8aHKbBIQRwXDkUNZqOy8JfP5LaZjspgHHB71qFH58G7CLvB49vkjbufvgOY4VLSAyVWKJsGDo5+S5lt0ZYikb7m6ukKXR7T2KrSqg7cJcFotykoAMWW6mnI19xi1202l5PUVcRbS6Zn4XkidGbidLqIItzcLOl2T+XSK2taYrNeIQsnD4VPG+wcsbiasVztabR3ZUGj1O8RrD6FS2QY+olQw3nfxowTTOGbrXyKkbR6ejZDkksBvvpeaofL85WsUWWc8HKApGu2uSr/TZT67QVB0sqpCkA3yNOLB6UNIc8Ki4N311wiVwx/89E+pyinv3gYMDhOyMOV6ktMZyyT5km9/8R6pavNP/y8/+7erobQKg7auc3PusVzNGfRNgsTnNniDNDY5Od3nxz/6koF7RFnUfPv9DeZQ58EXbR598pT3b7Z02vvkWYahO7TMAz579phBe4BUGjx42OL9q3Ok3OTTxyPyYEdH60JWkIZTpFRDTA10McXRRA6HY/b39pguPGJxTp7vcMwma2+5mqNJIvPpqnEuuhUvvnrPq+dv6Pf7VJXLV7+Y8PzrK0bOmD3rmJv3BV//6+d0hRFfPnhCEoS8eP6Wft8gjzw264B2e8zDT9oMx2PyQqYi4f3799zebEi2Oa6m4AgKemagxCpdXcfUQKwl8lokLaK7mBafPC8p8hJNF9F0G9GUyQhJ45A0iLFlAVOEve4+qmSS1QV+VhLkOZUCYVLS6XaRlWZB0QyXNK9ptx3yrMIwbKoqQpVq9gZ9TMMhjpsRSlHtEMUM1+mQpU193GqXIKodBFkiLSfY7ZQijxFKCLYbsjSk1XLvtV1lVqDKBkkgsFvUZJVEhoSfBKwjH9V0yKuCQq4QDQsviZhN18i4rLcRZZ3hOjIIOZKls4tjAs+nSmPqKuVmcU2lQOBHqKVCC4PNZMnicsrs/RVkInUMdVwRBBEiLnUuUZYCURYjqgLzG5/dOuX48YjKSFBbGYtgiRel9Ho9nL7L9WaCZSvY7R6eX+L2NYyWgK33oB5SC11MDWRVRWsZ1MoOz8sYjbuoakGZF1zd3LDbXFIlEr2+zdK7xdI1eo5GnHjs4iuqJEMtVM72P0dWSrzFhsV1gDVUqBWFIFiRxwpJkrPzpgiigHy30HvRlsvltyRVxGi8Txzn5KnK3uhhoxsVoMgifC8mCDeIWs5itQOlppZCxnvHGKZDFG+wHZPBaMT5zUvQSxTN4Pr2HZ88fYzneVxenmPaBoomM5lOOT+/JA59smLHbPGe9faGsopptRwmtzNefPeS1C8JfZ+O22I83EOsZbztjjTy2G5WUMhYuoRrtCjzgkG/RRIX2KbBfH1OmkBVqBSZQhZZlHnN5HZBWUbIgkzH1diuNzx4cIogSPR6HaqqYn90iFjLjAfHPH30DFVSKSqPw6MxILLeTnh78Uumq1esNxOycspq9xrHrXBdkSCYstlNmUwvKOumHlEQatLMQxALDN1FUQ0krdHjlYXIZru4v1d1jKYZKKpEUYWIkkIQ+RRVjigLZHlCmIb38Uye57HZNHrzxXpFlhX3jKGqqnfAq4Vtu/fd0x9ZPuFOQ9g4p6W/o7EU7vvRocm4FEWR7K6j/GNFoiRJaJqGrutNl7sso6rqvUGnKApUWbmLf9IRqpp+p4uuNCzlR7Co6zrtdhv1LsOz0+mgaRq2YSILImLN/YhflmX63R6WYbJcLgnDkCRstKTb9YYoCCmLjNDzEKuSLE3R1RLKBEWU8LcxWVSTJTlx4lNWMQDtXhtBEMiKFN1sjlWnK5BXOa12Hz9eIsglRSpT5hqKLFLnCS2jiV/KKGj1uqiqzma3ptWxefLolLW/xAsKxvsjTo73ydMSTdYQhRC3VbDZTVmudwSRj6q5KLLBarVB0xxEWaSmRhJlJFkmDmL2hnvYlgJCxeR2QZxG7B8MSLMQRZKRpCY5wNAtJEHAUHVURWG3WWE6BnEeUdYFr1+/RpIFLFOjknJ0u0Ut5VhuH8M26A0dXr97Tm/kMOyOm9avfIlraWxWtzhWhSFrnBwYHBwKKLXLYCizWL0njRMmVyVe4FOLJf4qxtAKVFOkrHbUWkBVy7SHMqLk8V/9v/4LJsuAq8k1g94Q29CweyK9scsXn/6Q8cjmcL/Hk4cP+eRJm5ffXWJozffq9jpGkiO6fYuW3WNxW7C3t4+sbnj9/C3rzYS/+sU32I7K33z1r5jcTBkPelimzmbp8+bdV1SlxHQ6Z7WekpdbBEzSKMexW2y3Ae8/vKYz7HF45vDh4jlpqnG7SrBap3TMp9hGH1k18cIlcSrhZ7ekJeSFSpbX+L5DUq/YBhPmC480C3jx8iuSOEOmKct4c/2BUlGxzENU2UESdfyoRtVFojhHM3SyrKBCpK5LOh2Fdltns13hhwmiqrA3HmC2TBIB3ly+4WK+RpQFnIGFn8WsNiphnDOZXaLIJbWw5Ltv37BYzDk/nxNHIt8/P6eWBLxkwdvLc+a7HUs/4NuXb4kzgeurCZPriJOjRyBoXE/mLPyGvLGNMaM9pzEyZo1MR5WG/PrVSyo5oT8a0G0fEccCvX3lt4WJv/3Iu+qbXExuGHf7lHLJPLzB6Lf4+c9/iaNr6EaH+fU53tbj6KTHkz/6KUVhowga0zfnVFVFEmcUiYlQaCTpBW4HPvvsD7i4+g4Kg3HnM1RDwXVrhGEFoo+iFWw3NZ39Et2oCDYVlg6PP32MrndZLgNs0+arX58T+VNWy1vG3WM6dp88u2S9XqOpLrZtkpcKsTehpZscPzzANET8bYSc6ojkaKJL7nXIpUt+/6cPmNw8wLJ2xLsYVRpyO/1AlFX0xhr+bY3lxpw9HBOscnStpi4lnpw9YL1ek8QFm3nCwcEDJK3i+fdvcFoyNxOPipKqkKlKmV24JktLgiKmzGJkQUexVeI6YZ34XK22HB3sM+wPWSy3mKaFqkooooEo1miKgFvVhFGC2zaYTC8RBQ1RqOn1bcqsJAwrZuGSWi7JswqtVpArkyqu2K62DAdHrHKftBaZn6/YG1pQZATrALdtk+Uehu1SS02FnW5IjPe1JqPRabFZZdjdPlcf3uE6JmWpMLvc0B7pKJSQ1dRl3dS4BRlVqVCVMueXN6iajCw2GtBo5zMctVnsdrT2WhiKSxkmRPOQoWtjjlosVgskUUIqBURRQjdsihokCRS1xm0P8OIrVsE5h4cPma4umK+uWSVrXEkgS+qmFcVJ8aKYm0VKkQvYlkUWga3bFEWIoSQsN0vypOAnp4+5WAZE2ZLD/aeo6Yy+MeLnL3+D6qgcHR+wnF6jWSHb+ZZ1vMQcjmFbEa0rZE3AMWR0ocbQpoT2BjfrQgkVJbl2x4qWPnHRp5RT8lJAQKAGIj9lLapYZhvPC+k6DlJVUtchlqvTHnWoFh77o2P8ICNLC2zXIvTh02df4u1u8HYRve6Y83dLFE0mrUS6gy7L9YInJ0+5vrkkSXMkVUEUa6xWF1noEoUhUq3geT6D/jH9fpeLy7d8OH/D55/8kGGvj++t+eTJM777/lfYrZIvPn3GixdvUEQZxxCgzllvdlR5zd7oENOsGjPINiEM5tRFjajr/MHff8puG7BcznE6GcPuM+bzHXWhYLdcvHCNbkpEiU9VSBwMjpDqCtdukeQhVanQcgziqKDfG/Lg4VO+e/Uzjs2H3FxMEYBey8ayDDRdZe1fkZQ+RabgSBIvX12iGVDhc3b8BSJt8qyp+sxzD3Jw3b9l+lqtFuv19g60NU7lIodcrCmKEkkCELAM6y57skIQG41kMxrOEEWRbrd73zm92TTxMU31XX4fyaNpGlVVkOflvenHMAyKIrv/X5Ik3Y/Pm1F6imFYd1mSDftZVQ0jCo056+86yT+62j863EWayB9LNyg0jazIKcoaQQBNMxBrWK1W969nWU3cURAEOI5DmReEYdjkXbZaAPi+TxiGTQe6aWFbGnlRkKcFAiV1KTRgS6gIvBWaKiDLIqA3Wu9Oi5oSw5RRFAnf36ELoGoSeRGSlTmWfsB2ndNyTMRKYbstEB2FKgUl90glgeUkREXmaHBC7efEUYRaCyz8a26vBU6OTnHt1Z2BrE9ZpeiagZQWROGOyMvZGw7wgy3L5Yo8L5EkGVESidKAOBZRpBG6qiEgczA20FSJxWJGf9C+Y9oV4nSLYWrUZUFEzW695uB4H9N2uLi+QFZNbEEnK3Z8/c1rdNdENUxubzLybItj6Ky8Lbqus15v2Dvr0T/oYA9EdkGArFqUgsCjJz9Bs+d8eHuNaVoIyorAM4jTCFXrsH8y5PzyPdvFmvksZTQ6Y7Tv8u3zn+H0DDptmbb1AEcd42dTfvfv/T1ub9/SNg54cNohDU1OzzosJtdcX52zP9hHpKbl1vz6m18wW0wYDvf49qsZn/6ozz/7v/81//Df/1OSYsFm4zAN5ihqxZOnR+y2IQejhwilSqdvMJ9/ha4d8+jslBfPv+PRw2doukiaFdRlgKSYxMWSJN2RZBLr3QeioEY32sTijKQKWcw9ZpGPZhdIos3zt2/oDnp4Xo6my3jbDW0roa2bXJ9vGQ+GROmGwnPQ5BHXl1c8/uwz8jji7dUazUgwidHkDjUKklGwiwKyqkYzFNJ0h6SWiKJKlYkMOz3CIuf1u285PH3IZr4iKzNkRUATS5ZFzTwNKOMlaW6yf9bl21+/45Mnn3O4P+Crv/4bvGCFttzj+OgEP1yyXjXZzJopYZiw3lwwyySErObJ4UNkTaROCgJvQRZHnL95yZNnT9DymJoSWVR5/+6a8aiN7W4Jwi0Pzp6ymFxz/t0Fh0d7fPHDhyjm3+mb/bcFKJ+/+p6yLjD1EWtvwnDcIlrmfPrZIzaej2E6lHj8zo/anL8S8NId3i7m0yePeXuxRlY05tMFbsskKUK6gy6mIyNYE9BSXLdPupDwliFX7y+xTYPNqiCOPUQlJZc0ygxkQaOIC97++hbbKtn5U07P9lCAzeaG8X4LTUy5vHrXnAgKCUkykbQCLdHQJZWWqeC0Dd68nvHq1TVf/vgEpUwQJZtCWOIYhzh2F+HolpvLBNvq4hoGsnLKZDlntfQYjVXCXYltyRyeqFSRim7J+EVEpUuUAhi6SVJEJLsIy9awbY0srjBMmpYTu02VB1RFhqM6hLlCXqbMvBW64aC2TGxbZLfeIWglo77Fxg/IZQNFhqwQiBM4OX3I5HoCUoVhWBwdHROHEVWR4nRkFNlEyiJm61vyLEPRuhRVTX/QjFdUxcYWcuIopkpzilBnsw44OjiiqFPq6iMzA67VozvoohsG3tYnTRsHbBImqJpAUeYkSY0ui9RZQVnLaLKIadTIus5qs8LRTSStT1bG5FWGDGRhjlBLSKpGnK7QUhVF0PFjH6NtMk2mWLWAaUnoLYdhZ8i7qwtCKcRBJCsqgjCmEBVqSWa7ibG0GlGsWUwTJKumjkRawpBrf42uVLTsLn64IUhmiPIhO39NWRpUQsIuXdHuSUiKglK36Nslk6TF69ev+eMvnvLqNxcNgJ+lSOUlR8c2v3nucTo2cFSXzaZmrytiVCId3SIQC46fHGDrJr/+zSWua2G2JOIIJq9vwBToDzWiKsRPcgRRpypLJEFBxSQJSsjXWKbBbLbEUCQsy+D46CG7IKLf6aNqAl54SZrFSCIMBgYvX/6So+MDbKPL5fkHDEvHbnfYLWfYFiiqyHQ6x/d9Op0DltsbesMWUZQyHvUoyxBN0rideXz67HNMS+fy8hrbMigL0FWdQFBYzaecnRwxuT3HetBhbzSkFhKMoYvv7Viv1zw6+ZKdv0ZXXOIoIIhDBp0HdFtjvGDNi1c/5/HDH+PtQFWP2GyXhEFIFKxp2/ssl3MMswlwTrKcINxwe3HDcDDA6bqsljNUzWa9ueXg4IBvv3mFFy/pdg8Z9g0oK5aLLdt1iqJ6xHHAcnVDr7vP1r/EjzYUdeO6ns5uUeQVSRohyyaW0RiBdP3B/TlxOlkThGtarR7UkOYejtW/G2nnWJZBFAcEcY5QVyAANAxrGIYYRjMe3263DdOo6JiGjarJ94adPM9JkoQ8T3Ech6pqWMibmxs0TWM8HuN527saSOU+nkdV9XsA97EqEWjaZwzjfiQtivId09m4lBVNpcqbrM0a7iN/RGoQhUY3WteEYQh3zKuiKAiCQBAEjZlJ1+8Bpuu6xGkCZdEEsYtN7E9RFCTrDUUFsqyg600VJVWBIslsVsvGIKiIVEiMx2MGgwFFmSBLOpJyB6TLANORsYwu5e6cq8tLzo6fsVk2kqg8ExiNB/Q7LSa319Q6WLaB27PZzFasixWarZMFAVbPogz3WM0jiuiSvUGbOlfRBIdtmGKYJo6WUgY1+/tDdquAsipxOgaSlFJVZXM+03UqoeL125cYaou8aCpTZWraro0oilxd31JXCk5LZLtdIt99n7M4pyxrbudz8qomSQt01UJVJMpyjVirTasWQwxXZLfZoCsmgqqwmWfI9gLEmtffNWxor68CAi9f/5K6Crh9n/LJ04eUsczrN8/R9UMefXLMX/7yFxiOiHts4Shj8k2KF8c8GT9gIW8ZKT0Wgs/eWZ9+IBH5K/YPjpAFh9VkhSis+O75DuItVS6y2U1YLVM+efYZN+tf8OL1t7Tb/4jf+ckjwmKK1Q2Zrr9ltV0y2Bvy/XcX2OaA1WLH5NpHkyVGI5OqVNBEkTLNODpps5hsWM82SJKAZrQRhRxdM4mriO1uRpzrFLmELPeZ7d4SZFu6zhmmuWUVrCgjBd9bsnfQxg8SdN1CVVUenvapypjp9QKxNIh8lVqquJqe42p7HI8/p8w8vGmAlNXIUs3BoctqNkW2DBxLZrPwcLs2cZhSV5DmCVUsIVQykZ9h2w7nt1Oq2yn7oxab64SB2UbFoihvCbKajt3HsD2++f5njA/22G19cs+m1zkm9H9DVmyYTnMMQyfyUx4+fka7o+NtV4hSwKLIOT4YIEkKaRpjmjbv35yTZxVPHp+RhAFVoSBWMt7Wp8preqMR33z3gu+/P2evrXN4NCT0ZTbLDWlS4nZ+ew3lbw0oVc1l0DvFD695OHhIUYq0Bx3ev7skK3I+Hz4hdxRy4QPjnsKDo6cE3Zw0vWH/YESvs89isWaz3tHv9xjuS2R5xV//9V9TFCqBv8J1x4iCTbjrsd2sUNkjrlaIKFy+aJitTr9FpKqE8RWGtSDwSm7nM7wgYzg8YjzsMbu5BKGmyCXSpMJwArxZgeFYhKnJ5TTG2kUIlUVelpzfzrBlmxKZ5aYktubYmsXk/YIk8SkTGUPRcJwWZlvn/fsZll0j1iXddpvuQOTlt++p6DOfJ/SHOoqQEieAVBPGS7rtAVUFQg3r1YSqLshEqWn0MGzmKw+36zDbRDiGS+oniEKJaBjUUoGWi8w/TJE0HcmUiL2YdrtNLuS8f/0c02rj+Vscs9f0AUsS66WHIFrsdjGKAMQiX3z6Ey7evKMUUsIkxGzbxKWPU9j4uwkPT0cIcsX46IDlfEVdyXQ7YxI/R7dldMUi9FO83QpZ0ZnNl3Q7InUlo+kOQllQiQWmCnklYLZHaHKBqsDtfIVlW6iIVGVKXmWEqd900aYabsfl/O0NrbZOvIpZ50v2D8/wo5AkSlEljSDNqeScVb4giUMkVUesRebzObkIlSySpwqm5pKUS+JkR8fpgeQiBiWqKnJ04LIJPCo/pes67LIQtBi9FWKZNV4aICgjEGTiKOI8OscwVEQlxmqVbII1qbbl+HBEv7dBTfoYYsmzTwoWFwVf/PSH5NWKX//qPZUp8g9+9Ig/+8U39L885XYy5+DApSwUlvOQL3/3CEOWWewW7JY6Wj9ENfTG6UqNKMInlUkSL5GLGqPWqUuH1WyKbVW4xZa2ILFefMCWav64s48Qy4SBTRnE/KA3QPIlXry44qc9F1EsEDcbukFJVwJvt2E0tIkyHWERom1jpE2AUcrIsx0nrk7LFumXGsU3v8YajPgdqY2h6aTzNXma0RNNDKlACQ26mYPydsVZ1yGKC9Qoo65UFgV05hfIgs1wVPH69TseDcaMOw4fXl/S1w2KdUG6/JojbZ/xXovn76446/VwUplRrFHqJ4gyFHWB1C85v3jDjx89xt96bN7c8HuDNu/fX7F/eEb84gadGKuMsbYLeqWMZYlomsLkeo4gavQMhRO5T7rwyLKUE3tAHtWk6ym23QKhwg+2tNs94vCakTPESK7vz4lxlHNs91CqCM+7oiWqWEaGIEnEcQSTBQPHRBIloihA1RTqQsAwYmoE8pUPgsCepNzlKHoISEhyI1f6qJuM44SiLCFo2EtF8RGWO/r9PvnbC7qKjOxnZFmjq4yiCEEQUFQd7gLSq210H0OkahoA+d3I/WNm5EemUhCEZgwmNsxERU2eF6iqQlnXcHdXJBlvt8R2HMqiYHQ3lpckid3OI8uWtNttiqIgjiJ028JQVM6MFmprQBAEZFmBJIkEQYgsyWz9LaqqkWUiDzv7WI5Fp9NCMxXyPENWDKhlyjwlC0OORYXJ22u6bYE9U+PAGlPOpxyLOlKtsIu29M0CYbZC9XxMaUQ+SyhDn2NNYxNFhOGOttFCmCR4cc1DVaGl6MxfzHg26BOubzCyjL1xl80u5MvuCAcDTy6YLTzGmcbHRKm6BkkW0AyNm+USpWti2/Zdf3iL69tr0iJlOBzihzmilhJ4IS3ZIowTTMVgOp8hqgp1pVKJKav1BqlSePj0hDQOWCwWPHjo4nlbPK8iU3fEkYBuamwWEaZZU9kOie8xDWPiUODhA4nNdkG7e0Bdq0ynAYgxqmYyW8wRJIHFdo4iqfzJP/6C3TTgu2/eIHe6KLXBZLtizYzoZcaPHj3i9TdvefDsRzh7EplcsbiMONwfss1ijk+f8ObNJaJYM1ufs5ornB7+lPPzc0b2M0QNJEXg7ftz8sQhTa9I0oyCBWXmIUg5O6/J9bR0F+oW8+U7Wu6Pefy05NtvXiAhEfseYRShOQ6et6bINPaPetye31JWWxy7T7GLSbxb2spj0kgjq2o+ffaEq5tXHAw+Y768IgtSSkHi6v0V9mCPMs+ZLy/oj/qUhYw1KCnqJdObjGG3RyYWzGY7bLfb6KDzktvZLb3egLwoyNICahHNtLi5npFFEafHp9QFKIpAGSlMbxKiCPa+PODV82/YG7usZyG7aoWuOXTaJtvVjuWlz+c/qEn8Ere1h6pG1LnCZpVTFyKL2RV7e48oTYU81amjhCqC68WKsqx59InON795yaC/j5U4LCZLBGmLbqjsHTu8fH+JqmtM5jfIUsWg3yermxSbSnSZrgMGe+PfGlD+1qac//x/+6f1zgdBq2lZKoWfoMsKeSVhtzsMW2cU4poPFx/YeSFhkIEYoRsyea4TemtCv3HByXIzUpElgzgJSfMVuiqhaYNmlILF1dVV08FbVoz2xrx/PSXKFmhmB7urEsUbjg8/YbPx8f0dPWePs0dH/PznP8O1unQcF8vU2HobTMsl21bovZptHIOUIZNjKU0223y1outoKIbIcrdpBPTmAKmGJN1Q1DaaoKE7AoZxgNUW+Oab99TiBklRiJOKfruFQIVau2RpTFmmTe0aEZrWotVyWe+2SKJKWsQN25BDlkR0Oj1evr+g3ZeYzxeYsoumSojA1o9w+i5ekBJFJZJWY1oakiRTk1GLMbKiYVVdZvNbZFGm3xsSBgG6rtLpNL29YpYjWjKyZHH+ak7LtnDHCpVWUtYKx/aA95dX1BK0ugaKKrLb+GiqgWGZiDVsdwuSOEeSVXStRZJW7Lwl4/GYYLmilkVMU8PPdwhJjq64pJqCbUnMliWilKMpFUkQcjg+wQsiMiKyMkDIbFquSRyk5EWEgIzZ1onSgrIuUWoZTVSJq4qWbeFIIpv1gtawRZ0XXF15tA4tFtsAMVORhRzTkakKAUe3UPQauTDI44i5v0K0bQaDFuFuQ5TX+KGHUOj0bJNYi0graAmQBzK9bosoSDj5tI8oecyvS/Segesq1NsIk0PCKubkxOH585eYTp/TswOGVp+//K//hoSSk/0B309mOK7Oo8cnSJXF7e0trYMWHy4mZP4Mf9bGE17yo9zkf/bSbUwPaYYsK5RFRV1Xd9pKmbqqKIsCQRARRQlBBEmSSZMURZGQJJG6FhBEiarKyLOKGhFBqlBVhSwp7rRuNWUFkigiAFVdUVUFsqJQl/VdcDeA2FwNIVAWFYJAM26VGyeyeFdxWNUFeZqjqtpdfWSBLDdxO2VRo2kqZVVTFE1vte+F1HWO47ikWXQfwQMCZVlQlQ1LZzsWWVpQ34XdFEVGXqToetPIYpsWSZpS1xVV1YSN1xQUeYkggICIokjkRXG/8Fd1jXLXNCPLKmXZ7OOyzJBEGUXVmkpFUUQUBCRJpaz+tilHAGRJRRAFyqpAQEKWZLI8wzDMpmZQEJoLA0GEuqbm4z4VqaoSUZSg5l4P+XfvNVBVzSj1PjxdEimL8n4sXVYl92p5QUAUBOqa+7DzjzdRavrCBZo4KoE78CgKUDds5Md2nvrOsPPxcdCM8GVZoqqa91DXNaLQyDKaix+an0XhPr8TuGNCa0RRAEGAO8Ba19wdV9wbhJqcT6F5/bu/1Xf7rNkcAYSKsqqp66YiUwDyPEMQRKCmrOq7w1RAEkXEu+9BVZXNc2geJ4oiZVEgqTJIIkVeIiPcxRbVlGWFLMnNcVLdAWhFBaFuusZrEO5AZFlUSPKdeaqu+d98VrM9GZLEKbrWJDHomk0QrpBF8MMUp9PBDzIKtoRewunpAzarDUpZsdxuWWx32FYXSSpwbYO6UKjqGFMX8YKcvb0xvr8jjRWefLLPZhsi6wJhvCPwCxSxz9HhmLX3BlmWWU1DHMcizmYc7T+gZe5xcfmB0yd7zNdrrmYTzh4/YSDqlHnEdBfwH/34D5kmN2SoSI7G21cvEeqEv/ej/5j9schf/+Kc/QcD1psdy9spf/w7P+abF78hTxyOzsZcT1+SJCrD/hF7hzWX51t++OWnXNy+5sXLc9qtEcPhmOnkHbKkMp9nPHt6xJvXH9jfO0KUd3i7BG/p8umXHc6vvkXT9rBsmavLy0aXvIto9RRmky2H+59wM70lw6M9VEkikTyumdy8wDB67B18gqrmXF9NiMIVVaGRlzKDvkbHaXNzuWCV+jg2eKsUUe5x9nBMHK8p8pJSqDk/X/Hg+AR/F7B/dEiQeGy2S6J4g+koWEaXzWxFt9PHS5fEIewPOtRFyduLd+wfPSLY5KAWnD04IsllluspphZBaYCskJdbdPmQLIwZDWUsI+Pt64R2V0aVDHRVQxR0BqMhq9WKL7/8ARdXL9msluzyDVkkItY6tu3i9hQ002B2PeX06BGSFDPo7+N5NWm+5bvfXPCP/vGf8OrVX0ElUtUCutWmO95nsV0SpWs0pc3//n/4X/5WppzfGlD+D/7Hn9ViMuaHX/4e0/kWzY2ZrT/w8OBHbFe37KIZVSVTiSo3F3OGvRYCEZcfApyOiucVhMmUbrtD7FVUZU4t+kS7Fm5bIgg8slSnOzJwrDEvnr/GcCLCnYnpqKTRgkpwkXTwkx0CKopcQV6jq110QyGvUrIipy4hjQMenZ0iyzbBDqa7S9pum9HQ4erikqoWiZKCvdED/HDOXn/ELglY7hZ0Oi7beUyeZjz99IDtxiffpZh2j8G4c2duCVmut5RiRVkqqKWIZucgimRZSpYUlInMwXiPLBQI8iWSmjPcHyCKJrc3C3abLfvjAbvtlloSieItCDmyZpPXJmHkUyQBaZAiqTKmbZBlJePxGFWh0Zq1HLIyJPENkrhAUWuKAjqdFgI5k9sltqOhKQqoMmmSkQUJj06PiYuIRBDZ7VLSJMB2DPI4w9Z6+Lsdo7FLVhZECYhqhShCFlVUdUEcpVAJDAZ2o/8SKyy3h1ClrHZrOmqL3XxO6ZbkcYIkjpAQiNIIp2vTbdlkfopYadzONvT2NbI4w3UsgiCnVgp0zcaxdW4ur5CMmqQQcJweRZRgazLH4xGLxQLDcYm9mMTIeX99Sd8FU3SIE5O8jBBDBcmMQFLp2V2QRGbRkjDZcDo+5dWLGR3HhVLmZM+lauu8vD5nTxdR8j7uUCf0K/LiiuPhCfPolnkk0m3LxIuUH//BMwS5wixknr/+nuHgIadPHyBOf4W7ifgXkxRdeIjYFhDygLSS6YxdWnLFzW1IkOcURUayCajVkn/HHfPf+xcLKiqESqYWaqqyQBQVoFnQREGkKAsU5W7hrkoURW3G5JJMVTaLeFEUCKKEKFXkeUktiEiiiCJLlHnTeFJRQn3XsIJwZwCBsqibzEWpvl9U6xpESaYqyzsAWiNLEqqiIisKQeBR16DICoqqNgHLQgPCRJSGMZOgLEHXDILQa0BmUaJrGll+N+otsnvAJEsKiqoQ+BGSXNOEzyfIsohhuEDzPpIkvXPagmFqxFGKLN8xfpWEfJczKSkyZdkAxSZYW7urDKzu2k4EROEOpMtCo4e8Y/Cq+m9BWl19PHfeASRRQJFlRLFxbJdVefe3mqqsECURWVLuCL76HoQXeQ4IzeNrkBUZQRD/DaAoNLgOQRAaAKw0YAdq6juQJ4lis013j20Y7mZbmq1sftfgLfGuf1pAuKfX7p7zEdDeAc6qqu6BavURTN0xmx+BsCT9XX/nPRq8u0CpkWXpHrQ22uCPKLa6R5YfG4Y+Auqyqu6PMUkSqeqKIs+RZJE8T++BeZFnDWBFRBCa86+qKnefSXPRVNcldS1A1XwuoqRQV1UDMsXmwqsuC0RBRBBlqGskSaDIyzsHfoUgSNR12bxmmSOJ0v1+EAQJ7gxU/5OzDO/YRDcUppMViqJh22Yjc7JU/CjGcrss1h6KllKVEoqkUZWwZw+4Xa4QFYvp7BLXtug4NqpWU1Y6WZ7Q7qmUtcxk9g7X7GBaAxQzwHX3mV7MyIuEZ89O0FSTMqvZrCK8XUq7V7EObyAbksdwcNRl48/ZeBInT9s8+vSYL/ce8t/8828ZfyKzmsyQ1Jp379dsZz5nZ2e4I5F4KXN6+BBrYOK2FWzR5MX3b6jVAkW0qIUE1cqRRJvZLGI87CJQ4vshtm3y3fMbfvCTHqZywGxyzsF4j++fv27yV4WAPDM5PDxm679nMalxXIuyWnLxYQ5ai/3xkJoc6oK6TLEdBVlpTGCbqU6svyeMLQZOF1OsGibwtMtyseHl2xtmtzsMq2TYO0K3YqgVpEpjOd2g93Ucu+b8/QLXPaa/5zC/vWJ66+N2B+RFzWF7H4EtnXGfD5dXKFKBUNVEaUSpqHQdCYEKP40JdxYn4y5JvEM0JG4XEyzZ4npxze/+5Cnf/GqN2bOxrZS1V6MKTemBomVItYMpiyznc7qtYyp9xs07j8ODAUUpMxh0EVUdSTQ4PBry/LsXzMMLXHNIWQSkScXZo4es51t6HR1bttHtFkEIYXKJrNg4dhdJislDKLIt7nBElcg4TsqLd5ekYsb48IT/wz/57ZpyfuuRt1QdU2sBf/Obv+TZw88IvZx9d4y3vWLhf4DaZjHzGIwcTCshTmyoZfp7Bq+/m1IJNaZbUac2ZSawWt2gWSCKDTu02tbYtsTsNuW2+BbqipZ2QlzfoKKitz9hF14R+Dm27WA7Jrt1SZCG6K7IcrFBUUpsc0AheAzG+4RhSZ42eWiq1sJPNgTXK6glyhLCKCctAoJ4zdWkJooSRkd9lsspsqAjSho311seDQcs8hLDFHj+m+cIos7J2Zg8TmjZHWpRoqwCbFVntS5xui5+vqTdl5uaLwSEumKzjPH994SRCFREcUCr7SCKOparU0k5i80SSyqpkpBsUyCoBt09E8VooYg6YTClqnLWy4IsqxrTkS6g1H0yeUFRGvjpDiWXMA2dspIAgU26Rg41aiQOHxxyOZuxDRMyoaDbMdGlHDKLIqmo1AXdvkyeZdzOdggqKBrotYiITSmpSFLNaKhwO1/gFRkPzs5I8kbALigywdJDa3VQFAW73wj0n798wXivxcneEFmwSUWfjbehP3IpoqSpkdNEyiihSiUkReRg/4T5ZoUiCKR5xHZzg67ZyKpKVEYoeovZ8orhoMtqscLVSigsKt1hG80Ytjo4bptFUKOUNlWWMT46wKXLxeQDCjZHvRLBCunUA8rKIkp2lLuCUrVxOzWz6QVdZYTeGnCbJHhFSS0kKOI+yqAg8ROm8yWWkXE41siKjGxxizebYp7+Q75seVxsdrTaPV68v+aHz54S3Ap4+jW+sGE4fMjbiymfP3vKyl/yT9+/4Y+sFk8SBUkWEKoaQdbJi6xhYO5YJgQoypK6AmqRQiioRYmqKhsKrpCRFYmirMjzCsSG4QGBMrsDO0JFVYEkNECsFirqutHpCZJAUeQINABQkWSgprzrs5ZliTovEGiq/TzvTuwtKVQCJHnSsDhVE2ItiRJZnpGXGYKgkpcxjmORpskd+1qjqgott8V2t0UUBdIsQtMk0jRH00TSLEfTJKg1jLtwf1lSKKSaJKmoSxFNkxAosEwdBNh5AbqioqqN4zbLckRBRJYathcKoEYQGna0LAChuXCikhBEibLK71y51f058Z7VE5r4KoEcSawpqpQoTlFU7a6msEKWRPIip64LBOEOWAsiRZFR1XXzGYo1kqggCiJVXd6xlALiRzbxjvFtWLea+k77KAgNLmu2rb5nPP8umLxDkVCDKAp32910TzdcqNBcwNyBugagSncs9R1LJ9A8R2j+p3jX5FRWDWiURPEOIzZMuiA0B6kkNeykUENdNUmO96BVEP+NbazqEvHueQ1ybbYsy9K7Y065A3VQ5BWSBILQgMaqrhpzkSxRUFPnFaosUgglQl1TCxKiUINQNUz3HfMulM3XQhQ1sixFUSvKqqC6C7IvyvIO4JcIdcPUS4KEJKrUdYYsSVRlsw9nbQ2/LyFUOV4oso499FRlb9BHyBWCMMHpaORFjiI3Guk0zVGMxsG/DBdouki/b+G6J8yXS7SORp6m1HjIlsZy7WMZOrrapxIlNsESfIH1+pJR26VSNX79/Qe6lsNqm9A9MVEcgcuXG7BVVG3Hg8fPiPM1smFy2DZwnTXXLwLidyLPfjwk9iKs7j7z2xVGq4WuCEiCiMUxmrvm9NFjVGlL7nWQugGKXoItEW+XbDY72u4RliUj1jnb7ZS6aLG/t4eo1PyjfzTgv/i//lf88R/tU1cy1/MpvfE+UbLgxbfvce09vN2K6+sNnfYQahHD2uf3/ugJ7z68JU18NKvm9tajO1J4e3XL0f4Jq+UFWarTkzrYQoYuKJRCjmyHLJYeb9/fkIYVe8MOw32gLDD0Dlkes15G7B23iPIQfyrTNruMhhrb6ZSqEJA0HbGs2euPOTkd8e7VhjyEtmUwXV2jKi6j8QlZtiNNUzyvoDvoEMs+C/8WqTQxahVVNJDkknHnlOVUYdR1uVhfY1bHPD4e8eLXz3EEByGLWe7eEW5tPvv8MeOjLrIpUwUWptrCdBLCdIMm94iiivUqwNR0VN9gfjPD6eQEoUUYbWk5LnW1Yzg4otBsNC3FDA/QWhIvnr9h1NPZLGvaY5NNuMMwHEQdHu4PsJ0216vpbwsTf3tAKUob4nyNqZ/w8s1bDvYlZGnMh4tXPHx2ysW5h6KmXF+ugYwsnUDewrJlWt02n3/2gF//6gNhvaLCwN+pUNsIakYc14SbCksREEixXRnbGfL+zQJF00FVuXi3ZHyiIooFZVaj1AZpNGHQddgsl5imjqpC6Kd0u3ts1h7trkKS7VBUg5ZjMp1myLKKYZhkZUK7b7Hxdmh6G8vUKKqQMMyxrQGSmFBUoMgWgmyRCzuitKCgZDPb0O/bmLJN5PlkUcVeRyTfeRwPu+yCnEHriIubN1SVSrfXIk4z9va7+J5MqYUIgsD+/iFx4lMU0BV6zK5fUYkyYSbQbsFg1GEXJ9RyiqkbFGmBiEQal+wND1huFOaLKS23z+V0QpyFHJ3JSFabohJZ76ZojsZgPGK1qFE1g0IUWXsecZLQsmyWmyXe7RLDtDFF0AwdxJIgKMnTFFHQ0A0VuW0S70K0OmXUtwiDFpezBbJs8vnZE2bL99SSwHIJEhrtfovIn6NIKXmao8stPvvkS/x4imm1UUSLt++eYzstZL0iqnPclkWQhhQljLs9FLnmzfNvOezuc311werGx+2LpGnAzbpkbU04OXqGiEKeVbi2iRDK+JsYqoTjgxMWkwm6a2GbGpqo0bJ1Lq9miLqKqlgUSHhxiKmUaC2TyXqDIEXsdVoM3A7r9Zp+u4tU5UznU8JawukOqcuE5XyBJlfI4xaqnXB+OaPzzCVNttTSGMn+B6TSMQfjmu8vfsb+gYQpFwhlRXsgMl+5HA4tLEvGvRF5cfGSJA/pd0z+5z+Eg13JQHVp6yZlWTHbXLL1U9pum7xKKBEY9ges5lN2u5KTxw6mbfHh7RWWLtIyhywXG/K6JM9kdENml26xbBdH0TCQWe3mSGoXTVSBirXngSzQ7bbQVIlgu6CWa3ZrAUOS2BsPWWzm1HWJrZn02n06gz7v3r7GcRwkReLyfIpiyLg9l9CLePxgH6qcxXzF8eEDrqYvURWNg/ERslAT+D6Pjj+jrgUkWef4tMXrd2+4vpmi6YdomsZicUNVS8yXK46O91gtdzimhSTVaOIeDx/s8eb8a3qjA96+OSeKPTpdAUPrUbNPv6vz4eqKONeoCovb6zm//4c/4Pb2BkFOabfbXJ9XaLrI4ydnvHzzDULdIi19RMFErGrWqxWObd2fE+OkarRI4x6mLHN1/i3IMis/RtUd2l2VPIZh55D56oY0DdA0EV2X0TSDKJBQ9QJRkDD0PpIEkqijyAZhkKBqCpZp35k6VKgltt6Gfr+P73n3EUGKJDfShbJs8i8N414LWVVQlgWq0uQ/qloTQC6JIggCeVUiC03daJ7nFHWBafytk11RZIqiuneRy4p0B3gb8GkaJlVVEgfRHUhSKMqm71tSlDsmsmFKszxDQKTVanqri6KgoqQsK4S6Ge+H8ZqajCBq3POKqlIVNYJYY1kSptJnGyx5/+ENEg6GJTcRT5bMdhtjGAa2LeGnJWZhYOs6sZCTpTmVWJNXKSIaQtUA5iTNQJKI4gjLsjD0Np4XUAslimKRZj6a7KAqFVEQMewcECcRLcdB0xx8b45h6Gw2a2TD5bXkU+UWXdcg2xSIMdSOxMqPcRUoapkq6hOnE1StQJU7CGSYlkyxTUnyAtdqs1pvkRVwWipVmSFUsDd+zIfrl0ReQV0L+H7O3vEBuhRR5DWb3Q1qWLGdlljSmHZb5uL6DeX1Po6tMjo6odXrIqkBpimwul2xt/8ITa+JQ4vDwe/Sb4vsoh2llPLXP3/LflfDcTMM8zGoDq2eRuR1uFlcMnAFdv4MKZe5uZ5z/FmHm5sZD88+J4ivOTn7MVdXKdQyoqCwC1aYlsrL50v6/SFXk19SVjL/8N//Q/7iz7+j7Tzi6ecwn22R9Q6tbozlCoiolIXA1fUFWaJjugJBsqY3PGC81yIJIM0rth6c7Fl4uwjH1djtJnR7PW5v1njelM6gxemxhapobNc+i8WMwyMd1+lQ5B6uPWZx+Zqz0xMEOScva7p9ndqfUNQFFCKKNeHF60scs4uf3OAHFWksIUpbev2HvLuccXW9wlAcijJBUyvCbcx+f8jeyMF7F0ARo4oCaRShY/Lpkye4kkmc+hzsjzjYP6Sq1uiGjH12wNGZgRff4ig9/vjv/wRNK/l//4uf0eod0u5YvHl9wXjcJctjDGWI0Qvxs1t0pSALKhRnTbgu+W/e/EtOHj5BKmy63T4tq8v+wQ5NVKkHJYvdOw5OP6EOJE5Gn/Ih+g0FEra2/1sDyt965P0/+s/+3frlq7eIcs7hwTGSHHC89ymaeshqPUe1t0wnIarcIgg9ltOAk+NHXF694Pi0x+Urn+lkg2IkKKqEIJTEsYEkSVi2jm3nLOcBIjZ7+0Nubm65vlnjtlQG4w6CWLFdwWozxzCbUZ8sKqiaiaK0cByN5eqKLJHxgjktd0gYNO/NbtWkJey8Bft7h/hRiKzU2E6LNM2wbJ1gu8LfSpSCDGJI17Wx3AHeLqSrmlxdXaBqErrhkiUSg2GbqmpYkLLK2FyHHJzukQs+sqqx2a05fz9D12wOH7hkcXmnCU3J8hCRNqarYFg6hqGzm1X8+vvvePT0jNlswaBvs17GxEWAbMrIKAz7AySxJvJyAGRFJI5SnK7KdLPBdnRKAcJABWlNWW0xhHEzYgxKLN1islqgmRqDdpfFYtWM3OIAq9slywrarsPNzQeoNKrCoN3vUhBiDSTsysHbrQkVn+H+D7h49YLPnjxis95yPfEwVBlDVwl2CyxdwG5r1KJEEGVYyLTcM3JSWu0Bz59/Ty0EmEaHkpxCTnEsF28dcHpwCmXBbrOkqCv2D47ZrEOiYsvDZxZxVOD7c1TRYr0sWC4z2rbI4dmAqw8B6+WGw+MBeQ0iBY7aYr6aIgpNXp/jdtjFG+LM5+GDR7x7/5JR/whT6lNRswkvqHx4/PATlqsNcbzCNE2iPGS+3VLUKpat03ZkpFoliSTaXZ+Jl9CWh3ScgmHXxdCOeX05QSq2KF0wdZvzV5cIcs4nn/2QNKmI45RdumRgunx38wFJMXAkG1WxmNxM6do2mqUyuZoiWxKKapHFJavdFZWsomodXLvEcFoYiowh66wmGy5vbmn3dB6ePSGOlkRlyXY9J8s0ikpB0gs6Wgu1LtltIyzbZb328PwQtVMz6B9iFCPWu3Nk3YNCp8wEev09VpsVru3QdVskhcfKv6LvnuKYFkG4I8lFvGBDy7VxrKZOrvAj8iLm4cMDFstbBqM2t7dzdKmFpZoMB3vkWc3hyR7fff8r/KBCFDRarRFb/5pKyFjOI6I0YtAfsVzNMFSTk+MRhnJK3+xxs/6KIA9YRxdkcYmjH/DjL38XRQ/5f/yXf8Hxkx6mI/LhfErg1fzkxz/EixboZuNaliWb45MBy+UGoR4SRkuCZI0idlisF5wdj8iC5P6cuF2n2C2dsgq4vdqQhjmnT7rs4pokkLC7OlJdoucWg8M9iiRiG25ouUOmsxfkmUCr1SbNU/r9Lt62QpBNdM1qxtqiRssZN8devEUUVQQJqEVUVUVTVWRZJAkDJPmOxUS7DzL3vO1dRuWda1wCUZDvQtHN+5ihj25vaFjlLMsYDAb3jTpN1aR4b7iRJIntdn2fj2maJqHn3xt7PpYJfRyHf7zJstw4ye+c6ILUjLk9z6Mui7tqyIp3F7/GdV2SuMJ2a/IyIEsEXr38gCoKtNoORVUx7B5g2hK7YE1e50iyTbQNsFsurY5OvPFom0Nm6ynz7TV7B2PeX01pmza62ISie3GCn0SYtoWh6dRCo+XcbjxkBaia2CjKJhzbkF0s3UHVc1TJpYgFHNciiC8J/Ix4F9AZPULVc969PGdv0KYQRUbHp+z82yauDR9dMzl/H6AZYLQyVNUlzULKQsQyTFp2i9l0xf7RMS9fPqftmlAXJFkIiDjumKysGOzt8erta44OuxR5zs3NCttoo0kJVekRRTqa0eXhY5tgI+GYDo4rkORTqkKjrGS22zWWXXI8/Iz2sM/V5CvWVxLtfZmO8oxV8oof/+CHHBzuUyk7yqTmw8WCcCXT69dMZlu6To91+oYigk7rmCC+xnFaJFFTK6qaFVfnO0Qx4pNPPscyHd68/xXrucCzH5xxfbmiLASMjk2VZ3z++RdcX7/kb/7mK3ruIfuHHS4uPhCnJRkxGy/kcP8xy/kFWZzw4MEDPL9CEVeUiUtWrBCEmPk84vjkIedX5xRFgqpZIFQcjM8o6x2T6QWPH35BECRMbyIW/hWOYdLqdJuxctdk6/mIUgdFKrFaFefvb3BMm/nCR1FaPHi0x9e//it6vT6WqREGCWFU4Lhq49LXNKpcpmO7mLbAbjNHlA3KOmB9UzM4arOdr/GSJf3+MQ8ffMLF5Xv6/S66IbNazgj9Ase1GOzt40cfyJM9Pv3BAzbBLUG4It22OBjuMzhIeP3NFbu0wtQKttuCo8f71NGK9+dL9o5O8NYFB4cGURqg1D2KIsIP5hydnHI++RXpwuSTx59TaRGrnc9Pf/SH/Hf/0//s3+7IW0BtWhPsPXzfx9vFPDrtoKkqeVbRNQdslksMM2fQP+XRWZtXb75GUVWurqe8fbPEcQ0k0aKqAr784if86q8u2Wzekyc2334d41gyx0d9ZpMF1x9uMFQbU1A5/+6ag8/3UbsLhlYX2zJIog3TmwxB3GG6Ce/exOwfq1Qk6JbE1luRZyKG1WK9yZFlmVH/iNl0RavbpiTkdnaDphqEyQYhBVHQ0UxQNZ26lImjCt/30aySbruFaUkkqYrRrSnLmm0woaiGKFqB1TWRHB9JUAjD5qQ7Hu+z2lxjG3u8+jAjTrfomsig8wzDzXC6Kre3Wy4vZ6R5AqrMcnvdCNzrNnUqcTAeUYslKBph4iOLAnFWUhYRhqFgOy3CaI1liRS5QJyliGJFGPnIos7wxCH0SlAltvMlhqbiOja71bpZeNwWYRgTL2aYhk3gg4CEqitIogJ1gixVbCcxtVNhOAKO5rC4eMfeQZsoD7m+XiFXAoratHuIGmSiTBi4iEiYakGVVfjBmtZAZbl6gx+sGPTG7A1PyHKf1fYDUipy1D9iM58iSRKLZUSr16IQMgb7KqIhsVzsODwa8cmnx6RJQF6kXL5bsLxRmdyuyROdRw8es1zfIGgaliGRZk3gclFFKIJGUcB6tmb/4IDJlY8itVlvfJZEGJpC5MVYso0oaXy4vMDtqKSeRJoXOEabJA1J/TWp2KXdspjtJmS1zuDJgMXNmttzn+7whPerc27iJUl0iVF2kcs1o46LY7coZY+b6wU9s0NbPePy+h11IqMpKrtgR5mt6RgWmi6zTSOSPMAo27iGgSbWTGYVZltDFQ0UuSAOZDZBibd8w9MnXR6cjbic3fLm/Q2PH45Y3E4QLRWpEig9n97YZbPZoFYKdtsgjQJGe132RmMm4ZSyigjCN8iKQM84YbpaICsK14sbOp0W6+0CQazZBGs0x0YzLTq9PVbbJXGR0+4O8TdL9kc6eVWS5Uv6gxHnl+9oOwOW8xQvCpFbLSarBZeTFc+ePeVy+haBMWHwnnY/JkwCJFFhsfaQVJme3eZ2co2igqLLxGlGGH4gCK9Yeefs7R8SJF1+9PvHWOqQlqPzz/+f/4pCCrGdffKipuU4uJZCEO7otnWWqwmnJ89w7TGT5a+5fpdx9rhAUWOsuke3Z9MfyXjrAG+V358TDaVLHu0Q1Irf/d2fMuod8vLXvwF5ysHeiM5gzGp5ydHZIzZeQL9vY3cE8qxEkSr2jsb4XsJ2FtNraxTFFl1XyYotil6SxSVZqaDWKZvdLbrsoNvtewayKAriOEcWBCRRIUkSICVJGjYRSUbWVNI8JUkSTNMGUUS3bMIwxDSbrvogCDB1jSxPEEsR13XvDT0fu8Y/usc/5lTKcsNol2XJcrmEsnGkS6qCgHC3Ldw3AH00CUmSQBhGDRitRaqyauQWYiObCKMdklyw86eYlsNis4BSwjL7PH6yhypqLBceg26L5fqCE/cYVTVYTNe0hxKymtMbmJAVHB0dsJguUJA5PnxKRUjHbFMWGbUhEpf/P9b+JNbWNc/PhJ6v77/VN7s/+zT3nHvjRpeR4azKLIrKEmXAIBASooTMAFEMa0iJIVMYwRzElJlVAksUoLRdbtLOjP7GbU63z+736tfX9x2DdR1mYIkoyVt6R3u83vVb7///e56KpoOuUYl3NUpPYxvuEMUa0zQRKIjDDFUT0BSXrhVIs5CybJHinK4O+NGXf8by8Ym6gzxrkBWbni2xCyMGJy5+HGApI77+3W+RbQdHLejIoSdwej6kKBPSMiVJPPqDMVnl88033/Gf/OXfo2sPys3PP3/N3luTJRWm3SeINwRxQm9ic33/kdn0lPX6HlN2qRoRVWqRFYGSGeefmeRxxLffXCGJA37wmUMU1UjimKx8ZPWU8fnnPyDcJ9jGCSdzmTi55OJnU7744TFfPptTdT/jr//Vr/jtN79C1juuvo0ZHcs8PTzy9Vudn//8L+mEWx7fbzk/e4ZmleRFn6LIOT39IdvdIw+rt4R7g8sXE1abJZKwZbdLePd+QW94eM0XpIRf/u0Nx/Mpv/nFLzGMhlGvj9ClLB49VMGlEWM+Xvn85E9/SF15pPucfm/IevMBSbLRlCN+8CdH/PqXO9pqxnRSEgQeJ9MXvL/+6rDuAdzd3WCYKrIw4unRw7ZN7F7LJm7x/T39yYDdLqbMGiQt5+WbAbunHQ8fM86OPmd+KqMotzw8xDjmkJ//9D/m8emaMj3A+L98/SXL7S1N15KkKaps0CkFq2XCcGySpQ0CCsPRmDRZ4oc7bHfCfO6iaQ3jwUtMM+Xm5gHXlbl+t+FP/z2N3kBG0z5j50cslj5hukQSS2RFpq5DXl/+FKMq+S//X79GHjiIrcB6u0XvSvw0o117xMmS56//PRqxJlhFvHx1xMe3CcezM7JkySLNuQ6+RdNaxNSiibw/Nib+8S+U/+n/6k1n2vBwHSMIBn13gCh1nJ2cYhg1QdDx+LBAtzTGwxOsXklRhlx/SJnMdMJtjuu61LnLzcMvcV2VJIw4PZkz6J1h9lTGI4vHe5+2khGFhqzY8utf/ktmk+eYzwz8IGa3FkiimPPzQ1AKggrLEaCFsopoG4eijBmPx9Rdy2YVYlk9DFVBMy02m4CyzVCMHEGG0C/QVBOtlmnFAxNLEoaInczSWzOdTiHO6GQTSUzZbiLcnkIc1HRtgYiDobfMxi5pXrML9mRFznDUI9h3PH81wfP23NysDpaJnku/NyCKcz5e3zA+lahrCU0bU+QxotCiSTrTyZD9tkCQWzopJcxrNFk6NPXihiTZ0es55FlHWftUlYJhOiCmxOlBfWbqPQYDmTRNiZMGRRBwhn32no+/2TGazhBVgyLOKeIdQ3dK0zSIUoskHXy+knSAJi8fU0anGnG+4/XlJfGuIwg8gihhPh0TizmaaKBLCl1XkKUlu11BWmZMjhyOByek+YYkybBdi6oEx5pgmgJpHHFz84nLZ2/omhaECgSVfZCzDdbodo3aGSDkjCd9PC/g9MLm5eUbpkclH79J+JtfXqGZEkeTS2S5ZLNboxjmARzczQkCD9ORKUuRrhGRpY7J4ITvPlwxmgzQDt+1tFXN8eiM0I84OT3n3dU3yHrH9dUa1z0wPes6RVF1JuM5fnR3wJoYcP1pz8iukMUTyjakP1GJUp1w4xMKKUPb5XSoUbUaYQNyV2JbHtfXEhcnL7i/vkV1dPK2pskK+o5LI4If+ehiS9PZdE1NtI+xXQdZ16hqFX/3QEGFYWk0hc7riyHL5R260yNMS/r9hiY1ydUKfxfiaApZ1aKpNnLbUtYNJ6MxTdfS5hm1bhAlCaXvoyoT/vP/9f+G/+P/9f+AZHTUNGiSii4pxFGApKkIkoytmMRezNFJj4f1kv0uom/pvHzxGVGZMem3SJqIv8loC5/Toy/pTQyWiy3efcOzF32SWKYRtgjIpIXPfH7M3c2W4djEi3ziOOPZxUvyPKUqDggc2xpxe3uL7EZQO0zHDlmSczR32W4Ses4RR0dTVsFbQGa7qrEdhZPjKbttgCiU7LcZbv+c8cRm8bjlhz/8IXmzYPmYY7sCUbZnt4zRNQ2x+Tde6jRNQUzo9U00xWQ8mhGvao5OFER9zHqx5XH1EceZcX56wcq/4v7pHteYYpoGcRiTpimmOUVQWrK0ZJ8+oWkmtC2O04dOpCoSNpsdujJlMr/AMCyKrMSynIMVJ8sP42FFoaoTBBRUVaehwzT1718aG3TDoqnLP/jARfEQGGXh+2lHkqLbDl0n4HneHxr3giD94TVT07Q/FGdkWaYsDy+aqnSw8bTCQTupadofVI2qqh6A65JMVmbf/++gfMzzg6WjrkvqoqTsQu6XXxEEAePJnKzwyZMGQx0wGB7WfsIwZDjs47j6ofSiWWRVQt5FWJKL4w5YfLzjsy9f0tQpfWPIIvIRxYK0rfDDkMFoyGazIQ1SZElHVQ3COCXZtwz7FpKSo6omIh1pcrAntW2LZSo0tUzfNWkqsAyFYf+ILMvQNRdBa6iSHbbVI+oSHvYbXKuHVDWkfk1Wl9RNiig19PsugZ9SNhENObJ0Sk/V2Ky2nJyc8uLVC/bhGtM+eNn3O58kSwERQVKwHJtPtzd8/tlrqiwlCnwmx1Msacg+eCJqWkTVx1VMVPo0ncJoqHJ/vaTnahhuyePTLSovePXqnMngiCxJCbKE84s+tx8CvniZ8NVXK/JSRpvU3D9GaHKL079EQkA3BB4fIn7+kws+3X7i9YsvULSCwOu4uBxyf5NAqyJZW0xjgO/dsl/LBxnEdklvZOM4DopW8+nqHkM9omk2pEmLIuuH8C3UPNwvOTl9Q1MVPHk+iuaQxfecHc1Zbj3CcM/p0QWWYfBwv6XX18jTGEW20U2H0WgEUsYvf/kbFEXGsjV8f49rjfn0ccFPfvoDwnhB1opIncxyHVA1Aj/4/ITlaoPlDDk7bxnYOo/XDpra58c/m/PNVx/Ic5Ef/fSCd+/vkPQGEYE0TfG95KBBVQ0kRHrmkMfdA6KgUtftQYmbqJhmRVF1fPH6DX/9z/8lX/7gZ8xPDwSMqt0hCyYXp5ds1ytW/ob1uuTHP/0Rit5x9eEj83mf3SLAshzG5jM+u3T53c0vuHje8F//1RpxYCNkGjfbj/QNhzAM0aQjfvonL1guFyTFhrPTz9CNjpfHIxIE4m7F6vqBF7NXPN1v+D/97//5v9uW99/7Ty+7NE3RVRFHe8Fut6OqCv7s7/wcSWgIQ5+ybtluAoYzDVGOEQWLppJZLG+xlSmrTYhli/zwp2cs7vYYVsIP3/yAd988ESULfvonP0ZXpnx4f8/8eMTN3Xcs1yvqvE9nB1S5jud5GLpOHhe4PQNVVdl6PmWbH0wN1YA43dB1AkXe4jp9FO3QRqzaCk1zaAXIcp/T8wvWy+CgLqxFkELsgUPXWHSVQCVV1E1JsctQDA1dV3GdIavljiQNMI0ermNSNT623efxIUS1U4bDMVKno6oVhi7yeJegmiqyJLLf5IwmNl6wQ9IE6lpkOO1R5Q1pdODCZbmPoqnEeUYtVExmY9I4w7IcJDoCP8c0NMIwxDIHiFJNmwvEuY8gJZjGGdvtGtMWMHQXz9ti6haDocP14oFakHEM9w+qt3yf8XR7jesM0BUdZ2DSCSW6ZeB5Iaqi46oqYR4TNzWS2tAkcDQYcX9/jz3R0NQxe9+j6Tr+5Oc/ZrlZsN8uMWSdNCyQVANFKQg9kV7fxOnJrNdLZLlGk8fkVYcktgxdlzxrqGoRy7HoKFgs76jqDkPXEVAQJR3LlonTDUczl90qYueVXL6+ZPW0oOsSzi8ueX99xWTq0MYu6+2Kk9Mpiqyz322xDJPIT3n15hXL1YqmLYlCH7URaUsVa2gwGs2o6oRKzFluKkRxw8idsl5vmU1PeVqsGM9FFMnFchvisCIPY4zeAEnq0EWdtBbppJKchLHropGy20p0msrZZzt2jxWp94auWyG0EgUyD08rXl68YBd7bHYr+rKKM9DIMombmycujk85OTrlYfHAarVlNpmS5AE4Bbo+RSkyhKrCGUzwE580zRkYIvuwJqXCmchsHgVsQWE2VVltS2x5zOPdR84vbGTLpjeaUqURVS5iGzqLPERQoaoaYi/h8uKc7e6JXq+HLFg0RUlZpvT6Fputj6qJtGWJqvXolAJ7KNG0CuQqFjmWdIJg7vDWW/47f/bf5/zygq++/j13q7c47pCbO48g9XCsSxBzovCO+fQ1ktAQJUtUaYSqV2iqy+JpR9wVjPsWAhn71RbXntEbGtSlhm3bxLmHqauYuk2vryFKUJcKfrBFbGW23p7JdICuq0ync46Pj3l6XAISNw+/o6sOJpuHh+gPd+LLly8xdZWnhwX9ocJwpDDq/4DXz17xt7/7lih44uWrSzZ+RuLtubp7i2aZHE9f4W09JEEmincoukAnp2jyOftwhWEqhEHGdHxBU5UoWk0cx3StjGUP6TqBqmqoyoZRf0gcpwx6/YPKUQJds2nbjjCJMU3z4NrOa2yrD111CICqfghBuoooQhyE6IZK00kHDJNt/yHwid+34SVJomkOpp6maTAMgzSND+Dz74tAdXe4a2VZRRBFqrL8fqQOingoVwmCQPd9ezzPUxRRIssS0iymkxMWqw9IskaRgyCWVGWGZY6xtD6y0mJYDWWZI0kaWQKK0bLer0ECWenoOSeodYPmqnz37Qf+/Kd/xj4Lubt94uz1BVefboiihLEzJItCFEVin/hIhsrcPqaqM9IsQOxkVMUmDCNcV6VuMqTWRFEFmjojTSrOT4+QGPHp0zXz+QRBKsmqCNd1KZoadI3jZyMePlyj5VCrOmUVUeYKTt8kzTyKEgbjEbtNSLHPODs7ZbNa8Cc//1MeV0uKuqLtStzeYee/LCq6VsF1BiRZiuvYtGVC19ZUTc1kcEnWLLj37jG1Y2zJRpYCyko4rJjYEvt1jGQkSIJDkjS8eHFC4C1x+kd0osfuLmE8Pebh/leMj85xrB6i3CIaKh8//JIkGnF+fE6U7hFEFVNJiKKG0bBPmlT0HZfP3pxzfX2FzJTpqcV2u+L+7i293gWD/pSqUlDMPaFfMJwMeXzYI0ggqzmBv2U8vMTzPNyBThQm9Poz3n37AXt+WOFwdYU4yGglDUMfIpYVvb5OkYeUZUqeapycnICYsN3432PVmgOWThBQ9YIgSKBxmR8N2O6WxGXNwJpTtTlZ1tB3R6TFnlqpGbgKlqiRBg5vPv+Mttuw3URkicizF3OizOf9uwV/589+wGJxz/IpZXY0oapjotBHE0fEVYSqQVkc9oZ122VgNaz9kDZXMVUN29GYTk6I0xBN0RGViPOj17z78CuGkzG7XYKsqZjGgOtPv6VnvsRyGuI4pk40mjzh5AcyJ8cq/+U/uELuGShCRlyauDbc3T8y6M8Z2DZ1faCuOI7FZ89+QLr3+XATMD0v2TwsMNUZx89N/nf/+T/4d+vyDpOSutY4P/kRo7H7PRdN5ub2Hcsnj/V6i0jDaKpguzFZHiJLGlkeYRojnKHK2cWAn/zJT8lih1dffEbVKnz7NqIVLaLQ5R/+w3/OX//Nv2LlL/j1V98QphInF+ccXer09AmXl0do0hDHbvnz/+AzZsM37LYhotyRxRqKIlFUe2zbxTLGSKhIgshu7dHUEAQeT0+f2G9XmKbL5nHHbrVFERskpWXYP0cWHRAOQXK1WhOGIYPBgLrKaaoWz9/RdSKt0FLUDft4g2xCqe9QhoCuEFcRimGy90KCMGc8t9GkAZqqc3ZpY9s2R7NXqFpHr29SpgKBv0USFYRG5Oh4gmHMsF0HZ9in6mQMU6OuOvI8J01jyuLQNvU8jyxtqJqQppCwzCHjkY0iGxSlD0JBmR8QGnleYtsug8EYWRSR6Xj31VdUaczzy5dYuk7T5iwWjzj9HnXb0nQ1vb5DI6VgHF4astUaS2xYZgtivWYTZGzDkIaK0/MZeZ7z/v17ojQGRcFwJyxWKUVVIMgRRZlQNw1+nJDVIqJuIGkqKFB1NUmRoxiH8X8Y7XAsF8WEWmgx+zJ1J7HNHxAMi21S0eoqs6Mpstph9lX64xFe7GMPDBo6agLcoUUQeOw2ezabLbY9RTVHxFnO/cOaRhJ4+exzzqcXjMY95s+G+EFAmZXUXYkgSyRZQVrETGcj/GjL/HhAkTds1h6Gdo575LAMajBUkrTB9zqiMqfUcoyBSEHKU7DDcmwuJjrr7yRuflPRlxYcjed4UcBic4/lyDwsP1FXOfP+FFUwSfwcoSuROoEoTNnvtyiqxHw+ochyNNGkyFvSIiaIUuKwI08lNssNVSjhLzuk1CbfVShig6O2nB47KCqYqkYlbbl84XB0ModWIMtDomqDO4TlfksndmRFg4CEZaooWktcpDw+bWkyAUXqKJuCWuiQRA3H7CNpGrpjIYgd+53IzW1Kg0AS1xSVz2L9yLMXXxBnKf+P/+ofEqUVo+FLVpsUp9+j17fQDFisH7l8/oYkqvn6u68p8oqqLuhamdVqRduVXJw9I88ypsPXWPY5+zhFtQyCxCPJC6q0oy1g0h/i7wKWS5+N5+HFG+LmltX2O3x/h2FY/PY3X/GP/tE/oMhbJLlmPnmO485QdZOzk6M/nCTcEwceqqQfJhXaKWka8+vf/4rp2EBWKx4el0Rexi7YUJQRQQRmX+fmbkUYCHStBE2ftjhitUhQBBNZhMloSFsdXvv2u8P9VdUpae5RVD5xukJRUqrWoyNENzuyYk9dpXjBkrrJcG2N5nuDSFXHVPXBM65oMlmRohsGumEBMr3hCEU2kGWZfr9/QB19r3xUFAXD0Oi65v/HsCMSRRFdJ5CmOfBvRuOCIFBWh5LPv0YCtdXBF/6vMTuyAHkSosodnZCCkCArGddXbwmCkKYuGE9N6rrF1Me4lk3b7QmDFEFS8IKU5cojTQrqJkdEQJcdJGVKGIb4Ucb9/Yb+0OF+/ch6syMs1ggFDHQHuehQEQ+A/jLDNXU0sSMK9wThFlnSmJ3N8MOQ8aRHlhYo8uHHSZkXSLJA37W+3wutubw8R1JqZoMB/X6PfZCQpxFCWvL49oY6aemPx+iyTV2DbgokcUFVyActcNYid0Omx86Br2r2uL9bYlkWTZOy3+8J/PTw4itLB+3lZomlaXR1S57nhEFKkTf40ROdoNLmJkXS0nYF6aEbQxAvgA7FrPH2JXmdEKQeD6s9knrE8u6JMjM5enZC2mYM+s/Z7hKKsuW7jyE31zG29hLLVEjLOzRNJYoEVNUlSLZkeU5TCwiSwdPDnv1+S5Q+cX3zHs/zMcwxpi3x7buvSIuSolKoWo2vv7mnFUC2axabgqozWa53tCisNksM00FTTfI2pkgNqDVWjwvAxM93JOUGsa0RpJJef44gGTiuxv3dA/ud/z1GT6GsUixziKo4lJVCVYuYjomiSkxnPfb7g/2qrCIUrSGKNoxGM5raxOkf0bQD7IHF/fKW2xsfWZARpIRf/+4rnjZ3DPsKm6cllqbj+WuqOmG3y9mHNV4SoakicRzRdCl+sKcRDne1YQ+4eHZ6kBWUHabRx7Udjk/GFJnAt98+oRsj8sDi+HjE7fUVghAgiR17/wZFtUCRCNpb7JnLp48x/+f/y9/Sn+lslylub8rAUgg8kc++fIasNwz7I1RJ5/F2TZWJfPj2ih//xX+L/8nf/wt+9OUP+MGXf47ZVwm96N8eCv8tf3/0DmVRQ5OWLJ62qEqJ6dQoeoGmjrF7IEl9omRFr2+wWiTIis3d7YqulZhMB4zcGR92b/nbX/7XIEiYK1BFm4Q9bSMiWCaudc4+zTFNgfF0jioauH1QtApBOELVRDTd4OPVO/w4ZeWv2Hg1Vk/F7qnsfZ/R4II0DvA9jyTM6Dl9Pnv1kg8fV9SVRVlWWOaArhKpqoY8qagsAdvtsdw8MJgMqWuR2WyE3rN49+E9mRxTSxpFV1HmCZICI1dD7hSG/TlZtuXu7Y7ZicLDQ4LtKmzie6q6wLZdagT84Jbj41N0vY8fRJh2TbQQGM8hyz1qQWQbPqGLLs2uoe3GiJqF1MoE3hJZbnD0MUGS4Domo+GA1SpEtWWqMsWyLeo2xTQGrHcPGJZEXpnIioHThxoRRdUxOoijArFuDwUV28BxLNI0xh3qqLLKPuxI8xQvDjEMjb2/ZhMtaTuNthUw7T5xNyQOn1CkFlt3EBWFNMvIy4zrb++RTRNFlVmu7nE0Bc0RyJuMy4s3JKHDbh/iDKeYloKX7OmqGkmQCfcRveGArA2I6wRbMenKimiXcn46RaQmTt4zfWYRBht0o0IUbBR5QJqmB0uQaKGaEkXZUuVgyh15XmLIOmEQ4LoOG39L0ZRswiX2sOXJ2/Lq/Ec8PL3DHErc7D4xm3xBm+TcLq948dmPWT4kbFYe/Z6A5ViUdc54PKXqZXz8+JE//fPnfPH6EcOS2MQNjRAzGOh0rYSemXzx5RF19RM2yxuG2kvcV+fMny2odxa6DaPpmOj2idn8lN3eY2o6lGFIbkCwzLGGNfP5MXGQIyoNiqjQNCk//smXfPO79wiijCRIxFGLqVeEUYZlDDGouTj7gqfHDXkrMEBDJCBNC2TTxFShUDukXMCyzlis3rN93DCYTImiiqBIMcwORRwg1AqaLLFcbZnMzwg3AYrcUOQRhtXDDwLGgylpWBAXJdagpctltk9PCJpCEuiMdQMv3FJrPVb7GKnu0xk6b+/e0xsohFmFRUvPnpCmCV88f0ae50haSX8EtmOSZxk95xm7zQ3DmQZZyfHoOcv1NaZjc/TihCLv0NQpuiJjWDrnJ5dsVmtu7q8YHg1Ii4aWDnc4pW9bgEDkS0xHM7xgwXqxpREDJpM5vaGNZSsY8uQPd+LD7RWzmcHnb77k17//PfswZGY6ZHXM17/8RKdIbPxHTubPiKuCz7/4GZsw4pu3v+Ho3KBvVez3JdPJnJvFey5eTcmihii9p2s6HMvFcqCsRAytj4DCerelaRqqIqetWvK0QpZU9nufumqZzx3+6q/+Ca/ffEm/77Jc3R92GKV/HQgFhoMRkiQQJ+Hh5bM7+LN1/YBh+teg8bY9YHiapsE0D8aXKEr+oFr81yPxOI6/t/polGVJXbeEUYRlOei6Tl1VyKJM2zbkeYqmK2x360ML1rTZre+I4h26rtHrDQlDgSKH/brgaHqJprQE/p4yFRjPhyiyzHT6jDjyD6IFScKxesRJQVKkXB4fsV7ckNU5qmiTbwMUXUCzbBb3jximRn9wCCKqbdIfTSmTHL1qSMuSk6PnVF1BklSYroQoKdRtx/npG+Lohjwz0EWTjho/XGGoPcoSulYmrlJ0qcURTTS3oyorHHmKX+/ZbCouLkckcYsiKUT5mvnREU1TUVfQ61d4XsPQFTGEhrTYs/70gNsb8PLyc1arNZpokoQplt1g6A6aCpv9Hbo64ui0TxDsyQuwHRtLGSBLBqKcAhZJ4aOikqYZZXUINattgiTL1A1sggWmWrBfLvHqFNEaYxkatWCgqiIae7wAzkanaEqf49GMm6ePlKVMWfXR1Dlx1PDi1YS20FguHzD1Y+L0CbFJiUNQZJuTk1PK9iPr/Qfm2jl3i1vqpiB4iCjwKBOJo6MBbd7Rnw+Q8gFlrBJ3S4a9MW1XE60bdCZUUo2uj8k9lZeXAk9bD0VuaGqVRhBw+xplUaPKY2TliTTJiQMPdwyyrDKfnaEoBjefAk5OLS4vvsCQJAQtYrHYMR0+PwTlMmS3s+iKgslkgqnqPHiPpKmEZQ7QDBnTHJFEAWldocmHzslisaAsBExDJc9SkmpMUvhoWsvF8ZdE1R5/n5KHFbWRYRgaVanz+PhI08bc3d/Q6+ug1NSdROCFJA10XcPHtwuEpkd/YLF8CnjaPpI3FaJQ8sMf/ozfff1IUZWMxhLhvqLrUuRGgGzE3fsn+sqGd9+uOP98wsp75Hh8xj/7R/8Vhjzg/umRH/7JawRRZLvK/t0HyjrI6ASVrEu4fdrRH1j0LAMvTLGcPlm7BlXj6naPqonIck6S5ZydH1GVJUEa04qgmxqjsY2mDKmqhvdvn3j1+oiLS3j4piXyFUxboC4Tjs/6+OGC9Y2AoPyWzWbHfPIMx+rhByvcgcVzYY4kd2zvI9JMIdw8Ier1Yb/z9ISmDMkqka7qMBSR0cShbAJ0Y0oSdswnMwYjiVaTORFmKJJMLlWgyQQ3O2xVQ7YV6lIkKQIkKUJCYGTPkaoxRRUQJzVHpyOaTqZnSOhWha04JJVKmu3QpT6DeZ9dElLud0TZnuKhoN+bcvewp+1qNEHC0UbUTUq/P2G1iJBqmThfM5r2iBOf92/vmZ0orNcxuhmR1xqSLiO6IoUqMNNHbNcRjSJiGwKK7JLnKQO3jyRq+OWaipxJf4YpQxTXWIaFpMtMXYMwyhAkDdN0yfKGvC3Ji4QmFFB1F1kt0BqRpDaJsh2WotCpBmmXoVYdjmvS1QJJ4HP5/A1x6FHUGopp07YxXWsidQ2r6gq7tAjCDl1SyaINQieiGiptpRK0KcQqjtRHbDJqEmRZZL14YngyZ372Eq1tiOsFdAO6WsCduKx2n3B0gd0mJhEyqlJhaKo4vR6a0xGuPV5cPiOJUxoVGqlj7ycIXYtZD/kn/+L/id0f0aw8Bn2Trk64urtmMDhiu3jAteeYncR05PK4f6SuJCQ6jF6Po6OGq99/pD+8BEnisx8+o2/bbJYbZEXg0/uGYK/z1//8nxF7Iv+L/+wLvn37gcerHV88O6cTMspySW/SEe582rigU/vskohWklAdA8c6Yb2JkC2ZVhYJ0y2irLMvEoy5RRaFaLaBMsoI/JQXz8d4UUOVFDyFD+huy2fHPVTTYLdJGTgCHSUXLz5nH6worZKqKfACH0EscFURXetj+zHT0ZDFx5CkSDm6sPjy7JLf/O6K3miO6mr0OWafxYh1j76lI2sZV19t+OzZG7woZDwYUzU+ChJRkVA3LdmmoMnvkLBou5hWTHhaVUgYtG1DFDSk+ZbZ/ILF/T3Hz0YUAYidgt7r2KcrBNfGD7egxszNF3Rqy6ZeYjcvKVKPqEk4085Z7VM2dkDctLS9Mav1jpPxnKCViXcyqq4hk2PrOUleMOrPMLUChAFxHCI2AvtNjK7/m5b37GzK3cMdUfo1YidTRhE3e59g49OmBtasI2klkixmE0QcHw14+HRPv2dTyBkfnjzmrkvBiroruflwT12vcftDBA46v7oWyZMaU655etgwm4/xE4+qbinrmq6WkFWdTx/ec3I65+b9FWNnSBll/LNf/47PPnuBM7CQFJnQX9FJNWm4Y9CfIUo1WZOgahZlq1PGBa4NadpimA5lFRBGAZY2Iq4b2jpG1CSy4lDO6dlD8qyCVkFUZLxwg6VZqJJMv2fRdgJFUVA1NY2okuYhStfRtSp541HjsfBkoqQkjhpCLyfLO159/hm3j+/QTRG7DyIiU+2E8ajHw/2Gu9sF/8nf/Xv847/6J3QCBGnNfr9HqEUETWG5XpFXHYPhiGCfILQCtqLRpRpR5pOIGmVTIlsqdVvQFB2LYIkiG0ztHpATBwkAg56J0BYMbJc8jqlrA8cqiOINjVijiAdTTM9yyOuYcCVw8tLk4SlkfDbl/dtbHHXE+Ngiy1t8v+L0/ILl+gl3oCPIGUKnc3SqEvoNwark/GTMzW2ALMbIZkpTjnH7CpEvYpguopzhhxUyMlkc4PR7NLlPFsgoncv5xTF+/ECY7DF7A+azMY+3TzSJSGO3RGWKKjr4kX/oFigSkb+glhQM08Ltm0SJSlnFxJmHpho8rtaUncjxvMfHd5+Yz4/5tP4WL8gY6kMUKSGOthyfPGO/LciTBT13TCt27KMGvVSJ9im7/RJJlkGuedw9UIsi46nDdpESxzmqYyD3ShRVpypSyjogbFoMPaKNOrZ+iKFIaJaA7op4QUOTtpimwKqUiEIR2xYxTJko3qLoLsu1R1PLOLbGYHTJ4ukOaoOy7Lhf3nH56hTVDAiDBqG2aMQC2zqh7+oYUp8saumZOmmwxVZ7dDkkSYAqqKRpzKg/YWCds/PuCTYetd0S+gFu38TzQvoDlzjKibMcxYkY9vqsNgvK6YYk3KObfRxBY7uLmB3rxPma4cik7x5x93TFahvRs3ss7u6QRY2zo2eUg4q4Eth5W7rIQpRj2hb2eUK+vMeLVqimgBdU5EWEBmh6TteZBNuIy7Mh202A7SoMTYedH/O73/weUSiRNAlJlym/WiE1DW3V/Fsz4b/t748eeYutia4olFlNsGuIPQHbOGWzivjFL/8pNzd3CI3BeDhFkVw0ZciPvvwhbSVTlAmyojEcjHCsY+gM7j6l/OoX3/H5T2Sev5zwi198x95PaToTWZdArlksN6TFFkVPsM0ecRhxd/+OrNiRZRl53jIYW8RpTJAUbB49RhOD6WyErSt4ZU1mNDx8uGMbR1SSToNFHMvkqYLlqkhuRq1WyFpK0auJioxl5rNvn8jdDNs1sI9HaGJA367RRRdbOEcoLfL0iSzJ6LsWs+MjmtphdmFQdjWSkZMWPpLQY7eO8MOA9XZLEEcIgoSmGbQczAqGYRCXGVmRMx72yMKY2chBkErs3oioKLGNIYosEnoCo9GUh9slXSkQ7zqKPCDcBdx9ekTisFxflAJBGOMHO+5un7h/fECoFNJ9zGazJK3kQ6mjKVivtiz3W5q6pMz2CEaJl3l0pYDcaIiKSBa2eMuOutGJ/YDZ3MF0LarcIgpa+r0xUVAS+zkn8zNCLybPGnStT17UdJWA1fX4zTeP5JsaVZORhhJW38RSLChldLmHYpTUeYYoZNRiiSRZFLmOYJqIjg4KGIZBGFdo4hC9sTBkg6ZZsPcT/KIhrDyU0qKvODRNw9M+YLPZoaoqlmUgySAJLXkaY6gGUiPRGwiMphKWVSB2Lk1msFzeYJqH5rVUO0yGHdNTlaUX0Ql90CSi3CNLa4LIx+mNaBuFx4cVi8UTQRDw9BAReiY/+PKcr779BSId416frA7Y+O85vRiiDcf4sY/QNnRRjdBENGJKo0ZIiszm0aP/fYHp2YXF5cWQ7WJFvgexULn65i3edodYGKSbDElwkXWLWMhpaoUyAy9PEDWLvOjQZQmJDIQC5IZ9sqeUWtbBhlpJGJ7qtKpEKdZobsfwyCDLOsyexPzMZTKZYJkOQ2dAm6Ysbt6jGz3asqAon6irlr76jL415tPjLe5YRFQ76lahJqLuWgajAYZTk+YZu+ATdSvhBxGKKhJHKUVWkBV7prMhsigxPT1msQnQxClVobFf7/HDJ3SzwU/2hFlAXRXUzQEqvl2GePs1opqz3GzYRhs6reL9p29puxpV1fH9kCyKCfc78jqgqjtWSx/XOkJUbL79tOPbj090yCiSircNGLmjP5zH6xVlfGgJv/3wiaLr8OKG4fSMZ1+8QnUdGgm8JKI/GfC4XoBSUIstT08hPUcjTRr8VYreOPz481fMZz9EpM9k7JLne4SmxjUtYj9GEkpERaZqOhzH4c3nLzk6HmNYCj//+c+x3cFhb3JgMD8f8fLNBWVT43sxXdNS1ylZ5ONv1zzcXOHtd4Q7j/uba57u3hMGtyzWK5abT0TZE59uPzIcjWmEirTcY/UMREGgzCsM3SLOAqouQjVbtvsNbatR1hKdVNI0ApIg09Q5dZGTRSs0gUNjf3tD2+7JYo+mbKiKBts0sF2Jy+endG3K0XyCaYoYap+2VPGCO/72F3+NVotoyPyjv/or7p7uiJuCUmzQbA1dE5j2xti6hmEIZHGDpttsvT1VYyCqORfHz3FrC6EuacKCLhcRZYMvT35MXzSpc5U0zuk7I0b9KXfXd1R5cdgJFgKOhyf0Budo5jmqZDMem4zP5mSCyPRoiDaWaCWd88s+vp/y7NULnHnHZHZEmSeouo2itsT5lrys+ebdFVnVsfM6bh+XjI8GxFlLUpSIqkGDhd03CJKEfRgR1zsaxUS2euiOgSCWBLstqtwjyUpUTeDpboMs9lAVgSgMuHq/oqktVEtju9rS5DlNXTB0bY6P5uiaQp6kB4GIpfLwsCTJ9pRVRtulCHLC06OHro4J9h3DkUsnhLSNQJandFJFGJZopkyedQhqw+Punk+3j4gIzPszaBR6I4cvPn9OR0metHSlyW7n4+0jVFVnOtcoioI6L0kCH7ff52n1gC7K6IKEt/fp2ybjkY0mWwiVRZknKGpJEiaUiULX1uz324MbXtbIs5rj41MURSPNSoqi5OWrC3q9EXV54M4WRYUiu8RJSNGEKGZ9sHdVLUFwz/nxCel+j1BIdK3I3d0dimJgWSa6ZbDzdwctcQ6qMqUTBAS5wR1Y1HVBHBRoco9hf4IgKGR5jWkO8TyfMldpa4ssj+kak9HQRVJg48UsNyF5KpAEOQ/3G2RDw+z32e0bNvsERAHFcFjtH/h0/568rTEEia7KiYMQuTt8fyf7BEFIydOKWqiwBj32ZcpTsibXSj4tF+SVjKaOMPtDnPGUTnLZeSp+oiEa1v/ffPjfOFAeHfVQFYXl3RbbAFPXeXq8x+6HvHj+jMvL5zw8HHAeg6FNXgTsgxVZFuN5O1abG6KwYLP7RFUKjOci/73/wU9IQ/jl3/6Ktu6RFiVB9ohhK8znM86eXSDLE16/+HMUueazF58x6k1pcoXBYMTN9RO/++pXqLLLT//yOec/HvHmyxdMZjZFr8Z8HqCOEqKuoj9RELqaPA7oGxa5X1PFCpoyII5bwl2BkB8+sIoh4wVr0tCjkRRWt0u6JkTBoIxFeq5NEq85eeay2cQEUc314zv87J5PNzvysuH6doeq9fD8LUKXUzUHv62u67SNSJwUiOJhyb2qKpyeS5T7hGlK0zXskyWS1FEUBbbRIywzND1nbCl0ZYio6+gjG9XaMZIN+sYIx3IR5YLJtE8YNvj7GsuyQWxoahGxa2jShhaIyxxB6ZiO57g9GxBpKhnTGZPkGWUd0DQltm59HxALRiMDzR6hyOBvF2RVgyinDN0Je28NbYlpaAxdG5kOWRJwrcPou983acWOzy8vmY8HOJd96sLn6ekT05M5s/mAMNgSZQGO2mfkjojyEkGEuoMuBce1yNI9friiUwSMoUMlF/jVkvVmz7PzIxbrHZPJnM/O3hDvQvJKJqk8yjpFEDt830eWVeqypef0KeL8UCigIIlk2kbBMARkRaAoRLarkl7PRZMgC1Oq0kRz+jhjmU7s2Ec1ZbGj7Ryq+tDuE1sNf91yfxWhyCZBsuHT9VsMq8fJ0TF/+R//GQ+PnxArhTfPPuf1D1+QbDVsbU7b6pimjlRLiO1B/3b2ak6R12zXOx6f7snSGFWRGfRdijxG6CqOpyO6okVsBDpB52h6QU3HarEErUHSVLImwx3KNHLEizfPaVoFRXFouhJROqBgtts1eRown0wJvQ339x8Q2wp/HfHs/AhZbLn+cMtmtcW2LORGRGwFsjJBl0XqUiXNM8o0Ophb9AxZMYiSGF1zUQ2DJC3xw4TTi2eoikHTNERphCy6KJLJbD4m+R5sXSQ1utaRNBnIAl3TUCOhigZxlrLZbNCtKUWekNYZfXdOlVS4AxHLMZGwcFydwWmfb2++IesywiRCMx0k9dCeRTpgYkpUgiLnw4crbj+8Y94/4nh0goHBepGjqTYf3l/94ez2G/xwx9dv32HYUwRtxHQ8wun3cIY6ZV2TxDGCaPN0syaLSnpGj/XKx9J6lFlOmhVcPHvN+fkzlt6GVi7x44A4zfCjLVHmo2k9Hp/WqJrD1f135EVM1ypcf3jE92LKtuTd1Vvef3xLWMYIKtwsPtHIhxfIOPNZPN7RVBmO6TAauhwfD6HOSEMf6gJDAUvv8LaHNZL7+98R+Bu8/Y443pKmW8JoQxgk0B5c8CIShqGRFwmGoaCoHZ1YkWYlaXzYodQVGamFIvXZb94TxO/YbW8I1iWmckxXVxhGhiAWTGZTqsY/KBJrgcSrCLYBT483iCgMh0Mi2aNSIwQl4eRkwHjosnh6xJA1bM1BrUCtG5qopc4TmsLn7GjO6XRKsMopa4HjoyFvnv2A48GUkTkm3UK6DTntnzEeOkwGR4z6NrNxn9Ppc4b9E3RdJS8q9smCKNrRc1UauWYTeWx3O8o2J2hLMhq2XsrTckFRqmy8lLgOeP/+gWFvzur2AX+3ZmBZKJ3FrHdKsA2JdxWaYCGIKrt9yvzsBD/ckdcVcZEgqQPOLp9TJRneyscwNASxpCxzFH1AUAossy2rMKaUSlA6HHuEo/UQq5rc32EJJqezc04ml5Rxy3g4RugkEq+iziSyMGcfLlDNDlnSQJDpWpVPHxeo6kFp+bR4gNbBC0Lu7q8RBAHdEim7EssaEWd7wixDs4copk4cegz7AyREJFlH1w+FM011UVUdVdGJo4woTvHCFcPRlJeXlxRxC0JBkbWk0ZbF/ZK6qfjy1edMhwPiKCLwYmRRQpd0yrilTAt0U0fXdTTVQEAhihLyoiArqu/3Kk0UUaYuD4rXXq+HrtkoqoGsqmiyiSio3N88YesDRiONLI54cXF26B1IEj13QBTUJHGDLLqIgkoYH3Zcvdjn7PKC0fA5i4eEyfgYEFBVGboCWalp25o8L+kalaZSkSUNRRUYzWWazMKxjtl4Pm2noGsNeVpgGTqqYrK63+FtPNROY7PYIlZ8v3Y2BqCMS4q8oukURFFFbg2OJ6e0TUUYtWz3HldXVwhVgy1rVGGF9xRR5RWaWRD6NUVe07Q5d4+fWGy2vLt+/HcfKKMow9vtmR8r/OSHP6JvDg4Ih72E2zNJq5TL1xcIksNvf/cO3Rhxc+OxDTbojsFm53F9+4AizqDuAznv392w39SMx2MUw8ceWFy8nKPIKrJkEfoVQjPmt1/9Dav7gjgKDoBQd8r2QcFUjjg7OUczM6YvFdxhj7vrkG3o0+otZZVBI+Ie95mNThlORMbzBqcHqt6SFgvWq4/omkh/5uD2FCSjoUwyTMnhB5cvGdo2ciLy8tUXTMbHDKcGnRSy2nlc34cMjvuoPZk4l9BslSQNSFOfIk+pm5xef0pZGSia/j0Q2AcOmKPxcIIkgmWYZEnK7KSHnzQ0mkLWFZStRN80CJZ7glVMVQwQrI5a6pgMbJo4QW5cGjSSeIfpgqzWFEWGJClMhmdoQp/RYE5WRNRlieUMaZWGIN6ShjmLO58w2mBpCtbQZDg/RmwVxFomiFI8vyRYFphmQ9sUZKWPgkbsaRS1hG2rSHJMWdYUZYrvbQi8EKETMVQFUWoRBQlT0+nNXDpyarXjcbmjSiQMuc/7j9dkZYDtCkiSyX4ZsLjb0hvOSdI9iiihOCoZBZp9MH1kVc1uGyJUErrskOQ1qecxUabE65Cnx7dEaYpsSCS7HAGTvGwRZIHV9pE0C0jCCFUySeOOtnCgUpAFnaKoWG7WmD2bl28uWW23KKZEz7XZbnyaLuDtb7ZMBlOOjy5pS5Nhf0aRN8RRztP9mjwCVx+ThgWbzYa2lFCwKcqGh+WCYF9wPHzN9jHi+tsPbHceg8mUpuvwvQzL7PN055HEJZ0g0FARRhFBkPHh0yOiYrLe71BMFWtoc3d/jaRWzE/HqLqEVHYIVcd0aqMOazTFp2syVqsAL4Lr2y1ZUVMWGa4u0tUNVQGKaBDuG4okQhEUbNVBbAyOen3SsKCMSmbDKWXV0rQtURIiqzZiV2EZM0ajAWnhg9bhDi3qEjabDafHJyAUtKWIrIggSCShgKIbLJ4OZYt+f8jT4z3LxY5ef8zx6RzVMpA1E1nsc/fwQNX4VHJBlCaIqo5t2CRpipwrfPdwS9sVHBl9BByyqiTdr9F0h+V+zWq3RjEUBEEkLUsEzUCSBMIsQK4ldNdlW0S0mowit1xdfYdkdHx8uGYfh2zDLYu9/4eTtTKFILCLU4Ki4n55z9Pa49P9iq8/vOXqbont9FFtG1WX2WwfCPYNqn6waKxXPsO5y7fvvmW5uWEbBixWTwj6hrQsyEqBvOrYhCHWWKRVSkxHJkoiVM0kK1q8IGDv7/DDHbqtcnI2J69LEBR23h7EiqKJMB0dx+7RNhKWZYFQMJqZ6I7I55+/odfrYZsWs2EfR5HoGzoj10CoU4QmxtI76iKk3ysxzZgsW+Dvl6wfd2RRx34bUBUlaRKwXaxoW5+2rJAEHV0vKcsARZJRRRPdkPns1XOyoCTc5jS1jOu6RPGa4WCKKhkIrYEpj2mrkmHfpswlppNjYr+giBtmwznD3pSulXj24g1Z04CqUkklWQPuaIZkaqSVgGYODjB+9xTkmCd/x/LpAUPvUaYNI01jdnxKHAWoisR06qCLBm0O5yfHOJZLFNZomsbWL5AVjTBbkbctgjhkNJogGzV+UiC0DVHk4YcRZZNSVhlpmDGaDlA0mcFYoigjkjRkOOqTZhE9tyEMl7RVwvXtDV6wZ7PeI8gdkqSQ5g1hmJNUPrtsy+jIRRIh8vZQSRSxTJ21DO0+JQmSrvL1N+8IozWyUCK1oCsy2+0NeZqxXm5J4hJVUmmrGkXQsU0Hy1JoapGqzlmuV2y3W8IgwzanB7yTAtPx5FDUdOY0Rct07LDb++yDnNXuAbQEUXDIEhClCj8J8KKMk7NLvCBgtdqSJyI9x0UgpueaKIqGrKmomossy6RFjmaYJEmCJElMZhNs28YyLDaLNfe3C9qqxdAlLNXk/e9vmIym2JZMXQlIMjRdTN3kdK2It0+wHBtJrvGDNZtdQttJdEKDaQ9AUgiiAKc/ObjIW43JeIYsSWRZR5zF7P2IKEnIEp8kTNjtHtBk+TDlslqS1ENRFI7nM9aLLVcfnsgSyFIYjedIKtiuSZqoKLKBaXdYVg9FbdGtnH5vTN3k7Pd7JLEl3lV4mz1JJHJ6doSEQrBfUBciZR4jUNAVGV0pYSkDYh/yLKGoKyRTAw2iMiJO9nRtTVlB2TVUZYrQZhiyiNGpGJ2BrThQSURejKHIyLVAk1UMLIs6q2ly7Y8OlH/0DqWmC8yOe0xnfb763W8IvJrBYMBwOGYyOSVpl9wvHgjDlOnpkMfNPUleYpgyO6/g2fmIxG1RJYmi9DBkDaGVePXZCADb0tltQwynpmtE/G2EYojcPdySJykKMs4wJIokDEskDkSiICBJVMYzmW9+fY3e6kT1ljToMFWJKjcIGw9N1Hhc7xjNVdpGoG1LJmcDRA9EqUPTW7ybR/aqgKsNkHsKkxOHQgxx2oZyouFvc6IgoN/vIyBx+eoYy3F5Wt1RpSM0Vyb2QwaOQ10ZiFJG4OdMX4zRnSmfPl3R6/XQZIW6qHl58QKq5nDp6gZW3/4e01Oz3Hp0XYlYhNRGfTXNZQAAzmhJREFUiiKIjIYOXhix8RsG9jlxEOPaLl7gUbR7LHtI2XSUmUzexshqzmxkEwYxZSNw+eyUXt9mn+yJi5po3aHYLeOhSlSCqY/xM4+P1x+hlhFbhcuzOcuHRwyrYtSffK9nS5lNNZplRRwUTNxjlo+3CAK4ro3YHprm/sOWyZGBrKiUBci6iNRU3Gc7/Djly94zcschbyouj48Js5i2zRiPx2RBQM8YEtUR/d4ETRkS5AtKoUJXNXY771D4MGwaRNI8o9+3qLYOf/F3xmx9j5tPGwbDU/abJWOnh+EYUHesNx5BUDGfjwjDkKZpODqeIIsWnZOzXa9QDR1ZNxBUCJsFXpARFhXvvnuPpg2YdFOen6hQVqyelvSNEU8PD4hkTCYjBGnKz3/yc959/ZEyEykymaRtEM2cpC7ZXi8YTQw+bRbITXgARjsdnz49IHUSdDJZ7ZELJa9evCTYBiy9BbZtIak6imbj+QltKyMIBo+PN3RSx+mox+rujsFgxPnRjOvtAkXtCMqKKpIxHYuuExComc2HRPEOVZLRFZM02KDqGmVdYVkTstpH7EwcQUfRLHx/hRcqzI7GvHh5yvt313SNTM/RmR8fUSQb7lcZHTWGJrBN91Rdjlg3uO6QYOshCTVdLXByOkVA5uFmy/llj7JK2C5X/OlPfkYcrpiOL0jKkPvFCk11uLr5iuPjYy4uptgGnD0/Yf9ks9guccwBUVKCUDJ0XShLLHNMKziouBh2Q1TsyIIEU1Q5Gs2gFGnbFt/36ZqW3sChrUp2m0cUw+RxE6GrDbUj89effk0jgpyLjMdT8iT8w53o1SFlWeEMhyx3T1ycnRDlEY5hcb9+Qrf7ZHXFd/c3CG2O7pg0okTW5FC19Hou19dXjOwpiqhSZwVS1zEcpuyXV5we/4TRaMDHT1dIuoThWJRRwXTSQ9c1NB2aTsAwNY7OjtltI66vbzk5OWO39xEFmbIs8X0fXVHpnw6pmoxKqCiLlDLOaDuBve8jijK7IGQ2GBPHAUlYIUoKklyz9/dEcUu/3yfPc7K0QtccbNcizVqCYHN44U1j2jrHNQbUjUfRRGQ7mbLM2HshaRxxfv6Gvj1HV1yq6p7JzEWQW5arPa4zZLcpCKN7qCVevHiBKLSEkYht5RRFxnw4JK01vnv/Db3BjP7RlM1yw9FoiJIL3C1uUdUeF/M+5VbFGduIwhZFyjg57fCXDWktokg2uzBmfjRm7Ji8u/1II5Xs9hFlLnFxdoIk13z3/gO73YbB2MEPS6q64TGIQCkYu0cUGfhJRJSlWEqPQa9HpTekaY4kCXibABGZnR6QJR7roGI0nNCKInWyRdINSqFl73u41hTDaqnygLzoME0FEYc0SXlq3qJqBXXT8entR8bOBZrqEFc+qtKgmy2WYbHdlSR7H02UqbIEW3MYTkekZchqE3A67qEKCpv9I0EcIUkSX/zoBX6wx098ilJCVQYo0posbWlFAc0RyXMRXVfJ0oogfGI6nTPsT3i4XSKKLobTYtoSYRijNB5n8xHXN2+RJZPTsz5REWA7OlJjYDo6itKiqCJllWEaNnkRs9/tMQsBFJGebRPuanoDhbvHHarYIucqoiMitho91yYvPZoO3rx+RV1lRFGLbhsEXgwYlGV5KCfNLSxL5fFxSyN2BN6eo+mE8XTO1adbBqPxAXAvHEgcTSuRZhuSKGUynDObuuSZwPHxMV2bUiQCph2w2z3RSSVVB5Zt0pQCtj4iznc8fzmjbjLev33EsibohsZm7fP5529YbT6R5gk3n1ZMJ0c8PN4zHo8RBY2HzRXHwgRNSCiClNmFSRgWaJJImQi0Wsg+NLh8fkEpq4RRRl3vifICAQvV6lA1jU44sE6nM4PYC3DUCVW7Jw50RMdB0seswwWimqHICUKn09Qygb/m9PwMTbComxZdk7Ct3h8dKP/oF8qqFXB6NqZ+wX/xX/xv+V/+Z3+f12/e8Hf/hz9htX1gH98TFwHnl1MWyzWSqFOWKd7epxNybq/XhH5NnK6QlQZZbTh7LtC0h4si9BLaTmS1TvnNb75htdrw+999xPMSLLPPdltTFBbPX53z+LjFGRT84IfP+PGPzxgOFXa3OU/BNdqwRBeBVEIWGmRJwLEHvPlySotPVReMR6fMxhNMrcM1De4+romlFtG0yeSC8+dDPrz7jmRVs13k3H17hyqY9G0bS9OwNJ35xCVPKsROYjy1CMOELEnpOwNURWE6neL2nQNayN8zGk3JkgTbMnEs+4Ab6BoEgQMrzlC5u9tSFAGz4ZSRM8GyD2OWTqlYb3fI2g65ga57QDNqOjlEMQKyCLJij6w39IYWlmWR5gm3N3fUlYjbn2GaJl68Jy5iBEHG1GXGgyHT6ZSB61AJCapc4ZoGLTL90ZCuTHnz/AXPX77AcmDQ0zAVizBIcAculiajKhXDXp9OOGBGBAmKImM6H4HI9+xOA91QqTpQBAnFOBh4hrbAsKcTJD6d2LF42iM1DUfHY6IsoK4j0rzh0+1HmkRmZp4SLivCXY2uuAyGU8azUwaDE0b2iNkZvPz8OSezH/M/+5//Xf70Zy8QcgdDHxL5FUXaIEkS/YFFUcZYtsFwOCbNGvxgQ5FVOGYPTW+ZznqkaUsUNTi9Caar0QgWrz6b4ZgVYi3zzS/3uEYfXddpqhJRFFmuN2y3Hr///e9p64rB0D5gqPKWNCvYJXuytmS5CdkkFcsiQRkY9M1z0lTFMHuHgogiYhk2cZwSRhG2ZjFwx4zcOVVe0XMNXr284OH+jq5W6E+PKKoWpVHxdxtWwRNR4OOFGZZ4RF521FXB6dGYKig4HV5SxECnEucFtx/vKLMQVW0xdIXZaIwq6myWO5oyp5Ia+vMRolLz26/+JWEYoisq5ydjyiJku92SVhv2uwBDHZBnNXQdNCmJH2NILn/nx/8Rx8enbDceQRCgaCJ394/olszrH7zk22/e4thjrq4/4kd7kqLFT1Nmp1OERuZs9JLQb7m+fWDlbxDEkrhYcHLm0BvPeTk5oc4gUxq24bcIXUwYw2KVUAUprmwTrnyezc94uL2haSpsXaMBHjZbdss1ZZAdrDAt3G82tJ3B2fQzZEVDUh2SPP7Dias9SCKIApLaUtYdjZCzC3coukGW5+RxgahITIbH6JrLLrwhS7eIoo2kSjS1hG4aJGVHVpeoYkLudwhthVDF5MmWNy8vKBODQW9EUVSoqkpepFiuQydDLXRESch6t0BAoa5rurZEliRENE5ml3hexPXNB7LKY7NZHJrdqsXeT/nuw0dW2xVxFtCJ0CBy8eIVk5MxG39NK3bsIo/fv/+K3W6DKIrc39+y2lwjSCtMy8M0Q3Q1R8Kgqnc0TUAQPLDc3bAPNxRNzGZXEwYFu33M/eMNdZOy2caEYXuwR7UJSbrHtofMT2ZUdUpZZ7Qk7MMVSVKAolB1cHb+jNloThM0uJ2LWtsoncXJeM6bl6cEG48uzVDY8+VnY54d2zTZE0Mt4uVUohByzi7nZFXB3cYn9DIU/QizZ6CaLVe3N7y/vsJ0ZKyei+eXyIqBOxJohYAsLViu9xT1Cl1SkFqXho40KthtUyRZJ8trjo8umM9OaWsFZAnEGlkTSLKa6093iLhUqchf/MXPGI1bhs6A0WDIy+cndGVLldYockdvINGWEiN7zMQdIRYFlmp+/2NdJatzPl5/e3h51Fxk1UFWD2pMbxMitTKWarHbxcRZxHg2ouWgYf3mu69Zrh8oygPb1NuFjIanPLt4QdvWqKrMcNDn7bdXNHVO1ZRsN3umgzl52KLrOnG6Q8FEU00gpcoLjqcv0E0NUWvZ7X0QFERZQDFa6rrlxbMvUCUVWRRpK+j1TVRZQhA6FssNR0cn5HnJarsBSebZs2csn/aYjglCjiAIWEqfcb8PXYamqTw93WFYKm3b0u8PGA77bHdPfLp+i2WZ+EFMf2xjOTZRWFKUGbZtstsGrNZLYlIeoy2VUqOYArsgpKwjiiSgLA5rYFESM5wMaRE5Pj1nOJhAazCajlDUlrJKKIqIqkkRJSiLhjgqyAuVJFtyenqOoVvMT6ATAwb9KS0CcSLw8//gPyROM06PT5jNJEI/YHx0ULGKrUSadYznQ7b+ArdnYRgKsmSiaSBJEQoKV+9uqJOGgdlnt9yRxRW/+9VHdMni5NkpZk8nyRMk2STNSlokkrpBdmwG0zOyOiGpPPIyIm8youKPN+X80YFyv5d5//GG33/7e/7RX/0r2i7j+HTAw6NHq/po0oCBc8R+u+dPf/JTXj9/yfOzV1jKAFsZH6CiZUwaSzSlwd3Nhv0u5buvF3i7nP2mwHR0ZKNBVU2++fo7isbD6fW5/hRgDBvc0RG//PoXjCfPmJ6abLwnOiHD1OacTI95/uon3H8KsHSFTjQoqo6uVknbitX2Hsc4YmheYCCxfnhi87QjiwrqqkCoLZQmwxjKfHz/gecXn2EKNopo8aOT12hmRt22bDYFlj0mikuCcEeHxMPqiiJOeXbyhqKq6LQcpaciWQ1h0KHpkCYleZ6jmxqGqZHkCV4YcHJygu+FJPsER+nTc1WyPCAvK8omIyszkFrGfYWXz36ELKv0exaudrCcmFqfsm2gUUiTGkmAqkjpuWPGszmDaZ9tuoWuRhJN4qCCssJ2GrykZucXlNWOLG3Yr3KW9wmu1aerEzS5Y7teEicBqqqiqDKzkc5wck5WZHz+YsR+E9M0EaOJxfRojqKqZGVA2cTouk4Y+MhqRZMJRGmHmTtMjBm5LNMTDcogR1d7iFLDbHZEGcaIksrsZIqmaOyjLfbYoLUrlvs7ii7B6ZuM3R5i2RDtAqxSxLZt7LnMh/sHguqRx9WaYF3wP/0f/yVlnkGZ0rctZFFltdjxeL9BRMDz10ymNqPRCNfp03YFVVVRFw2qInE2P+N0ek5TaczO+0RRx2bt4cUhr3845vn5S+I0YDQxaTuBtrFpOoNWatknT2SlR91WWJZBJoIgSwxNizaticOMJEloupI2adEkleVqQxCFpBE0ucJ2lZIUNWbPIq8bnpYeedlSlBlZ4dPrGxzPJziaQVMcvLl+kLIKQlzdZLtLCDcB89GEYa9P8j325frmI0fHY5pK5uG+QGxBRmTzuEeioEg7Pnv5jKP5nCLL6A3GtFKHFwY83ftomkZNTtvWPC3WFI2Aa/d4+XLMZDAg36ZYgo6lTmjKiOP5Oednr6ABzwvIqpq8qSiajk4SKboOP92AIFC3HYZl4/RMJK04oLvKjOPZBZpqcTIZc/XpI0US44fJwVveFxmOHO6Wj3h+yX7f0mgF++qG1WbJZD7l9OIc3bB4+/ZbkiKhECrypuJp47OJMgRUqqRi1NcQ2pLLoxlKAtnCw9BlNotHMj//wzlxzyAR2S92mLpBmKxRBJPQy3HtIeOegyPLWHqDIUlsFit0TWLkOCiigiCqGLqOqAjs0j2y7FA2KqunHmI7w7J1kmzP9c17kjRgs39kMppSlBnr9ZKszKialqTIubr9RNsVtEJBXiYIHXRlSx7lSIKCo7mkUUoaRzTlQfuWJuWBJdm0yKJIHEZstzvW6xXv3n9NkkQH13cnUdYVaR4RhBV1o+D2jkjjg8+8yhVurnZcX61R9QZZqdBUA9000fSWIAoZDGf8nT//Ae5YxDA0oiRFtXRUzQBRQjVF0ixgONZIkgLTtlmsPTpRwouXSKJG28DmYU0Rp/QtE/KEngyO0RLmS0ojRrcNvNij7lQ0y0V1Bry/TflwDxj/Prdb+OZDBILA3/zNL4gTH9SY4+dTcnGDrIl4UYxstqBmrPd7hqMZpm1gmS5VaaEzwRCnOM4MUWqRGxVNMFDMFrUn0QglZRVT1BVJFTE9dRjPXKaTCTp9Ej/meHLCq+dHhwlRkPD1b65I9i1t3ZJFFYoo8ez8OaqWMBwbRH6NIuvsohWypZKIK0oSRCSoQKlGHI3PqQUd0Shp5ZTR+JSmFBgPTD5/foyY6yiySRwVgIQggOM4HJ++pMhU/H1FW+k4jkNbNfRcmT/56WvmkyPyJOVofMHR0RF5EdE1HWKlo4g1SbxHbnusFmtO51OapmIfhIxHcxRZxN89kRcxsmJS1Dnb3YqmbWkbEX8X41gup8cn2KYNtPQdg2n/mKbNqEuFZ+dHmKrF3f0NRV2jaBJhFCAikEQe+80WXTNIiwjdHNAIJZ0UEYRbtpuIQe8UXbNpyOhaAVkQ8X2fsmhwLJfA3/H6xXPGvSECImdnZ9ApqLIDask+9NlFG+5W99w9LVnvPX7xqys265JPVx5haKJoY4ajCfePH3H6OmFY8/VXVwcrlQSKLDCf9lD1iPvbO15/9iWO45AXEXXb4vRc9uE9y/Ujbn9O0ZakWcPZyQlSJ/Mf/Xd/zvzZCHdkoToNztAliFKSYocoSui6RpbEdF3H2dERumSwW4do2gDN7DM+HmJZA2TRpK06lg+PLK/vMESLYJuDCJvgiqj0KCiRVAVEHdBp+KOY5v/NAmVSLRgdOez9B/7mN/8fvnv/L9hFN1zfvefm04IoCHEtiYE95P76E6G/J/ETjqY9qCvSNMUyDM5OL1hvnqgrCV2ZYBljTFtgdNKimgKNkNN2Bkcnz5ieOTRCzPyipWhzvn73W/7iv/0TdBd++6sloQfrdXZA44z7PP3+gYnlIisufXOI3tkIYk5ZJIRVTCW36K5IS4qiKLx+/YpGynj+xUt+/PqcyXMd21RJM4u6VkGvMJqMuk3w44ZOUWnkkKjcstrkWEMT2TBY72Jcw2G/32LZI0RD4Gb9NWVtoegNvucdWu7jMZqmEEUR0+lBv/a4eCKMI5RCos1LhK6lIMbPg8NTuqvRVi21oLBbLRgqNrVn4QUhraRiWg6u3EdXZaaTM+LoMOJCEtFNFS9eE2QZbafStRKmBl0hUOYShVDSqAGOMYRSQ1Qc5scnNFmG0irUlYGsDon2PpY5ZDAYsH0MEQUV3dQYOxckUQtSjCRJeLsUWXJQNBlVlZFEnSSuqcuCri3pwpjJ6Qinhr5p87TfMJqMcXWTtm3pOTZn83P8KCSrfPKso9NqOkWgyGrUTkYSRASxoaky1psnTFtBNzv2yT1ZbXL/tOf91bdc3X/H2rvl//5/+8ecn4lQCwgdrJ4eOTuZc3E2p+0qppMRy+USzarRTJnJtM/x7ARDs6mrkDzZ8nD7W8oiBbEjq5eoyojnX8wRFY2qDhGxWG8CVENCtQvOX02I04Q0bVlvI3rDEWme8OStEVDRM5PzwTEjU+DVbEy6bdENBbFuaSKFl89e4ScJrZGTVXt6DoRVSNT4tEbF+cvnyPqAolIxTBtBqdC7Ck2ViQsfVJWNVyKXMvOjE2S9QChFJoMxnu8jyBb7sEKUNaomxjJq5qdzTGeA5cyxhha90Sm7ZMn9aoGfVQi1yH61oSt1xoNzDMumk0oeN1viSqFTJLq6QahthDanr2YUUUJWNAx7x+TZjn/2L/7f3D5+xB0aSGqHIAnIhkKYJgcnuGUTxSWKIhClPtudRxj6nJ5NGI3O+Kf/+F8xGQ4Qm5QXL4/5yQ9+jGE77IIIV57wr775BcMzC0vPQIypdQt3fMqXP7zk4s1z9L5Fo0qYfRfJVjEGJmvfp1FUTF3GNG2qRuZxt0YzDNIkpKpTsHP8XUkabPny8vUfjiu6GEIBZcV2neEFPkESkrUZG29HlHnf+8599l7KcHxKWmiUNZRNjqYPMFWBNo/Iki3LTzvulylWf4rW63H9+MRmWxMkBc5QIUhybGuEJGoIYkNa+qRViBfsEUQJxx7SH1sIQofjDJhNjjiaTuiqgkG/z/H0GV3Voik6aRix3SwY9AyGfYtBr4/UKaRhShTuEam4+vieYW9MkVX0rB5np8es/QVpuWcXPHJyMafX66FqEkenNq9e91H1mrz2eHy6Zb2o6fePOT2bk6UFkhqBGNIgcHb+kqPTKbrTIms5lq2jqyPiqGI6t/D2CUXeUlYNPecZs+kp09mYi7M5vb5NVKQUbcnR0RGmrHPcG9OEBUWesF0GyHKBqitkmcbGT6lkg7BuUF/8gNGXr5nNjukPTKo6Q0bjw8MNZd0QRx11J1K0JffLB+IiJIh8+sMefrhA6SSaTgCpwJChyhrunzZIukyRhXhFyS7c4/ZmdJ3GJvD56t03XN28Z+gc8e//h5+h2iV5UaMYNm+vvqWVWhRTplEqstynawT8fY5jzUHes9ttaIoepi1iigZSM6DXm6DoBrPjPoIWkVQFrQmnxz1UWSRLIuo2YDp3GI+n3N4vGIxdmjZnNJqhKAqyUVBUHo49xNAG2LZJEK6ZzWYgFKwXAf/sH/+O85NLJEFBlCLevf8OReozmbkUhYci6lS5gKUPcE2D9WpBnNQ0Qs3j6hMKGplfMx1PqLsSUemQFR3bMfD8DcPBFFEUSdIA2xoyGPTIgxiahiBMsRybtmjo6gYvCBjNdLIyo+8eY5ku5xdD2i7H0HuomoGklZRliWvPyLOGNAvRjINzvs6VA5i+aPH3O6oiJ4sqYj+gLEMs3WVsG/ibBVmYUWQy/aHDxk8pUEEfgWRiuw6el5AUJUmRs9lkRFHD9c09VWVgmibL5RLTcJBFBUmQ+NnPvkCS9xSpjusMuLu9pUhsjo8usXst+72PO+jz8dMVqqUxOR5z8WKCLIuIQou3D7lbRUye2ewSn9vtA7lcUikiS9+jVWqqzkRSK/pjFVnvGIxtWlmhEEsu3gyI24z9akkR55iWgzWxaJTDyD5PBZpCJCsyFo8B/g7qSqDtCrIk/XcfKH/0ozGvnp8yGV0w6B+BaPDuwzumsxEnx3+KLDpMp3OSSCDOUmoKRMUizWryRKXNhtSlQlVVqIZM0+VE5YZd9o6wiMiVmnVwxde/qggSj4YMKjgan7J+akn2OS8/O6MSW767usIdzHGGKp2iIPQMrq/XTCYj2lajKFK8/S2GIzOZHpHvGqYjlzBf8v7+hrfvVmiSQNcZdLrOJtvyt99csX4SWC1Dnr+waVkTbANaVFb3Id7TBkNoGdpT9tsEWVVYP8ZUqc9Q7aMUBm9OfkTPcVGqivnwlJP+lMA76M/yyGc8skhjCdPqsdlfcXX1gbbyycOKZRiT5SVFLhJlAmFREcYRRSIS7HQsUUMRHEbzAf2pzXQ6ZtCfst6EqD0ddzgiTvbIes78+IQkSUjaAj/rGDod/Z6OLJQIjYRiqnSqgCjmSKJG0sBnP/6cy1cTtts113e3CFqDpinUVYFmGiSxT5QWFEoFVcnJ8Ixfv/0VhtNiOSZtrOLoPRQDdEMlTWIkQWYwcmllmdn4nP7oCE3raMSOJG1p85Iyz+mEli4VEduWiJw6qyjTjiKLmbpHiHWLpQuoroJpO1SJQJO3zIeHMkzY5jStwu2HLev1msFYRBc0fvKjL/kf/f2fMT/9jKPjc5bLBZLiHBRxdUmW1qimxPmLF4zGM1RVQpEG3F35rB/WaIJAnHiYrolGSeeVTIZHuDMRUxHQZIl9uKeut7S6SpTlhGHN+nFNV1R4ccA6XVN3OusixulUsk0OVk5cllSljKx3iG1BmyVsnnxev3lGXgSoYo0qNKiKgKYPUOUeVV1wenYEnYLtgCw2xFnMw2bN4y4jC3OSvGZ6eoxlq+gXLk2TYmqnCGJNKXSIiowktpyfWawfH+npNqoGVdwgI/Pi4hkKKlmaEOxrZNGi79pstj49U6NtAgwHyipAEBSKQmLSExi7ffpujzRN2WxrtlFHVWaQpywWCz6+u2PrB8iOROhrJPuUPIkh13DlPvkWiqBksbxn5RUIqHRxiZwYlJnP7fI9J69PD+5yb4WSiaSxgCWWnDpD0Bq0TkIZaDiKzlgzCFOfyVDi49UnNFfmcbWl7BIko2M0m7NfrEAVmbkqs4GBomi8ujiDsEBqbbJSYHqkk8YicZLh9gyypvjD2Xp7VFVmMFIxhBhbcoh2FUenA9I4RoolXAvkWmUyP2MkKfzs5edMh0fkyR5Z12klnYfdAq2e0VQp3j6iEzqqWiArJPZBjCSrBPuEgXnMp+UnNvme++SJoAzRVYsmLg7GlK7DVmwsVT18nmqVPIkwtD6tkGKPOybjZ4R+ROAVnJ1+wWA0QxJF8jJHcjSc/oi8FokT0JUZnucxHh3R6/UoEwupU/D8NePhkPu7R3beA+vdgvdvb9l7EU+rW7K0pixLhmOFKpf48PFrnpZbluvgMIk4d/ALn28/fktWrumagrdff0AzWgztYNpxXZGXL0+RBImubbm/WxB4OappgSJQVjVhnnK7XxHUHTkyPfsUwzURFZGulamKkLaseH56gtyWRP6CeLfErCvKSkQ0JBRtxO+vf09RV+RRTlEUdNIeRIH50TPGwx5BsKbIY+hkJFtAERXqWEAUK0xtzuXzIUNDRPYHOKLFxL5kde8hCwF9o6XNZZJE5P3tO3792285nb2gTPY0ccGLi2cMega63aFZCvbRGONcYJfe882n3zG0z1E1ifFMBWyqqkJRU3RBYrFdo/cdbN2gUyOqSkBzZHb+mvnZFHSNXezTCTW7XULRlVi2xj7wSVIRMR/Skw12i1viqOLk7ARBEViuAyStz+T4HMN0WK02nF8c0R+5vDh5BmVJU6lopkzPmPFs3McxWnqTCbqqcHk6Z2RriIJ/IIU4B3ZvlxbsNjGmpRBHBYalEWc7WgpESWK7C0mjFlU3aMUczXIpi4bBoIftmoyP+3jxjqpOUc0dhiWy3yfMT47J65i2Brk0UGSTNIqROwXDtNlsVigCpHmBoYg0DZjWMegFz16folk6aVaQ5gmbbEfV2oRpRtdlPFzf4/TGB/MaHU3T4QxMZEtnMD5B0cbERUJeNuz8HUm74p/+86/RbYnJqYnZM3jc3vHh+j2t0FILGTvPx9ulvPhsQtkm7MI9iDJ1qZL4e1aPN4R7D9t20YyO6YnGtx/eMZnO6asG3nKHWHZEmz0j20KXJNTaZiR3OJqOVB26K5qtoaoqXdeB0HAyHNEbmNBpGLrKdOBQxx1KJ6GrGmJ7jlhJGI2BYzjUXczReACN/kcHyj+6lJPHLn7dYGgy7hgE6bBr1O8N+OabBZKY89tfvaXKWobDGUPnFKr9/5e2//iVJk/TLLFjWpuba7/63k9HREZGVmZVV3VXdU/PNHumh4MZkAsOwMVsCO7mnyK54YYbggCBIcjWXaIrK7MiMyNDfPJK125aKy48UbVNAsXFD7iLu/CNmb1m7/OcQ5Z1VEWFJNWcnXuUeYemZ0wmYyZjgfHogu2qoSxq/H2JO4TPPz+n7XJUaUZ4iEmzmH/6z3+CIot8+s0dA83D8WwWpzd8fBciSxH/6A8/47tvEn78M4VaKLm7q5mfOHz77bdcvvLoIxc9zRiPaxJRQzIVpoOUH/5tw/zmmjvpPY0gsfdLBrOAsuipSrh8c8Ps5BJTMWj7jIfHR569OOH2bknT5oiCjmNaZH6CrBTs10vmoxOiJOcQbenrBs+cc9g/UdQQV0sG7pT1psabDmnbln/5Zz/m5c2C2K8p6phGMsgzuH37SNtIiKZPkR+zKptgRwe0bcupdYFqqAThgSIxEIWWydggTyosy6CtSzRRRRVl3r5/h+tZuCMbWZZJ04Q8z4mKhLOTMzbLDXHUoRkti5MRE+cVy9UHpjObtjZI8gO2ZnF6NqMoOrK6p+9qPNdFVgZ4E5WyjynrDEXzCIM9trM6fhVVT1ivHzEsmzLniGrpUsbTKT0tu/0GQzJo6540Szi9uOL7798y9uZInUIeRHS2jmHaJNkWa9zgWAoD94Qog0OS4VkSitYhNgqS4JKEUA8r+jagLBXGCw9b/wxFrOmQWAcHqjKmw0FoIUkymrZit11hWQqqqiPLQKex32b0nURf5cShi+NZPD2tUUwB1dQRA503F+c8PASYs5LNwadXNBYXZ9SpQnX4QN8M2O8jbq6vME2bx7u3SJrDetkyP5lQdinDy4ay9VGlhh+9fs5mU2F7LaYtE+wl8qonXJXE8Xt+9OWM+7slpjzCW7gobQeoVFmH1kDWFfhPEWoqMT7t0eQh6/WSJMk4O79mv9+j6AN061hO8iYtSRrT9h27Q4gsmTiuhaZX+NEGy9OJohJLmwMFoR+SxD4Dz6DMG3RdJ01jEGREucLzPKIoIksL5qcnRFlAU7esn7YIDWgDA7kvKEuJfmQj9S1KLxOlGvNzgbPTaxw9Jy0TwjymTgIQBVSl5XQsQVbQHDpk1eXyxxafPn5DTUQdDKhbg1aomUgDbOmCP/nHl+RpwsPHb/nsZy9Jig1ZvOX54kvirOftN3f82R+/4rv373nz5pI4m1MBSRSgtFPqruR0OkAWG5T079/BW0nmw2bJZDzHPV/gKg6//u2vSMPZ8Z43Nthsa8o6I0zfURxazm9+xte/fcf5+QVVEZKlMV1jEkUSrz7/CfeHe8Jggz29wZAdlukjvdGSpiVtWRAHa5KkQzME1FYi2WeMjVN0VMrKJ6VBV1UGU4GhPUJSeg7+jsFoQt0U/Ognr/mn//w5/+7f/JybVwaHfUWbebieTH5/ICwjTi7nbLYP5AUMZycE4Zr9YYMkmswWZwRBRJjm1F1KU/UY6hjNDin6iE4UyMoa5A5FdjkcNrx58zmCaJIUK7b7ivvldzx79RJFFek6BUEasTg9RTF6RrbKbrdHlQzqtiVOAk7Oh7TkGLbEw3pJJ0sE4Z7p6Jxvfvsdzz6/YZeHWFpKlxWkZcBkPkPXJywfIz7dPjAYDJF7kdl8yN2HtyzOb3Bsk8L3MdSWupUYaKcU6YokqOjdijRNoYOT2TP8XchifsNy+4SuSpimQVOLNI1KGjeYpgpShCR3tELIH/7JZ2z3dzw8rTEMm+ncRVELHn67JwlTnl3fsN/7KKrO1cXnrNdLyrokD2LqukVEIolD2sZgPJ6yfHjEdaaojktLCZ1Onmz54be/xVE8FAk6OmLhiICJyek7CVtRWC83NHWNIXYookPfJIzHHv52TRb0eO6c8URjv99yOr1AEASEvsK1VF49u+Tp7iO90CJJKm2TM3Mv6VOVu/VHRlOZ7SrnfDDm028OjE9H+PUW09LpyjGaZpMVOR9v72h7CYSWNDki8zrhCMxeb56YjOekaYhAz+J0QFknHPyCoTukqRVkVcN1e7IswxtMieOYIKgQWoNoX+NYNooucLt8pApari6uybINh8OB85NTVg87DNfENmW+u/3I1fUr4qCEPmB2co1hKvz6V9+jDQssq0FsBAzdIS8zomCL0OVsNzV109GUEXkgcfXZlO1hS55L+LHP1c0E33/CcmQsRycKM/paZjq3OQRLRoNLPKfjsDzgxwfuVzJZU2BZBmoj8eLla8Z//BWOZfPh/dcst3dcXN6w3YQ4us125fNY1MxHczxvwu5wS7RumI0sFguX6KDjtw2pX6OrApohIRYytjFjYBiUScJi4fH0FHHYZ8iyTJHUnJyckZYFstwwmkxIdzVVI9Cj8/gQIwjGP/xAmcY+oihi2CKCkGK7Jj/76f+Kn//N14TBjjLRKZstmqyjqqdEkc/j0zvevPmcvs3QnY6KnKLNMfUZo8E5SfzD78DaNTefG/RVj1BC/jv2outYqFrKj9URf/qn/4Kvf/E9vfAdX355g6I6fHi/4fbjI6qpU0yfuLwc0Ss2+31GU9f4QczwdMzIaRmcj/jmFx2qOMEY+DgXJT/82ueLP5ihOyZ5NsBzZUzNIFpqOHrJeD6jrkRaGoIoRNMUFvMLDn7Aw92W8XhClh0YuWPEScs333/H4swmjDPCNOPk3EM2Zqx2Wyauy2xkUaQKsphxMp0gig111VMVFd/+5j2aLpCXAr0YMBvrfPFG4/LmDb/49ffHLBMiuqHy9LhClRXiwGe/XKJIIpLrYhgaURKz28XY3hDX0inzBkOzUYUcTVeoG4v7u3t+/JMXCMgs7w8sVztkeYcqeSB0zOYTNqt73IGJIutE4Z7xfMRuFeJObWopIy/2uO6Ak8kpnx4ecYwOXfeIgpauSbm6PkcUDARUFFlgl4d4oyG3n1bIqoJpiORZTV3kiLLGYDFiu1yiyiL+8ompY9O0DUGacnK2OJa3igN5klKVUJoN9Clp7lPXOZI4YzGb8+HDJwy7ROg63r6/4/TGRJYHtPWKNz/6EavNe1x7jrRVWO0a6q6l6yvCuwNFETEej0nC/HfrzwpBqBk4LrKqQGeA0LFaPaEKFqqpERxSXG/Iw/q4/snDlImi0SQiRQWFmDDwLA4PJaYNo3nJh4+31I1GwYbxaMHD3Z6o2tPXOp7TYlkSUg9ZkiGrEnGUsT34yKpJVj0xGCl885slst4wmUgUpcJhf8tkcYogFvRygywevdy5I+JHIYuxR5bumUzHbLaPVB3MZjMelg8kScJkYNM0HWm24+zS4fE+gO7I+Lu4MWkqBUMdoBkVfrhmv+lQVR1dNQjjDONMIvAbLEsjrwLaXkbRLTTLxjRtijomK3e4A4Pt+sCVdwXKkg4RqVU4eXnCYRlydeLgDjvi8EBbirRpwcBqCeOCJFtz9uySx+CBn/7sDVo/YJOWZFXI1DvnH//shh/efc+BFK3XeHFyiW6OeMj+luXHDc8//4IukKgCn6G1oNMUrkcWjqj+7todkxc1htOhST0X+UtiP+PyykM3B+y2t4iDv0doZNsAXfIQehvynqgK8UYzhC5DEHLqXkNzVZRII00aHMflabND0yEKlwwGVwTbiqwq+elPviBIbxl7NoY2oahKknrLdDpltymo6g6/2tLkCnRQZymirDIZujTVga510KUJceFj6DauO+ebH37OxeUJ89MbwmyJa4mstnfHfGnfkSYVFxcXLG+XPH9+TZht+MM3b9jtMrLLEWneEaQRWZFycn5G1xfsdltevHxOHMd0gkqWpHjjlkroSPIOVTE5OZ1TlhF93zBfeFRtge9nzE7OabMHZMFgd7dFZ4CnW+R5z8n5hJ1/x86vUDWL9XaFY49QdIswqkjzCkH26eyWwz5gPJ3Qty2fv/mMJE/RBJ0o2hPHGePZKZrl8u7D9+iKgWU5GLoDXU/XHtV5VZGTVQmedUrfNdDLaHJKVesMLY+DfxxCFTzaqueLz59TtTnbrKaqBFzHRdUEJBWCIKYXhmhuzTZc0go9O3/HdH5K0XSkeUsU7+n6iovLOUWecnv/Hc9uXrHZ+vjxDlUT+HT3jsXwAmdogNDSNBGiKBDuO9pcIqg/8uzN52zufMrywNn05GgIEhUs06ESSsokwVYcJFklSlKG43PkpkboN1Rljq0bdHWOjMtk7BDscixHZrf8SCse0XZiP8BWB/hRhWxKDB0bQez48PGR0WCCO3eompKXZ5+hCgdenqjML3LmnogwgKeHgiTrkFUD1JysSHENj7ot6SmYzp5T5PC0estoNEJRdB4eHjhZTBGljsenFbrmIAgtWRXStDlda9H3DXHRkFQNRS+yzZeYisbQmPLDx7dMp1NU00YkpSXH9FT6sCNNai5vLjm9mPLtr79mPLbo+5D5YsLTeoMYGFyNF5ieRtfKiJ2MrEh8un/PYjEjziOqBBxnjChtccwxmtSw2XzgsO9RNBvdaNgFG4ReIkkiHh4SHGdAX5ekeYrQFyjCFFXruHl+g3NQWD6s8aYGF/MzxM5GUwT2wZ66irEMl7oQ+MWf/0At+aiWysmzK/L6Eal32DzseLyv+PzHDrd3PkHQc/N8hrTOuLmwedisaHAZDVSCMGLtF5Slz6k+xhvodF2DIPS47oQoiNEVk05oyPOUKGtos47nry4ItylZ+vurF4W+73+vf/wf/sfn/Y+/+pK2smiaDt0S+eHj3yBpBQ8fe5Kg56f/6IIyM/nw3RrNynEci6qWaPsOdZCwXxUU4YAf/1FLWxjsDncgiZj6hMG44sP7O1x7wU/+4DPSsDo2okcd332z4uxsTpxXLO87/qt/9qe8/eYb/q//l/8n/9P//Gfs/Iy8AUj58O4HXr/4kroTaanolJRf/nLNs2ceeaKgoDKfyIyGE37xq0/c3Fzz9V+/5c/+8UuyOmG9zhm4CxSpwff942pA3xPuW24ub5BlkbTIieOa1WrFaGhj6S7rwxLXsehKhY4cy9WQFJUk9RGxkEWJlj0Dd/Y7j2qH0JvEYcvT446hrZE0awxzhCSaFEmBJvdMxxplodCQ07Qwn5+w2uxJ0xzaI7pJUxX8LCfNImQZilLBtl1UvaWqCtqqpEViMnXJE5mq7Lm8HvPdt+/I0prF6RBdFajLFrqjo9fQOyRRQ1NG6GZP19QUZUfVSqRNjiTVZEHG5dkLbh/eMzJP0C2JMI2JUp/r62v8XYTrOLRdgqpr+IeErhWZnwzYrjeYpo2kHkHVsiyyXi8ZDwf0fY+umWw2B1RZxXEckiyhFRO6VkIVHPbbgNmJiiybHIIcZ2QRRj7eYEKefcLSLdrSRJBLFM3BMhS6ruHF6zd8ePuBPCvpBZlDENKJEkot0rQlru2gKQZhkNK2NaoBPRWyrHC6OOPtD59wBhZir7Ncb8jynIurGYqmMxtbvL29YzDWqQKJ9SrCnmZImGS+hj1L6BuNIpDRDZksLzEMC7FvqPKIJOkwnRpJEslTgfnZhCJv+fRpw2imIKkKVVUc4ceblsXpAMU45q2mIxPL0Enagl2453JyzuPHLZIq4bo9ruLQ9h2T+YzVdkValBRFwXA4pigKZoMrHpffMppYRFGBpnpcXJyx2i1RVZl3b2+5vjwjiwPWywxV0XCHNmXWcnI64nF1iyq52I5KnOyoG4HxaA7CESB892lLTg9Sxfl4hOcs+Nvv/5bJeMDuoUZ4LnI+uOJsqPHw/UekvqPvLR63a/7wT39Eug8YTl2eVin3Dx/47/67f0ERi/jJPXnRczbz6LKEWh0gGj2bxx2fP3tOXdf8zerX9JHI5fiE/faJTCipHkFzxkhGyuh0ytSast1/QpE88jJjNh9y/37PdHpCL5Wo+oJPH/8ajL9f/1RRy3a1w5tPkMWCTpKQRAPHUcnSHWVZkuU9bSNjmQ5iD6oFAhrhas1weIYuitgThdFoxl/88j8xd06Zz6+oupz17nsEwUYRx+TNHsPq2G4imkpmfjIgizMWo3OS0Gc08iiqEEmB8ACWrZBnHVmx4+zsnLbW6fsGP1gf4yOagKhqnJ1csV/6rNd75uczRiOR33z9ic++fMO7d99xfnmOgEHXS0hyhSAIvHv3ibYRce0BQbBjNBqRxhmGccRSiWKHow8okorRaETVp/gBiGrObDTksN8fXeBtjjXQAZUgykApMbQZbZUThAd+9OVXHA4BoR+hqjKqoVN1BbQdVd4TBEuuLy5RZAdJa7m/fUen6SiKQtOICGKPJAGdiCqpTCY6WQlxeKCsQdd1Rs4J6/U35EnJxLlClGr2fsLizKMqRSxjTJYFvHj5mn/3n/41umchiwp13tFTMRzPEJWGpobA92nEjNl4znZ1YDgcstmtCcMQy3AxVYskKaAv0XWd6eQEQRLpRYEkCVDkhrPzBe/evkdRNCSxRVVV8kQ7RsXMkKYCrfWoxApbNxAFDdHQOGwCzLFOW4bQajRdjqgpaMKxGJaXPlnVMhkcbUrn55eEhz1FCpajkTQr6sZDFPbs1z0vX/yIqgroChl/X1F1BxStxtTHjOYGvZhTJRli06EbMaJdYA8n7LdbthvoBIMwibl+9hldVxH6e4pMwh5UKJJHEork1ZK2bRmP50RBhGXpdAhomkaUJqiqiCyrjAcL6FoEqSBMK7oe9kHGj//ghl//6pe4xpHnHB1iJM1mYEukdUxdwfn4nLLOUS2F7d0jUZQxPzklSg/IioaiGyjygE4OjpndoiCJSyaTIQ/3Wzx3gKmrCI2MJABqgMwcSW2PsbKspRNq2k5BUAskUSEMchxLQ5RaNFWmqgo8x8VxhghKg6mM2O2X3Fy+RtVbvv/tO2QFXMtDljRGEwPLHvPtD7+hwwXlnqe7mvPTKe5YI9pliK2AJJVs11BWMrMzBaQGx56QJzW90oJUEG4CkrhGVkHoKxAkRMlkt9twejalLCuyCIqyxXVtZEdkdZdR9zne0CLxj2SUd3/1/vdq5vzeGcp//k/+d6iSSZo9gZjy//lf/pzdeoeq5Vxfz/jpH70gTQq8kY2gdBi2hmHr+EGOZPRIcsdg3HFy3jPzXkHj0lcuJ/MRwS5jNjrlz/7kv+SzL274+P49htmQJg33t5/46R/9hCyrCOMVn/1ozndvv0XSRP6P//P/xOz0BN0TORz2HD5V/Kv/8n+LKp8Q3vtsnh5YbVKmzhhVMYjTO24uBb568TnZpmI+GGEpOs8uZkgOnJ2d4Q0M0uKekhq/yiilNfvDcXXn+z7rzR3r1Tu2uw/YrkTTNGw2GxxziNh37Hcr8jRBaETqrEHuwNV1TF1mPpkR+QFdqzCdaRiGyNDzuLo85fXPhozHN0ePdh2xeDajMVtazSWIK5KiohMlNoeAqu3oZQV14CJoCoY3QFDAHFjojsFw7KFoR6Wj4wwQFRlN8TAMg7qNcDyN9x8/kGUVp6fn2I5BGBTEacjrN59TFi0Pjz57P+IQbfD9Ix4prxOGowFqqxNtUuYnC5I6RjUdZos5st7iODJxkLFaPSHKOWV9wNRlmuaYq5K1IyBfElzyPCcIj7io1XpL0/Wsdj5+mLLzI6qypUekrBqqpqVrj0wuUc84u/DoW5W+UXAtm916j9A35GmI1C8ocwXNlMiK/jisBgWHKOev/v3XZGmJNRiy3sUIgoAjWBi2hSTKxzhAnqBpGo4zQJZVRFHkbL5gtz02ZKu0QpBThkON68s5o8GEvhf47dtvUYcWT0FNrkpUao5suIiiiawUyN2cIpWYzGzCMGa5ijjsY9I0Z+JcMV2oiKpMmBWYQ4sojinrPbOZQZmIlHFMuq/ooiEvr0+xVZliX9ElLeHKp4giNg8PdEmLUAh0dUVfV7RJA50FvcZvv/meKMpQJRVdNsiCAkNyjlDp0ZTdKsdf1wydIfd372nKjKe7La5hYWgNTSGwmE44ObewHQXdkrm7X6GoPaIIpmFjmDbmwKaVe+K84PbjGmsioHkK5+cL9N5l+W6HrnuonsKzN2ecmmNMpef204o0h8ddSNo2XH++YDidURQiJ94J7b7i0j0hXkeoqopqhpycqpTdCnOsoKoeaRLw+uUJimiw2foMnQmX4xMqQ2V2OcTVBC5ez7DPZMyBRSWUrLcfEBsdd2BS97BZ+wyG+rGIVZUIYkzd1UhJ+fdHEjh7dYHiwGO+xnJcijolDg/EfoVtLBgOh7gDHUUVMIcivdSRZiW2baPaGd5MpGhr/vabr3GHFusw5OPtr6BsUfohrVxhLRoMW6JOe4Yjk/mZRV7m9HJLQYFkWkR1RCWI7PclSC2qnVA0e8aTE/xDTS90JMWGMN+TNQlB1JMUDb/54QfWvg96xtP+kfeffBaX52x3BWkOHz6sWa33bFZrokNHlsg0dc504hEccsTepClFdG2AqupkaY2pOdDLPDw8UFY5h31AHO1Ik4iwrOi0BtGskSyJIM3I6ZEdhU38RCEGJGXIeDpi569o2hjbErDNIwqmPER0Wcl05KJbMo+bRxqhpWk6NMPFc2TyZIlty5iqh6o4jEbHskdVC+RFRNkK6JrJw+MtitYxn10y8iZMFgOswYjF5ZBKiOnlhn24Y7nzuV8v0YctcX5A1RUmixNOn1k0YsbTak9a7WilnOHg7Gg3cxWKMiI6+JxMTlEFGUu3cFwNRe8ZeCa9WJJWOyS5QhIbFLXj+w+/ZbM7iiH6uiGParbbRwyrI0kE0ryjMzIE3SDLS/zDA33ZM7ZdqiBEQIGuoqlL+qYmSffUXUtTyTQ1dJWBImhEh5g4zFAVncOuoK8tDF0h2jZQqbz95pbV/ZL3775FEhsuLxZomoaf3fPx3XuKg8B2s2SX+Lxd73h8cvn0/YCPv7U4rA1kScMbmjw83FGWLU3doaki3uDYbC6KAkWVEASBqjyqRGVZI8sDZFnEMBRUTaape8Igoyih60z6WEHpNMS2Io99mrJgs9qRRSW65lFXCZJmMRw9Q5McqiImrgN2oc/16SvOLl+SFR3n58/whicIvUoSZWiiRlu0GJqJZRlEccBw6tBSoagdii4Q5jsen1Junz6Q5TWBLyBKNUnms9+FCP1ROeoObAzdxbEGJKmPrtuIkkNWJhyeItabOzpJYBns2UcZ63XC5ekZctcQp0sO+4B37z6y3QVIkkTXmSymbzi50NClljqLEJuO3WPOi2czfvJTlzLNKOMM3/f59ps7dpsUUVDIs5YoCpEVkSxtOfgR212CYljUvQA4BGGKYSqUTYu/DpEkAdc2yCIfWYHJ+P8PHMq//pv/yF/+1b9jt9uTJgHPnp+hcsr2saHtP/Ef/s13fLpd8rh8xx//2WfYhothGLz60RhBbjmdWogVTOwTNncRY2vBT350hdj0/OiLM4pEIMm27NcHzuef4RlnfPz+I0VRsdx8i6QUqJ1BcoiOir+RRdCvud9uePvulnIjMBws2G7v+Prnf8X16y8ZOi8YSpf89GdniKS8Pr9Ebgd88+0T7lRmYjmonc/zVxpCp/Huhw8IbYjUyRRljEBLFpRY4hhd1/ntN2/pWwFZMqkrcEwPSZKpu5LtbknR5AwXBcOJy3K7ZzqzEQWNJN6S+immrNIXMLGGkDfItYomZpxMZfIkR+s92qTls5sbsjBFMQwKsSSTAvKsRpV1ojDBNh1MXUdXdNq258OHDyAI6KYDkkAYr2jblKZuqYqeKM7xXJPN+oCqGaRlQNEeePFmRpIHPN7vSPMjj+u773/DdOHx7OYFg8EJTduSlgmbzYayFLi9e6DIfDTVJM1bdtGWNMtopIDReIqujjFNC8tUsO0Bod+w3QREYUoQBGRZwmg4w7ZdDn6Iqjmouk0jgqJpDByXkTNA7gUWixlZlbIKVsRVQlKVDMZjqq7mfrVDUkbcLe9JqhVlWdNUKnXZ0PU17sDAP0SojKmqljjKiIISxdTpBYUoiPEck8lgTFEmJElE3TdH44gh0LQp+/2erlVoapnQL+iFlldvTkFoUTSZ8XiMpij88N0jd3drHNOlDSSKZUV3CLHQePubHe/fLmnrDvqUl69u8PfH8L9jy3RtjaEpfP7TM1RLJS1rLOeMJBQxZAdXn/L4IaKqJITe5Ox8TtfnbHdLZEVE02ug4sX5BZZlIdc9E3XCw1NIQ8V4qKCpNqomkmUZTdMQ+uGRD9tCkeVQt+RFyuP9gbaRODud0DYJSdhg6WMcW8e2PR5uCxaLBaJcEkcVcdhw++mBIhMwDZc0TXlabtkcIsq2xY8D4iRDkCRkUWEwUOhFn7LdYU1hcmKRFwEPD4+0YQulSNdKvPzsDW8++1OMoY479PjVr36FO4wIio/09oHxc4GH6GtEvcMdjfhwvyTJFO4PAdOxjKTGHPwlt9+/43H9HsKEzmjJH/YE6xptfEnQ+dgTAc0bMDCH9GKDY7m8f3eH7ZnomsXjpx2iICMIJsvVhouzV/hR+XdH0xVs3WVgzehzkSKviLOIrCpRFBfHHNLWJeOhCxxv6GXR0fcN9kBl528QBIsg6Gk6kbZt0Q0Vw7Vo5YZObvHcMf5uz26/QlJkNNEk2G2RRZnFYkaS70jKDXXjs3uKEBURRTPYrzugI0m3OK5Okj1hOi6aOSQvdCTZIopTsiIkylZUbUnbKoidyHa1RmhF6HqGA4+mqlEUBdcdoMkOi8WCroWBM+bZs2vmcwtVaRHaiizKSYIG3ex5/aNnOOOe04spi0ud6XxEVu0xLRlDt+haBQEVUTi+yM0nZ2x2KwxDoW878qxAkg2SsMIwLGoyGqnl4tkly/1HsqbC9DziNODdD/fYzpjx4JwyVSiSGEksSNMtcRwTJRlx0tBUClmZIikVJ9NTHp8ONG1P29cESUorVvixTxiWKJqO4YLpdnx6+oGsKrHtCVkmAh0Ptwkf3+/wRipl0WEpJ9iGSRbnR51rc9Tgnp9NGXoOZZZiuTovXl5RVDmiKFLXJXEco8oOfaNRZSq2riH1HZo0Qm5Nzhdz2qZh4p2zmA4RBAvNhunZgLY3eFp+4vrllK5VqAsBRQVJUug7BdvRyPIE25qgqxoHf0kSZVj6EAGROF0RxXv224QsCklijbIpCZItRVlj2lM2+x3b6J7hqcPrz77C8FyWYUTSy7SKSV25tE1BGvpUpYCARN/a1IWBIAgk4YHJ2GOxsNhvAlZPS+omYzabIcs6TdOQZyVxGKLqJquNjyKb5FmNKAnotkSWh5jGAMVuCeInTEfmh+/v0ZUFw+EJrdhSyjscU6QoCrI0ZeCqxFlIFJRIvc7T9pFWANszMR0bQeyPX4KFjqYqGDouSj/ANQbUdc14PEJVdKL4QJYlIGlo2pTpfM7H2wf2+z1xlKPJQwQawiAjCnPi4GiNOuxSmlxEElWyrCDNC+pWomgKXNelLmpWd1uGwzHbQ0Qt9LRiR4fM2DP44vkrLCWjCitcY8B48BJTs9FNjazNOX0549cf/5rlNqaTAyaLATPP40dvzri6mNO1R12pqDQ0QkMrqiRZTd02iIpA1VZ0CJiOTifWZGmO0KrIsoosSiiygYBEU//e8+TvP1CK6g5nIJJEDX6w4b/4F88YzxSqVCP0Oxyn5OS8x/Favn/7H1DUCEkMcCf33LwEUTD47/+H/zWnVxJvPvd4/aWCJI5ZnNxwefmKqmlBTDg7vWYxO0PuTH76k5/hGNdstyn0Is+efUX3OyDt3V3IaDRiee/zs5/8MX/8p19wcz4iK7b87B9/jkLL5dUIZ17x9d98zec3n2OpM9rGRnUU8kqgV0vGsysG3gsOyxVtUVHmApZuoaEzG4xpoymOrpMnGUPnhDoX6RsVSx1Br1BWoGkq7nCEpIg8f/kTttEOzZJ5+/GWvb9muw/QbBVBlTFth6w8oGlDJMXAGmgohsTMviHMVpzcaOhmQ1MnyDSsnpbQqsymZ3StyOniDH9/wN/uMTWVNE7QFJ3R2GW/2RP5MUPPRe4F/E1EGBR43hDTkBlYI/pOpW4EmrYnSHx6CizL4uR8xmi6QBBl9vsdqiFxcnZKFFeE6Z5DUrNc7cmqCmzQPI3dPoDmqNK7W614+/4Hdnsff5dy2Ke0rYwomozGF0dsUd9SFRptI9L1Kefnl3StQhyHSE2HIogomsra9+lkiazK8RwTtWsoUwn6o2M0K3J00yUtQixXo0Xj7HTOYjblbHFCXdRHO01SkhV76rzHUVSapmV3WJOXBaokoss9eZrRCA2qrjL0RsxmC6qqoqhTFE1mu10jCAJlU5GVAavtCsPQSJKOx8ctH949kuYJttxgKxBvd9h6g1RniFXPF88vOR1MsU0FwxLZH3a0+LRVjyK4TKYSY2fCj3/6nB++XdGWIp5pc7lYYFk9eZkyOxszmns4IxPXc7EHCoomEoQhCDK2XZNVKbtDQFtrqLqC7FY0qsIu6lENaNvjUNDUImnaoqkObdtydX2KN1JIEwXDlDHsEssy+PjhE4qiHPE0eU1eF0znE7aHPXcfI5ID5GmNobp89mbBZtngBym+H6EoDnGWkxYxvdJieRJtVaLVGpmv0BsC22RLsE+5WrxgNLSp7RgsCUkSefjhLUq3QVN2WLbOaDLAUifcPmzQxga9buJOPPbBlsO6YjY8QcLg+nLEbvOXJH4OnYXlwOXVnLbQCT8daI0W73yGWDWUZYlQmQytC354t6GqbWStQTI6mr5hMBhwOh9zfTNhvdzihymCICHaxt+dyA+4+/YdfdZz7s5IkoLhaEyWStRtSt/HOOqY1ZPPwV9SpA1ZFuN5IzarlKZSiZKYuo/pqpRo21CWKXlds4mW7OMnVo879suStgLdMojiGG8wJ04y4jSiqnsM3cPSLVTVP2Yv8wOdcDQdWdocWekI4oiuOQ4XmqYRJxvKrEUTT/nixZ8iVA6ebXDrf0JyDRRXwZ3YaIaKqgn0vUCRJ3TSgf0+Z7dN+eyLK0QlY73aYBkmhgWjscJ0MSTNIyaLKcvdnjA9xlVW2zvqquKwDcnCDNcymU4t2jokD1L6TMPShxRJRV0WdH3L/d0SWTbxw5hvP3zD8PSKspNwB2NEQcM2bRSp4fJ6QtmEfPvdO1R9RNt1CHKOabfkRUKc+SRZjNiLKNKA6FBS5RlNF9L1DUFYEsYNRVWjWyadIJJlCZJg43nX0GscdjUtBXVds90/kBcxr16e0dQhQiMiCCXBfkWR5EfxwmBM37dsD0u2+w2TkwmCILA9+Ago/PDDO7I8JooPPD2tqMoeU1Vo25Yo7chqAdmR0R2ZrgXLGCDUGvoA6iZhdyjxZjZXz+f8m//0SxT1iMfJ4oom60njnLKANCnwD9ujgEKskSSBzXbJYnGKrPaIaoRjaWwefVShR+l7zibnzMcLxkOT0VhBV3vef3/H+jZD1wYMpgNkzeHp4cBwcErepEhuh6w4bHZrBKHFdWwsXcAbyvi7lL/5j3tG7pzri1e8eLkgjnKKvGU8ssiyCknuSaLjQBpEazRdoipbFEXh+sUF290DKSUfVvcUbUZaFkRpRN4ECIqCpJh0vYzQd6hUFE1K1rWYmout6JhTHUlpcFyVd+/eEcU+u/WGze6W/XZLXde0bU4axii9QZl2GLoHrXKM+dnQtGsOm5iB5eI6x+euSE/X18jCEE3SkCWNNEvQ1SPbc7v0abuSwz6k10SKuudwOBBsfdqyYzLRCOMKxXPpW1iv1/z2h7fcrzZUnYKqWeTZHUqjsV7VIE1oNY1N+Ugpq9zuPxEWNYf4I7ebb4jKmNXhgU7K2O12qKKBUPUYCDjOgJPTEXkRkWYVQRwhyFAWFX0v0LY9ogpRnHI4RKiWhWz8/urF3ztD+S//21f9P/3nX5FFJXEWs1iccnv7kcnC5uPbkNG8RBBV0gzGY4FXL8/IEoH397eUfYcinCB1IhfnJnGQ0jQRjnOGpAwoyo4ofIthtxThiDKu+clXU5JI5/3yHfo4ZjH+DD/d8u5txOvXl7SpzH79yOnliDyX0awatTlhs31LXZeE8QpBNfiwWuEMRT4bvcJwztgF9yhyhKl5rFYVr38y4ttfLdluP3J2eUZWQujLCEKPJGg8fFojyR2uPaCIewQ5YjR2SIuWJM+Ik4qmzWgykSQuGAwGpNWOxcmAOEiZeROmgyneHDTF4/7hHV/+6Blvv19xcX3BZHRGWeXsDxG32wTR2/Lx24Ai6mn6LTJDDLmhVu1j6SYIaeuS+XyOLMuUZU4nwD58QFPGXF9esN9ukHqZupSx3AGlsCPdVyRpyngxAVnju+/fIUvw+etLomiLqnp47pwij9AVi7YRMCy4u9+CoFKnHTcvB3z/6QOt0mEaOl0sIHYtvSDTSi2eI1OnAkKn4Y110qyizGuGQ42HxxDb6Xh+9Uc8PtzS8MRk8oyskmjlmDqrMHWDru4oiwJZPmZPVF0CuScNOjS7Q1F10nyHZcwJgy32QCWOBTxHJU9K6Fo+/+INh0NBUSTUXXgM1gsiRZZg6Rpt3xFlKd5oSBamFGnG2Ysz/ENC5Mc4toprW1RVS55nnJyOaLua+9UBco3z8zF3yw2S2OCaA4q+xNJkmrogzRtEuSbZVoynZ9RCwOoWXrxw2cchw4nA3buCPNJ4/uoU/3Dgq8+eM11o/Jt/+1fEccvA1XBslaJqaRUBezRB6lI0yaCIVAxdIk4TdENm/bRnPlXZ+DG6ZKFaGlVfoNomh7og3gW8HJ1x2ISMpyfcL7fUXYusCJydj/APd5ycjnhaN5wsPB7u32HJM8LIx3ZdahIcb4wgQZGENLnAbHRB4EfYrsDDwzs854rH7QZJKanKjkbQsacem82a8/k5Q8dGsUr8xz1yZ9GpBaY9QBZrBE0A2SYPU5K2IApaLhcLkNf0vYEqaFzMZgjdkLS6wx2KfPi24PnLGTIW46nGu48/Z3LqYgkitx/WGO5nhMUtVVKAaCM6x1yho/eUtY7cl8TCirF9RtrIuJZGXaq00Y5Wy1AHDnN9RrLa0Ukat093GI6LpepI1d+/g4dpgmRoVH2MYqms7isU26OuYtJgz7/6Z/81v/jF1+RtiTuW0fWSJLXoKxdd7CjFR0xFYxPk2LJxLIKZCsHuA2Iro2gWsmaw2yfcnJ8g1i1B5aOoBlmZo5sKRVZiqQptWqD1LkHbMZvqRIctL59/Rp72fPPNbxmdQZrIqLpNHgccNhGWLeGYU3RBx5t0JEnDcOKRZjGW6bDbhgioDGcidWHRtTIFW6gVvvrqKwauydPDmjQsaJuSKFlydn4DvUYjxpRtTFFK6JrB0+ojSSgxWjgMHYOxO+Hu9hF7YLI9rBF7+Ed/8gf84uvv0GuJ6WhE3CdHfZ7iktc19/GK57OX5PEOakDRkWUZXVMQ1Ir3j2uyaImuOSxOhiiSy6dPn3j27AVBvKFvDRI/ZXF+QRmH9KVC2Ymcnk6Q5YJ9lCNLNYIgMBqO+fDxLaPBAm84YxPe8vh0QBQGmE7CbhUy9AwscwRA1+RUObSVQJ7niFKPqks0XU8vKAwGDohHCHxW+GiSjaJpiGpIEuf4q4qL8zMcT2GzzkHtGC88wqCkzgpsQ2LkziirmtatyZKILK5wZQ+pqdhHW+Tc4eTZnMMqpu0y7ImHvy85GQ3wDw/IkkGSNZiGhKz0jIdXzBcjvnv/F1SZyu0HH1WzMRyVyXjGxv8N19dX3N5u6eQCRZ5Quk8Y3RkmBo51zIKvHjfcXF3SVx2VWPLtbx45v54iqSmypOE4Qz68/YBrzvHGOnFa445bnp4i6lJB1RKG7jNGk47/8Jd/y/n5ObLSkiQpp4vn1E1BURR0tUml7KFTiIIQXXTpmoKqDtHMKbo5QO17VPm4+SnFivv1ks8vnnE2n/Fud0+895lMPfabgrrJUYUBricRhQeG3imyUBH5BYIKQbGh71SqOEOWXcRBhitp6MoCVa/ZrhNMR2a9TnHcE/bhHW0j0lYgUHBzc8X9bUjbFbhTka47As4N0yIrU9bLnK8+/xxV39NUc7aH9wjSALqGga1Q1RsU4YS2kxhNBeqqY2IPCZKUVbhF0ASefXbO+28/4Cgml5MLXry54T/9x/83nWSAeNRstm3Jx4cHpFZGkCrKPqVH57CDkTdFVWqaOkORdfq+p1Nbon2IgIzpjZBFiW//zbf/sBnKl1+cEudrvn37gY/vd6RxQ1503H66R9E61o8wnV1yejbm9bM/pk3OSH2BrnB4fKfyi//8LY9PH7n99MRmlfPbb9ZUhcCvf/nn/PDbf0sWSCjtK3aHPc+/nPObd0t6rScrD/zw7S2rzQciP0DtFcTaJAxDirYEYYggnvDs4o9QHYXAT9juH2gED3MyYTo75YuLP6HodYospUs6tncCdQMfPh341//6LQ+bDbY7oG56Nk8NQbimbWQ01WA0FRCo2a4f6aQYxVGQbZei6wjSLZpVH7Vscsfl6wnuVGA4PsEYjHnz5pqTyYTWyNmvMpqywTEm/OqXtyi6guXYhEnAr775jlrZ0HcRmw8HxFrh7MRj4V1iqy6m5XJ1MkDpGmR6fvTFZ+iaTJmltE1zBBePLpjNjg3etpbRTIvBVKBtQ+6/X9L1PWpvYMgC4XaNljV89ewUmRZDHKAoCk+Pe4qiIQ4rDEPj228+QJ/iODGmoRHsO4a2gFkZTDQDTZAYDocoqozQtfhhR9GZ5G2N7+8JNhsuzucYtkQnNcjCkPXTPUWZoTsXtJpMK8cUSYTRaYiyQimVGJbMyLa5PLliNJogygLOGAx3RE3HdDzH0XVUzSbyM4a6QFWDYJioA5f379aoSsLAKdDECXGzR1J7sibAsC16WSOXcsJ4jW1ojC5miPToGizOHNqmo+sERCym0ymPjwcOUQxlh+UccyqqLGKaJlmR0uQhdVYh9zYmx9jC69c3jNwBpjjg9WuHvMnQqInjGNeDz746pVGf0PF5/XzKN9/eIatwcq5jj0ziqiNtGiRFJwlCir5FLBvEGsLkgU44gmdtvcEdn+AM5kiqgKwJKKKBKbh4mstkMSVtGuZv5pTGHllOOD31OD1X6JoUST3BTws6LWGTvcPUbeajGcPxnIE3OWa5JJEw1Hh++RrZFoiyhK4JKJqcth5xcrag70ouL56zuLnCm1joZcKJJXF65iBbEnW8Q3J1UjVEFxR+dP4KTVWx1Uum+hBRMjFFAUsraaSSvJNQDZEo3LNebRGIqCi4/dSjWRLr9C2nr+DThx1bv6ZVVDb+AFG3qfs1eZ3ww1OE7FhE+5Aw2rMJQjbbW2zX4sRaEG1K8l1DVSgU+dHhXKciGgofPt1TSzLoLYuTE7o4Z/XwSC12f3eaGvJod1wdpwa90LP79Al3YiGKJpbqkKYx5likFiK224rdLkLRduT6BhqDC/c5M3tE3dq0Tc9me0/Xq0iijjcYk6cJmtogygKtpLL3D7R1R5uK2P2IKq6RJZVOEsiEEKXPWa3WqIpDsEoIVzsWJ2cgKxi2AUJOViTQWAynp3hjkb4uaLBYH0L2fklbVKw3S6K2pNMbHu5CirznxbMX2LqDZ+nU9Za//ov/zId3S4q+RBBbLMdlMTrjfvOJZbhitdkQJxvCfIfujHjzxQ1SB/E+5+H+lqJd87j9CIhczKaMxQqzMVB0j7DJoATdGtJIBreHJ8Ync+KioKEiy1p6Kirh+CV+H7V0NVijEY5j05UKUVzgDiYs7+8IlgFVUeMNJtzdLpF0l17NqcuIWogRNAWhAVVU0DWLsuuoaWjoCYOUzcOGtsxIsnsoWpBL0gDCTUORN8i9QRjl2J7LYKqh2x3z8QhN0hiNXA6HHUXc8ri9R9ag7EK6viUJcmRhxPWrUwSj4YdPHxB1uLi4oC9bsiBAkUVETeHT/hO+sCJY1wjNhNHCY5sF5G2JrY4QdI28Duj1AmMks10tGdgWvZwxXrjohsPp6RmSoh591aXPcvdbBu6MNO4YjgakZYAg9OyCBwaDGUWukaQttu0wnkmwkZm7GoosE8c5mmoxmQ7olJR9kVA2OSdXQ2zvaA4LQh9/GxzJAyOJjf9A1aZstgGDgcPpuUfT1+z9WyJf5sXVM1SpIo8S+sJAajV2yzWH9ZoyC2nKijQNcQcT4iTB81xoVZoipyp90mBDFO4pu4A8brEUhbvNLV+/+47M7xidTilvCybWiE4SGM50FKljaCzwsz1VKzOYWPj+nqHmMrBqTNugayWGyhzFUKjbiCQoERsTU7FxBzCwLTxzhNBGDFwVy7JZryIWiwmqKpLGECcNeSixfMjxgxTTVdimaz7dd5RVRleI5HHDaDbl5OQ1A+uG4cRB6EUscY4k6viZT1602KaBLU7QSh2j0RmZOvtNShR3iPqEqmzRxIY0KpDFAefzOYZhkNQlTWNTlS32IKbsNuRli+M4OOaAqm+wNJWqz0HWkOqOKFn/vmPi/w8u7zqmLUcIUsGrL6Zs9weWDymGesZXP70iSysebw+Eh4Svf/OfuH/6NU/rFfv9luXuF3gDlcl0QN13tMLRC/7p/i2qofP89R9QlPDXP/85bQPbdYJ/iPmrv/oLwkOOIszoWoH9psQaqBz8NaYjEuxFfvnLX+NHH7n98JH93RrLGPNnf/jfksUatzsBc/iSTWeRxjpff/Mdq8MSUZeJ8ho/2/Pdh2/5tLyl6yCJesLIp28NonhLnD0i9BaSJHBxNaTrOmQRDvuYJC4Yj0fQaUxGUxxH4PJywnzucHbuUBY7Qn9FGK7R1R5jNCBuSk6eXTA8ndNrMr999wPv7j6y9Xf86rs1q2CLpNsMxh5hsiWII1BKWqHB0rxjJmc6YLPeEYU5VVNj2jpt3yHWDev7PW3RI0oZaRqyWRV0vcrNqwWqaeFMVbKmIi17bl5fYAwcklykk3pEoaJMCiwTRLni5z//W2aLc4pC5+6uxp31tGKIJHnoTsPqMeDNZ8/QVQNVTrA1i2Dr09QxdV3S1AqONyRKYlbLPTQdA89E0irGc422jbn99IH9JqWtHJAL4sSnKWWGg0tUXaMSDuzjNXHSkVUFXZ3gaApN3R0Hqt8B0ItKYDT02G1X1F2L5vbEfoQquWzCOwx7QNmmdJJOUXWoQk2TxLSVgGE6TAcmt093BGlAlEQ0tGw3IbvNlijaUFclD59yZrM5+cHB0SQGukbsR8hah+nOEFUdy52gDSpaAd5+uMPP9uSVhGrbWNIptZST1zbrsCPpduxXCc5U5+e/+ituHz7RCwqqbpGmIaomHCH5okTX5KiNiuF6tGZOVdf0VY+iiIxnC1RTQpJ6Xn9xRVmL/PaHJ3q1peoL1us9pi2gNzpVKZB1HbXs06sZYZKg2Mc3/LbriQ8apjlBdWA6c8hKn0boeXxYke1XZFGC2KgIQkEr1nQo/OGf/DG7ZM31xXN0UcRRTGzJoykkLq7OyasQ/7Bh5l6Q7zO6TGK9zLh//I7FzMDUQ5Z3T+ySR8IkRzdsNEWkiWvqXck/+dmfsosLvnn75zw8rkmKT5hORZlp/PrXv8TyfE4uRJ7e7+ilCG14IDzsCe8VLkczot07Ul/gdOLRVwKWOWS5WxMFAwb2lD/66g1mX+MMLO7v9xQV5FVM03RY5jE/GEcRhnXO6fVPUCX5746pgTG2OTk5Y6rMcDoJbzRAyAtGrsIu3BC30Lcm0TrBUSyeXc7JaBkKElfTCx7yBySx52Ju46dbbMnmdLxAFiUEKrzB8XcWZUqcLxkMLPK0QOoN9kGAbkNd5uRJzWjsIas6ktjjuh5huubZqws810JqDRzDQGgkhN5hNOuYzlTubw9ohk5eJkeYvCTgDDy225BwlXLqXPK/+e//R/6bf/mvOFuc8cd/8F/x+Zsvubv16TsJ120xzQ5R6ugFgU/Lv0XXVfxdhNxZjK0FfVERbWKCzZY02TKanSE7HlFT0+sCqlMTJSH/9//bb8nyhrzas1pt2Kz3fHj/iH94gkYgPkTkhz2iIuPNJijKMTN9e39HHmWcWh5fLb5A7RXypkJQZGxzgibOmHrX6KJA08XoZkvXisiSiTfRSbKUjx/XxPmWqA64f1zz/tsP6JpDLYmEfUpY9iwWz5hOTtiGawz1hJOzayrBJ/IDRsM5X33xJZIgoMoeY++awyEgCAL8Q0Fdyez3Ptfnz+k7AXoBWQDP87g8PyNNc/K0oSk0yjLkENxSFBmuZaPr+rHUopgIqU0RrulKn+WHgIHuYhsygwGcndtkeUJVCpj6jK7VePvuO/KsIw4E8nrL4/aWrCoZnVhkdUnZQl5JvPrxKYpdMRgMmJznzE8VNM3g9vaegWPRdyXLxwjZmLHxU1rlgKR0PN490lYCcqdTZnvEXsY0IUm2jLwJqqgxmpgMxzrL5Rpds+hamIym5FlFltZYxoiuldjvImTBoM508lRAUXtkpaXvVLK0ZzqdYssGA8NC7hrEvqarSy7OTxl5Q1zDYnriUVcRqZ8idj10OpqpIIkCdAmrxwcaSQJRQhNVyiznEOzpuo626MjLhCjNmc2u0PQJtrnAdRyev5hiOy30MqpmMplf40wsVrtHpMaiy0rkPmQ2PSXPOryRSprsSeKYgWuR+jmuYXF+Pefly2d4zhRdk9iuV4idSNckGNoIz9EYOi4f377DUHQcc4iuGmw2K3RDo6klirJF6lxWyz1/8xc/oMgadSuRZDF/8e//LfkuQShEFElkOhsSHELiMMFwZXRkujxFbBUcYwqdQNuVpEVE0e3I05gkPqDKFm1TINEj8vtzKH/vlff//v/wZZ/EJZ9/+YIPbw9YzoCq9KHXUJSSxfySf//v/zOvPpuhaBWPtxHX19fs0juiPOL5xRWPHx548+xz9kGN5mlstil1nnE1m/Ltx3skWUAURSShRdc7yrLkD776E5K0JM4LntZPDEcOaVKjyw4//tGX3D8+UTY5ZRpjWi6kNWVX8fZhzyHdoQ1durzAFE1UwaBrQp7dzMlyn8e7FFnJmM8mHOIERRkTpQeKQqUXIlQF5N5DUwWKLEHVBiAX3N4ecAcTZLXAcRyatiDzC0zL5uzqlOVqQxDu+PHz1zRpj+LAJgpIkgTXG9D3HR8/vccyTEzT5NnVM/78l99gj2pCv+FkcoKpdSzvM6Zjj76DJMrJqxBv7KIbA5K4xHJM/GBD29Z49oim6Xm43zCde7RdfXQFWxZBFGMYBpYl0LSQJQJNsWcX+FjuHNPoEbsOf6VycqGSpgJ1KWKPVJ5WWwQ5p29zxhObLKuoc4HFZE6VHn3u7z/eYeoSYeDhDEUOYcDp/BTL6vnV335kemKhqxqWZWFZFtvtFtM26Pueomooq47LC5unVYLnnqAKLWUVU/QNndjT1EebjiZCV7d0qEiizMB02W739IqEZzvkVc42DnDsnnFvsN0d6Cc1gjyi9XOGi1OqqGBkmWRtQiPK6JqDoUvsN1uqRuDx/g5bn2IYLa5zRJXUTYpQO7Q16KpM1TTk9QFZceipmU+viYsnPOuK9fYTRSZhmiq9lOLvegZjldQP0T2BzSpGEnXOZzOCpUwmrFA1AdVUjxda31LXBaZuQ68giiJNmzIdDullnaf9ijPPRepM9kWC0LVcnE1Y71I0TaGsdVpBpWg23K4+MRxOUcoaq5RYtQWWJ5NlFX2b8erZF0RFwuruDsOdUUY5tiFzfnpOlqQc4hWyblJkEi9Oz+iFhB/e33N2ckZWpZycXaMLKg+7j8iCTHjYopvGEVciKzjDEZ2qs7zfMMgVfvnN93hzl5PFEKnvkIUZV1dTfv6Lf4s6dFi+jfnszTMUryTPGpTG4E/+0U/52x/+hm+/veXZm0uaqqcsNoBImTSYuk4nF7w+e8U2O7Ct32FqE04HM9YfAvK8xLInXJ5N+M+//DXGwELRh7QIZEmLIMc4A4jThs+fvUY2FL75/jfogsXV/DgUxH7A1eUpjQzbNP37l+xmh6uPeLzN+aN/8hP++j/8FYrVo480ykOHGPUc0pjWBFUTOZldkZUrRtYCoyt53O/ZxgWGmNE0NY4zYzjyuL//iDOY4TgOiiIRBAGyKZIkFYpqUwU9k5nJLr/F0F3KoIS2o+nBsiz6toJeomkSxt4MVVJ4fIpwJ7B5ypDtHks1iMMEy1KQRJUgaQn9HcOTGY7cstqH/LM//EOSfUjS6uRJTtvAZDSkbHxqqaEuSixdoGw7dNFmG0VobgyyQl2JnM8mLB+2ONaEsvYpS5/z2RfERcsm/UjdZ4T7jsVgRB1nXM9/zLZcUXQJs/k5+8cnesOkThJU00FxOoq8Re5rJt4lcZMQJRm2qjMyBwhiiyzL3O2XSIaNq2uUh4CffvHH7Hcx3737JYOJyW5fYpoqmp5SZgZlL2I7JkUWke4SgjDjR189o20F1puAy2czbm/vGU2HZHHKZr8j3Hb8wc/+EFnd88Ovf+Dlsz9iPjbZ+DGrzRPzxRipEYjTiqysqLtj+S4rW3oxom87ukpkthggYrDcPmLqQxxL4HF5S92o2NYQWapJkxLH8ZD0EkcbAxF7P2A6vyFKtkzdCaqgEMQ7hqNzyrIkDGMuz2fohsTTg08a1ShGSFoXjLxzkmxP04johkaZtXR9jmXMOJm5ZGXOahlimjp9l1FXHWleM5oMiMqEuo05bARmE42BrSG2KkJvE8Y+p2czdFvg8fEeS/MwTRNvZLDZbNgfClRVxnIsdLvj4/sVkjDgZDElT0qqqsLQO2RFQpCOrXQ/jLi+OUWUWvaHA2mS0AtHpN14NEOWOzbLPUXZMT+Z0VYFwSbDUE1W4R3WaIbjWcT7EkVIKfsOsZUYjkfomsTHuycsQ8HAoVcbajpkVPL8SDLI0hhJUqjqHk2TkPuavKjRbANnYFMW8Pa7W14/e8Hj/Yq8rbHcGlFq6GuDPG7whmd8+/ZbFlcaN8+esVzmVE1IVVXohozQQJ2DJMiMJzY9IqZs4NgGD8sdnjeiKn3KvqTIJGT1ONhKSsXN9StkLeL9uwdoTDBF6iRE0zQk0+b+dsfC82hbn8flAXdgEcQRdSfhDm2quoPOQFMl2rbGVUXyqma52uE5E+qippF7Nr/e/8OuvJOoRVbhh7eP1D3YroE3mhAnB6oSbu/uuby5IUwSirrhyz/4QzZ+wmaTUxUuT0sRa3BGUovEhcom6NiEEavDio9Pn6g6Ec2YkGcqiupRVDJnJz9mte5J0h5JE9ju1hwC/9jA9R+Iy08c/Ee++/5ryq7g06cfWMYbNsGBLA0Rqp7+KWBYq3RtQNMcqNuC79/dc3sfoJo2ljflh/c5fe9QFg1ZliHLFZqik8YCbR8TpxmGbVE3Jf6u4/r6GkHKUTWZLAvYbyIMw+D09JynxzWi3KOqKkl6RP1stzF90WAoOtE+5MO7TzjmAMt0aauWj+8/MDAmdHnD2XSCqYkkQcyLm1PyYsmPv3rOyfmYs7MLFotzuq6hbgsCPyFPjlaFw7pgvbljNDaRRY2qkAj8hKpJaeqOtkwQkFk9LcnCNRIquqIiiRVdLVAVMmUbMRgM6fsazSxRZAj2EaqocnX+jDJVsA2Xs9kVjikg9cfg+cQ7Zb/puXlhIQo9pq4QhiHff7/k5sUAx/Q4O51SlAlRnDIczdFkG0mE6cRmPNN5fAqYTD0Ms+Xx8RHdGFA1HdvtFkEQKAqZNMsRBZm+EznstgTxBkHr8SYWUb7G1gTGnoumGqRpj6KNQTTI4hLn/ATBUBE1hbbryaMKS9QRq5Z0GyF2KdF+jaWZnMxVrq9HiL13zENpFW0X0bQFjVCQlDmmuUA3BdpSRhFrkn1KEq4ooopnF1eczy/IDi2SkJEcdvRqi+U6jEwPU9YZDWx0vacRFKyxQV3XCGKLbRvkWcP+EGMYOqalYegD6r5ntzkgVQ59J5KRkTQ1ddMQ+QVhmrPdxQxHLne334DQY5kmdC2ybjKcjqm6CuQaVRxiqFP2QcvOz2jrYwP8dGFh2i1pVlDXLaPRmLIGTW5ZHh5Yp2vOn1+y3W9BkbDMAY4mI9QtSV/TCzrhIWegD1AamfX7JeXjnjNVoRYjXnw+4uUXM8bXGrviQNo+cff4AcNWGUoOP/uTVwwuDR5XAVUAVivy7/6X/xfhujg6frOSNJJYTK+xVJe+cvjs1Y+5mD9D0zTSPMO2n7M/SIT5nunpCMedc/VsTiS0XH/2hqKtqdoDkiSgmg1VV6C7LgPHI00q7h7u8bwRs+kJH27fgZgzn87IspaqCHEHyt8d3T1nG2S0Zsjj/h2vrq/o8ozb9ROfwi2SZ3NydYakyFStjB+lBH7EzfCSp11BkGRU+5b52SWdIqKpA4QenKGDovf4fk4WNUThGgkJ0xiTpSEX52MO2wO2cUpZ90hCx3Q0psgbVEmmb2oEOjTLRjWOYG9Tl2h60A0RQRaQVZuh4zKezNlHOeOpzdnVCYrUo8kK7mTEh3dL3r9d8en+jt1uQ5bHlEVEEIY0UoxhCwRBgWnYVH1NkMZ0sk2cdnSty+3qiV4paYSWXoaqV3j/6SOrw3uypGViXSPVFUNzgmOdkbJCNeDm2WuSPCbJC5Bb8grsoUXVdBiWQ6toVECd14xsl7LN8auEfeqziXLsiUtZR0htg6XBh9u3/Oq3v0Y1JLbbWzzHRWhVQr9BViwWsxF5HmPaLpORy6uXl1R1Tpr4qEJLE7fE+4SmKmiyY3zp/GTM7vCAoTu8eH5xXAs2sN08ohg9RRNyfr3AGzoMRzaS3FCUPnmb0dKh6hov37wmSWvCyEfoWk4XC2QaNk8tuqLTizFhkOJYLvQVcjMkTXKWTzmSOCBKUnqxpBVsglSh6kxoBarSZz66oMxlojDFMAzMQUvTSlyevcZ2TDRNYTASSPMK2RSRTIVKilkeNoTZnvm5DYLMblcQpQkIMl2nIxIT+w1Dz6Cn4BD4SPKCxdUUUdXphI4862hqCUWVqZuK3TaiFw10SyevE5o+5927H3Ach9l0xH67YTIa4hgGjjXC0sYkQYEodViWRhjk5CnI0hGJV7U1eVUSxBFZVaPYBoqpolkqm3VM02i4wwWvXr1i4OrYmoUkHNFbnmtguCZtJZDEGfPTCRdXMyShQehVLM2jLLrjh6t8R1ZGDMcOstrjxwWSpnNycU3V1KxXAbcfNww8k+XuA87YwLArEEr8fYMsWaimwNa/5bMvn9F2Ar/9/hNVdyCOcsqyxnM1irJF1W1mixFpmpJlBV1XU2bHv3ugFwTaXiGvE7K8pypU6CWeHjd8+rAnDCHMfVb+ElHTqDqRQxLgjlyKtKIre148v6ZOGobGDKkBXRRps5bz6QihDWiygrqQkDoby9BxTAV3YGAY6u87Jv7+A2Xbtuw2GUUu4QxVojTh9m7Nmx89o2pqRK0mybds1gX+XuLu6ZGHpzVd76IaKvFjwsNdzuM+J2sCxOpAtfaZOwvQVJbbR+I05PLmnLppCKOCtGh5XK943N6zegrx3Btk0Wa3rXE8B39fsA9CPO+U7Trj04claVLihym90DEcjnHsozR+oCyg1VEVB9eZMh1fo6oqXZ8xGPW/q/YniKKKbZvEcYhpeOi/g63meYui92gWrDYfEehQhAGK5CALCn644xe//GsOwY4w94nSA70Iq90j++BAkVRYqs3qacv1xTPqSjheaJ1Mlle0eY7KkDwpyZMSxzxjvdmgmgYPq0eSMkJWVe5uH49Qc6Flt13StzDQJyRFjCjpqJqFrPXIEszGpzRFD2KFrko8PK5BFBiPLYajASfnF/QU1E1Jh8h4YnJ3/8h4bGFZIrosczodQFtSRxWWbDJzL3hxfUUSNpxfnPDwuES3dFRLpu1rdtuKydRjcaZzcjLl+voa21Y4HA4oiojraWR5SFVntG3DfhdQphKGbrHd+Lx7e4dhGPSUWLaKaZpUmUBw8PEmU6q+pykrLEMnzCJUW0UQW4IkRUJCqgUyv0czPTbxgSz/HfRa6Pj2F98T7WJETQZRxDJHJGnJLvaJKzAdl/OrCR1H3WKa79B0C5k5NSboMogajqHTpDllBIPBcXVEZ3DY7VBEjdFQZ+8/4LouuiYg9jq6bZHVKVXX02sKqSRiXLS8+ekJQ2/KwLVRJAlN03EcD1GQMSyL6eyEsmp52qywlB5JKmkEmX1yQFEa1us1+yigrkUMyyEND5zNJ0iCyHy0wDFshlOPTBJwBlOynYXRG4zH0lEdKlZ4hs3PfvwGqReRRYW2T5Clkq5pafISVa5JqgO7Q8R659PJNYIskKQV69WeOA7p0o5gu+F0MSUIE+KqJhEbVkWIXzXc3HxJXYFhmHz3zQpNdbGHLaopYGgepm6gaj3LuyeyoEbzdO66DQ91TJAWyJLDfr2iqp5Ik4rpeMTFpcUf/+Mfc34+4oe7e968+gKjgRfjM1zhnOvLLzAHHnfbLWKnst/vkQWd87MrwmhDEO45O71G6lWmowH3d0vaSsK1Legr6ha2uwBRqulLic024e139393DsuPTKegtvD+z+8wWgsdnTN9ysvJhIltsPxwz1wd8HJyiiULnNoz3q3e0ckicu3xk5++IAxLBvoMyTTIa4XVNqIiOz7Ie43Z+IS66NltH3h2c44s9VRlTt8LZFFN10pUVXW8UXc9QtdTFSWKbNL2EmHUsLhwkAQT23UoS4mijLi4GpMkNZo55NWrFyh6z5vnL49tdqOkVzXWfsjwTEfWBZI8Qndl4iwmSEO2YUhDj+VpiKrAcG4RhwW2Nmbg6tCLCIpKlAbEcUtRNghGi6gYdGjUvcRscUFWJ4h2TaMJlE3NcrlE1UDWZGTTwxhqFF1I3UgU+xS5FhDrnjxKUAT47PUzDFWhyBLMukXwYz6bPWOoT0mygrjYkTUBZVlwMv8xVxeXuK5MmoaAyH4bUGY1sqSjeAaHMKGINebzOZ1asvFDvvjiS9ReYTKYMZt6nM6nWEZPnpa0jUwrHsjrHMNp2e99liufj3ffs9m9J8sjPMdDVUQ6uQAZdHvA3/7ql7StQFVVDD2PoeuyWRVcXnooas9hl6PrFlVV0ZQQhT59o+I6Y05O50iCyHQ6ByUk7T7ijASKMsbzPIqiIIoikjQmL3e0fYRlTHAMDVWo6dqGtu4QBZk0KylKhaIUqPqcpOr55rtb4mrHs9cXzGYX+NEWPz6wmD3n6vwCd2BSVRVZrLNcfUCSJN58MUNWWooqRzM0OrFE0rqjKrhvsTyV8XxMWbcYuocqmdDXOKZCHqWkQQoN1E3Oy1c3qKpOWda8fPmSoqhI4+p35ZwG0zRBlIiTHAQR2zbpKbm8NDl9JlKIj2RFR1tAlVScn81p2995rDURUcxBbdiHEV0t0VNT1RGKkHKycOnagvF4imV6pGmKYkLZZWybjLsgJq6gE2tG4xKhrjk8CtCnuK6A0NrMF0ddqKyYjE8cajGlagUU2cT3fSRFRlEk8vIo8Wi6mqyJyPISQRDo+oLbx49oloI5MFB0i6Ls6HuBssqwbJEkLEnikKqscRyLuhbwVA1HVEn3AULb0VY1+/2eN28+Z7444+T6lEMSMT45pxcNwiDj/vEBb+yi2hKj0RDdUdltch7vdyiyiS67//ADZdn2jGZz0iJjvQp4eHggL0J+/td/Sxj69L1xRB/UGQ/LRx4ePyDrFY6r892vP1LIT8yemWzTPbu4JiwqVE9jcOIh6Da2McLQdTbLFVUpoKgD1ocdUeoTRQV1LrKYT1FVFVmBt9/f8Vf/+Vf0SEiKSNZCp+vQ9eRliag0pNGehIRNvsJ2Jdq2ZOjOce0Brq6jCypyYzG2J7Q0pGWCppq8f/dAR0dRB0d7iW0hKhJpBo49Yj47Q9NFdtuPhIeYgeNRtwKyKGG7A6IsR9NdNocDUR7TdgJZXrE/hJwsziny40NAkjUOUYJmOHR9wWaVkmcdadYSxgJh1rD1G777+ERe1Ky3K+q6xj8kFHnNdOZhaBKbR5/hxOTy6jlF0+MHx69pdZ0iCAK6ruK4HoIOk8UJsqYSVxGdLGJYAwxLRzcENF2izEvWqwfERiGLEj5/dcGr60sGusXLqzOGjs3b798xcCd0hPRShag09KKBqmt8+dMJqmJSVyKm03F7e48gVnS9TNv2xGFE29akeYIggWpqyLKMv9/SdBK1AKOFw2r3yN3dPSIKsiwyHRuEkU9Li2MbyLJK08m47og8KdGlIZLgokkWdQ693HHz5gxT09ksP7H87hazrGizhDhJubx5yf3mnrhLSAB9OMOeD1HclqaSOD2z+fzHBnlaEMc5hmGgKAKaLeBMbPQBKLLFaDzA30bkTUpa9+yijF0cc4j3DOcO05NTRFUjyVOqWMYyYTayeP/djv0+JQl6DocAgZa+lSjSjvFwyM31JQIK3/zme/aHLSP7AhmdXsr41Xcrus7FFhU8zQbNZDQxUUQFqe9YLEaEQQatiCq2pGlM1jScLU4ogxBdERG6EsvKMRSZ2fgEpVNQRYcibajylCgNKSof02xQFYeBOUTrZWS5Q1V01o9bvv7mF3x3+468E1FMlYubK+5Xt8h6Sd+lOLKG3Rm0QYnvB0wmlxRJgFS1WJKMq5+gCDoSJVEecLeNqHA5mU0Z6R3RZs3AHmE6CvefVsh4DJxTntY+adazXgr8n/7P/w8+fvzI2eKK77/+SL7NaLKI8XDK3/zyN4yGQ55fXHIzOefl/IKXl5+xuqsxLJ2bq88pEzg87Xn4eMfAPTIR98uYKsupi5zp+JwyK2n7DbrWoTnW352qM3j4rsDVBlx/OaYQRf7Rf/Nfc/nqhjcvPkNBw5nqzF6f83G7Jyt9xK6n0RsO+x0DT6ehIg0PoCks10s6SWA6e8Z2m2I7BsOZwy4MEZCYDG3SOGS7TrAsjb7zoQFFclEUhec3V5RFhmZM0R2J+9sNdSWy2T8QJjlllSKqBqPhHEmSqIueqFhi2A1ibVPUFZPJCEE+PvgmVypnLxwMVcceSty8Oidu1ly+mKAoFgNvwWjh8bTfEKcZsqzSJjFnIwe1L0nDCKnXaUuJuqyZz2a0eU9dNOg6iGJHlnU0fY4o59RpS1c3pElJ1yrYnkJZtQwmDlVT09KiKwa2YhMlEbUu4PcJ7z78gOMMQFbAaTBcl19//YHLs8+ZTa6o+5rR1MWbTZG0nl/88i95uN9iOyeEoc98OkBTVN5+/y33H2LOz8+PK9dK5ebsNWWa4dg6A3tE1fXcXF0QJzvkvqevOi4uLjA1m124QlFFvMmcHpPtIcVyDPzD8sgdLFVsw0BXTNZPO0SppcxSFrM556dn/OVf/jnOUEeSVTa7BEO3ybMSWZbp+prJdIit5/Stz9PDe4S65/77iKbQMfUBZdwyn4+PL2tezbOXI8oyJy8SmlogCFf0dYO/KTClIcGuRVdFNFXC0MDUVZ42e/wgRtIEyrzj7du3SJrP5Y1LmvncPT0h6BlpEZNFDldXI1xHIdqveffNB9pGQBQFZFlGUSzSrGQ6n1CWJXXVo4gj0ui4aWoasLQBQ2+BZdhcXpxiuzKqUrNa32OZA26uX1HXObLSMhg4iJ3M0PGQe4k8ysjTgr4FGYFws2f1lNChYA4GLBYLLk+ntFlGFh+YT2cYuoqpqdjGgDQvsB2FrktQLOHoe5d6ZLHHMGwOm4zx4IzbTxs+fVhjmZOjvMHuGU91hM5hYD1jOLFwJgrIFmksIWsCTVeQFBvSKiWNO4qiQJRAVUQsfcrANRGljihoSMtjjMb3A4IgYrPfoFkqDT1FU7HeLWnbDkmQcWwXVQbD6jg9m/H85QLTMqBrEFHI8460ykAWKLOW7fKIXgtDn1//8lvSMOPZ1QIdEPOemW0ytz2qoEVtNbyZQye3nJwtMHQXARXPc/7hB8oGgaIu0I0jOFSSFNpGgt4hyrZ8/91bri5fMB572IaFbZzTVC15tuL64pLx+DXBsuPUXvDqeoI7FrFHHm/fb9jcp5zNFqiYrB9X9HWF0HdstyvSNEJTbU5PLkmTksOuYn94wrSG2PYpT48RT5snKiVlNNDYpD4CGkKjIMouUqZRFCKrzZ7Xn9+QNyXrXcihCCiVmDt/xbtNTNMWQEdZlyxO5wzcGYapUOQt2/2W5cpHUY9+1Kbr8A8RlukxHnrUbcD5yQWz8fHCAQFdtwkCn6LNEEWRrKoRZZksy2jrhqaqEBoRS7WJgwRdt5lNz5CxURQFSW6R5A5BENA0CVN3KbKKJIxI05iyLJEUmbqrsWyV2eSS1faJol5jmjrT8Tm25aBoGpIGTVGiqAJV3VLXLWmesFwdyNKaJEpp6xJFFtC0Hs9RGQ9czmYL7t/7qP0I21X44bt7ijLFcXVkNSOIQuIkoyVnce4S+CFFVhGGK/I6Ic5iBFVmH8Tw/23vP3alWbM0PfAxrYVr9y1/fUScEJlZkkWyKBoESAJEA7yInvaV9CX0qG+gG2gBZINNFAokq1IwMyLOiXPOL7d2aeamtVkPPJCccJCFrJrtB/gmGxs+8O37s+Vrrfd9RYEkSSjzkqaEvGy53zxxCLeEhw2mMcLyZbyFxuenR9pWxXV8sjwAucEyFdJDjKWaZEV6ylyWLKq0IjgcEdoCQelQFAHXMyj7nDLLUTqbprCZjOcsLmZcvl4AMt//zY+UWYAqlpwZI2gqHt8f+PAXBf/9f/dn/OmfJjj2Ay/fHpkvCzxDIokC0jTl0/3PHOIMQc9oSgFTNzBdE0VXaIWCtIyYzBfsDinv399huQpSpyO3BnGUEq0zXp+PuZxfkzwNNP2OYxSDINH3IIoSwzAQHQMUGXRNpu0yBGGgKnJevXmJKA9Az9XlJaoCitABArKmopgSsiUTxQGqKFIkFbqg01cJr99ZzBYmtnaBI7lQiSiKxNPthk6o6Zse0xhh2hMcd4yhKtzebjGUCWqnoopQVwN9efIStcY2RVEzHy/pWhvNuaSRLcbnVyjOmKDJGdyabXyPaA5Iss7MnWDrOlGcEecFVa2yPFuhti0LU+arq2tA5/LFW2zbRRN9fvXdL5nPDIxxwKtvJ3RCy9tfjlFHIa77jq4qqZsE2XQ5lhJpP7AJb2janKf7O/7qx98SVR2PT1vmcw3yEdkupokSVs6Cq+tvGLDJkxIFBbHVeXl1xe3dD5wvLvCWC/K1ThHt/u5EySOXl+eoE43J21fkbUiXJnx4f3PqDKYJ5tjm4+Yjk5nP3HTJxJ5wt+X64hzJrnh42LBczRB0UBSFoi0oqwRNdDFMiS8PPyMrOoZh4FlT2qZks384jb/DAV0ZMCwR23Woq5T1LqBBRlAl/JGDbWpM5haK5NNUOWmRMnFneOaYMmtQZI3pxCTdbzgGFb//8BNJ2ZKHAuPJnLOzS8hl3n31EtQWxVbR7B5Ds/F8B0EVqWoBz3fRBIvvvv0lfT1w3DRMxi9oeonlcslqvkLCx7R8VgsduRc5n7zAMQcEsafMZLquQ5Y0JFFHFHTqsqCqU4qs5OHuCc8ymE4npLQEXUReRMgKCLbMwz5gOllx7Dq+rB/xLiz+8qf/hbKHxfgFpm6CJrA55Bg+jJc2F6+WvHw3JooijruW2crn66srbm9/pBVC1vsdnz89YpseH98/kpUtm3TL7hChOjpFVqKKAsExpG81WqFkv0/xfJ/JYoxmmJjmye/18fERkJAbGbHWsGUbx5QxzVN3OYpOnUVRUkiKksXikpdvrnnz9Zj50kVTXcajCQgdVdUhCS59XeBZBpJQkURbdKPiDz//yMP9Fk3VCXYpZSoh9g6q6HN2vkBRDdq2p+0qvvnqBUXSoQsu767fEm8zFNmkH057gMMwYOkj2uqUsHO2nGAaOrI0IAkib78eIUsO09GKosiYjC5IwxRFktF1nbruThO4tMDQdMJdjNTKnC/OkAUZ3/LwPI+iyJgtphiuRJrUmOYZVxdf0Q0pPSmHfY5jniMIEr6/pOtlXNdlPBvj+ifRkiJrNGWHP1qRHCuO2xhRLGnqCjqRpX9Jl7SItUocx+wPT4hij2s67PYhu/TI3Lex9HPCY41tjXFdn/Cw5fX1GWfzGZau0RwKdp8zDrsISY8pmhzdGuMtTJI2xDJHuCOdOBHQzBG2q1DXNX3VcTZZ8ubyEsfQCDcZwtBSFzUyGgM1kmAzn8/xRzaiCIKogyTS9A1JlOJYKsHhibbWeLrLqaueYxhzWLckEVxfTZF0laQWaXqFuswwDAHH9pFkDUXq+Phhf1pflDQ8z+Ff/st/hqS07MIt86tzkqLk6bDDHsPl9ZTx3KEj/vdfUMpaTZoXiJJB10IYRPRDQV7uKTOT1eKCOMy5u3lCFFrCwxqhNSmOAqaqEIY7VAMEteNxu6cqRdKsRdA0rLlLkhzYPx7xzAmOI6OqNQM1mqYRRRH0A4+PP1K1O8ajFbajI0gt3kzAtidoTUVZJJyNx4zGU7zxGBWRs6sl53P/NPp6uqWsTyq/PAnY3+8hlymOEcHxwGjiM5l6iFJNU8lIgo2kgOvazJYOeR2Q5y1ZWuB5Hm2jcHd3x9DIFGnHw+2GMmuRBJM0rGhqEAQRBhVRlBGGgb6t6aqaNisR2x5T0vF0B9eSmY01rs6XrKYuI0di7vtMvSmWapInFUXSoCgGy8UU1dROC99FheXLPD59Yf2wRkCiLGvKJqcdcnoGmn6gyTJM1aStaoa2w3PGTP0Zvu4xlAKaZOHoFobkUCYS6/WWqswYeWOS9AFNG2G5DsuVh2FolFVOlgeYpk0Q1hyjgDQtUZQS27bQ1RGWY3KIjrSAZolYtszIM1FVlSCMMR0fyxwz9s8xLYm8KkiqAsGQsEcOgqCwWCyQlJIoijifXTOUPfP5HEkxaIsOoWlQVZWZP+YQBNyv7wmSJ8YTG0NWifcBujrD+2MGc9O15NmRie8wnU4RhgFDGPAaG4+G//P/6T9GqP+K3/3rJ6LbKYYKqmJSFj1Da2EpM/R+jqUYWLrD7foTgy4wmYwQBIn5bIkg1STJE5LYMx3P2DykmFKNSE2KjDqySNo7ZL2k6x4o8vYktEpj6rYijnKEXqCtE1ZLH02WMD0Fx3R5efYrgu2OtDwg6S6yopIeYvK4JM8CZFXhw+0Xrr8e8813LzmsEyzVYuL0iG3P2H6FZul0rcD+IcDWFbpeRRRavLHBbHpGlbUIisYxlAn3LYNYYZgu5/M3RIeYiT/i1YsLpv6IkTfCMTRsQSPY3mCaDaIyEMR7bF9jdX1OVA+kfcom3rIJSjSj58uXLzxuN2R9h6irHKKaqpGJq4T7+I7HY0xbymjDwC/enXN3/5kqN2hrh8eHjiCKCZKAp01BnG+RXIXx5ZxSTDH8nu9//FsGQeDDzQPbY88uijhGEauVQ5UXFHmAYjjIlkEq1Nw9fubu4Qu6ZqHKBlUWMx9f0qYGQ9tx+/GWUiwwjfnfnZmzpC1S5NQn+SKQlUf+p7/6n6lSnf3mC/WwYXOXUQc1up4TNx0PxxRdUFGUniwqsUYKcd3RZBWDzMnA2qq4OB+RpweKsmY6f4Oqa1QF5InIZHZGXu1xbbANG6QeUWmoqwHXW+BMFTR9yXyukWUbDN0lSRIMVUPXbJo6oUpz8qRC1zyabCBNNqwWE+ISbH/CyK94uHukKmpWi468SBlNZvS9je9NOOxCdtsbBBSaRqGsN3iOzT7MCY4lx+KBYxaRFCW7+A7dMlGMhliIkUwbuhpxKJElg6pRiduORGjoBo2R55OnAV1j4DoGXd3wzds3iI3Ejzef0DWF1XiMZeqogkj0tKWICwx7xK/Oz/Fki3ZouIvfE1Yb4mNIk7c4uo9sF9j+mMVqQVrsCeOIvofV2Zj5fMHo3MC2Fwy9RlwGFGpIJmeEdUk5hCiejuk6ZIWE7ejkeUpZitRVQSeISJLC8RjjT9xTMXTYIQkuqmLRDTHRNsdWRnz1+ltW8zGvX10y9C1DL6AoCvXQ4I9HLM8d9oc1cXja9TOshn5oOLu4YnF2RV1pTCbnfPvdL0Ee6AeTvnMwTAnb0agriaenJ16/m6HqNbIsIjMiTkv8yQWqOiJON8iyiKrlRIeAKhHQVAfL9JHaGWNvim2LZFHNdLRg6GuUBgymqOJAkSc03ZGyq9Btn1bMcWwVkRZDl4mTwx/DKCpsy6CpUoQhQWjB1adYpstm94HVlckx3aFoFqo2oGqnL9SqojMMcDweiJItrqfxsA5RDItjnrKP9oiaxNN+TZKWzBYXWJ7JbGGe1PGyQt8pfPPtG4LDE12ZIw8i5+eX9Ej47ojd/RNt36I5Bn3dU7YhWbFmu7uhaTMcR0HXRWxLpasTfM/BG0lMZiM6EfZBxH5bIA09Q1lzjEPub3dUTY6mOrSViiyojJ0JtiZyd/ORMsmRBZGhUVBVFcOUOe5Lxv4Ew1RQVJG7uwfKoqNpWu7v71kul9iGimMsYCgZKE5pa46HZtSIQs3xkHJmrPBkk29fvcWSDNRBRTdOscnL5QVfv77AMSxeXV1TJxn3H/eYso1j6wSHhKfdB5zJgKqZ7HcJd3cHJMH8919Qpkdo6477h09UTYU/sgnDA3QqI9cnyzpubj8w9h1kNHRJxDZERHrqoqSOavI0odIUok5lqCSaMCAPNjRJzjAoGJ6EMZIp6oEoGXCdCW1XoqkSdRuRFi153SJqJ+Pxpskw5BGW6VN1NnluEhxihD5AGhSKIWEX31IUGefLCVdn50g0BPsNQZBT1hrL5ZLzxZixPafNRdYPAW1hM5uelNKyYAEQJw0SBnWxRhZqQGQb3DGa+MiaShomXFwvuVwY2I2GrQ/MJyvG/oheytA8nYdwjzMfU9Py6uuvmV54KG6D5CgMkkyS7RHEhraVacSSRhAYhgHHFpGFjsXFKzRvxHwxRelPthedJBFXNYNeMZ44qILOZDQlawp2+ZqmDymCAtPyUZsYW6lpWh15ENGHHFVo+fbbFY5u4lg+b9+s+MV3b7m6uESg580bh1+8+44sDZmNDegtNsETWZPjjl/wsLulbWuk8rSncswr0rpEt1uCw5qRabKcOORhhlDLhLuGtkpOopXw9LfVdJGyBrFqMfqaIesp8gjD6gmjmG1Q0QoeotmhaB224lCnPb/+06/ZHyPqqkJQTNIixpvrpxxuY0SWVCiazPJ8zsg/43CUOEQteVszqBpxWPLVa4vJwqYYav7b/2bOzcf/J3/+rz5RVhrnr6YoTPnL//FIUh558eqavN3TqgGT8zPWQYBqnLzr9kkCoo4/EunKnqZ0aJqGKDmSFxn1QiIrC351PsfzdY6Fy+PNFtcSERs4n57h6A5SL/LV1xPyPMWx52RpgSBLSLmIaHk01Di2hqMLiKVEXXTIsktZVxi6QtcWTLUJxeeK2x8e0HQTup7HxwJNhcf1e+Jwz3Ru0Uo145GNZcpoqkm8z2nLEsORoU+4WHogGixHC+oyxH9pMp+uKJIGyRaom55OSDFnFrvsgDlxT2KGoOT41BDfJ9z97iN1eURqFcowpR8qClXCOpuzWr1iqGsmKwdJKSmLjDpXkLCwLRnLEVldn3MoUjoKLt9N6GOQmoCmF9gHHbqtsw4L0iymzyVEfUJwLNjtM8zRmCwMqLuK+09PtFVJlWU0ac43b8+RhQNNe+AY5+RJyYuLc8aWye7pFseb8eMfvuc/+Zff8vPNBxRFYTyW0dry7w5ZR9kO+BOTOHpEsSZIrYjtdOy2R7JG591XrzhbzLl72FKFNWPbZZAUBFGlqAvSoKRKWpqups4OqGKNMJg0sUiyazHsCVW74dXikiZNoREp0h3HuEaTbaouR9dHKI1J3Uborkge7GjznH1SIfsaxzIjKAMaXeXYBTRiT1H1iJpDEGTUdc5jHJwiH4eEiicQWoamxXRMNknMw/0nwqcHHEknPzaIdDSGyn3ymesLC9dYUfYpSVzz6+9e8uY/+iUvXp4z9wxKuebn339iE2xxFBWNM7yZw6f1Z0o63InI0DaoiIzHPkWa0fcwWBp0ImIjkQciiqXhOBITf05bSZiShSYOqJKBJAnUTcFTW1F2MsOgMbfOaauebMjJDEjyEGlQ6JqaxXhGE/XEUYk2UdBnMmWd8tvPnxEkBctsMVyR8WiFr8hY1NA4VFmNqIqIosjYv8R1HBRFQZ/YlGGJYjnk9Z6b9R2HIOfs8g0UNYfDHZPFkuWZgzCu2QgPlO2YH393w6cPGx7CI4Ut4c2XWL5NsE/RJAfH1hGReHX1DcvJgtuHPZqk8N/+V/8FF+cviY4lh6cO356hKBpla9FJHdt0Qy5IRKmIb88pioJjGdMOKVn5AGTsHyrEpuPwlJBmEbMzDcuWMEQFIe+Z2SqmNqBbNU/b91jmCHOsEkZHri+vUHofyxDJ83u6NkYSG+p2oE5F+qJBGQZkteJ4jFk/bTlfXJBkIY5rMh2N2W9uUVD59D4gKgoej19AsOlrAUWqiA+nkbbtqNi6QnHMMQyNoSkQ2oFu6FnvA2bzM1BbykGglXKiuKSuS+KgRZN6kDLSvKdTdCxP4hhtmMzGdGWOokrotoqvq3x+uiUHRqs5ggab3ZZOENgeWtK2QHYlxL6nyVtcG9JAw9JMlr5F33ZUqsyg6QiyzWQyoxdaZEVi4k2gltkdMkzNpO/Bn2m4nk3ZpxT1wHfvvsFzBIZG4Hy6pG1bLFegKlJmswW1uOOnn3ccwwOK3iOYGubSJCgGjlnDIAisFhcIWsM6OXC7vkcQXfSJxT7IqJOM+NigmlPOXzl8udlhWEv28ZFO0hj5C47bW8QO7F4jPUagwOXKQRmqf/8FJUILQoOIgiL3qKrOyLtGkhSSJDvJ5HsZQZLpBomy7ajaCsPykVSFxcWExWxGto3p8iNpUlANIMoW8iDRyzKt1LANN9T9QNMOFGXG5dUZddNz+7DFMGyaWmC7idBUh74dsGydODmiqy3ffjdjMVuyvivYrte8fvGOvtVZrmyC4wM//O4RQ3cZeWdcvzjj21/NaPuBrEhJihjN1FgsZjRdxNP2nq5r0d0ChJa+y4iiCEk6hdYXacaL669papkkzlisZtzc34Eoc3+/pygTiiYkiWv6vkaiRxkkqrTAdWzCYEtdtmRJwc3Hj2RFydBbaJqGqknomk8/1DTdgc06ROgHDEMg2K/5ww8faOqBzdMd45GHLAl0mc5ydoWIQBhsUUWJOmvQFRNdM3nabIgiia7rEISCKBCoChnH1ekHDW/kU9QhURGw3j/QShHuxKDuJJ52a1oEOlngp08f6Lqeuu5o2pyL5QUvr6+4eLVENrZcnS357quXfPrpBy4WV5ydzygKicMh51jUHJuMoquYTVzOpy4X8yldX/B4d49pilSRRRomUBkYssrEn1Gn4JuQpjmHvOVv3n+PNa3I6wxnpvDu139KeAgwDIOqlpieLfl4t0F3zJNhqyUgSCln1x62b2CYEkKdcT4VmLkqphXyj77L2X0WMDqbV19PkWWXnz9EvP9QM7+aYJg+WfmE5fb49gRa7RT1uDrj6THEVGxWSxvN0AiimGpIOMQ7TEflzTceTTiAqxPrPZ1QkoQP1MpApek4zgJBNFGNk0oyr3IMR6YXMxRDpKMirRpuHz/RtDmmKjG0MrppIukqfX/yIYyimDg+stmteXpa03UdhmFhmRqGodA1JsvZ10ync25uPnFx/hp6g7at0U0NQRZo244k6mkrlcenDRcvzlicrVB6gWAb4BgqZ0ubqulQZQW96jEakTo74Dsq08mMfRjhznyKPmI0MfEdj2YIGc/GCKgIqMRxyOF4y2xlEUZrirzl8vqMl29WiFJP1wukWcnN/Q3rwxpnYtH0ErImIygCXV+iSAPnZ69QlII0G/j8eMfj/Y8UaYZny4i5iCoZhE9bfv3rdwS7iDo1ePfml9x+SmiahvXjPXkSYBgTFEUjSk/Z1+OZSZodOO4L6lxlF+1xR0vyqvu7c/3yimHo+P3v/0DfQl1kaNoIVfaomwFEgWOcs9kGTKdTLNdhMpuA0FA0R6YLF8PQMHSdqsxxLY1sW7I/tMRJwC+/eUfTxXS1wPZhQ1AEJGmNqisIokRRNuRVQFmlgAnooDRo4pyxoaF3DdmhJIsTNFUmjkKqdYpQx1iyBHWNVLXYqo0h98j1gDS0JEGB2Ns0VUbRPCEpGpqxOt0BQouAjj+yCMOIphTI85ztPqJqO9RRyLYI+PbqW4ZBQDBq7NrlV2++4/52h2k47IMb+r5FlipEOUeSBSS5R5JkWjGnd7f0eockiCThDmtk0Ksxu6dPuGOXuD4QV4/0Ssc2qFjva2x/xNPhM5ubPbqp0ZUJWZUwNA2GYzB1PA5BQiMLpFmPKAkEWYLre+iDjivLdI1AcH9AV3r6WkBuJc4mY1zDYTLyGNoCRx1RRQOLyZiq7Kmbhrx/pGhTkrJhMjOZTCanrr0z5vExxF04LJZTFAnOXlwhCwN37z+cLKYmDtdfvUBWVXRDZr/fn94nx8Nxdcq6IEpSFF0izrdEcUYQFny4+y3/+t/8f6n6BFFtyMuSMH8iae6pKdntYlYX56fPT1tgajKGUtMNIlmWESc51y9fY5gSZ/NrLha/4GJ1gS2PuFjMmU3HqLpN2yq0nYYiSgxlj9RoaFJNm6f0zZ58r3C9uiKLDuRRgiT39H2PaSnomgetQpVnjNwppqGjiB11maOrCm3d0bUyhqbT1RnSIJBmBxzbIw0HPEfD1FRURUKSFA7Bjq7NsQwbTdGZz0aokkQc5miyRZbE0Opcnr+haXMO4S2PTzs+vj8wm15g2zbt0FFXFopS4uoVE+0Mw5iQNBmILnUpoEsKUp/j2zY/v7+hagrEQUfqBARBwPZPImM5ygjzez4Gp9+ps5YyF5AklbJo2O0ObA93dAT0YkGWtiStgGqbtEj0nYgmW+iygm+qxLsYWYbt/hbdkMizHtfTaKua3dPAZGzhGC62eY7UaeS7kIfvP2J0BhI6hzTjw08p0/E1TaNj+SI9KWWV8f3vd/z482cMQ+APf3vg4fET690T9kgkL0OKvGW+0NAECanTsJUxnm2RlQGt1P39y8S/rw/lf/5/fDuk6clHqkgzZMlAEU8B6013ZBgGmkpGNXpcd0xZ5vRDQRK3eJ6COx5Rxi150ZL1B+g1bM+mLAaoW1R7iueLuLZKlsfstgGaaqDIIl9+zrh8OeLh6RHb8nGsEaosEQb3nL9csQm2SJ3KxWpCtA9IkozJbIEgyWiqTlVmPG22yDJUhYztqGi6TNtDN+QokoPlSKeElEEhzY7Yls9PP3/EdkUEdFRVRhENNFXm6eELrusjix4iIq9enHO/vqHvJMrmyNA5+DOZpjntoQiiTBgEtLXCeOLSdAXj8Yztekeep0xnIyRJwDRc7m5u8EZjBFGmG1KyNGHqXUCXU/Q1QZQzG6/IkhTLsZFlle36iTYX6YUGwxRp+g5FNpAUaMWaIIyZmHNUs6SuVQzVY735zOXFBfQaZRviWDZNLZ7a/J5E14pkR4HV8pKqTMi7EtfTaeoO3ZBo257pzOX29hZBkBhNLcJgw+XiDW13wLUXWOaSH7/8Ww67AUmSMF2LNE9oqpqxvUSWa9K0pu0EBlEiSh7JY53RbGA2eUHVHAmiksWZQxKHxFFBUxUIjcbFas7hEDJemAgGlIGAZng8bO+RRANd69EVh6oMGGjoSgHdlwiSCkeRuB6P+M23Fzw87NAcCccvyDYJXWdwlCs+f9KpsohSEsiLiLHjIQkWTdXStDmqJnJ9dclhH1CWLb0AaXak6QZevHrJersnjjN0WWQ+mWJoJn/YfKDuGt7N5ixHSz49bjBtC0d2KJuQrkso8p7VasH+kMIg0vYd4/GYuio4REeGrGfqzajkBFlVoVH4/vcfmXhTrl9OqbqUT58/IcsWpuFxtppxOOxZLGc01cl382n7B3RLJE8HyrLENAy6rmJApS1qNEdBVFo02WE0PacpEh6Oe9ZxzOvpElmK2fUlF7Mzjp+fOOawmvV41js+3KwRLQHH1QnXBybjJWH+xDD0+O6SNK4YugJBrBmNXbpuoKoq5pMlSZKhywaCIJDXGVWd4Y9m5FmDLIuUTYEpKQRxyvnVBRICf/jwM2PXQcehGB7JQ512COlqEV0y0S0FRZUok4HgsOOf/ZN/RNfIbHYhgjYgKT1hmNBKNV3b4OgWdBqSJBCFO2iGU9qIbxIEAV37v92Xl5fX3N/cY+oGy7MVYRyRp3tm52+xVJXb9RcGoUc1ThnLHgtKOadKMtpOQFElmjxHlHUGscXTPdKg5NA2fPduTl8JPCZbLMFlpJpEJORxQzPklFWDrZ/EB22tYCsiuqeySY/MTBEhrwnjhkowEAQZXZN5eLrn/GJK1hZMnQlyP7Dd7ZFVE1EryOsCTXZYjFeEhzU9BbY7IoxCmqZhMr1EE0xW8wV/8bf/llppcRyHKo6ZTHyOeYFvCCz9CUUGqSCgiDH1oUeZuCebr9LC9RW2u5CqPomLFM3FsT2eHveYMw9ZzBBx2R8OuJaKJUy4v3vCGKkoQk6UpQiijG3PCYM9tmOwnJ2dog0rkTiPmboe611KEaesXo2xNIO8zonrlpHs49kST/uAvMhYOHNEoaYeFGRpwDV1srQ9dUuLgmAfcnHt81d//deY1gt8y8MfWdzfP2C7J/uustWhrpifnVGXCnm+Jyt6bMVmuhQJthvO5q/YZTlN2hIGCfOXLnk10NQptAqqZaChUNUFut4hCyJtqYBYsFx4fPp5iyCBYRvk3QFTnnO1uub77/8NuuHTKR2OpfF0n56Uwn2Pq3lczn2i6AlZs3HcGfvgSNsVuLaKqXvYuoOmaRyjHfsgxvYGHh+2+KM55+fn7Pb3bHcPTPwV7aAziCFirxAfU3xvjK7rFHWEbo6gS2DQ6dqIrnNo2pw4CZiOz3EsA1WBrj1pCVzXpSo7BKVioGToZQZMuqbguG+5frHEtA3u1h9w7DGuM+anjx/wXZOsLBAtEc1w2D1ELGYjdsEOzxhz8cLk/vYBeg3PlVnfZ1SZxX/2X77k+x//wIf4gLeaUNwXvJhM2AQ7mmFA6gbG51PC9QGh6ZBEHcVWGK80gnXLUKvURUCvGFyOHdpU4g+bj5TiwD/+05enpLlCRlEbkjhFVA18c0xX9piWztM2wjBNJLlHMRQOj49Y4zGKoqCWBYKq4fkO+/BnhMHG8ydsgyf2WxiPDVzDRtUkTOVAn8TMRxW6bGHaFrrdcNgr3G80elnm5vOaF6+uicqT+8P1xRW2MyPP7/npD2vaoUWWReZLi/U6QlElijKgRUSUNLK0xrQUBKFB6GR+969u//36UCZJxn4dYqgWVdGxXe/QzYEsDxGEnjxPEcSO8/NLBFmgajpkxcMwDI5xyO3tlixt0VWNkXPaN3varMnyAFkVSasdu/2Bv/yrP7DfBVimgyI6ZEnJ5cUY2xwxtDKWbiELJWWyZTFbkh9LmqzHUjTysCdLSkRhINgmOJZNeLwnT2NG/hzL1nnz1RyGnroSEHqRq7OXVHnB00NwinMsGtK4oW4bXr58TVurMAgYqocknt6uy4tXZGlJ3cSMJjphcjglZEgwm56dRpqHAEN3EKSTCs3UTGxT5fHpCUSJIAxpuw7LsdFUA1WV2R8esV0XTTMoixRhEHl1+TVZWpHXDZ7n8fLlSwxD4/J6Tl9X9GWNbSgYaodtmsxnZ/Q1lFlJnmX0fYs7cTH9jqKOTg/mKmE+94GOY5TRdRpFnXJMtuRVzHods16nZHVEkNxiuCaiCLquUFYJgtidujvbzyTp8TTGrDN+/ct/hq4rmIZHlrb8+Z//OZY2Jz2WFGnB7n6N3A4YqsHjfk08JFRSTUOMaB0wXYfxmcLrd9dE5ff0SoQ/ttgHIWEykMUl55Mlr87n7Dd7HF9BUzzibUqvS5RdhSJqTByV1XSJLPaIqkavwnz6gr600QUN19EZhJ5//T//JWnZ8LsfNvzr/2XD/+d/KNkWCpki8/Y3CovvFHShZeyeY1sebZvTti2vXn6FZTkMgsrt/Y64TOkGEVV2GGqLImpoqwxDE0+dGMVjHx3wPR1FUEG1We/3xLsjtqrSDxVRINB2Mq7v0A09htXTSzGryzmHaMf+GKHoJzsoXRpo8oFeGMjznHevz3n1asnh8EAcFIycCa7tc768pi4zLENDEASSbM0xfsIwLMIgZR/sKMqYpmlOvp9Ny2Th0gkJvj9GxOLm9p4PX35AcVukoWNiT9GQmJljdg8R4XHA9k2KUkdVdUxtQKwbdvd7irzn890jUV7RdTo///yJuq5ZLs9wrDF9p5ClNYcgZbvfkOYRQXxAVVXKLMfUFeoqo606guBImu8QBY2uFRhaGFqNkT9GV0fkWYDBhKFvcfVzFpMxs+UYWTkZ7R/3Fb/87gWOJ7DdblD0AlmAvpUokwypF5n6C87PLjEsnaKOGU1HeKMliAOaaiCKoGry351gFzDyfM7O53RtztA2ONYSqRd5enhgaAcsy+Kwi5FR6YWcNDly2B4YWoksadB1k9l8RJrUKKLD8mrOfCxhWDZRWtDGPVVRk/c5Vdzgj3o0Q+Xq/BXDICEIApYl4Hgy73++4bjfY7kj0tZgn7b0OqhOh8BpJzUIHrGtEcc8YbuNWUynfP3VS2x9QlcKbB6OtHVxSsASPYqkQVNPO1RJ/AB9zeHwSBonLEYjDusDrjkhSVJUxWIkL3h8LDmbvMXtJUzRQ1IVsqhkPHJYzWeYukJXC5iaT1ODIugokoTvWiRpQLwvyY8RiqQjDDLzkcnFcs5oIhEEAUXeYxgz4rTk3S/ecH4xZn/YMAwDddzQZi1llfPN9QW13PP1y69QFYs0KfFkg/HYJy8qhqahrRvc2YheU6mqCsv2adCph4pByE8/8yziusD0lni+jWE23D984eWrs9NufOUiDQqKIBIcn8jrGyy1RZQbim5HKzX47oK7LzvqJsUwZS6vx0TxniQNMUyZqgqRapEsyLCMmrqMMU0T1SgRBImPP4XkZYbrykTFDmSXvO4p04Kvrl5TFwmG4pIdanzb4vx8zFfvXrFarUAZEHWZ9eaIpqiUecJoavO4PiWuPa3vuVt/oO56ltMJjjVneXFFVXZEQYowSIjYaK4HfYBQ1xy3Oa5toMg9TZfSNANVGVCUA1EU8Lg+/HHHz0FVTIr8SJHWdJXJfHaBZVkUVY5mDZRViCwNtHVDntW4vsr19SV0Mm3dMB1NybIU27b5k1/9I6qyRFdPE7iubBi5CqY+cHm+xB837J5yppMFmimSpAL+xGR21rI7RCzm53zz6g19UuG5UyQfnKWP7S2ZTGZ0TY/jOCzPZsxWOm0Ddx+yUyfVLbm/X6PjsokrYgSW0wnX8xFNLTLUA55lIrUGq/kKWzOoioKyKGjrjrNLE00/iXQUQePi4pyubthtQ84vzxgEgaxIUcRLbN8nKWKyVOD6+hrDlMiKBEGuKI46M3tEeN9y+ylnv8/49DGlyksUOaBIHnn3doShq2iyxHQ2pxMb7h6fCHY9Td2zPHMwLYWff9qi6j1tIzAMAseoIK9yRPW0bldXHVnxH2Dk3Vanbwx3XzZkacViNqYqEkwb4qjCsz1sR+PpacP26ZG6bkiTDlkTkRQFEZleFKiGgq45tbltzcJzTMq2RxJaojBmOVuiyjrxMSc8HHEtlyjaUhY7Xl5eIA8Cuiwx9kekYUaV1nz31RviIKSpc2TBpClN8rzk86db+lZhNLbJsh3hrqQrB5qiR2ZAHEo+vb/HdRzCoOTThx1JlNLUAh/ff6SpE2RRoiyPuI5FHMdUVYVt2/gjD9cxeXy8Iwh2pHGO6+gokkp4CBj5Lm1ZkxwK2ipBKAdsTeHV2YqZPcFWdX759TsMSSHahsRhiix6uM6Eui0Z6FBEha5v8XyDxdk1ZS2gyDq9ULI/PKBJAjQdhiJgewKOa/CwfuDsasnF5ZSmzdFNDdt1yIqOpnFohprpXOUYNARBBFJCXsSUWYuqynSNDOhUTXraATzm3Ny+R0Rit90iiwptPdC2HY7tMZsu6DuoU51jvObu6YYvNxnHPOXy9Yg8KXl9ecZi5rOaTWjKCklUcUdjyrIkLwvKJiU/SpR5ytnS5O7TkfVDjtyPWT8eiLYDWmnw9sUZxyiiEisET0OfjrkLHygHgzA80HUNsigw9A1FnqCoIsMAdd6BXGC7FZYKbd3QKDWHSuMuKDnEPd//PuflqzHdtKQ/6mz3EovRhBdnMxaeTR6LzGdLbNslzzraTuBp94BsygyyiDUyUK2el5fXDKV4El01oEkmVZ3wmB1J8ghJFFnvE94/rLEn/umfti149XbCbO6dOvutcXImeIz48Q8fKfJTioQsSUz8CcvFjL7tiY8B8/kcyzLpmhJVsTENnZHvUhdHDK1lv4nxrBVJVBMnIU2bksQ5ZS6hSjauM6FvFQRBYLqYExwKtg8wNB7H45EkDZEVD0sc83olMrQb6nSEeFzy9HlP3YoMRY9uOiiygdCCLotMxg7zmYehGQytxDHcMx77yMh8+PGW+FgRBcmpi2LbSIrFZDJBkns220eEQUTsddZ3T7iWjC6LiIPMYnaGpgls1ncoosx8PqfqYtyxjTRoqK6AKreneEJF5PHjGkXw+M//y/+IzTrk3/5PHxk4Jark5Y44PjJZ6miyQ7gr2G4DdodHFmcjguiIoGootk4RB4zsMTTD3x1RrHBdha5pSZKM5dkIQY+oijX0A6qsUeQR56slK/+aPBWQ2p5X15esVg5tn9J1HZpm8Pr1SyxXoRFOE4A//O4TdVciazKjqYVpKUxND0nQMG1o2yOS2KHLPWPbpipbfvOrbzmbLDjGFarjIGkGVXOkySPaXGd15aCIE5JjQN+nTPwV9VHi/e8+EW4DbHHM2fmc2/sHuu5kdSIIAu4fPQOFDqIgp8oUXMtEHhRczeUXX73EVDVeXM54ip6YTGbMziaM/TGGouLYIKQd+XqgKCL22w3ffPUKRZTQVBVNlXh6eMRxFC58m+lIx7EGSDKUQkSSRIoqJHyIMRUPU3aoypSRq7J9fKLJZCxlhNx1dHWFY/mkSYHtmFxfX9J3ItvHANexqKqGh8dPNH2P7dmsVgt2+0dUScN1deIkwNBtjnFEHNcwGOR5SZlDLwr0cknV1Fj2aQJlqxd89fprLmYGQqWiyCNUtWW/STDlCdOJRVH1fPm4RUJiZs0QjZLBCJEGiT5vEFuTkeNjywITT8UzXQxd43g80jU6mqwhygVnqwtszaHYSei1z9T1yMqIfVwymZ+RFXuiJKbsTk4nx32B50MzpLTdSS9wCG9YXumkacL5xUssx2CyWBJnHZpj0Jk5RVmzmIzwbJW6jRDUDlHVEKWGi7OvmIy+4+03L9BslaIpqVqBppUpSoOqAEk2+NWvfoWqymi6xGjscHa2xPNGIAh8+PwHkCX60yYuluvgODNkWWc298jTjvHMxXINRFFEE8eoqszn2/c8PNwyHY2wTBVVVpARsHWNLD41T6TBJM8T0rRlv2kxLJ1jeuAQHgnDiPv1hu3PD5hJhtGUJE8RUiFhCD09NdE+JCsf6EQ4xgN5OjAen7qyZSkynrxjvtKplZTKyLHHMnlYsf1SocsyeZYg9Br7zRZFkhj5Bo7dUxc16bGiUSoU2yApc0xLxrJ0rs8u8XQHuR84xilhlBMnFYOoIAgieVZjWh5pVXAIar5sE7bdgLJ4izl+SYuPYr1ksL6mYYI3usbxzxmGAVvvSLZ73v/ulquZy2ghYvotN5/X7NYVwmAwny0JwgeqqkYVWixFItofKTIoqwbJ+PtNsf+dCsqL8wVpkgAt5+fnCMJpCdrSXL776s9gkMmSI/EhZr8NiYKIzcOWLEtOXkauwzE7UFQJx2OC53ho6EzcJZJgEq6P+IbL2Da5//yAPCjIYsvQ9Qi9QBrnyAgYao9ry/R9hq5JeNaIm5+fMDWXyXhE2zfE8RHdEKHvCQ81P/74SFMPDI3E+z8EFHnMev2J7dORts6g63h5uURXDBRpQFU7VvMLsjSl72pW0zOaOqQucgzD4ml3jyzDMIjMZkt6OnpaHHvOTz//gGXrVOVAkhzpWom6bsmLirIukFSFsm4wTZPb+zuiKMKwLGRZxfMtguOBNM+YL2eIssFun5JXBV+efiYID4THPXmdIMsyQy+TJhVp0dMJA01Xk0QHouMjaRVwfrmgqhrCfYIuW6fIStXjmIRoqollOtiGR55mmKbJ5r4m2Dd0Q85k6iKJNkOv07Ytx+MRUVBQZJMkzhmGgTwVMC0X2xmTly1PjzsU2cKyZcLjI0VRMJ2avHt3jmWq3K4/MOgKmmkgtS1KLaH3Crbhs/QnyK3AEDt88+pb/uyXv6avKy7nBouRzEQVkAeB2aXP3T5CkHrCzY6RriA3NY6s0RcVktjSDAJFn7HZ7YmDAFuyCYOYIhHp2wZJqam7HtGwqYSYKCmY+BVf/dMDP/3tmkFSeff2NU9bgb/9qyckzQdxR1FmLJdnrHe3yJrI+inFc1+iyiMOu0fiIMRzVYRhIA8lwl1LsE8o0gwBGVMxWYxG2KLCyHbpuoaqKKmOBbttyGFXIssqmi7RtRXT6YTFdIE0qMhij9BJuP6MH99/QlFEHMPguN9zCELSKkFWTFRtYORPePv2LXUbMfQyum4Qxnsc52RhUtU5k8kEAZ0oTNBVERD4/e9+JDgcWS4WGLpOejzgODaz2TmWbJMcGsKipLVhn3/En3tojoU7kaDt+Ou/+Es0XWZ1OWI8O6X0qAOYAiwXE968uML3LDxXx/c03r29ZLVYYigeqqSQpxWe5SMJIrIocX93R1O10As0VUlTDqyfvmCoGlIPXXs4WT/JJZXUYvgD9shB1FqaXGK/SblaXHE+WxLm91S1hm2Nmc5c2magHxxUXcPxfIoypusbdtsj49EZ+0PB2cULKiIUB1zThrbCtrS/O/PpDPoOQexQNZH1eo0gDUgUxMkddBnH3RGld/n+b3/AG+nMZguSY8rj3ZrLsxf4/piHhzvaXuTLzR1xVPDh8xbLnpKXB0S7pRVqNg/3SFqHKmrkeUrbBuiGgDC4PN2EaLrMIGT0TU9Z9Ow3n7hcaOi1x+EmxXdlgijDmtqcXSzouoG7zUdq4ch0eYZqi5xdzFA1gb7p0TSNusqRUJAliZk/Iz+a9E1BVR+ZjEcogsjZdM6Xj49oos36yx5UiTeLKX94/6/4i9/9rzw93WP7FqbWY5omVT7gmDPubm5pmorokGAoJr49IQkqqghG+oRkW+M7Fpqi8PmnR8JNzNBaiL3OcmYjdyld2lDuWsRqQO4kbHXFeHQqQlbjFX/5+x8RmoHf/vgjVV0wmYy4229I8iOmYeNOxzT9wJuLM6ogRhxalrNz+q7k8nKKpDSMpgPnK5+2EgCRRsqx7RmCIFPVnDqAwynRxJsYWLaBJLi0DQh9ga45DFXFxQuN8wuTOoYyqykLgf1jycK3kfseoYWhi/A8+PT+kaHxMTQDU5epc5GhkxnkDR/fP3I5vuJ6bONoGj98/sR9FlILFkMrYc1Melnmcf2EIkOWJPj2DFVS8WwFyzH58vlAnA6EUUhHQ5af7tNjHnAf7rl93PL4aUPfdjgjn0NSYJgeumwSJCHTsyWvfvGSN7/4M779p28R7QHRHEiKirKpYdDIy+xkFZembPdr8rIjrxIGNWc8txBlCd1w2O2PdJ3GepPRtiLHKMAfL6i6mDRPkGSL4BiSlyGqYqGbA2lRUBQVqiKgyjJNOWDqE7qmpW81bEenKGPa/rSe1HQ1k/GcskrRDZHpdMTQ6VR1jyobDFkJZUnyR1W6qjgcgph9uKVpc/raQBFNivKIZhfUw5E+b+m6hIfjBkk3qfOUodERsVhvHhmNPKIg4+5mje/ZZPmeKN5SZQfi4IiqCByCLX3fY7s2n27vyMoCSVRp2LHZHbm73eKPVeq6ZbuOMCwXx+9paPnxQ8Hnhz0/fX7gf/z/bbn9nJJFJb/73Q0ff7rlh9994ePHj3x+v6csQ5bLOWWVsF6fRL+2M+HyxYjv/sTm08f3TBc6VdnQ9dDRMZn5SJKE506Q/h2kNvLf9xcn7pzdY0qW7FitVtTNwHgyJdyW7DY/4XoaFDJir/JP/+Q/oe0rojShGjL6XiU6xiwWM4L9AU0SieMYy7CRKhlHV7HP5izHLwmjJyb+ClnQkQ0J07So3RRJUmj7hNlkTtu2hGGJruvEyRZNO2WFTqZTHh63tG2LKEvUdUk7tGiaQZs0lHmBa5mYps1kcklTD8zmEz58+pm2q3n96hzHNYnThKYGEYvXb88wTZcP7/8Kw7AQBRldN5GGHtfx/pjHqXJ+Mef9H+5Zri5Oo5J8YKBjGDqKTAZFRNB0OkFANQ124Yau69Atj6zOicMYlA5BE1BFk10YkwQlrjOhGxoEucGzPA77Lf5shK643D88sVyds48D+q6kymJm0wm2JVM2PUEYIyoqrqlTFDt8VyMI15hWSdcZVHlDmlTYxhxVG3B8DUUXGY11jseUN2+vECXYrgfSIicte6q2Q9dEdEPk9nbNeKrDIHHxyofSoYwLBHHDq8sLbP2KONqw30W0rc6r118RpS3hcc/UG3F/F6MrJqvJBZ+/X/P4cKCODaIoYh+vGXnnqCosJhJiZ7OLEwxTQZNVujJF6Hts/Zq2LVFsEWEYkGSfti+oO5XpTCc+SBRpgTueYdoDQRij6gZZVpKGOUPf4boqf/ILj30QYWoqLxfXhHnA598+ok4tbu8+0dYKktRx9/geTQdBUJhN5owdjZu7jyi9zpsXv+bx4YZDEOKMxpR1jjMyaZualTvFsWCoKrpqIDvGnF9MT2knWYnnyRz3FVdXU+hk4qhAUw3QG1xXQRLPSLKIokqxJw7HLIFKRlcMSqFANGUc0yBLO/LiZIJb1ANvvn1BkD2g6yfxRxhENE1Fnh1xbI+67GjrjsMx4vxixdncRFE0do9bVMnFdRy2+xu00qbDIpNbFksd07Bpyo447knTjP29wOWZS9vt2G8ahsHE1h0s+RT9qFlzuiZHVTouzqeszhbc3HyiLDqKtGcynpClMnlS0zQ9y+UEwzEAEd1xmApwv7lDEDtM2WDhL7HUns8//Yhpupijnh++/5nz13OySOL8+oyqbjEEEdnoKOqMr755S5btyYsEf+LS7Hui4sDuw5HJZAQdmKZFFB2RNY2mrwmjR1Zn32FZLruffsdkPvm7O1HTNAZJRtNl9ocNimJRtzL7uwRFnTH2lnStwW6342zloRoyd3dPUCtMpiuKtOb8bMHt7Sf6fo+mGNRJzfnqDYop0pQyZZLR1z1hcOTs4pw8358W6JfXKIrO5rHEd05fJP72bz5wdfEVT/Ej4WNBvL9D9TVevlvS1glNXdN1G5KHGWJscLFwMYyQ/fYzi4sJeRozVBJ/9qe/4cNPn7GMOZcXKw7BEUl0MbSC6XjG/fYDlu1TlgmiKPL0EOKP9VO3OK74+dOWQZd5+eKMTRwSHDUsfURpVEz1OfcPt7QDDIOAZy/I4hbP8fj55w9cvrjm8cuGMhaYzD1sS8OUKmy7p9Ir9ls4FkeavkPtDVRVIitj2lbl812LY9Zcnr/kb377F+gTjzIP6eUB3XZJo5av370mfFqz3j4xkRaAwPppz/WLCz7df0J3YlzHQSh00qFCMiyqKKZrY2Szp6w6UGt0zaZuO1oxJylKBsmmGAK0QaKvTGYLmbYqoVlw5o1Zrz8RRhXTM4NsJ1JlAtfXK16+mHLY7elUiaou2K4LBCRevL5iv98SrGOgPt3JYcXVu7eMbZemCTHtOdP5Cs0RUCSB88ULbsP3eP6EWolxrBbXmhMkEWkZoipjBn3Et7/+FeeXZ9x8/i0/f/+/8vXXL3g5XxFkJZQF/kRDkVU0XUTRRZqi5eqFxdDUPH76yPruicenislixGhu4HsWQbhDkwcESaLpBpKsoqx7xKHF9sdsgxBFArUa0DWPvm3xfIemPuWzu+6IoReomxTEjihPiWPw/DPckciQOmw3AbIO1+cvqZojt/ePaCMTTVHwPY/P6y2t0KDoAv1Q8+rVOUlyJAxS5mMFsZcItgcczUNXXNAbFMeiESvSJGfoXRhyJAyqJse1fIZBQJRFDMlCGCQUB3YPIX09UDcR3eBielPscYzSa6TJnunqNJ62XY+2stmsa87PL6npCcMHwkOMoQpIQsOg9Hy4/YmZYbINQ16/WFFUKnmTYpk+uiaxKx549fIb2r5EqFtMqWeyktg87uiFgus/PadTJf7m8Wdk1yWvKxSjo6wbikwkLfbEqc5tICIIMr5v4JgC4iARHzMMbY7QijjumtvHEHNs0wkNs7M5ZZJR53/vMvHvX1B++binyjtmkytEqcPzLYTBRJRyJo6NaWlkSUHfCmwfIwwHbEdBrG1E0SVLUtq2x1JGiOTEccTl6pwu7/AssKwzPvy4ZjwTeHE9IzgmlHXD08ORy8spP/7hM4vlya1/u8k5HhNmM3BcmyxqGY/mfPn8gKoYiHJH3bWIKmiCjGkq3N6HXF/NqYqSx8cKEQPN6Pmrv/4b6krAdS1EevpWIznuieId8+kVRVqxffqEpvqI0kDfg21OWD98wbIMOmrKpiI4nlJjFHVJnKXYroIoGKhKjSK5FF2KLEhkccHH9Xv6oeJs9RJNMxEkkdnkimMaUgkhZR5jq0sMV0TQU/aHDKFvqIWYqu4QBp0873F9g6R+xJhY9JnG5eqSqe/y8PgZbzSiqBt2+3tkoWZgSivFdFWNJEmUQ0qdN5iWh2EnbPclpu1TVRLvP6xZnU14fNrRtwOyUqE6Aj01dT9Qlw2Ga+O6NrrZkiYFT7setVapgpSv3s4Reonbz7eExx0j30F3Jgx9hdyCqQoc4j3eeEaR1PzNv/mZqs4Yz+bIek2cC8i6ztNujy5pePacdZHiuAp11EGe409eUtY1QVlSShFdKDK0HXQhI2/JZObQDSWbYo1jj+jajAGTtKyxapUqERnbMnVToBgRM0vnp33H25WMZ/X88PsDQzgw/8Zl+3CPbk/YbQMc30I3JMosxtQs8vSAb5mM9SliV1L3Ry5eTwiTnInl0RcK97cPzHybOu9oqhzVnnHhjWjrHeVQoM1c4jjAc85J0hhFbjlfLRGEU0pQ1dRkaYEsGci6wyCLTGwJR7Go845+EGhalUOyR+41DPO0LqKqUx6ebqgb8RQykCaIwqkLL0saddWQJiWm7nJ+scTydeLtDlV1Oe4j5guLoo7pa5OmLagFGV820Xr/pBhUHQRFYVvkzM9XrNdHlv4UeoGqSTA8mzgrUQaVw37NIGTM5iNs2yMKa7pWwbYtZjODp/sDiqJwdXXBdvdAz2mM1ok90f4eAwHf9yky6Ak5G5/z8fsvuOYUR/cpDhW/+cXXtI1Dq93jT1Q2m4ioa9ErE1FvuVv/yOYu5NXrS5qmIWu/IMgWhuOhWTZDX5DGES9evGQfxDzcfcCwRBg0fvvDb5kvVvTD/6Z4DMIdiigRRZDEJeOpQdElOL6DN9YIwgeuzv+E//q/+2/4v/1f/y9s9wdsZ8SLr674/fd/geNOOAYhkiAjSyKqVFGmNa7r0mopWWfgSRJdXTK/uqQXBe7udrgXMzbrI5Kscoxi5uNr0rTh4vIr4nzD+unA+WLO2YsprdlR7nPao8j53KBIU4xKZ6z3iLVA0YBtjKlLmTjZ4DgrPv10wDHmfPOLK/7mtz+gaT6CmDM/V+iL03i/6StcZwIDTGdjdNVDkSQEQ+HmcMPq6iWuB8M+QlIEltdnxMGWXmywHY2q1uiHGt9dMh5PORw/I1LTtS3GSKZTSo5BRJnBi6sLfvwxwdd9Rn6D0FsUfY5n6nSNzOPjE7/8zT/G9kUOhwfocpzZCEEwaNoC15Qph5Y8jdENgfnsEsNWOAQRXdPylBYIuoDjT2jR2B0qtoeARjndRUtzhDOuiOMSTRxI0y37fcDVq1+wOYTkZU+jtET5wMLQqcqE6xdTjk970mSH1FoUqcS76yX3xx/pC4Ox5+EtNfLCQJWntENOXJXkSYFmd6w3t9zc7plPbPI4RRIuuDq/5ubwmU+Pd/z6T/4xd5snJKlD7gACVMtH6lwkQYNOJU9A6CNM3+BuLfCP3vwLXn7zS3qxx7ZdPMfBkFt0teH2MSBvZfTGAbPiKU1R8op5PeFydI3Y1jw8bZEkC8edEaZbJLckSE7PltliiuMofPzwmeXFJZZlEEpHkvxAdCiZz3wUWaQuSppKwTB0svzIYjnlGMTExwhZsnA8mcftHVWlYBljtodbRhNoDxa6kaFpC3bbmFZIMMwRRd7huwNpdsDzLao8Jc/kU5c0q5ElB89Z8viwxbY7omNCN+npWxevGRB3Ha5q87Tb4sxUJFmFRsNRW8KoY/Vyiq7L3HzYY+DQ6wVi5SL2PX3bowkCcfpEgkS73SJqCoosEWwKvFmAoWoUKdyvDyBbp6QeV6dqcs7nZzzuU7qhQOw1VpMZVZnTlqDKGqOxSpFmjH2bQ/QDdelioTM9nyHoBWWj8+t3X/PTT79FdEaI9oy8CPBdj6yuSVOwzYGyksmTlrEjYo1thKHmL/7NE//8n/xj5GHg8fEHNG3EyzdvkMR7yrpBGBSyOCLLQurq76XHAf4dRt6CNGD5PWfXp0V3x1ry9LSjbCoUC35+/4XZdMXF1YTJXMMwHYTBxtItouABzxPZfUlpqwTTcJBan/v7e5y5xNC7yNjUQ4lh2RTJwHqzwTbGqGJP1TasznwW03d8/ryh7TNkQWfoZSTRRFEEZKOkrEPOzsb4toMuDziGh2mM+PRhi6a2JPEBe+IyW41J2dP0BXUroenQDzWWrZOkIVEUYhou9zd3fPnyiV9+9w5dsvFMG8+QMESZxXTOcZ9Cb9PULXVZIKsNj5vf4poTJqMZSZQSRTFVF7J92JHkGU034K6mvPz2K2RRQtZhvJjQk9L1FVJnMHcnKFpKU4scthFXExtFMdAlg9HIpe07bm9uyKua+fQVjuCgyAKyLlDS0EqnvM++7qkzgTxqENUayzBxNAWtHzO2Zoy8KaraQmugSzZd0zJZyby7GmMqEo4hYpkqpqAidQpCp1McwRdUfvyrO/qmpwkE1tuUci3SVQXO2EU159zcbqiqiPHEYZBkHB3SUiFSakrNZhsmhJ9DwvBAQcloNsUey0zmIzRVhEbgcuWiaiWa4TC2xmxvE5IsAtkiTRNEpacVSw67Ek1WmM19RpMJ7tjiGIS0ZY8sSRjq6cFS1iEj22UYfNq+4+13r9D8CUvvjKZSqQ4D49GS3X7Dw/sHlq9HdGmLqo+p05y+Etk/rnl9+Rs00eWw27PZPqFILmERsIkPmPqEJpGQWpOHj2uqpODy6hR5uIk3yJpBUe1RaKiGU5auKah0hYQmN7RlQd8cabuGpikQhAFdMlCRWI3HFPmRu7tbtk8BxzTj/ecv9JFIs7tnKBUkQaSOS7rGIMn2VElJHiRIXcPIneBYJq5t4tk6ChXz2QTPVxmKBKNXT19WipDFS4usL6nTHtdUwFJ4e/0SFZfvf/vXaIrKIdsyVCXxTUX8peRPvv0O2zWZLmS++erNqQOvjigj+D/8V7/Cd8fEIaiqRJVGJ7/FYeC43dJWCYqicIwKNsGeJCspq5Sh69BlB8P0MRQbf6pzsVpyjLZcvBlxeTan7RJ6aeDxacfPH/6Sqsj58vkB27ZZTCfUZc5hH6MYc958/Q2KrfDDT1tUVkRRzHLqMtJlirRGUQea4kDfrLFGMroyxxggSyvapiCryr87baGSZCV34ROWM6Fpodi3vH37ls3hSFpXjGYy/8Of/9+ZjE87xFNtIFx/odUUZF9GkCW86ZjDdsP7zxsEraYdcqKHCDkJOB/b+IaOVEO4L6hEieNTzna/w3Z0VtMR3rmDUMv0TUyZDvzi65e4FxJd3bL7fIMh97iXOobWMLIVvPmA4ZgIUkrX61hOj6q2yLrKIPRYVkfbh9w/3ZLnMboSMfZMPHfCPtwzHtksJjPaXMEybHzP4OraY+glrqYziq6jTBPChwrTGfP5y0eyvMQ0XXTR5WxxxXI0Y2r52I7FevuZ9dOB8cWCJD/QCyqqOUO3RCajEcdDiesrCIqMrEFWNPziF7/kfDnGXqkszmYU2YZ4HyAJLjePW9IoQZFUFEUhCEoMEaI04uY+Q1INmkI+rQ6IcLZ6gVDLzOdTlLSlqhMMGXQUrL6nKDLCXYXZiIhtwy7uUUcTSqFFckvSNiCNj4hFhy4ISLZJ0yuM3XOuz+YImoQ/8tgEXxhb53z9zRneROPDT2toROo+JMg3REHJ9dVrFrMlx/WWF2djmhYmk0uiZMM2WSMIDeOxjdJWSHWC71lYhoE45OT5kdl4SldkSELLIb+hFgfCsOFf/Iv/nn/+X/zXbKI9xfG04hTHCd/96j8m3A+0SBRljDnRqescQ2jpCokwrUiEjE3eMr66QLBlyq7kq+8u0bQRuq4jDwU0Be9/WGNYZ1Al/PDDj+wOe46HHEO1iQ4B9BbxsSE43lP3OXXbst2EmOYE23V4++4FgyIh4qAZIvOxwUCH1I2QtFPW+ubx06m71huYjkjaxDSCwj46kOc5fTPQNTW+5dGUCWVx5PxsyWQyoetszl99haOOMBGYzc5JxYZeFtCMnqbqaIuOuqowrRnTicdxE1InHXmUISgDyQ78mYA/tvjnf/JPePPtBXrfom1LKilBNUWieI/li1R1jSzL/PKXbzEV65RKJjqoUosli3z8+AWqgu+++oa+6uiVjiTrMHUbSZKIq5JCkIiLmDTuaaqayzcXqNqR9797ohNKcFqs2Zz1fUyxfWQ+dSiEkjAP0USF8eKSs+VLltMRQjfw+OkROpnl3CLvEmzPR9A0JjOdLK2xjTG6oXI2n5Hsc+azBRL/AQrKrm9IsgJDt+m6jqenB3RdRVVl0jTj7du3qKqKruskSUIaJ/iOS1WUOJZNcIi5vJ7y4sU1oijjewqWpRMdM6oy4xAGaLrC8RgQx0dc18W2TZIkYbPeYZomd3d3mKaJ5zmoqoplGQjCQBiHCILA2dkZl5eXLBYLdF2nqipMTWW5nDMej7Ftm+iPy7mSJDGZzEiShMVsybff/ArP84iCkMVigWUZ+L6H73o8Pj6iqjKyIjIMA13f0LcdSZLQ1g0jz6OuG2RB5fr6JUVeEUURmmawmJ+z3RyYL2fYrsN8PmcymaAbKpPRCFEU+eH3f+Cw2zHyfGRBRBJEbP1U4Hm2Q102iMLJQ1JVdY7HI47joMkK73/+ka5uYFAo8oq7uztkWaXve9quZDxxMUyN2dSn7UquX1zQ9y3zxQTD0BCEAUEc0E2L8WSGabm0nUQYZgyCfPLprCrKokEcZIqqQJRUrl+8RtdVBElhPJ5QtiWDDFVdgyhjmCaL8zPqriUvC37+8J4kPaBILZI4YNs++11CHJWsVgs8z2M0GtF0LVmRM52eYixHoxFt27JZ71AUCVO30GSNskqpipKhHRiPfbIso65rsjxBkDh5i6YRV1dXHMKQuy8Rf/vXj6jSHNc6Q5F8bj4e0KURQiez3uw4P59h2g4//fgRx3aRRYnHx0fmi+lJtDKeMJlM2Dw+0bYtqiyzmM7Y7XYAeJ5HWZaomgFwWsmIY+4fHtiHwSmHXlKZjE+Z9K7r07Y9WZrj2h6SpOC5Y4ZBoq8H2hbiMKYoCuq6pihLvnz8RFWUzGcTkiRhtpzh+y7j0YLD4YAkqYxGE0QRFosZvu9x/WLFm7evuLq6QFEkRiMP3/e4uL7ANHV0U+Pi9UvKrgK5ZzobU5UNTTuQFTm6q3N2vkDW4ePHn1kuZ4gSzGYTRmOHf/bP/4zX7ybIOlRlz/2XnJ9/XnPzZcsglHz9qwtuvmyxTIeentvbex6fduz2AVESo6gGv/nNb9gf1hyPR3TdZBgGhmGg70TyPEUSVcqipy4lskTk5lPEhx8PBFuQhgVvX1zwz/7sN7x5cc3l+RmWaTIMAh8+3WJYLkMvIMoDvu8THI7I0oBl6iiKhiQMf/SYlWiaBtOwaJqOMiuYz5esn7a8fP2KquwQRfnvjqLIrNdbNO1kB1ZVFZZl8eHDB+I4RlEU7u/v+fj5E4cg4HGzJu8GBlHBdUbc3dyz2+3YbJ6QlZ7Z2AEE9vuTy8VivkJTDYIgYL3eMgwDLy6vUGWJ89UZbXuy7QnDkI+fPrEPDpyvVlxcnJFGMVIPy9mSvukJDwFdc/pf7BgYTcZYrgOyRFaVxGlGM0DX1zi+xWRysguazmcUVUlTd2iazupsjmVZtF3N6mKKbinEacIhDGj76mQvNBpj6CppEuHaJk3TEMYRoiSzO2xpuhLLtVieLejpECRQNY1jHLLd7Hh4eqSqTmlkTdOQpilpmhLFR+o/rjOtt098vrkBUaAbepAl9seQ3W5HXde0bUueF4zHYyRJIklzfN9H10/3Z5alCAJIknQykTZMvny5RdIsBFnDMC3iPAdJJMkTFqs589WULEkQ2h5dkaiKjLZsGTrhj6r402v1fU8cxydRhG0TRdHJBUUQTjvMVUUYBnR9S9+3rNdr5vM5oiCRphlN0+G6Hn0PlmUxn85QZQXxjy4jWZFSFhXDIKBrNnnWoMgm2+2ROAmZTCaslleMvDPms0sW80tGY++0s9cUWLrMh/ff89u/+Qv+X//v/wdDX6MqEmLfEQQBkiQxGo3+GLEsEUURRZYTBTFxnNK2HXefv9DkJbZmIIsK9/ePJ8Nuxz2Jibru5NZQlsiyjG3bPD4+MogDru+flM6Kiiirp/AOx+HLl09E0ZG6LhHFP0Y8KwpFUWBZFooq8eLFCwzjdL8WRcYwDBiGgWVZNE2DZZlUVYVhaEiqdLLmaiqCcEtexNx+uaGtanRV47DfUhclVZkz9ny6ukFRBYJgy263QdMlXNdmGDrckcV8PuG7X71muVjRthVJdmC7ibFtl36omc/OGU00EFqaWkAWHaKw5P3Pn09WhVZH13WsVitUVUUQToVanp80CdvtljzPEUSRqmpo64627anrhr5pGY19vnx+z2GXM5uPoG+4/bzG0FRevbzk6vIFINI2PZ43YjKastsemE2n1HXN2dkZs9mYJA2RZJEg2KMoEoah0fU1bVeQZRmue/obXl9fUxQFsvIfYIdSlkW+/vpbDvuEpu0Yj0eExwOKqpJlGb7d0/cttmWg6yrDoPH9D79juRqzPDtjMh8RHvYMdAhIaEaLZ3iY+oTtekOa1jw+7nnzenHKrXU8kiQ5RRyKJWmaUtWcxiVVhqZb9EPHZrdGViREQSZOc26+3LFe7xlPTKhOHn5xWrFYeuRZTJyH+M6INAuIwiO/+PaXeLaGMAh8/nSDpmkcdhts22Q6nTP0NVmWIYsakgB5llDkLZZl4Ng2mqwQZQf6TqTrhz9+QDriJMCxT8WQLJn0QJRE1BUUfYquDFidDaqMbZisFmeEScbV+QVDVxOlEdPRGFVpyI97ptMpZVBTVQWSJCF0AlVVcX1xiaGrtMce+p5gt+coKKxWK969ueLHH39AYiA+hmiqwubpgelkSV3nFGWCZWuMJwv+8PMNtjsg95AcczZhhD+eUHU9sqLiGRbHY4xuyPz88R5Vd1mcWzw+7JAcF1URKKoSWYDf/v57LEujj2JaBgZBoOlEJEGna06WRouJx4t/OqIzQTFViqiiH1rgdMk/Pq65OF8wnU55vF1jmiZ9k9G2Lb4/wcegamGzf8QQNSRJQpQFJrMxZZWi6RLT6QRJHDBNk/lCJCkS8jynoSJJNmi6Rzf01NKBxdhFtVz2+4DgEGG5YxgkxmOftqqp6xqBiosX55R5T9+2qH/87AuCwGg85e7pkTcvXrB+2pFkOZqmY2oqyD2qKdFLpzFJWZZQNQgWaIpKeDwwurg47VK6OkVe47gumqYRBFvoJZarFYJwGvsuVzOK4pTt3XUdZVPydHfA8z3ubjfI51NkRWAYutOlJbaIosB2t+bh8Y5vf/EWQRiQZRlRFnBHHmmZUXUlo4lHcIg4BDHL8yWKqVH1JVLX0xYV/+I//SdkccKZM6PqKsqoQFBgNDo9JPqhRpYMzs7PMCwTWR9o2pbf//YzL19coOkSmq6iTXyORUVHiqJZPNw/0fcNsqywWSc4boskKcRBiqJVPG1ChgZM3cKwNH75q7e8fnmO7UqUdQl1S5aH/OKbNzw9heR5RXAI6bqeLK+xPZeur3j//j2i0jNfTBi6BmqBtjk9gKJDjD9yEcXTjvegDmRZxvEYI2gVddXRlcnf3YkmOo7rc/nyjDIpYRBPF7B8yjKWZZm6a4mzlFbpGZ/NKcqS7c0dsuVgqRZ10WI5Fp5hoMsK212CIikIvYCum4iiTFnUvHjxhul8xmb7ha/evSHNCh6f7jk2B/70P/0N9dWAObJxbYfPNx8Z2g5DNUjKjDzN+O67X/Hpxx8ZxAFJkeiFnmMcUdYgtB1V1+JOx/R5RT80BGHMMY5YLmx03aQoKvqgYx9uGIbTZ9jzWpomOylxtZNY6OlpR1INaPTMJmPCwx5ZFkmShKeuJYj3aJ5GkmyQJdjFBQM1ggSyInN2eYGsKtQthHFIl1Zcnr0kb1Q+PXxBtU1EQeeYHJn6Pp++3GJJFpZj8rR5ZOxOqeIC13Yoqortdk+WFbiewW4fsFq95BBskRBQ/5ijvd6vOfvmNVmV87e//5F3f/oVybFkMp/x+LDh7evXHMKAh6d73l5eEQUZIi1F0aKKMsE6wpmY2IZ52k8uShx9dNqxFQc0zWA0HZHFO2zbJTpuyPOcxWJCXiSYpk4YhqeiklPU78PdI1fXS7ZBiKHrjFyPSlIp6oHRaMTTZo0o9Ax5jaZ7tE3CbHpJJZyelYvJOaOrOaPpFbph8cMPP9D1PV1R8jfv/4Ifbt6jKT0jT6GtMrbhDtfx6EWF/X7NcYgR0KjKEt/xaZqOzeMaAYUyr2iaGseV6duaKE4p8xJVO+0V97qOUkPTNHRdxzCc7hrDEFE1KMuS0WhKnhccDkdG7hRJAtd3OK4P6I5JUebIrUInQC92uDOdrgXbtDgG29Mz2T69dpJkpGlKVicMVYMgSVRtjeNYFEXBw/0T8+UCUQQJg93dDe3Q0tEwmU2RBAVTcZFVgzRPmMzGOLbNMT5iuRZlm2FZGkWZ4NgSnz/doGg6d/cfWO8Hzs8XLM6nfAkidusY15kSRxm2c1Kf111HnGik2YCmF1xenQEFXiVQ1w13d3cY9cmXVpNUZFlGQGQYBMJ9jKXK6JrE5vEJ39cIDj3ffHuJbQnkSU+aphzDClGZUHYlmqbj2BbHxyNNB0EQMJvNiKI9ogRNVaOrFp0goOsqCB22YwEKYV7Qdd0fn8GPzC9cuvrvr/L+exubP/PMM88888wzzzzzzP8ef/9e5jPPPPPMM88888wzz/zv8FxQPvPMM88888wzzzzzD+K5oHzmmWeeeeaZZ5555h/Ec0H5zDPPPPPMM88888w/iOeC8plnnnnmmWeeeeaZfxDPBeUzzzzzzDPPPPPMM/8gngvKZ5555plnnnnmmWf+QTwXlM8888wzzzzzzDPP/IN4LiifeeaZZ5555plnnvkH8f8HKjENS+oXcSUAAAAASUVORK5CYII=", "text/plain": [ "
" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# show the results\n", "visualizer.add_datasample(\n", " 'result',\n", " img,\n", " data_sample=result,\n", " draw_gt=False,\n", " wait_time=0,\n", ")\n", "visualizer.show()" ] } ], "metadata": { "kernelspec": { "display_name": "mmdet", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.16" }, "pycharm": { "stem_cell": { "cell_type": "raw", "metadata": { "collapsed": false }, "source": [] } }, "vscode": { "interpreter": { "hash": "26395be4d8bd6f462fe6992ade267d864a329fc5ba918775a7fc2edf93f1463b" } } }, "nbformat": 4, "nbformat_minor": 4 } ================================================ FILE: demo/video_demo.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import cv2 import mmcv from mmcv.transforms import Compose from mmengine.utils import track_iter_progress from mmdet.apis import inference_detector, init_detector from mmdet.registry import VISUALIZERS def parse_args(): parser = argparse.ArgumentParser(description='MMDetection video demo') parser.add_argument('video', help='Video file') parser.add_argument('config', help='Config file') parser.add_argument('checkpoint', help='Checkpoint file') parser.add_argument( '--device', default='cuda:0', help='Device used for inference') parser.add_argument( '--score-thr', type=float, default=0.3, help='Bbox score threshold') parser.add_argument('--out', type=str, help='Output video file') parser.add_argument('--show', action='store_true', help='Show video') parser.add_argument( '--wait-time', type=float, default=1, help='The interval of show (s), 0 is block') args = parser.parse_args() return args def main(): args = parse_args() assert args.out or args.show, \ ('Please specify at least one operation (save/show the ' 'video) with the argument "--out" or "--show"') # build the model from a config file and a checkpoint file model = init_detector(args.config, args.checkpoint, device=args.device) # build test pipeline model.cfg.test_dataloader.dataset.pipeline[0].type = 'LoadImageFromNDArray' test_pipeline = Compose(model.cfg.test_dataloader.dataset.pipeline) # init visualizer visualizer = VISUALIZERS.build(model.cfg.visualizer) # the dataset_meta is loaded from the checkpoint and # then pass to the model in init_detector visualizer.dataset_meta = model.dataset_meta video_reader = mmcv.VideoReader(args.video) video_writer = None if args.out: fourcc = cv2.VideoWriter_fourcc(*'mp4v') video_writer = cv2.VideoWriter( args.out, fourcc, video_reader.fps, (video_reader.width, video_reader.height)) for frame in track_iter_progress(video_reader): result = inference_detector(model, frame, test_pipeline=test_pipeline) visualizer.add_datasample( name='video', image=frame, data_sample=result, draw_gt=False, show=False, pred_score_thr=args.score_thr) frame = visualizer.get_image() if args.show: cv2.namedWindow('video', 0) mmcv.imshow(frame, 'video', args.wait_time) if args.out: video_writer.write(frame) if video_writer: video_writer.release() cv2.destroyAllWindows() if __name__ == '__main__': main() ================================================ FILE: demo/video_gpuaccel_demo.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse from typing import Tuple import cv2 import mmcv import numpy as np import torch import torch.nn as nn from mmcv.transforms import Compose from mmengine.utils import track_iter_progress from mmdet.apis import init_detector from mmdet.registry import VISUALIZERS from mmdet.structures import DetDataSample try: import ffmpegcv except ImportError: raise ImportError( 'Please install ffmpegcv with:\n\n pip install ffmpegcv') def parse_args(): parser = argparse.ArgumentParser( description='MMDetection video demo with GPU acceleration') parser.add_argument('video', help='Video file') parser.add_argument('config', help='Config file') parser.add_argument('checkpoint', help='Checkpoint file') parser.add_argument( '--device', default='cuda:0', help='Device used for inference') parser.add_argument( '--score-thr', type=float, default=0.3, help='Bbox score threshold') parser.add_argument('--out', type=str, help='Output video file') parser.add_argument('--show', action='store_true', help='Show video') parser.add_argument( '--nvdecode', action='store_true', help='Use NVIDIA decoder') parser.add_argument( '--wait-time', type=float, default=1, help='The interval of show (s), 0 is block') args = parser.parse_args() return args def prefetch_batch_input_shape(model: nn.Module, ori_wh: Tuple[int, int]) -> dict: cfg = model.cfg w, h = ori_wh cfg.test_dataloader.dataset.pipeline[0].type = 'LoadImageFromNDArray' test_pipeline = Compose(cfg.test_dataloader.dataset.pipeline) data = {'img': np.zeros((h, w, 3), dtype=np.uint8), 'img_id': 0} data = test_pipeline(data) _, data_sample = model.data_preprocessor([data], False) batch_input_shape = data_sample[0].batch_input_shape return batch_input_shape def pack_data(frame_resize: np.ndarray, batch_input_shape: Tuple[int, int], ori_shape: Tuple[int, int]) -> dict: assert frame_resize.shape[:2] == batch_input_shape data_sample = DetDataSample() data_sample.set_metainfo({ 'img_shape': batch_input_shape, 'ori_shape': ori_shape, 'scale_factor': (batch_input_shape[0] / ori_shape[0], batch_input_shape[1] / ori_shape[1]) }) frame_resize = torch.from_numpy(frame_resize).permute((2, 0, 1)) data = {'inputs': frame_resize, 'data_sample': data_sample} return data def main(): args = parse_args() assert args.out or args.show, \ ('Please specify at least one operation (save/show the ' 'video) with the argument "--out" or "--show"') model = init_detector(args.config, args.checkpoint, device=args.device) # init visualizer visualizer = VISUALIZERS.build(model.cfg.visualizer) # the dataset_meta is loaded from the checkpoint and # then pass to the model in init_detector visualizer.dataset_meta = model.dataset_meta if args.nvdecode: VideoCapture = ffmpegcv.VideoCaptureNV else: VideoCapture = ffmpegcv.VideoCapture video_origin = VideoCapture(args.video) batch_input_shape = prefetch_batch_input_shape( model, (video_origin.width, video_origin.height)) ori_shape = (video_origin.height, video_origin.width) resize_wh = batch_input_shape[::-1] video_resize = VideoCapture( args.video, resize=resize_wh, resize_keepratio=True, resize_keepratioalign='topleft') video_writer = None if args.out: video_writer = ffmpegcv.VideoWriter(args.out, fps=video_origin.fps) with torch.no_grad(): for i, (frame_resize, frame_origin) in enumerate( zip(track_iter_progress(video_resize), video_origin)): data = pack_data(frame_resize, batch_input_shape, ori_shape) result = model.test_step([data])[0] visualizer.add_datasample( name='video', image=frame_origin, data_sample=result, draw_gt=False, show=False, pred_score_thr=args.score_thr) frame_mask = visualizer.get_image() if args.show: cv2.namedWindow('video', 0) mmcv.imshow(frame_mask, 'video', args.wait_time) if args.out: video_writer.write(frame_mask) if video_writer: video_writer.release() video_origin.release() video_resize.release() cv2.destroyAllWindows() if __name__ == '__main__': main() ================================================ FILE: demo/webcam_demo.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import cv2 import mmcv import torch from mmdet.apis import inference_detector, init_detector from mmdet.registry import VISUALIZERS def parse_args(): parser = argparse.ArgumentParser(description='MMDetection webcam demo') parser.add_argument('config', help='test config file path') parser.add_argument('checkpoint', help='checkpoint file') parser.add_argument( '--device', type=str, default='cuda:0', help='CPU/CUDA device option') parser.add_argument( '--camera-id', type=int, default=0, help='camera device id') parser.add_argument( '--score-thr', type=float, default=0.5, help='bbox score threshold') args = parser.parse_args() return args def main(): args = parse_args() # build the model from a config file and a checkpoint file device = torch.device(args.device) model = init_detector(args.config, args.checkpoint, device=device) # init visualizer visualizer = VISUALIZERS.build(model.cfg.visualizer) # the dataset_meta is loaded from the checkpoint and # then pass to the model in init_detector visualizer.dataset_meta = model.dataset_meta camera = cv2.VideoCapture(args.camera_id) print('Press "Esc", "q" or "Q" to exit.') while True: ret_val, img = camera.read() result = inference_detector(model, img) img = mmcv.imconvert(img, 'bgr', 'rgb') visualizer.add_datasample( name='result', image=img, data_sample=result, draw_gt=False, pred_score_thr=args.score_thr, show=False) img = visualizer.get_image() img = mmcv.imconvert(img, 'bgr', 'rgb') cv2.imshow('result', img) ch = cv2.waitKey(1) if ch == 27 or ch == ord('q') or ch == ord('Q'): break if __name__ == '__main__': main() ================================================ FILE: docker/Dockerfile ================================================ ARG PYTORCH="1.9.0" ARG CUDA="11.1" ARG CUDNN="8" FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel ENV TORCH_CUDA_ARCH_LIST="6.0 6.1 7.0 7.5 8.0 8.6+PTX" \ TORCH_NVCC_FLAGS="-Xfatbin -compress-all" \ CMAKE_PREFIX_PATH="$(dirname $(which conda))/../" \ FORCE_CUDA="1" # Avoid Public GPG key error # https://github.com/NVIDIA/nvidia-docker/issues/1631 RUN rm /etc/apt/sources.list.d/cuda.list \ && rm /etc/apt/sources.list.d/nvidia-ml.list \ && apt-key del 7fa2af80 \ && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub \ && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub # (Optional, use Mirror to speed up downloads) # RUN sed -i 's/http:\/\/archive.ubuntu.com\/ubuntu\//http:\/\/mirrors.aliyun.com\/ubuntu\//g' /etc/apt/sources.list && \ # pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple # Install the required packages RUN apt-get update \ && apt-get install -y ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* # Install MMEngine and MMCV RUN pip install openmim && \ mim install "mmengine>=0.6.0" "mmcv>=2.0.0rc4" # Install MMDetection RUN conda clean --all \ && git clone https://github.com/open-mmlab/mmdetection.git -b 3.x /mmdetection \ && cd /mmdetection \ && pip install --no-cache-dir -e . WORKDIR /mmdetection ================================================ FILE: docker/serve/Dockerfile ================================================ ARG PYTORCH="1.9.0" ARG CUDA="11.1" ARG CUDNN="8" FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel ARG MMCV="2.0.0rc4" ARG MMDET="3.0.0rc6" ENV PYTHONUNBUFFERED TRUE # Avoid Public GPG key error # https://github.com/NVIDIA/nvidia-docker/issues/1631 RUN rm /etc/apt/sources.list.d/cuda.list \ && rm /etc/apt/sources.list.d/nvidia-ml.list \ && apt-key del 7fa2af80 \ && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub \ && apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub # (Optional, use Mirror to speed up downloads) # RUN sed -i 's/http:\/\/archive.ubuntu.com\/ubuntu\//http:\/\/mirrors.aliyun.com\/ubuntu\//g' /etc/apt/sources.list # Install the required packages RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ g++ \ openjdk-11-jre-headless \ # MMDet Requirements ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 \ && rm -rf /var/lib/apt/lists/* ENV PATH="/opt/conda/bin:$PATH" \ FORCE_CUDA="1" # TORCHSEVER RUN pip install torchserve torch-model-archiver # MMLAB ARG PYTORCH ARG CUDA RUN pip install mmengine RUN ["/bin/bash", "-c", "pip install mmcv==${MMCV} -f https://download.openmmlab.com/mmcv/dist/cu${CUDA//./}/torch${PYTORCH}/index.html"] RUN pip install mmdet==${MMDET} RUN useradd -m model-server \ && mkdir -p /home/model-server/tmp COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh \ && chown -R model-server /home/model-server COPY config.properties /home/model-server/config.properties RUN mkdir /home/model-server/model-store && chown -R model-server /home/model-server/model-store EXPOSE 8080 8081 8082 USER model-server WORKDIR /home/model-server ENV TEMP=/home/model-server/tmp ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] CMD ["serve"] ================================================ FILE: docker/serve/config.properties ================================================ inference_address=http://0.0.0.0:8080 management_address=http://0.0.0.0:8081 metrics_address=http://0.0.0.0:8082 model_store=/home/model-server/model-store load_models=all ================================================ FILE: docker/serve/entrypoint.sh ================================================ #!/bin/bash set -e if [[ "$1" = "serve" ]]; then shift 1 torchserve --start --ts-config /home/model-server/config.properties else eval "$@" fi # prevent docker exit tail -f /dev/null ================================================ FILE: docker/serve_cn/Dockerfile ================================================ ARG PYTORCH="1.9.0" ARG CUDA="11.1" ARG CUDNN="8" FROM pytorch/pytorch:${PYTORCH}-cuda${CUDA}-cudnn${CUDNN}-devel ARG MMCV="2.0.0rc4" ARG MMDET="3.0.0rc6" ENV PYTHONUNBUFFERED TRUE # Avoid Public GPG key error # - https://github.com/NVIDIA/nvidia-docker/issues/1631 RUN rm /etc/apt/sources.list.d/cuda.list \ && rm /etc/apt/sources.list.d/nvidia-ml.list \ && apt-get update \ && apt-get install -y wget \ && rm -rf /var/lib/apt/lists/* \ && apt-key del 7fa2af80 \ && apt-get update && apt-get install -y --no-install-recommends wget \ && wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/cuda-keyring_1.0-1_all.deb \ && dpkg -i cuda-keyring_1.0-1_all.deb # (Optional, use Mirror to speed up downloads) # RUN sed -i 's/http:\/\/archive.ubuntu.com\/ubuntu\//http:\/\/mirrors.aliyun.com\/ubuntu\//g' /etc/apt/sources.list # Install the required packages RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ g++ \ openjdk-11-jre-headless \ # MMDet Requirements ffmpeg libsm6 libxext6 git ninja-build libglib2.0-0 libsm6 libxrender-dev libxext6 \ && rm -rf /var/lib/apt/lists/* ENV PATH="/opt/conda/bin:$PATH" \ FORCE_CUDA="1" # TORCHSEVER RUN pip install torchserve torch-model-archiver nvgpu -i https://pypi.mirrors.ustc.edu.cn/simple/ # MMLAB ARG PYTORCH ARG CUDA RUN pip install mmengine -i https://pypi.mirrors.ustc.edu.cn/simple/ RUN ["/bin/bash", "-c", "pip install mmcv==${MMCV} -f https://download.openmmlab.com/mmcv/dist/cu${CUDA//./}/torch${PYTORCH}/index.html"] RUN pip install mmdet==${MMDET} -i https://pypi.mirrors.ustc.edu.cn/simple/ RUN useradd -m model-server \ && mkdir -p /home/model-server/tmp COPY entrypoint.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh \ && chown -R model-server /home/model-server COPY config.properties /home/model-server/config.properties RUN mkdir /home/model-server/model-store && chown -R model-server /home/model-server/model-store EXPOSE 8080 8081 8082 USER model-server WORKDIR /home/model-server ENV TEMP=/home/model-server/tmp ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] CMD ["serve"] ================================================ FILE: mmdet/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import mmcv import mmengine from mmengine.utils import digit_version from .version import __version__, version_info mmcv_minimum_version = '2.0.0rc4' mmcv_maximum_version = '2.1.0' mmcv_version = digit_version(mmcv.__version__) mmengine_minimum_version = '0.6.0' mmengine_maximum_version = '1.0.0' mmengine_version = digit_version(mmengine.__version__) assert (mmcv_version >= digit_version(mmcv_minimum_version) and mmcv_version < digit_version(mmcv_maximum_version)), \ f'MMCV=={mmcv.__version__} is used but incompatible. ' \ f'Please install mmcv>={mmcv_minimum_version}, <{mmcv_maximum_version}.' assert (mmengine_version >= digit_version(mmengine_minimum_version) and mmengine_version < digit_version(mmengine_maximum_version)), \ f'MMEngine=={mmengine.__version__} is used but incompatible. ' \ f'Please install mmengine>={mmengine_minimum_version}, ' \ f'<{mmengine_maximum_version}.' __all__ = ['__version__', 'version_info', 'digit_version'] ================================================ FILE: mmdet/apis/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .det_inferencer import DetInferencer from .inference import (async_inference_detector, inference_detector, init_detector) __all__ = [ 'init_detector', 'async_inference_detector', 'inference_detector', 'DetInferencer' ] ================================================ FILE: mmdet/apis/det_inferencer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import os.path as osp import warnings from typing import Dict, Iterable, List, Optional, Sequence, Union import mmcv import mmengine import numpy as np import torch.nn as nn from mmengine.dataset import Compose from mmengine.fileio import (get_file_backend, isdir, join_path, list_dir_or_file) from mmengine.infer.infer import BaseInferencer, ModelType from mmengine.model.utils import revert_sync_batchnorm from mmengine.registry import init_default_scope from mmengine.runner.checkpoint import _load_checkpoint_to_model from mmengine.visualization import Visualizer from rich.progress import track from mmdet.evaluation import INSTANCE_OFFSET from mmdet.registry import DATASETS from mmdet.structures import DetDataSample from mmdet.structures.mask import encode_mask_results, mask2bbox from mmdet.utils import ConfigType from ..evaluation import get_classes try: from panopticapi.evaluation import VOID from panopticapi.utils import id2rgb except ImportError: id2rgb = None VOID = None InputType = Union[str, np.ndarray] InputsType = Union[InputType, Sequence[InputType]] PredType = List[DetDataSample] ImgType = Union[np.ndarray, Sequence[np.ndarray]] IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp') class DetInferencer(BaseInferencer): """Object Detection Inferencer. Args: model (str, optional): Path to the config file or the model name defined in metafile. For example, it could be "rtmdet-s" or 'rtmdet_s_8xb32-300e_coco' or "configs/rtmdet/rtmdet_s_8xb32-300e_coco.py". If model is not specified, user must provide the `weights` saved by MMEngine which contains the config string. Defaults to None. weights (str, optional): Path to the checkpoint. If it is not specified and model is a model name of metafile, the weights will be loaded from metafile. Defaults to None. device (str, optional): Device to run inference. If None, the available device will be automatically used. Defaults to None. scope (str, optional): The scope of the model. Defaults to mmdet. palette (str): Color palette used for visualization. The order of priority is palette -> config -> checkpoint. Defaults to 'none'. """ preprocess_kwargs: set = set() forward_kwargs: set = set() visualize_kwargs: set = { 'return_vis', 'show', 'wait_time', 'draw_pred', 'pred_score_thr', 'img_out_dir', 'no_save_vis', } postprocess_kwargs: set = { 'print_result', 'pred_out_dir', 'return_datasample', 'no_save_pred', } def __init__(self, model: Optional[Union[ModelType, str]] = None, weights: Optional[str] = None, device: Optional[str] = None, scope: Optional[str] = 'mmdet', palette: str = 'none') -> None: # A global counter tracking the number of images processed, for # naming of the output images self.num_visualized_imgs = 0 self.num_predicted_imgs = 0 self.palette = palette init_default_scope(scope) super().__init__( model=model, weights=weights, device=device, scope=scope) self.model = revert_sync_batchnorm(self.model) def _load_weights_to_model(self, model: nn.Module, checkpoint: Optional[dict], cfg: Optional[ConfigType]) -> None: """Loading model weights and meta information from cfg and checkpoint. Args: model (nn.Module): Model to load weights and meta information. checkpoint (dict, optional): The loaded checkpoint. cfg (Config or ConfigDict, optional): The loaded config. """ if checkpoint is not None: _load_checkpoint_to_model(model, checkpoint) checkpoint_meta = checkpoint.get('meta', {}) # save the dataset_meta in the model for convenience if 'dataset_meta' in checkpoint_meta: # mmdet 3.x, all keys should be lowercase model.dataset_meta = { k.lower(): v for k, v in checkpoint_meta['dataset_meta'].items() } elif 'CLASSES' in checkpoint_meta: # < mmdet 3.x classes = checkpoint_meta['CLASSES'] model.dataset_meta = {'classes': classes} else: warnings.warn( 'dataset_meta or class names are not saved in the ' 'checkpoint\'s meta data, use COCO classes by default.') model.dataset_meta = {'classes': get_classes('coco')} else: warnings.warn('Checkpoint is not loaded, and the inference ' 'result is calculated by the randomly initialized ' 'model!') warnings.warn('weights is None, use COCO classes by default.') model.dataset_meta = {'classes': get_classes('coco')} # Priority: args.palette -> config -> checkpoint if self.palette != 'none': model.dataset_meta['palette'] = self.palette else: test_dataset_cfg = copy.deepcopy(cfg.test_dataloader.dataset) # lazy init. We only need the metainfo. test_dataset_cfg['lazy_init'] = True metainfo = DATASETS.build(test_dataset_cfg).metainfo cfg_palette = metainfo.get('palette', None) if cfg_palette is not None: model.dataset_meta['palette'] = cfg_palette else: if 'palette' not in model.dataset_meta: warnings.warn( 'palette does not exist, random is used by default. ' 'You can also set the palette to customize.') model.dataset_meta['palette'] = 'random' def _init_pipeline(self, cfg: ConfigType) -> Compose: """Initialize the test pipeline.""" pipeline_cfg = cfg.test_dataloader.dataset.pipeline # For inference, the key of ``img_id`` is not used. if 'meta_keys' in pipeline_cfg[-1]: pipeline_cfg[-1]['meta_keys'] = tuple( meta_key for meta_key in pipeline_cfg[-1]['meta_keys'] if meta_key != 'img_id') load_img_idx = self._get_transform_idx(pipeline_cfg, 'LoadImageFromFile') if load_img_idx == -1: raise ValueError( 'LoadImageFromFile is not found in the test pipeline') pipeline_cfg[load_img_idx]['type'] = 'mmdet.InferencerLoader' return Compose(pipeline_cfg) def _get_transform_idx(self, pipeline_cfg: ConfigType, name: str) -> int: """Returns the index of the transform in a pipeline. If the transform is not found, returns -1. """ for i, transform in enumerate(pipeline_cfg): if transform['type'] == name: return i return -1 def _init_visualizer(self, cfg: ConfigType) -> Optional[Visualizer]: """Initialize visualizers. Args: cfg (ConfigType): Config containing the visualizer information. Returns: Visualizer or None: Visualizer initialized with config. """ visualizer = super()._init_visualizer(cfg) visualizer.dataset_meta = self.model.dataset_meta return visualizer def _inputs_to_list(self, inputs: InputsType) -> list: """Preprocess the inputs to a list. Preprocess inputs to a list according to its type: - list or tuple: return inputs - str: - Directory path: return all files in the directory - other cases: return a list containing the string. The string could be a path to file, a url or other types of string according to the task. Args: inputs (InputsType): Inputs for the inferencer. Returns: list: List of input for the :meth:`preprocess`. """ if isinstance(inputs, str): backend = get_file_backend(inputs) if hasattr(backend, 'isdir') and isdir(inputs): # Backends like HttpsBackend do not implement `isdir`, so only # those backends that implement `isdir` could accept the inputs # as a directory filename_list = list_dir_or_file( inputs, list_dir=False, suffix=IMG_EXTENSIONS) inputs = [ join_path(inputs, filename) for filename in filename_list ] if not isinstance(inputs, (list, tuple)): inputs = [inputs] return list(inputs) def preprocess(self, inputs: InputsType, batch_size: int = 1, **kwargs): """Process the inputs into a model-feedable format. Customize your preprocess by overriding this method. Preprocess should return an iterable object, of which each item will be used as the input of ``model.test_step``. ``BaseInferencer.preprocess`` will return an iterable chunked data, which will be used in __call__ like this: .. code-block:: python def __call__(self, inputs, batch_size=1, **kwargs): chunked_data = self.preprocess(inputs, batch_size, **kwargs) for batch in chunked_data: preds = self.forward(batch, **kwargs) Args: inputs (InputsType): Inputs given by user. batch_size (int): batch size. Defaults to 1. Yields: Any: Data processed by the ``pipeline`` and ``collate_fn``. """ chunked_data = self._get_chunk_data(inputs, batch_size) yield from map(self.collate_fn, chunked_data) def _get_chunk_data(self, inputs: Iterable, chunk_size: int): """Get batch data from inputs. Args: inputs (Iterable): An iterable dataset. chunk_size (int): Equivalent to batch size. Yields: list: batch data. """ inputs_iter = iter(inputs) while True: try: chunk_data = [] for _ in range(chunk_size): inputs_ = next(inputs_iter) chunk_data.append((inputs_, self.pipeline(inputs_))) yield chunk_data except StopIteration: if chunk_data: yield chunk_data break # TODO: Video and Webcam are currently not supported and # may consume too much memory if your input folder has a lot of images. # We will be optimized later. def __call__(self, inputs: InputsType, batch_size: int = 1, return_vis: bool = False, show: bool = False, wait_time: int = 0, no_save_vis: bool = False, draw_pred: bool = True, pred_score_thr: float = 0.3, return_datasample: bool = False, print_result: bool = False, no_save_pred: bool = True, out_dir: str = '', **kwargs) -> dict: """Call the inferencer. Args: inputs (InputsType): Inputs for the inferencer. batch_size (int): Inference batch size. Defaults to 1. show (bool): Whether to display the visualization results in a popup window. Defaults to False. wait_time (float): The interval of show (s). Defaults to 0. no_save_vis (bool): Whether to force not to save prediction vis results. Defaults to False. draw_pred (bool): Whether to draw predicted bounding boxes. Defaults to True. pred_score_thr (float): Minimum score of bboxes to draw. Defaults to 0.3. return_datasample (bool): Whether to return results as :obj:`DetDataSample`. Defaults to False. print_result (bool): Whether to print the inference result w/o visualization to the console. Defaults to False. no_save_pred (bool): Whether to force not to save prediction results. Defaults to True. out_file: Dir to save the inference results or visualization. If left as empty, no file will be saved. Defaults to ''. **kwargs: Other keyword arguments passed to :meth:`preprocess`, :meth:`forward`, :meth:`visualize` and :meth:`postprocess`. Each key in kwargs should be in the corresponding set of ``preprocess_kwargs``, ``forward_kwargs``, ``visualize_kwargs`` and ``postprocess_kwargs``. Returns: dict: Inference and visualization results. """ ( preprocess_kwargs, forward_kwargs, visualize_kwargs, postprocess_kwargs, ) = self._dispatch_kwargs(**kwargs) ori_inputs = self._inputs_to_list(inputs) inputs = self.preprocess( ori_inputs, batch_size=batch_size, **preprocess_kwargs) results_dict = {'predictions': [], 'visualization': []} for ori_inputs, data in track(inputs, description='Inference'): preds = self.forward(data, **forward_kwargs) visualization = self.visualize( ori_inputs, preds, return_vis=return_vis, show=show, wait_time=wait_time, draw_pred=draw_pred, pred_score_thr=pred_score_thr, no_save_vis=no_save_vis, img_out_dir=out_dir, **visualize_kwargs) results = self.postprocess( preds, visualization, return_datasample=return_datasample, print_result=print_result, no_save_pred=no_save_pred, pred_out_dir=out_dir, **postprocess_kwargs) results_dict['predictions'].extend(results['predictions']) if results['visualization'] is not None: results_dict['visualization'].extend(results['visualization']) return results_dict def visualize(self, inputs: InputsType, preds: PredType, return_vis: bool = False, show: bool = False, wait_time: int = 0, draw_pred: bool = True, pred_score_thr: float = 0.3, no_save_vis: bool = False, img_out_dir: str = '', **kwargs) -> Union[List[np.ndarray], None]: """Visualize predictions. Args: inputs (List[Union[str, np.ndarray]]): Inputs for the inferencer. preds (List[:obj:`DetDataSample`]): Predictions of the model. return_vis (bool): Whether to return the visualization result. Defaults to False. show (bool): Whether to display the image in a popup window. Defaults to False. wait_time (float): The interval of show (s). Defaults to 0. draw_pred (bool): Whether to draw predicted bounding boxes. Defaults to True. pred_score_thr (float): Minimum score of bboxes to draw. Defaults to 0.3. no_save_vis (bool): Whether to force not to save prediction vis results. Defaults to False. img_out_dir (str): Output directory of visualization results. If left as empty, no file will be saved. Defaults to ''. Returns: List[np.ndarray] or None: Returns visualization results only if applicable. """ if no_save_vis is True: img_out_dir = '' if not show and img_out_dir == '' and not return_vis: return None if self.visualizer is None: raise ValueError('Visualization needs the "visualizer" term' 'defined in the config, but got None.') results = [] for single_input, pred in zip(inputs, preds): if isinstance(single_input, str): img_bytes = mmengine.fileio.get(single_input) img = mmcv.imfrombytes(img_bytes) img = img[:, :, ::-1] img_name = osp.basename(single_input) elif isinstance(single_input, np.ndarray): img = single_input.copy() img_num = str(self.num_visualized_imgs).zfill(8) img_name = f'{img_num}.jpg' else: raise ValueError('Unsupported input type: ' f'{type(single_input)}') out_file = osp.join(img_out_dir, 'vis', img_name) if img_out_dir != '' else None self.visualizer.add_datasample( img_name, img, pred, show=show, wait_time=wait_time, draw_gt=False, draw_pred=draw_pred, pred_score_thr=pred_score_thr, out_file=out_file, ) results.append(self.visualizer.get_image()) self.num_visualized_imgs += 1 return results def postprocess( self, preds: PredType, visualization: Optional[List[np.ndarray]] = None, return_datasample: bool = False, print_result: bool = False, no_save_pred: bool = False, pred_out_dir: str = '', **kwargs, ) -> Dict: """Process the predictions and visualization results from ``forward`` and ``visualize``. This method should be responsible for the following tasks: 1. Convert datasamples into a json-serializable dict if needed. 2. Pack the predictions and visualization results and return them. 3. Dump or log the predictions. Args: preds (List[:obj:`DetDataSample`]): Predictions of the model. visualization (Optional[np.ndarray]): Visualized predictions. return_datasample (bool): Whether to use Datasample to store inference results. If False, dict will be used. print_result (bool): Whether to print the inference result w/o visualization to the console. Defaults to False. no_save_pred (bool): Whether to force not to save prediction results. Defaults to False. pred_out_dir: Dir to save the inference results w/o visualization. If left as empty, no file will be saved. Defaults to ''. Returns: dict: Inference and visualization results with key ``predictions`` and ``visualization``. - ``visualization`` (Any): Returned by :meth:`visualize`. - ``predictions`` (dict or DataSample): Returned by :meth:`forward` and processed in :meth:`postprocess`. If ``return_datasample=False``, it usually should be a json-serializable dict containing only basic data elements such as strings and numbers. """ if no_save_pred is True: pred_out_dir = '' result_dict = {} results = preds if not return_datasample: results = [] for pred in preds: result = self.pred2dict(pred, pred_out_dir) results.append(result) elif pred_out_dir != '': warnings.warn('Currently does not support saving datasample ' 'when return_datasample is set to True. ' 'Prediction results are not saved!') # Add img to the results after printing and dumping result_dict['predictions'] = results if print_result: print(result_dict) result_dict['visualization'] = visualization return result_dict # TODO: The data format and fields saved in json need further discussion. # Maybe should include model name, timestamp, filename, image info etc. def pred2dict(self, data_sample: DetDataSample, pred_out_dir: str = '') -> Dict: """Extract elements necessary to represent a prediction into a dictionary. It's better to contain only basic data elements such as strings and numbers in order to guarantee it's json-serializable. Args: data_sample (:obj:`DetDataSample`): Predictions of the model. pred_out_dir: Dir to save the inference results w/o visualization. If left as empty, no file will be saved. Defaults to ''. Returns: dict: Prediction results. """ is_save_pred = True if pred_out_dir == '': is_save_pred = False if is_save_pred and 'img_path' in data_sample: img_path = osp.basename(data_sample.img_path) img_path = osp.splitext(img_path)[0] out_img_path = osp.join(pred_out_dir, 'preds', img_path + '_panoptic_seg.png') out_json_path = osp.join(pred_out_dir, 'preds', img_path + '.json') elif is_save_pred: out_img_path = osp.join( pred_out_dir, 'preds', f'{self.num_predicted_imgs}_panoptic_seg.png') out_json_path = osp.join(pred_out_dir, 'preds', f'{self.num_predicted_imgs}.json') self.num_predicted_imgs += 1 result = {} if 'pred_instances' in data_sample: masks = data_sample.pred_instances.get('masks') pred_instances = data_sample.pred_instances.numpy() result = { 'bboxes': pred_instances.bboxes.tolist(), 'labels': pred_instances.labels.tolist(), 'scores': pred_instances.scores.tolist() } if masks is not None: if pred_instances.bboxes.sum() == 0: # Fake bbox, such as the SOLO. bboxes = mask2bbox(masks.cpu()).numpy().tolist() result['bboxes'] = bboxes encode_masks = encode_mask_results(pred_instances.masks) for encode_mask in encode_masks: if isinstance(encode_mask['counts'], bytes): encode_mask['counts'] = encode_mask['counts'].decode() result['masks'] = encode_masks if 'pred_panoptic_seg' in data_sample: if VOID is None: raise RuntimeError( 'panopticapi is not installed, please install it by: ' 'pip install git+https://github.com/cocodataset/' 'panopticapi.git.') pan = data_sample.pred_panoptic_seg.sem_seg.cpu().numpy()[0] pan[pan % INSTANCE_OFFSET == len( self.model.dataset_meta['classes'])] = VOID pan = id2rgb(pan).astype(np.uint8) if is_save_pred: mmcv.imwrite(pan[:, :, ::-1], out_img_path) result['panoptic_seg_path'] = out_img_path else: result['panoptic_seg'] = pan if is_save_pred: mmengine.dump(result, out_json_path) return result ================================================ FILE: mmdet/apis/inference.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import warnings from pathlib import Path from typing import Optional, Sequence, Union import numpy as np import torch import torch.nn as nn from mmcv.ops import RoIPool from mmcv.transforms import Compose from mmengine.config import Config from mmengine.model.utils import revert_sync_batchnorm from mmengine.registry import init_default_scope from mmengine.runner import load_checkpoint from mmdet.registry import DATASETS from ..evaluation import get_classes from ..registry import MODELS from ..structures import DetDataSample, SampleList from ..utils import get_test_pipeline_cfg def init_detector( config: Union[str, Path, Config], checkpoint: Optional[str] = None, palette: str = 'none', device: str = 'cuda:0', cfg_options: Optional[dict] = None, ) -> nn.Module: """Initialize a detector from config file. Args: config (str, :obj:`Path`, or :obj:`mmengine.Config`): Config file path, :obj:`Path`, or the config object. checkpoint (str, optional): Checkpoint path. If left as None, the model will not load any weights. palette (str): Color palette used for visualization. If palette is stored in checkpoint, use checkpoint's palette first, otherwise use externally passed palette. Currently, supports 'coco', 'voc', 'citys' and 'random'. Defaults to none. device (str): The device where the anchors will be put on. Defaults to cuda:0. cfg_options (dict, optional): Options to override some settings in the used config. Returns: nn.Module: The constructed detector. """ if isinstance(config, (str, Path)): config = Config.fromfile(config) elif not isinstance(config, Config): raise TypeError('config must be a filename or Config object, ' f'but got {type(config)}') if cfg_options is not None: config.merge_from_dict(cfg_options) elif 'init_cfg' in config.model.backbone: config.model.backbone.init_cfg = None init_default_scope(config.get('default_scope', 'mmdet')) model = MODELS.build(config.model) model = revert_sync_batchnorm(model) if checkpoint is None: warnings.simplefilter('once') warnings.warn('checkpoint is None, use COCO classes by default.') model.dataset_meta = {'classes': get_classes('coco')} else: checkpoint = load_checkpoint(model, checkpoint, map_location='cpu') # Weights converted from elsewhere may not have meta fields. checkpoint_meta = checkpoint.get('meta', {}) # save the dataset_meta in the model for convenience if 'dataset_meta' in checkpoint_meta: # mmdet 3.x, all keys should be lowercase model.dataset_meta = { k.lower(): v for k, v in checkpoint_meta['dataset_meta'].items() } elif 'CLASSES' in checkpoint_meta: # < mmdet 3.x classes = checkpoint_meta['CLASSES'] model.dataset_meta = {'classes': classes} else: warnings.simplefilter('once') warnings.warn( 'dataset_meta or class names are not saved in the ' 'checkpoint\'s meta data, use COCO classes by default.') model.dataset_meta = {'classes': get_classes('coco')} # Priority: args.palette -> config -> checkpoint if palette != 'none': model.dataset_meta['palette'] = palette else: test_dataset_cfg = copy.deepcopy(config.test_dataloader.dataset) # lazy init. We only need the metainfo. test_dataset_cfg['lazy_init'] = True metainfo = DATASETS.build(test_dataset_cfg).metainfo cfg_palette = metainfo.get('palette', None) if cfg_palette is not None: model.dataset_meta['palette'] = cfg_palette else: if 'palette' not in model.dataset_meta: warnings.warn( 'palette does not exist, random is used by default. ' 'You can also set the palette to customize.') model.dataset_meta['palette'] = 'random' model.cfg = config # save the config in the model for convenience model.to(device) model.eval() return model ImagesType = Union[str, np.ndarray, Sequence[str], Sequence[np.ndarray]] def inference_detector( model: nn.Module, imgs: ImagesType, test_pipeline: Optional[Compose] = None ) -> Union[DetDataSample, SampleList]: """Inference image(s) with the detector. Args: model (nn.Module): The loaded detector. imgs (str, ndarray, Sequence[str/ndarray]): Either image files or loaded images. test_pipeline (:obj:`Compose`): Test pipeline. Returns: :obj:`DetDataSample` or list[:obj:`DetDataSample`]: If imgs is a list or tuple, the same length list type results will be returned, otherwise return the detection results directly. """ if isinstance(imgs, (list, tuple)): is_batch = True else: imgs = [imgs] is_batch = False cfg = model.cfg if test_pipeline is None: cfg = cfg.copy() test_pipeline = get_test_pipeline_cfg(cfg) if isinstance(imgs[0], np.ndarray): # Calling this method across libraries will result # in module unregistered error if not prefixed with mmdet. test_pipeline[0].type = 'mmdet.LoadImageFromNDArray' test_pipeline = Compose(test_pipeline) if model.data_preprocessor.device.type == 'cpu': for m in model.modules(): assert not isinstance( m, RoIPool ), 'CPU inference with RoIPool is not supported currently.' result_list = [] for img in imgs: # prepare data if isinstance(img, np.ndarray): # TODO: remove img_id. data_ = dict(img=img, img_id=0) else: # TODO: remove img_id. data_ = dict(img_path=img, img_id=0) # build the data pipeline data_ = test_pipeline(data_) data_['inputs'] = [data_['inputs']] data_['data_samples'] = [data_['data_samples']] # forward the model with torch.no_grad(): results = model.test_step(data_)[0] result_list.append(results) if not is_batch: return result_list[0] else: return result_list # TODO: Awaiting refactoring async def async_inference_detector(model, imgs): """Async inference image(s) with the detector. Args: model (nn.Module): The loaded detector. img (str | ndarray): Either image files or loaded images. Returns: Awaitable detection results. """ if not isinstance(imgs, (list, tuple)): imgs = [imgs] cfg = model.cfg if isinstance(imgs[0], np.ndarray): cfg = cfg.copy() # set loading pipeline type cfg.data.test.pipeline[0].type = 'LoadImageFromNDArray' # cfg.data.test.pipeline = replace_ImageToTensor(cfg.data.test.pipeline) test_pipeline = Compose(cfg.data.test.pipeline) datas = [] for img in imgs: # prepare data if isinstance(img, np.ndarray): # directly add img data = dict(img=img) else: # add information into dict data = dict(img_info=dict(filename=img), img_prefix=None) # build the data pipeline data = test_pipeline(data) datas.append(data) for m in model.modules(): assert not isinstance( m, RoIPool), 'CPU inference with RoIPool is not supported currently.' # We don't restore `torch.is_grad_enabled()` value during concurrent # inference since execution can overlap torch.set_grad_enabled(False) results = await model.aforward_test(data, rescale=True) return results ================================================ FILE: mmdet/datasets/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .base_det_dataset import BaseDetDataset from .cityscapes import CityscapesDataset from .coco import CocoDataset from .coco_panoptic import CocoPanopticDataset from .crowdhuman import CrowdHumanDataset from .dataset_wrappers import MultiImageMixDataset from .deepfashion import DeepFashionDataset from .lvis import LVISDataset, LVISV1Dataset, LVISV05Dataset from .objects365 import Objects365V1Dataset, Objects365V2Dataset from .openimages import OpenImagesChallengeDataset, OpenImagesDataset from .samplers import (AspectRatioBatchSampler, ClassAwareSampler, GroupMultiSourceSampler, MultiSourceSampler) from .utils import get_loading_pipeline from .voc import VOCDataset from .wider_face import WIDERFaceDataset from .xml_style import XMLDataset __all__ = [ 'XMLDataset', 'CocoDataset', 'DeepFashionDataset', 'VOCDataset', 'CityscapesDataset', 'LVISDataset', 'LVISV05Dataset', 'LVISV1Dataset', 'WIDERFaceDataset', 'get_loading_pipeline', 'CocoPanopticDataset', 'MultiImageMixDataset', 'OpenImagesDataset', 'OpenImagesChallengeDataset', 'AspectRatioBatchSampler', 'ClassAwareSampler', 'MultiSourceSampler', 'GroupMultiSourceSampler', 'BaseDetDataset', 'CrowdHumanDataset', 'Objects365V1Dataset', 'Objects365V2Dataset' ] ================================================ FILE: mmdet/datasets/api_wrappers/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .coco_api import COCO, COCOeval, COCOPanoptic __all__ = ['COCO', 'COCOeval', 'COCOPanoptic'] ================================================ FILE: mmdet/datasets/api_wrappers/coco_api.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # This file add snake case alias for coco api import warnings from collections import defaultdict from typing import List, Optional, Union import pycocotools from pycocotools.coco import COCO as _COCO from pycocotools.cocoeval import COCOeval as _COCOeval class COCO(_COCO): """This class is almost the same as official pycocotools package. It implements some snake case function aliases. So that the COCO class has the same interface as LVIS class. """ def __init__(self, annotation_file=None): if getattr(pycocotools, '__version__', '0') >= '12.0.2': warnings.warn( 'mmpycocotools is deprecated. Please install official pycocotools by "pip install pycocotools"', # noqa: E501 UserWarning) super().__init__(annotation_file=annotation_file) self.img_ann_map = self.imgToAnns self.cat_img_map = self.catToImgs def get_ann_ids(self, img_ids=[], cat_ids=[], area_rng=[], iscrowd=None): return self.getAnnIds(img_ids, cat_ids, area_rng, iscrowd) def get_cat_ids(self, cat_names=[], sup_names=[], cat_ids=[]): return self.getCatIds(cat_names, sup_names, cat_ids) def get_img_ids(self, img_ids=[], cat_ids=[]): return self.getImgIds(img_ids, cat_ids) def load_anns(self, ids): return self.loadAnns(ids) def load_cats(self, ids): return self.loadCats(ids) def load_imgs(self, ids): return self.loadImgs(ids) # just for the ease of import COCOeval = _COCOeval class COCOPanoptic(COCO): """This wrapper is for loading the panoptic style annotation file. The format is shown in the CocoPanopticDataset class. Args: annotation_file (str, optional): Path of annotation file. Defaults to None. """ def __init__(self, annotation_file: Optional[str] = None) -> None: super(COCOPanoptic, self).__init__(annotation_file) def createIndex(self) -> None: """Create index.""" # create index print('creating index...') # anns stores 'segment_id -> annotation' anns, cats, imgs = {}, {}, {} img_to_anns, cat_to_imgs = defaultdict(list), defaultdict(list) if 'annotations' in self.dataset: for ann in self.dataset['annotations']: for seg_ann in ann['segments_info']: # to match with instance.json seg_ann['image_id'] = ann['image_id'] img_to_anns[ann['image_id']].append(seg_ann) # segment_id is not unique in coco dataset orz... # annotations from different images but # may have same segment_id if seg_ann['id'] in anns.keys(): anns[seg_ann['id']].append(seg_ann) else: anns[seg_ann['id']] = [seg_ann] # filter out annotations from other images img_to_anns_ = defaultdict(list) for k, v in img_to_anns.items(): img_to_anns_[k] = [x for x in v if x['image_id'] == k] img_to_anns = img_to_anns_ if 'images' in self.dataset: for img_info in self.dataset['images']: img_info['segm_file'] = img_info['file_name'].replace( 'jpg', 'png') imgs[img_info['id']] = img_info if 'categories' in self.dataset: for cat in self.dataset['categories']: cats[cat['id']] = cat if 'annotations' in self.dataset and 'categories' in self.dataset: for ann in self.dataset['annotations']: for seg_ann in ann['segments_info']: cat_to_imgs[seg_ann['category_id']].append(ann['image_id']) print('index created!') self.anns = anns self.imgToAnns = img_to_anns self.catToImgs = cat_to_imgs self.imgs = imgs self.cats = cats def load_anns(self, ids: Union[List[int], int] = []) -> Optional[List[dict]]: """Load anns with the specified ids. ``self.anns`` is a list of annotation lists instead of a list of annotations. Args: ids (Union[List[int], int]): Integer ids specifying anns. Returns: anns (List[dict], optional): Loaded ann objects. """ anns = [] if hasattr(ids, '__iter__') and hasattr(ids, '__len__'): # self.anns is a list of annotation lists instead of # a list of annotations for id in ids: anns += self.anns[id] return anns elif type(ids) == int: return self.anns[ids] ================================================ FILE: mmdet/datasets/base_det_dataset.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp from typing import List, Optional from mmengine.dataset import BaseDataset from mmengine.fileio import FileClient, load from mmengine.utils import is_abs from ..registry import DATASETS @DATASETS.register_module() class BaseDetDataset(BaseDataset): """Base dataset for detection. Args: proposal_file (str, optional): Proposals file path. Defaults to None. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. """ def __init__(self, *args, seg_map_suffix: str = '.png', proposal_file: Optional[str] = None, file_client_args: dict = dict(backend='disk'), **kwargs) -> None: self.seg_map_suffix = seg_map_suffix self.proposal_file = proposal_file self.file_client_args = file_client_args self.file_client = FileClient(**file_client_args) super().__init__(*args, **kwargs) def full_init(self) -> None: """Load annotation file and set ``BaseDataset._fully_initialized`` to True. If ``lazy_init=False``, ``full_init`` will be called during the instantiation and ``self._fully_initialized`` will be set to True. If ``obj._fully_initialized=False``, the class method decorated by ``force_full_init`` will call ``full_init`` automatically. Several steps to initialize annotation: - load_data_list: Load annotations from annotation file. - load_proposals: Load proposals from proposal file, if `self.proposal_file` is not None. - filter data information: Filter annotations according to filter_cfg. - slice_data: Slice dataset according to ``self._indices`` - serialize_data: Serialize ``self.data_list`` if ``self.serialize_data`` is True. """ if self._fully_initialized: return # load data information self.data_list = self.load_data_list() # get proposals from file if self.proposal_file is not None: self.load_proposals() # filter illegal data, such as data that has no annotations. self.data_list = self.filter_data() # Get subset data according to indices. if self._indices is not None: self.data_list = self._get_unserialized_subset(self._indices) # serialize data_list if self.serialize_data: self.data_bytes, self.data_address = self._serialize_data() self._fully_initialized = True def load_proposals(self) -> None: """Load proposals from proposals file. The `proposals_list` should be a dict[img_path: proposals] with the same length as `data_list`. And the `proposals` should be a `dict` or :obj:`InstanceData` usually contains following keys. - bboxes (np.ndarry): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - scores (np.ndarry): Classification scores, has a shape (num_instance, ). """ # TODO: Add Unit Test after fully support Dump-Proposal Metric if not is_abs(self.proposal_file): self.proposal_file = osp.join(self.data_root, self.proposal_file) proposals_list = load( self.proposal_file, file_client_args=self.file_client_args) assert len(self.data_list) == len(proposals_list) for data_info in self.data_list: img_path = data_info['img_path'] # `file_name` is the key to obtain the proposals from the # `proposals_list`. file_name = osp.join( osp.split(osp.split(img_path)[0])[-1], osp.split(img_path)[-1]) proposals = proposals_list[file_name] data_info['proposals'] = proposals def get_cat_ids(self, idx: int) -> List[int]: """Get COCO category ids by index. Args: idx (int): Index of data. Returns: List[int]: All categories in the image of specified index. """ instances = self.get_data_info(idx)['instances'] return [instance['bbox_label'] for instance in instances] ================================================ FILE: mmdet/datasets/cityscapes.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Modified from https://github.com/facebookresearch/detectron2/blob/master/detectron2/data/datasets/cityscapes.py # noqa # and https://github.com/mcordts/cityscapesScripts/blob/master/cityscapesscripts/evaluation/evalInstanceLevelSemanticLabeling.py # noqa from typing import List from mmdet.registry import DATASETS from .coco import CocoDataset @DATASETS.register_module() class CityscapesDataset(CocoDataset): """Dataset for Cityscapes.""" METAINFO = { 'classes': ('person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', 'bicycle'), 'palette': [(220, 20, 60), (255, 0, 0), (0, 0, 142), (0, 0, 70), (0, 60, 100), (0, 80, 100), (0, 0, 230), (119, 11, 32)] } def filter_data(self) -> List[dict]: """Filter annotations according to filter_cfg. Returns: List[dict]: Filtered results. """ if self.test_mode: return self.data_list if self.filter_cfg is None: return self.data_list filter_empty_gt = self.filter_cfg.get('filter_empty_gt', False) min_size = self.filter_cfg.get('min_size', 0) # obtain images that contain annotation ids_with_ann = set(data_info['img_id'] for data_info in self.data_list) # obtain images that contain annotations of the required categories ids_in_cat = set() for i, class_id in enumerate(self.cat_ids): ids_in_cat |= set(self.cat_img_map[class_id]) # merge the image id sets of the two conditions and use the merged set # to filter out images if self.filter_empty_gt=True ids_in_cat &= ids_with_ann valid_data_infos = [] for i, data_info in enumerate(self.data_list): img_id = data_info['img_id'] width = data_info['width'] height = data_info['height'] all_is_crowd = all([ instance['ignore_flag'] == 1 for instance in data_info['instances'] ]) if filter_empty_gt and (img_id not in ids_in_cat or all_is_crowd): continue if min(width, height) >= min_size: valid_data_infos.append(data_info) return valid_data_infos ================================================ FILE: mmdet/datasets/coco.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import os.path as osp from typing import List, Union from mmdet.registry import DATASETS from .api_wrappers import COCO from .base_det_dataset import BaseDetDataset @DATASETS.register_module() class CocoDataset(BaseDetDataset): """Dataset for COCO.""" METAINFO = { 'classes': ('person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'), # palette is a list of color tuples, which is used for visualization. 'palette': [(220, 20, 60), (119, 11, 32), (0, 0, 142), (0, 0, 230), (106, 0, 228), (0, 60, 100), (0, 80, 100), (0, 0, 70), (0, 0, 192), (250, 170, 30), (100, 170, 30), (220, 220, 0), (175, 116, 175), (250, 0, 30), (165, 42, 42), (255, 77, 255), (0, 226, 252), (182, 182, 255), (0, 82, 0), (120, 166, 157), (110, 76, 0), (174, 57, 255), (199, 100, 0), (72, 0, 118), (255, 179, 240), (0, 125, 92), (209, 0, 151), (188, 208, 182), (0, 220, 176), (255, 99, 164), (92, 0, 73), (133, 129, 255), (78, 180, 255), (0, 228, 0), (174, 255, 243), (45, 89, 255), (134, 134, 103), (145, 148, 174), (255, 208, 186), (197, 226, 255), (171, 134, 1), (109, 63, 54), (207, 138, 255), (151, 0, 95), (9, 80, 61), (84, 105, 51), (74, 65, 105), (166, 196, 102), (208, 195, 210), (255, 109, 65), (0, 143, 149), (179, 0, 194), (209, 99, 106), (5, 121, 0), (227, 255, 205), (147, 186, 208), (153, 69, 1), (3, 95, 161), (163, 255, 0), (119, 0, 170), (0, 182, 199), (0, 165, 120), (183, 130, 88), (95, 32, 0), (130, 114, 135), (110, 129, 133), (166, 74, 118), (219, 142, 185), (79, 210, 114), (178, 90, 62), (65, 70, 15), (127, 167, 115), (59, 105, 106), (142, 108, 45), (196, 172, 0), (95, 54, 80), (128, 76, 255), (201, 57, 1), (246, 0, 122), (191, 162, 208)] } COCOAPI = COCO # ann_id is unique in coco dataset. ANN_ID_UNIQUE = True def load_data_list(self) -> List[dict]: """Load annotations from an annotation file named as ``self.ann_file`` Returns: List[dict]: A list of annotation. """ # noqa: E501 with self.file_client.get_local_path(self.ann_file) as local_path: self.coco = self.COCOAPI(local_path) # The order of returned `cat_ids` will not # change with the order of the `classes` self.cat_ids = self.coco.get_cat_ids( cat_names=self.metainfo['classes']) self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)} self.cat_img_map = copy.deepcopy(self.coco.cat_img_map) img_ids = self.coco.get_img_ids() data_list = [] total_ann_ids = [] for img_id in img_ids: raw_img_info = self.coco.load_imgs([img_id])[0] raw_img_info['img_id'] = img_id ann_ids = self.coco.get_ann_ids(img_ids=[img_id]) raw_ann_info = self.coco.load_anns(ann_ids) total_ann_ids.extend(ann_ids) parsed_data_info = self.parse_data_info({ 'raw_ann_info': raw_ann_info, 'raw_img_info': raw_img_info }) data_list.append(parsed_data_info) if self.ANN_ID_UNIQUE: assert len(set(total_ann_ids)) == len( total_ann_ids ), f"Annotation ids in '{self.ann_file}' are not unique!" del self.coco return data_list def parse_data_info(self, raw_data_info: dict) -> Union[dict, List[dict]]: """Parse raw annotation to target format. Args: raw_data_info (dict): Raw data information load from ``ann_file`` Returns: Union[dict, List[dict]]: Parsed annotation. """ img_info = raw_data_info['raw_img_info'] ann_info = raw_data_info['raw_ann_info'] data_info = {} # TODO: need to change data_prefix['img'] to data_prefix['img_path'] img_path = osp.join(self.data_prefix['img'], img_info['file_name']) if self.data_prefix.get('seg', None): seg_map_path = osp.join( self.data_prefix['seg'], img_info['file_name'].rsplit('.', 1)[0] + self.seg_map_suffix) else: seg_map_path = None data_info['img_path'] = img_path data_info['img_id'] = img_info['img_id'] data_info['seg_map_path'] = seg_map_path data_info['height'] = img_info['height'] data_info['width'] = img_info['width'] instances = [] for i, ann in enumerate(ann_info): instance = {} if ann.get('ignore', False): continue x1, y1, w, h = ann['bbox'] inter_w = max(0, min(x1 + w, img_info['width']) - max(x1, 0)) inter_h = max(0, min(y1 + h, img_info['height']) - max(y1, 0)) if inter_w * inter_h == 0: continue if ann['area'] <= 0 or w < 1 or h < 1: continue if ann['category_id'] not in self.cat_ids: continue bbox = [x1, y1, x1 + w, y1 + h] if ann.get('iscrowd', False): instance['ignore_flag'] = 1 else: instance['ignore_flag'] = 0 instance['bbox'] = bbox instance['bbox_label'] = self.cat2label[ann['category_id']] if ann.get('segmentation', None): instance['mask'] = ann['segmentation'] instances.append(instance) data_info['instances'] = instances return data_info def filter_data(self) -> List[dict]: """Filter annotations according to filter_cfg. Returns: List[dict]: Filtered results. """ if self.test_mode: return self.data_list if self.filter_cfg is None: return self.data_list filter_empty_gt = self.filter_cfg.get('filter_empty_gt', False) min_size = self.filter_cfg.get('min_size', 0) # obtain images that contain annotation ids_with_ann = set(data_info['img_id'] for data_info in self.data_list) # obtain images that contain annotations of the required categories ids_in_cat = set() for i, class_id in enumerate(self.cat_ids): ids_in_cat |= set(self.cat_img_map[class_id]) # merge the image id sets of the two conditions and use the merged set # to filter out images if self.filter_empty_gt=True ids_in_cat &= ids_with_ann valid_data_infos = [] for i, data_info in enumerate(self.data_list): img_id = data_info['img_id'] width = data_info['width'] height = data_info['height'] if filter_empty_gt and img_id not in ids_in_cat: continue if min(width, height) >= min_size: valid_data_infos.append(data_info) return valid_data_infos ================================================ FILE: mmdet/datasets/coco_panoptic.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp from typing import Callable, List, Optional, Sequence, Union from mmdet.registry import DATASETS from .api_wrappers import COCOPanoptic from .coco import CocoDataset @DATASETS.register_module() class CocoPanopticDataset(CocoDataset): """Coco dataset for Panoptic segmentation. The annotation format is shown as follows. The `ann` field is optional for testing. .. code-block:: none [ { 'filename': f'{image_id:012}.png', 'image_id':9 'segments_info': [ { 'id': 8345037, (segment_id in panoptic png, convert from rgb) 'category_id': 51, 'iscrowd': 0, 'bbox': (x1, y1, w, h), 'area': 24315 }, ... ] }, ... ] Args: ann_file (str): Annotation file path. Defaults to ''. metainfo (dict, optional): Meta information for dataset, such as class information. Defaults to None. data_root (str, optional): The root directory for ``data_prefix`` and ``ann_file``. Defaults to None. data_prefix (dict, optional): Prefix for training data. Defaults to ``dict(img=None, ann=None, seg=None)``. The prefix ``seg`` which is for panoptic segmentation map must be not None. filter_cfg (dict, optional): Config for filter data. Defaults to None. indices (int or Sequence[int], optional): Support using first few data in annotation file to facilitate training/testing on a smaller dataset. Defaults to None which means using all ``data_infos``. serialize_data (bool, optional): Whether to hold memory using serialized objects, when enabled, data loader workers can use shared RAM from master process instead of making a copy. Defaults to True. pipeline (list, optional): Processing pipeline. Defaults to []. test_mode (bool, optional): ``test_mode=True`` means in test phase. Defaults to False. lazy_init (bool, optional): Whether to load annotation during instantiation. In some cases, such as visualization, only the meta information of the dataset is needed, which is not necessary to load annotation file. ``Basedataset`` can skip load annotations to save time by set ``lazy_init=False``. Defaults to False. max_refetch (int, optional): If ``Basedataset.prepare_data`` get a None img. The maximum extra number of cycles to get a valid image. Defaults to 1000. """ METAINFO = { 'classes': ('person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush', 'banner', 'blanket', 'bridge', 'cardboard', 'counter', 'curtain', 'door-stuff', 'floor-wood', 'flower', 'fruit', 'gravel', 'house', 'light', 'mirror-stuff', 'net', 'pillow', 'platform', 'playingfield', 'railroad', 'river', 'road', 'roof', 'sand', 'sea', 'shelf', 'snow', 'stairs', 'tent', 'towel', 'wall-brick', 'wall-stone', 'wall-tile', 'wall-wood', 'water-other', 'window-blind', 'window-other', 'tree-merged', 'fence-merged', 'ceiling-merged', 'sky-other-merged', 'cabinet-merged', 'table-merged', 'floor-other-merged', 'pavement-merged', 'mountain-merged', 'grass-merged', 'dirt-merged', 'paper-merged', 'food-other-merged', 'building-other-merged', 'rock-merged', 'wall-other-merged', 'rug-merged'), 'thing_classes': ('person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'), 'stuff_classes': ('banner', 'blanket', 'bridge', 'cardboard', 'counter', 'curtain', 'door-stuff', 'floor-wood', 'flower', 'fruit', 'gravel', 'house', 'light', 'mirror-stuff', 'net', 'pillow', 'platform', 'playingfield', 'railroad', 'river', 'road', 'roof', 'sand', 'sea', 'shelf', 'snow', 'stairs', 'tent', 'towel', 'wall-brick', 'wall-stone', 'wall-tile', 'wall-wood', 'water-other', 'window-blind', 'window-other', 'tree-merged', 'fence-merged', 'ceiling-merged', 'sky-other-merged', 'cabinet-merged', 'table-merged', 'floor-other-merged', 'pavement-merged', 'mountain-merged', 'grass-merged', 'dirt-merged', 'paper-merged', 'food-other-merged', 'building-other-merged', 'rock-merged', 'wall-other-merged', 'rug-merged'), 'palette': [(220, 20, 60), (119, 11, 32), (0, 0, 142), (0, 0, 230), (106, 0, 228), (0, 60, 100), (0, 80, 100), (0, 0, 70), (0, 0, 192), (250, 170, 30), (100, 170, 30), (220, 220, 0), (175, 116, 175), (250, 0, 30), (165, 42, 42), (255, 77, 255), (0, 226, 252), (182, 182, 255), (0, 82, 0), (120, 166, 157), (110, 76, 0), (174, 57, 255), (199, 100, 0), (72, 0, 118), (255, 179, 240), (0, 125, 92), (209, 0, 151), (188, 208, 182), (0, 220, 176), (255, 99, 164), (92, 0, 73), (133, 129, 255), (78, 180, 255), (0, 228, 0), (174, 255, 243), (45, 89, 255), (134, 134, 103), (145, 148, 174), (255, 208, 186), (197, 226, 255), (171, 134, 1), (109, 63, 54), (207, 138, 255), (151, 0, 95), (9, 80, 61), (84, 105, 51), (74, 65, 105), (166, 196, 102), (208, 195, 210), (255, 109, 65), (0, 143, 149), (179, 0, 194), (209, 99, 106), (5, 121, 0), (227, 255, 205), (147, 186, 208), (153, 69, 1), (3, 95, 161), (163, 255, 0), (119, 0, 170), (0, 182, 199), (0, 165, 120), (183, 130, 88), (95, 32, 0), (130, 114, 135), (110, 129, 133), (166, 74, 118), (219, 142, 185), (79, 210, 114), (178, 90, 62), (65, 70, 15), (127, 167, 115), (59, 105, 106), (142, 108, 45), (196, 172, 0), (95, 54, 80), (128, 76, 255), (201, 57, 1), (246, 0, 122), (191, 162, 208), (255, 255, 128), (147, 211, 203), (150, 100, 100), (168, 171, 172), (146, 112, 198), (210, 170, 100), (92, 136, 89), (218, 88, 184), (241, 129, 0), (217, 17, 255), (124, 74, 181), (70, 70, 70), (255, 228, 255), (154, 208, 0), (193, 0, 92), (76, 91, 113), (255, 180, 195), (106, 154, 176), (230, 150, 140), (60, 143, 255), (128, 64, 128), (92, 82, 55), (254, 212, 124), (73, 77, 174), (255, 160, 98), (255, 255, 255), (104, 84, 109), (169, 164, 131), (225, 199, 255), (137, 54, 74), (135, 158, 223), (7, 246, 231), (107, 255, 200), (58, 41, 149), (183, 121, 142), (255, 73, 97), (107, 142, 35), (190, 153, 153), (146, 139, 141), (70, 130, 180), (134, 199, 156), (209, 226, 140), (96, 36, 108), (96, 96, 96), (64, 170, 64), (152, 251, 152), (208, 229, 228), (206, 186, 171), (152, 161, 64), (116, 112, 0), (0, 114, 143), (102, 102, 156), (250, 141, 255)] } COCOAPI = COCOPanoptic # ann_id is not unique in coco panoptic dataset. ANN_ID_UNIQUE = False def __init__(self, ann_file: str = '', metainfo: Optional[dict] = None, data_root: Optional[str] = None, data_prefix: dict = dict(img=None, ann=None, seg=None), filter_cfg: Optional[dict] = None, indices: Optional[Union[int, Sequence[int]]] = None, serialize_data: bool = True, pipeline: List[Union[dict, Callable]] = [], test_mode: bool = False, lazy_init: bool = False, max_refetch: int = 1000) -> None: super().__init__( ann_file=ann_file, metainfo=metainfo, data_root=data_root, data_prefix=data_prefix, filter_cfg=filter_cfg, indices=indices, serialize_data=serialize_data, pipeline=pipeline, test_mode=test_mode, lazy_init=lazy_init, max_refetch=max_refetch) def parse_data_info(self, raw_data_info: dict) -> dict: """Parse raw annotation to target format. Args: raw_data_info (dict): Raw data information load from ``ann_file``. Returns: dict: Parsed annotation. """ img_info = raw_data_info['raw_img_info'] ann_info = raw_data_info['raw_ann_info'] # filter out unmatched annotations which have # same segment_id but belong to other image ann_info = [ ann for ann in ann_info if ann['image_id'] == img_info['img_id'] ] data_info = {} img_path = osp.join(self.data_prefix['img'], img_info['file_name']) if self.data_prefix.get('seg', None): seg_map_path = osp.join( self.data_prefix['seg'], img_info['file_name'].replace('jpg', 'png')) else: seg_map_path = None data_info['img_path'] = img_path data_info['img_id'] = img_info['img_id'] data_info['seg_map_path'] = seg_map_path data_info['height'] = img_info['height'] data_info['width'] = img_info['width'] instances = [] segments_info = [] for ann in ann_info: instance = {} x1, y1, w, h = ann['bbox'] if ann['area'] <= 0 or w < 1 or h < 1: continue bbox = [x1, y1, x1 + w, y1 + h] category_id = ann['category_id'] contiguous_cat_id = self.cat2label[category_id] is_thing = self.coco.load_cats(ids=category_id)[0]['isthing'] if is_thing: is_crowd = ann.get('iscrowd', False) instance['bbox'] = bbox instance['bbox_label'] = contiguous_cat_id if not is_crowd: instance['ignore_flag'] = 0 else: instance['ignore_flag'] = 1 is_thing = False segment_info = { 'id': ann['id'], 'category': contiguous_cat_id, 'is_thing': is_thing } segments_info.append(segment_info) if len(instance) > 0 and is_thing: instances.append(instance) data_info['instances'] = instances data_info['segments_info'] = segments_info return data_info def filter_data(self) -> List[dict]: """Filter images too small or without ground truth. Returns: List[dict]: ``self.data_list`` after filtering. """ if self.test_mode: return self.data_list if self.filter_cfg is None: return self.data_list filter_empty_gt = self.filter_cfg.get('filter_empty_gt', False) min_size = self.filter_cfg.get('min_size', 0) ids_with_ann = set() # check whether images have legal thing annotations. for data_info in self.data_list: for segment_info in data_info['segments_info']: if not segment_info['is_thing']: continue ids_with_ann.add(data_info['img_id']) valid_data_list = [] for data_info in self.data_list: img_id = data_info['img_id'] width = data_info['width'] height = data_info['height'] if filter_empty_gt and img_id not in ids_with_ann: continue if min(width, height) >= min_size: valid_data_list.append(data_info) return valid_data_list ================================================ FILE: mmdet/datasets/crowdhuman.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import json import logging import os.path as osp import warnings from typing import List, Union import mmcv from mmengine.dist import get_rank from mmengine.fileio import dump, load from mmengine.logging import print_log from mmengine.utils import ProgressBar from mmdet.registry import DATASETS from .base_det_dataset import BaseDetDataset @DATASETS.register_module() class CrowdHumanDataset(BaseDetDataset): r"""Dataset for CrowdHuman. Args: data_root (str): The root directory for ``data_prefix`` and ``ann_file``. ann_file (str): Annotation file path. extra_ann_file (str | optional):The path of extra image metas for CrowdHuman. It can be created by CrowdHumanDataset automatically or by tools/misc/get_crowdhuman_id_hw.py manually. Defaults to None. """ METAINFO = { 'classes': ('person', ), # palette is a list of color tuples, which is used for visualization. 'palette': [(220, 20, 60)] } def __init__(self, data_root, ann_file, extra_ann_file=None, **kwargs): # extra_ann_file record the size of each image. This file is # automatically created when you first load the CrowdHuman # dataset by mmdet. if extra_ann_file is not None: self.extra_ann_exist = True self.extra_anns = load(extra_ann_file) else: ann_file_name = osp.basename(ann_file) if 'train' in ann_file_name: self.extra_ann_file = osp.join(data_root, 'id_hw_train.json') elif 'val' in ann_file_name: self.extra_ann_file = osp.join(data_root, 'id_hw_val.json') self.extra_ann_exist = False if not osp.isfile(self.extra_ann_file): print_log( 'extra_ann_file does not exist, prepare to collect ' 'image height and width...', level=logging.INFO) self.extra_anns = {} else: self.extra_ann_exist = True self.extra_anns = load(self.extra_ann_file) super().__init__(data_root=data_root, ann_file=ann_file, **kwargs) def load_data_list(self) -> List[dict]: """Load annotations from an annotation file named as ``self.ann_file`` Returns: List[dict]: A list of annotation. """ # noqa: E501 anno_strs = self.file_client.get_text( self.ann_file).strip().split('\n') print_log('loading CrowdHuman annotation...', level=logging.INFO) data_list = [] prog_bar = ProgressBar(len(anno_strs)) for i, anno_str in enumerate(anno_strs): anno_dict = json.loads(anno_str) parsed_data_info = self.parse_data_info(anno_dict) data_list.append(parsed_data_info) prog_bar.update() if not self.extra_ann_exist and get_rank() == 0: # TODO: support file client try: dump(self.extra_anns, self.extra_ann_file, file_format='json') except: # noqa warnings.warn( 'Cache files can not be saved automatically! To speed up' 'loading the dataset, please manually generate the cache' ' file by file tools/misc/get_crowdhuman_id_hw.py') print_log( f'\nsave extra_ann_file in {self.data_root}', level=logging.INFO) del self.extra_anns print_log('\nDone', level=logging.INFO) return data_list def parse_data_info(self, raw_data_info: dict) -> Union[dict, List[dict]]: """Parse raw annotation to target format. Args: raw_data_info (dict): Raw data information load from ``ann_file`` Returns: Union[dict, List[dict]]: Parsed annotation. """ data_info = {} img_path = osp.join(self.data_prefix['img'], f"{raw_data_info['ID']}.jpg") data_info['img_path'] = img_path data_info['img_id'] = raw_data_info['ID'] if not self.extra_ann_exist: img_bytes = self.file_client.get(img_path) img = mmcv.imfrombytes(img_bytes, backend='cv2') data_info['height'], data_info['width'] = img.shape[:2] self.extra_anns[raw_data_info['ID']] = img.shape[:2] del img, img_bytes else: data_info['height'], data_info['width'] = self.extra_anns[ raw_data_info['ID']] instances = [] for i, ann in enumerate(raw_data_info['gtboxes']): instance = {} if ann['tag'] not in self.metainfo['classes']: instance['bbox_label'] = -1 instance['ignore_flag'] = 1 else: instance['bbox_label'] = self.metainfo['classes'].index( ann['tag']) instance['ignore_flag'] = 0 if 'extra' in ann: if 'ignore' in ann['extra']: if ann['extra']['ignore'] != 0: instance['bbox_label'] = -1 instance['ignore_flag'] = 1 x1, y1, w, h = ann['fbox'] bbox = [x1, y1, x1 + w, y1 + h] instance['bbox'] = bbox # Record the full bbox(fbox), head bbox(hbox) and visible # bbox(vbox) as additional information. If you need to use # this information, you just need to design the pipeline # instead of overriding the CrowdHumanDataset. instance['fbox'] = bbox hbox = ann['hbox'] instance['hbox'] = [ hbox[0], hbox[1], hbox[0] + hbox[2], hbox[1] + hbox[3] ] vbox = ann['vbox'] instance['vbox'] = [ vbox[0], vbox[1], vbox[0] + vbox[2], vbox[1] + vbox[3] ] instances.append(instance) data_info['instances'] = instances return data_info ================================================ FILE: mmdet/datasets/dataset_wrappers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import collections import copy from typing import Sequence, Union from mmengine.dataset import BaseDataset, force_full_init from mmdet.registry import DATASETS, TRANSFORMS @DATASETS.register_module() class MultiImageMixDataset: """A wrapper of multiple images mixed dataset. Suitable for training on multiple images mixed data augmentation like mosaic and mixup. For the augmentation pipeline of mixed image data, the `get_indexes` method needs to be provided to obtain the image indexes, and you can set `skip_flags` to change the pipeline running process. At the same time, we provide the `dynamic_scale` parameter to dynamically change the output image size. Args: dataset (:obj:`CustomDataset`): The dataset to be mixed. pipeline (Sequence[dict]): Sequence of transform object or config dict to be composed. dynamic_scale (tuple[int], optional): The image scale can be changed dynamically. Default to None. It is deprecated. skip_type_keys (list[str], optional): Sequence of type string to be skip pipeline. Default to None. max_refetch (int): The maximum number of retry iterations for getting valid results from the pipeline. If the number of iterations is greater than `max_refetch`, but results is still None, then the iteration is terminated and raise the error. Default: 15. """ def __init__(self, dataset: Union[BaseDataset, dict], pipeline: Sequence[str], skip_type_keys: Union[Sequence[str], None] = None, max_refetch: int = 15, lazy_init: bool = False) -> None: assert isinstance(pipeline, collections.abc.Sequence) if skip_type_keys is not None: assert all([ isinstance(skip_type_key, str) for skip_type_key in skip_type_keys ]) self._skip_type_keys = skip_type_keys self.pipeline = [] self.pipeline_types = [] for transform in pipeline: if isinstance(transform, dict): self.pipeline_types.append(transform['type']) transform = TRANSFORMS.build(transform) self.pipeline.append(transform) else: raise TypeError('pipeline must be a dict') self.dataset: BaseDataset if isinstance(dataset, dict): self.dataset = DATASETS.build(dataset) elif isinstance(dataset, BaseDataset): self.dataset = dataset else: raise TypeError( 'elements in datasets sequence should be config or ' f'`BaseDataset` instance, but got {type(dataset)}') self._metainfo = self.dataset.metainfo if hasattr(self.dataset, 'flag'): self.flag = self.dataset.flag self.num_samples = len(self.dataset) self.max_refetch = max_refetch self._fully_initialized = False if not lazy_init: self.full_init() @property def metainfo(self) -> dict: """Get the meta information of the multi-image-mixed dataset. Returns: dict: The meta information of multi-image-mixed dataset. """ return copy.deepcopy(self._metainfo) def full_init(self): """Loop to ``full_init`` each dataset.""" if self._fully_initialized: return self.dataset.full_init() self._ori_len = len(self.dataset) self._fully_initialized = True @force_full_init def get_data_info(self, idx: int) -> dict: """Get annotation by index. Args: idx (int): Global index of ``ConcatDataset``. Returns: dict: The idx-th annotation of the datasets. """ return self.dataset.get_data_info(idx) @force_full_init def __len__(self): return self.num_samples def __getitem__(self, idx): results = copy.deepcopy(self.dataset[idx]) for (transform, transform_type) in zip(self.pipeline, self.pipeline_types): if self._skip_type_keys is not None and \ transform_type in self._skip_type_keys: continue if hasattr(transform, 'get_indexes'): for i in range(self.max_refetch): # Make sure the results passed the loading pipeline # of the original dataset is not None. indexes = transform.get_indexes(self.dataset) if not isinstance(indexes, collections.abc.Sequence): indexes = [indexes] mix_results = [ copy.deepcopy(self.dataset[index]) for index in indexes ] if None not in mix_results: results['mix_results'] = mix_results break else: raise RuntimeError( 'The loading pipeline of the original dataset' ' always return None. Please check the correctness ' 'of the dataset and its pipeline.') for i in range(self.max_refetch): # To confirm the results passed the training pipeline # of the wrapper is not None. updated_results = transform(copy.deepcopy(results)) if updated_results is not None: results = updated_results break else: raise RuntimeError( 'The training pipeline of the dataset wrapper' ' always return None.Please check the correctness ' 'of the dataset and its pipeline.') if 'mix_results' in results: results.pop('mix_results') return results def update_skip_type_keys(self, skip_type_keys): """Update skip_type_keys. It is called by an external hook. Args: skip_type_keys (list[str], optional): Sequence of type string to be skip pipeline. """ assert all([ isinstance(skip_type_key, str) for skip_type_key in skip_type_keys ]) self._skip_type_keys = skip_type_keys ================================================ FILE: mmdet/datasets/deepfashion.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import DATASETS from .coco import CocoDataset @DATASETS.register_module() class DeepFashionDataset(CocoDataset): """Dataset for DeepFashion.""" METAINFO = { 'classes': ('top', 'skirt', 'leggings', 'dress', 'outer', 'pants', 'bag', 'neckwear', 'headwear', 'eyeglass', 'belt', 'footwear', 'hair', 'skin', 'face'), # palette is a list of color tuples, which is used for visualization. 'palette': [(0, 192, 64), (0, 64, 96), (128, 192, 192), (0, 64, 64), (0, 192, 224), (0, 192, 192), (128, 192, 64), (0, 192, 96), (128, 32, 192), (0, 0, 224), (0, 0, 64), (0, 160, 192), (128, 0, 96), (128, 0, 192), (0, 32, 192)] } ================================================ FILE: mmdet/datasets/lvis.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import warnings from typing import List from mmdet.registry import DATASETS from .coco import CocoDataset @DATASETS.register_module() class LVISV05Dataset(CocoDataset): """LVIS v0.5 dataset for detection.""" METAINFO = { 'classes': ('acorn', 'aerosol_can', 'air_conditioner', 'airplane', 'alarm_clock', 'alcohol', 'alligator', 'almond', 'ambulance', 'amplifier', 'anklet', 'antenna', 'apple', 'apple_juice', 'applesauce', 'apricot', 'apron', 'aquarium', 'armband', 'armchair', 'armoire', 'armor', 'artichoke', 'trash_can', 'ashtray', 'asparagus', 'atomizer', 'avocado', 'award', 'awning', 'ax', 'baby_buggy', 'basketball_backboard', 'backpack', 'handbag', 'suitcase', 'bagel', 'bagpipe', 'baguet', 'bait', 'ball', 'ballet_skirt', 'balloon', 'bamboo', 'banana', 'Band_Aid', 'bandage', 'bandanna', 'banjo', 'banner', 'barbell', 'barge', 'barrel', 'barrette', 'barrow', 'baseball_base', 'baseball', 'baseball_bat', 'baseball_cap', 'baseball_glove', 'basket', 'basketball_hoop', 'basketball', 'bass_horn', 'bat_(animal)', 'bath_mat', 'bath_towel', 'bathrobe', 'bathtub', 'batter_(food)', 'battery', 'beachball', 'bead', 'beaker', 'bean_curd', 'beanbag', 'beanie', 'bear', 'bed', 'bedspread', 'cow', 'beef_(food)', 'beeper', 'beer_bottle', 'beer_can', 'beetle', 'bell', 'bell_pepper', 'belt', 'belt_buckle', 'bench', 'beret', 'bib', 'Bible', 'bicycle', 'visor', 'binder', 'binoculars', 'bird', 'birdfeeder', 'birdbath', 'birdcage', 'birdhouse', 'birthday_cake', 'birthday_card', 'biscuit_(bread)', 'pirate_flag', 'black_sheep', 'blackboard', 'blanket', 'blazer', 'blender', 'blimp', 'blinker', 'blueberry', 'boar', 'gameboard', 'boat', 'bobbin', 'bobby_pin', 'boiled_egg', 'bolo_tie', 'deadbolt', 'bolt', 'bonnet', 'book', 'book_bag', 'bookcase', 'booklet', 'bookmark', 'boom_microphone', 'boot', 'bottle', 'bottle_opener', 'bouquet', 'bow_(weapon)', 'bow_(decorative_ribbons)', 'bow-tie', 'bowl', 'pipe_bowl', 'bowler_hat', 'bowling_ball', 'bowling_pin', 'boxing_glove', 'suspenders', 'bracelet', 'brass_plaque', 'brassiere', 'bread-bin', 'breechcloth', 'bridal_gown', 'briefcase', 'bristle_brush', 'broccoli', 'broach', 'broom', 'brownie', 'brussels_sprouts', 'bubble_gum', 'bucket', 'horse_buggy', 'bull', 'bulldog', 'bulldozer', 'bullet_train', 'bulletin_board', 'bulletproof_vest', 'bullhorn', 'corned_beef', 'bun', 'bunk_bed', 'buoy', 'burrito', 'bus_(vehicle)', 'business_card', 'butcher_knife', 'butter', 'butterfly', 'button', 'cab_(taxi)', 'cabana', 'cabin_car', 'cabinet', 'locker', 'cake', 'calculator', 'calendar', 'calf', 'camcorder', 'camel', 'camera', 'camera_lens', 'camper_(vehicle)', 'can', 'can_opener', 'candelabrum', 'candle', 'candle_holder', 'candy_bar', 'candy_cane', 'walking_cane', 'canister', 'cannon', 'canoe', 'cantaloup', 'canteen', 'cap_(headwear)', 'bottle_cap', 'cape', 'cappuccino', 'car_(automobile)', 'railcar_(part_of_a_train)', 'elevator_car', 'car_battery', 'identity_card', 'card', 'cardigan', 'cargo_ship', 'carnation', 'horse_carriage', 'carrot', 'tote_bag', 'cart', 'carton', 'cash_register', 'casserole', 'cassette', 'cast', 'cat', 'cauliflower', 'caviar', 'cayenne_(spice)', 'CD_player', 'celery', 'cellular_telephone', 'chain_mail', 'chair', 'chaise_longue', 'champagne', 'chandelier', 'chap', 'checkbook', 'checkerboard', 'cherry', 'chessboard', 'chest_of_drawers_(furniture)', 'chicken_(animal)', 'chicken_wire', 'chickpea', 'Chihuahua', 'chili_(vegetable)', 'chime', 'chinaware', 'crisp_(potato_chip)', 'poker_chip', 'chocolate_bar', 'chocolate_cake', 'chocolate_milk', 'chocolate_mousse', 'choker', 'chopping_board', 'chopstick', 'Christmas_tree', 'slide', 'cider', 'cigar_box', 'cigarette', 'cigarette_case', 'cistern', 'clarinet', 'clasp', 'cleansing_agent', 'clementine', 'clip', 'clipboard', 'clock', 'clock_tower', 'clothes_hamper', 'clothespin', 'clutch_bag', 'coaster', 'coat', 'coat_hanger', 'coatrack', 'cock', 'coconut', 'coffee_filter', 'coffee_maker', 'coffee_table', 'coffeepot', 'coil', 'coin', 'colander', 'coleslaw', 'coloring_material', 'combination_lock', 'pacifier', 'comic_book', 'computer_keyboard', 'concrete_mixer', 'cone', 'control', 'convertible_(automobile)', 'sofa_bed', 'cookie', 'cookie_jar', 'cooking_utensil', 'cooler_(for_food)', 'cork_(bottle_plug)', 'corkboard', 'corkscrew', 'edible_corn', 'cornbread', 'cornet', 'cornice', 'cornmeal', 'corset', 'romaine_lettuce', 'costume', 'cougar', 'coverall', 'cowbell', 'cowboy_hat', 'crab_(animal)', 'cracker', 'crape', 'crate', 'crayon', 'cream_pitcher', 'credit_card', 'crescent_roll', 'crib', 'crock_pot', 'crossbar', 'crouton', 'crow', 'crown', 'crucifix', 'cruise_ship', 'police_cruiser', 'crumb', 'crutch', 'cub_(animal)', 'cube', 'cucumber', 'cufflink', 'cup', 'trophy_cup', 'cupcake', 'hair_curler', 'curling_iron', 'curtain', 'cushion', 'custard', 'cutting_tool', 'cylinder', 'cymbal', 'dachshund', 'dagger', 'dartboard', 'date_(fruit)', 'deck_chair', 'deer', 'dental_floss', 'desk', 'detergent', 'diaper', 'diary', 'die', 'dinghy', 'dining_table', 'tux', 'dish', 'dish_antenna', 'dishrag', 'dishtowel', 'dishwasher', 'dishwasher_detergent', 'diskette', 'dispenser', 'Dixie_cup', 'dog', 'dog_collar', 'doll', 'dollar', 'dolphin', 'domestic_ass', 'eye_mask', 'doorbell', 'doorknob', 'doormat', 'doughnut', 'dove', 'dragonfly', 'drawer', 'underdrawers', 'dress', 'dress_hat', 'dress_suit', 'dresser', 'drill', 'drinking_fountain', 'drone', 'dropper', 'drum_(musical_instrument)', 'drumstick', 'duck', 'duckling', 'duct_tape', 'duffel_bag', 'dumbbell', 'dumpster', 'dustpan', 'Dutch_oven', 'eagle', 'earphone', 'earplug', 'earring', 'easel', 'eclair', 'eel', 'egg', 'egg_roll', 'egg_yolk', 'eggbeater', 'eggplant', 'electric_chair', 'refrigerator', 'elephant', 'elk', 'envelope', 'eraser', 'escargot', 'eyepatch', 'falcon', 'fan', 'faucet', 'fedora', 'ferret', 'Ferris_wheel', 'ferry', 'fig_(fruit)', 'fighter_jet', 'figurine', 'file_cabinet', 'file_(tool)', 'fire_alarm', 'fire_engine', 'fire_extinguisher', 'fire_hose', 'fireplace', 'fireplug', 'fish', 'fish_(food)', 'fishbowl', 'fishing_boat', 'fishing_rod', 'flag', 'flagpole', 'flamingo', 'flannel', 'flash', 'flashlight', 'fleece', 'flip-flop_(sandal)', 'flipper_(footwear)', 'flower_arrangement', 'flute_glass', 'foal', 'folding_chair', 'food_processor', 'football_(American)', 'football_helmet', 'footstool', 'fork', 'forklift', 'freight_car', 'French_toast', 'freshener', 'frisbee', 'frog', 'fruit_juice', 'fruit_salad', 'frying_pan', 'fudge', 'funnel', 'futon', 'gag', 'garbage', 'garbage_truck', 'garden_hose', 'gargle', 'gargoyle', 'garlic', 'gasmask', 'gazelle', 'gelatin', 'gemstone', 'giant_panda', 'gift_wrap', 'ginger', 'giraffe', 'cincture', 'glass_(drink_container)', 'globe', 'glove', 'goat', 'goggles', 'goldfish', 'golf_club', 'golfcart', 'gondola_(boat)', 'goose', 'gorilla', 'gourd', 'surgical_gown', 'grape', 'grasshopper', 'grater', 'gravestone', 'gravy_boat', 'green_bean', 'green_onion', 'griddle', 'grillroom', 'grinder_(tool)', 'grits', 'grizzly', 'grocery_bag', 'guacamole', 'guitar', 'gull', 'gun', 'hair_spray', 'hairbrush', 'hairnet', 'hairpin', 'ham', 'hamburger', 'hammer', 'hammock', 'hamper', 'hamster', 'hair_dryer', 'hand_glass', 'hand_towel', 'handcart', 'handcuff', 'handkerchief', 'handle', 'handsaw', 'hardback_book', 'harmonium', 'hat', 'hatbox', 'hatch', 'veil', 'headband', 'headboard', 'headlight', 'headscarf', 'headset', 'headstall_(for_horses)', 'hearing_aid', 'heart', 'heater', 'helicopter', 'helmet', 'heron', 'highchair', 'hinge', 'hippopotamus', 'hockey_stick', 'hog', 'home_plate_(baseball)', 'honey', 'fume_hood', 'hook', 'horse', 'hose', 'hot-air_balloon', 'hotplate', 'hot_sauce', 'hourglass', 'houseboat', 'hummingbird', 'hummus', 'polar_bear', 'icecream', 'popsicle', 'ice_maker', 'ice_pack', 'ice_skate', 'ice_tea', 'igniter', 'incense', 'inhaler', 'iPod', 'iron_(for_clothing)', 'ironing_board', 'jacket', 'jam', 'jean', 'jeep', 'jelly_bean', 'jersey', 'jet_plane', 'jewelry', 'joystick', 'jumpsuit', 'kayak', 'keg', 'kennel', 'kettle', 'key', 'keycard', 'kilt', 'kimono', 'kitchen_sink', 'kitchen_table', 'kite', 'kitten', 'kiwi_fruit', 'knee_pad', 'knife', 'knight_(chess_piece)', 'knitting_needle', 'knob', 'knocker_(on_a_door)', 'koala', 'lab_coat', 'ladder', 'ladle', 'ladybug', 'lamb_(animal)', 'lamb-chop', 'lamp', 'lamppost', 'lampshade', 'lantern', 'lanyard', 'laptop_computer', 'lasagna', 'latch', 'lawn_mower', 'leather', 'legging_(clothing)', 'Lego', 'lemon', 'lemonade', 'lettuce', 'license_plate', 'life_buoy', 'life_jacket', 'lightbulb', 'lightning_rod', 'lime', 'limousine', 'linen_paper', 'lion', 'lip_balm', 'lipstick', 'liquor', 'lizard', 'Loafer_(type_of_shoe)', 'log', 'lollipop', 'lotion', 'speaker_(stereo_equipment)', 'loveseat', 'machine_gun', 'magazine', 'magnet', 'mail_slot', 'mailbox_(at_home)', 'mallet', 'mammoth', 'mandarin_orange', 'manger', 'manhole', 'map', 'marker', 'martini', 'mascot', 'mashed_potato', 'masher', 'mask', 'mast', 'mat_(gym_equipment)', 'matchbox', 'mattress', 'measuring_cup', 'measuring_stick', 'meatball', 'medicine', 'melon', 'microphone', 'microscope', 'microwave_oven', 'milestone', 'milk', 'minivan', 'mint_candy', 'mirror', 'mitten', 'mixer_(kitchen_tool)', 'money', 'monitor_(computer_equipment) computer_monitor', 'monkey', 'motor', 'motor_scooter', 'motor_vehicle', 'motorboat', 'motorcycle', 'mound_(baseball)', 'mouse_(animal_rodent)', 'mouse_(computer_equipment)', 'mousepad', 'muffin', 'mug', 'mushroom', 'music_stool', 'musical_instrument', 'nailfile', 'nameplate', 'napkin', 'neckerchief', 'necklace', 'necktie', 'needle', 'nest', 'newsstand', 'nightshirt', 'nosebag_(for_animals)', 'noseband_(for_animals)', 'notebook', 'notepad', 'nut', 'nutcracker', 'oar', 'octopus_(food)', 'octopus_(animal)', 'oil_lamp', 'olive_oil', 'omelet', 'onion', 'orange_(fruit)', 'orange_juice', 'oregano', 'ostrich', 'ottoman', 'overalls_(clothing)', 'owl', 'packet', 'inkpad', 'pad', 'paddle', 'padlock', 'paintbox', 'paintbrush', 'painting', 'pajamas', 'palette', 'pan_(for_cooking)', 'pan_(metal_container)', 'pancake', 'pantyhose', 'papaya', 'paperclip', 'paper_plate', 'paper_towel', 'paperback_book', 'paperweight', 'parachute', 'parakeet', 'parasail_(sports)', 'parchment', 'parka', 'parking_meter', 'parrot', 'passenger_car_(part_of_a_train)', 'passenger_ship', 'passport', 'pastry', 'patty_(food)', 'pea_(food)', 'peach', 'peanut_butter', 'pear', 'peeler_(tool_for_fruit_and_vegetables)', 'pegboard', 'pelican', 'pen', 'pencil', 'pencil_box', 'pencil_sharpener', 'pendulum', 'penguin', 'pennant', 'penny_(coin)', 'pepper', 'pepper_mill', 'perfume', 'persimmon', 'baby', 'pet', 'petfood', 'pew_(church_bench)', 'phonebook', 'phonograph_record', 'piano', 'pickle', 'pickup_truck', 'pie', 'pigeon', 'piggy_bank', 'pillow', 'pin_(non_jewelry)', 'pineapple', 'pinecone', 'ping-pong_ball', 'pinwheel', 'tobacco_pipe', 'pipe', 'pistol', 'pita_(bread)', 'pitcher_(vessel_for_liquid)', 'pitchfork', 'pizza', 'place_mat', 'plate', 'platter', 'playing_card', 'playpen', 'pliers', 'plow_(farm_equipment)', 'pocket_watch', 'pocketknife', 'poker_(fire_stirring_tool)', 'pole', 'police_van', 'polo_shirt', 'poncho', 'pony', 'pool_table', 'pop_(soda)', 'portrait', 'postbox_(public)', 'postcard', 'poster', 'pot', 'flowerpot', 'potato', 'potholder', 'pottery', 'pouch', 'power_shovel', 'prawn', 'printer', 'projectile_(weapon)', 'projector', 'propeller', 'prune', 'pudding', 'puffer_(fish)', 'puffin', 'pug-dog', 'pumpkin', 'puncher', 'puppet', 'puppy', 'quesadilla', 'quiche', 'quilt', 'rabbit', 'race_car', 'racket', 'radar', 'radiator', 'radio_receiver', 'radish', 'raft', 'rag_doll', 'raincoat', 'ram_(animal)', 'raspberry', 'rat', 'razorblade', 'reamer_(juicer)', 'rearview_mirror', 'receipt', 'recliner', 'record_player', 'red_cabbage', 'reflector', 'remote_control', 'rhinoceros', 'rib_(food)', 'rifle', 'ring', 'river_boat', 'road_map', 'robe', 'rocking_chair', 'roller_skate', 'Rollerblade', 'rolling_pin', 'root_beer', 'router_(computer_equipment)', 'rubber_band', 'runner_(carpet)', 'plastic_bag', 'saddle_(on_an_animal)', 'saddle_blanket', 'saddlebag', 'safety_pin', 'sail', 'salad', 'salad_plate', 'salami', 'salmon_(fish)', 'salmon_(food)', 'salsa', 'saltshaker', 'sandal_(type_of_shoe)', 'sandwich', 'satchel', 'saucepan', 'saucer', 'sausage', 'sawhorse', 'saxophone', 'scale_(measuring_instrument)', 'scarecrow', 'scarf', 'school_bus', 'scissors', 'scoreboard', 'scrambled_eggs', 'scraper', 'scratcher', 'screwdriver', 'scrubbing_brush', 'sculpture', 'seabird', 'seahorse', 'seaplane', 'seashell', 'seedling', 'serving_dish', 'sewing_machine', 'shaker', 'shampoo', 'shark', 'sharpener', 'Sharpie', 'shaver_(electric)', 'shaving_cream', 'shawl', 'shears', 'sheep', 'shepherd_dog', 'sherbert', 'shield', 'shirt', 'shoe', 'shopping_bag', 'shopping_cart', 'short_pants', 'shot_glass', 'shoulder_bag', 'shovel', 'shower_head', 'shower_curtain', 'shredder_(for_paper)', 'sieve', 'signboard', 'silo', 'sink', 'skateboard', 'skewer', 'ski', 'ski_boot', 'ski_parka', 'ski_pole', 'skirt', 'sled', 'sleeping_bag', 'sling_(bandage)', 'slipper_(footwear)', 'smoothie', 'snake', 'snowboard', 'snowman', 'snowmobile', 'soap', 'soccer_ball', 'sock', 'soda_fountain', 'carbonated_water', 'sofa', 'softball', 'solar_array', 'sombrero', 'soup', 'soup_bowl', 'soupspoon', 'sour_cream', 'soya_milk', 'space_shuttle', 'sparkler_(fireworks)', 'spatula', 'spear', 'spectacles', 'spice_rack', 'spider', 'sponge', 'spoon', 'sportswear', 'spotlight', 'squirrel', 'stapler_(stapling_machine)', 'starfish', 'statue_(sculpture)', 'steak_(food)', 'steak_knife', 'steamer_(kitchen_appliance)', 'steering_wheel', 'stencil', 'stepladder', 'step_stool', 'stereo_(sound_system)', 'stew', 'stirrer', 'stirrup', 'stockings_(leg_wear)', 'stool', 'stop_sign', 'brake_light', 'stove', 'strainer', 'strap', 'straw_(for_drinking)', 'strawberry', 'street_sign', 'streetlight', 'string_cheese', 'stylus', 'subwoofer', 'sugar_bowl', 'sugarcane_(plant)', 'suit_(clothing)', 'sunflower', 'sunglasses', 'sunhat', 'sunscreen', 'surfboard', 'sushi', 'mop', 'sweat_pants', 'sweatband', 'sweater', 'sweatshirt', 'sweet_potato', 'swimsuit', 'sword', 'syringe', 'Tabasco_sauce', 'table-tennis_table', 'table', 'table_lamp', 'tablecloth', 'tachometer', 'taco', 'tag', 'taillight', 'tambourine', 'army_tank', 'tank_(storage_vessel)', 'tank_top_(clothing)', 'tape_(sticky_cloth_or_paper)', 'tape_measure', 'tapestry', 'tarp', 'tartan', 'tassel', 'tea_bag', 'teacup', 'teakettle', 'teapot', 'teddy_bear', 'telephone', 'telephone_booth', 'telephone_pole', 'telephoto_lens', 'television_camera', 'television_set', 'tennis_ball', 'tennis_racket', 'tequila', 'thermometer', 'thermos_bottle', 'thermostat', 'thimble', 'thread', 'thumbtack', 'tiara', 'tiger', 'tights_(clothing)', 'timer', 'tinfoil', 'tinsel', 'tissue_paper', 'toast_(food)', 'toaster', 'toaster_oven', 'toilet', 'toilet_tissue', 'tomato', 'tongs', 'toolbox', 'toothbrush', 'toothpaste', 'toothpick', 'cover', 'tortilla', 'tow_truck', 'towel', 'towel_rack', 'toy', 'tractor_(farm_equipment)', 'traffic_light', 'dirt_bike', 'trailer_truck', 'train_(railroad_vehicle)', 'trampoline', 'tray', 'tree_house', 'trench_coat', 'triangle_(musical_instrument)', 'tricycle', 'tripod', 'trousers', 'truck', 'truffle_(chocolate)', 'trunk', 'vat', 'turban', 'turkey_(bird)', 'turkey_(food)', 'turnip', 'turtle', 'turtleneck_(clothing)', 'typewriter', 'umbrella', 'underwear', 'unicycle', 'urinal', 'urn', 'vacuum_cleaner', 'valve', 'vase', 'vending_machine', 'vent', 'videotape', 'vinegar', 'violin', 'vodka', 'volleyball', 'vulture', 'waffle', 'waffle_iron', 'wagon', 'wagon_wheel', 'walking_stick', 'wall_clock', 'wall_socket', 'wallet', 'walrus', 'wardrobe', 'wasabi', 'automatic_washer', 'watch', 'water_bottle', 'water_cooler', 'water_faucet', 'water_filter', 'water_heater', 'water_jug', 'water_gun', 'water_scooter', 'water_ski', 'water_tower', 'watering_can', 'watermelon', 'weathervane', 'webcam', 'wedding_cake', 'wedding_ring', 'wet_suit', 'wheel', 'wheelchair', 'whipped_cream', 'whiskey', 'whistle', 'wick', 'wig', 'wind_chime', 'windmill', 'window_box_(for_plants)', 'windshield_wiper', 'windsock', 'wine_bottle', 'wine_bucket', 'wineglass', 'wing_chair', 'blinder_(for_horses)', 'wok', 'wolf', 'wooden_spoon', 'wreath', 'wrench', 'wristband', 'wristlet', 'yacht', 'yak', 'yogurt', 'yoke_(animal_equipment)', 'zebra', 'zucchini'), 'palette': None } def load_data_list(self) -> List[dict]: """Load annotations from an annotation file named as ``self.ann_file`` Returns: List[dict]: A list of annotation. """ # noqa: E501 try: import lvis if getattr(lvis, '__version__', '0') >= '10.5.3': warnings.warn( 'mmlvis is deprecated, please install official lvis-api by "pip install git+https://github.com/lvis-dataset/lvis-api.git"', # noqa: E501 UserWarning) from lvis import LVIS except ImportError: raise ImportError( 'Package lvis is not installed. Please run "pip install git+https://github.com/lvis-dataset/lvis-api.git".' # noqa: E501 ) with self.file_client.get_local_path(self.ann_file) as local_path: self.lvis = LVIS(local_path) self.cat_ids = self.lvis.get_cat_ids() self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)} self.cat_img_map = copy.deepcopy(self.lvis.cat_img_map) img_ids = self.lvis.get_img_ids() data_list = [] total_ann_ids = [] for img_id in img_ids: raw_img_info = self.lvis.load_imgs([img_id])[0] raw_img_info['img_id'] = img_id if raw_img_info['file_name'].startswith('COCO'): # Convert form the COCO 2014 file naming convention of # COCO_[train/val/test]2014_000000000000.jpg to the 2017 # naming convention of 000000000000.jpg # (LVIS v1 will fix this naming issue) raw_img_info['file_name'] = raw_img_info['file_name'][-16:] ann_ids = self.lvis.get_ann_ids(img_ids=[img_id]) raw_ann_info = self.lvis.load_anns(ann_ids) total_ann_ids.extend(ann_ids) parsed_data_info = self.parse_data_info({ 'raw_ann_info': raw_ann_info, 'raw_img_info': raw_img_info }) data_list.append(parsed_data_info) if self.ANN_ID_UNIQUE: assert len(set(total_ann_ids)) == len( total_ann_ids ), f"Annotation ids in '{self.ann_file}' are not unique!" del self.lvis return data_list LVISDataset = LVISV05Dataset DATASETS.register_module(name='LVISDataset', module=LVISDataset) @DATASETS.register_module() class LVISV1Dataset(LVISDataset): """LVIS v1 dataset for detection.""" METAINFO = { 'classes': ('aerosol_can', 'air_conditioner', 'airplane', 'alarm_clock', 'alcohol', 'alligator', 'almond', 'ambulance', 'amplifier', 'anklet', 'antenna', 'apple', 'applesauce', 'apricot', 'apron', 'aquarium', 'arctic_(type_of_shoe)', 'armband', 'armchair', 'armoire', 'armor', 'artichoke', 'trash_can', 'ashtray', 'asparagus', 'atomizer', 'avocado', 'award', 'awning', 'ax', 'baboon', 'baby_buggy', 'basketball_backboard', 'backpack', 'handbag', 'suitcase', 'bagel', 'bagpipe', 'baguet', 'bait', 'ball', 'ballet_skirt', 'balloon', 'bamboo', 'banana', 'Band_Aid', 'bandage', 'bandanna', 'banjo', 'banner', 'barbell', 'barge', 'barrel', 'barrette', 'barrow', 'baseball_base', 'baseball', 'baseball_bat', 'baseball_cap', 'baseball_glove', 'basket', 'basketball', 'bass_horn', 'bat_(animal)', 'bath_mat', 'bath_towel', 'bathrobe', 'bathtub', 'batter_(food)', 'battery', 'beachball', 'bead', 'bean_curd', 'beanbag', 'beanie', 'bear', 'bed', 'bedpan', 'bedspread', 'cow', 'beef_(food)', 'beeper', 'beer_bottle', 'beer_can', 'beetle', 'bell', 'bell_pepper', 'belt', 'belt_buckle', 'bench', 'beret', 'bib', 'Bible', 'bicycle', 'visor', 'billboard', 'binder', 'binoculars', 'bird', 'birdfeeder', 'birdbath', 'birdcage', 'birdhouse', 'birthday_cake', 'birthday_card', 'pirate_flag', 'black_sheep', 'blackberry', 'blackboard', 'blanket', 'blazer', 'blender', 'blimp', 'blinker', 'blouse', 'blueberry', 'gameboard', 'boat', 'bob', 'bobbin', 'bobby_pin', 'boiled_egg', 'bolo_tie', 'deadbolt', 'bolt', 'bonnet', 'book', 'bookcase', 'booklet', 'bookmark', 'boom_microphone', 'boot', 'bottle', 'bottle_opener', 'bouquet', 'bow_(weapon)', 'bow_(decorative_ribbons)', 'bow-tie', 'bowl', 'pipe_bowl', 'bowler_hat', 'bowling_ball', 'box', 'boxing_glove', 'suspenders', 'bracelet', 'brass_plaque', 'brassiere', 'bread-bin', 'bread', 'breechcloth', 'bridal_gown', 'briefcase', 'broccoli', 'broach', 'broom', 'brownie', 'brussels_sprouts', 'bubble_gum', 'bucket', 'horse_buggy', 'bull', 'bulldog', 'bulldozer', 'bullet_train', 'bulletin_board', 'bulletproof_vest', 'bullhorn', 'bun', 'bunk_bed', 'buoy', 'burrito', 'bus_(vehicle)', 'business_card', 'butter', 'butterfly', 'button', 'cab_(taxi)', 'cabana', 'cabin_car', 'cabinet', 'locker', 'cake', 'calculator', 'calendar', 'calf', 'camcorder', 'camel', 'camera', 'camera_lens', 'camper_(vehicle)', 'can', 'can_opener', 'candle', 'candle_holder', 'candy_bar', 'candy_cane', 'walking_cane', 'canister', 'canoe', 'cantaloup', 'canteen', 'cap_(headwear)', 'bottle_cap', 'cape', 'cappuccino', 'car_(automobile)', 'railcar_(part_of_a_train)', 'elevator_car', 'car_battery', 'identity_card', 'card', 'cardigan', 'cargo_ship', 'carnation', 'horse_carriage', 'carrot', 'tote_bag', 'cart', 'carton', 'cash_register', 'casserole', 'cassette', 'cast', 'cat', 'cauliflower', 'cayenne_(spice)', 'CD_player', 'celery', 'cellular_telephone', 'chain_mail', 'chair', 'chaise_longue', 'chalice', 'chandelier', 'chap', 'checkbook', 'checkerboard', 'cherry', 'chessboard', 'chicken_(animal)', 'chickpea', 'chili_(vegetable)', 'chime', 'chinaware', 'crisp_(potato_chip)', 'poker_chip', 'chocolate_bar', 'chocolate_cake', 'chocolate_milk', 'chocolate_mousse', 'choker', 'chopping_board', 'chopstick', 'Christmas_tree', 'slide', 'cider', 'cigar_box', 'cigarette', 'cigarette_case', 'cistern', 'clarinet', 'clasp', 'cleansing_agent', 'cleat_(for_securing_rope)', 'clementine', 'clip', 'clipboard', 'clippers_(for_plants)', 'cloak', 'clock', 'clock_tower', 'clothes_hamper', 'clothespin', 'clutch_bag', 'coaster', 'coat', 'coat_hanger', 'coatrack', 'cock', 'cockroach', 'cocoa_(beverage)', 'coconut', 'coffee_maker', 'coffee_table', 'coffeepot', 'coil', 'coin', 'colander', 'coleslaw', 'coloring_material', 'combination_lock', 'pacifier', 'comic_book', 'compass', 'computer_keyboard', 'condiment', 'cone', 'control', 'convertible_(automobile)', 'sofa_bed', 'cooker', 'cookie', 'cooking_utensil', 'cooler_(for_food)', 'cork_(bottle_plug)', 'corkboard', 'corkscrew', 'edible_corn', 'cornbread', 'cornet', 'cornice', 'cornmeal', 'corset', 'costume', 'cougar', 'coverall', 'cowbell', 'cowboy_hat', 'crab_(animal)', 'crabmeat', 'cracker', 'crape', 'crate', 'crayon', 'cream_pitcher', 'crescent_roll', 'crib', 'crock_pot', 'crossbar', 'crouton', 'crow', 'crowbar', 'crown', 'crucifix', 'cruise_ship', 'police_cruiser', 'crumb', 'crutch', 'cub_(animal)', 'cube', 'cucumber', 'cufflink', 'cup', 'trophy_cup', 'cupboard', 'cupcake', 'hair_curler', 'curling_iron', 'curtain', 'cushion', 'cylinder', 'cymbal', 'dagger', 'dalmatian', 'dartboard', 'date_(fruit)', 'deck_chair', 'deer', 'dental_floss', 'desk', 'detergent', 'diaper', 'diary', 'die', 'dinghy', 'dining_table', 'tux', 'dish', 'dish_antenna', 'dishrag', 'dishtowel', 'dishwasher', 'dishwasher_detergent', 'dispenser', 'diving_board', 'Dixie_cup', 'dog', 'dog_collar', 'doll', 'dollar', 'dollhouse', 'dolphin', 'domestic_ass', 'doorknob', 'doormat', 'doughnut', 'dove', 'dragonfly', 'drawer', 'underdrawers', 'dress', 'dress_hat', 'dress_suit', 'dresser', 'drill', 'drone', 'dropper', 'drum_(musical_instrument)', 'drumstick', 'duck', 'duckling', 'duct_tape', 'duffel_bag', 'dumbbell', 'dumpster', 'dustpan', 'eagle', 'earphone', 'earplug', 'earring', 'easel', 'eclair', 'eel', 'egg', 'egg_roll', 'egg_yolk', 'eggbeater', 'eggplant', 'electric_chair', 'refrigerator', 'elephant', 'elk', 'envelope', 'eraser', 'escargot', 'eyepatch', 'falcon', 'fan', 'faucet', 'fedora', 'ferret', 'Ferris_wheel', 'ferry', 'fig_(fruit)', 'fighter_jet', 'figurine', 'file_cabinet', 'file_(tool)', 'fire_alarm', 'fire_engine', 'fire_extinguisher', 'fire_hose', 'fireplace', 'fireplug', 'first-aid_kit', 'fish', 'fish_(food)', 'fishbowl', 'fishing_rod', 'flag', 'flagpole', 'flamingo', 'flannel', 'flap', 'flash', 'flashlight', 'fleece', 'flip-flop_(sandal)', 'flipper_(footwear)', 'flower_arrangement', 'flute_glass', 'foal', 'folding_chair', 'food_processor', 'football_(American)', 'football_helmet', 'footstool', 'fork', 'forklift', 'freight_car', 'French_toast', 'freshener', 'frisbee', 'frog', 'fruit_juice', 'frying_pan', 'fudge', 'funnel', 'futon', 'gag', 'garbage', 'garbage_truck', 'garden_hose', 'gargle', 'gargoyle', 'garlic', 'gasmask', 'gazelle', 'gelatin', 'gemstone', 'generator', 'giant_panda', 'gift_wrap', 'ginger', 'giraffe', 'cincture', 'glass_(drink_container)', 'globe', 'glove', 'goat', 'goggles', 'goldfish', 'golf_club', 'golfcart', 'gondola_(boat)', 'goose', 'gorilla', 'gourd', 'grape', 'grater', 'gravestone', 'gravy_boat', 'green_bean', 'green_onion', 'griddle', 'grill', 'grits', 'grizzly', 'grocery_bag', 'guitar', 'gull', 'gun', 'hairbrush', 'hairnet', 'hairpin', 'halter_top', 'ham', 'hamburger', 'hammer', 'hammock', 'hamper', 'hamster', 'hair_dryer', 'hand_glass', 'hand_towel', 'handcart', 'handcuff', 'handkerchief', 'handle', 'handsaw', 'hardback_book', 'harmonium', 'hat', 'hatbox', 'veil', 'headband', 'headboard', 'headlight', 'headscarf', 'headset', 'headstall_(for_horses)', 'heart', 'heater', 'helicopter', 'helmet', 'heron', 'highchair', 'hinge', 'hippopotamus', 'hockey_stick', 'hog', 'home_plate_(baseball)', 'honey', 'fume_hood', 'hook', 'hookah', 'hornet', 'horse', 'hose', 'hot-air_balloon', 'hotplate', 'hot_sauce', 'hourglass', 'houseboat', 'hummingbird', 'hummus', 'polar_bear', 'icecream', 'popsicle', 'ice_maker', 'ice_pack', 'ice_skate', 'igniter', 'inhaler', 'iPod', 'iron_(for_clothing)', 'ironing_board', 'jacket', 'jam', 'jar', 'jean', 'jeep', 'jelly_bean', 'jersey', 'jet_plane', 'jewel', 'jewelry', 'joystick', 'jumpsuit', 'kayak', 'keg', 'kennel', 'kettle', 'key', 'keycard', 'kilt', 'kimono', 'kitchen_sink', 'kitchen_table', 'kite', 'kitten', 'kiwi_fruit', 'knee_pad', 'knife', 'knitting_needle', 'knob', 'knocker_(on_a_door)', 'koala', 'lab_coat', 'ladder', 'ladle', 'ladybug', 'lamb_(animal)', 'lamb-chop', 'lamp', 'lamppost', 'lampshade', 'lantern', 'lanyard', 'laptop_computer', 'lasagna', 'latch', 'lawn_mower', 'leather', 'legging_(clothing)', 'Lego', 'legume', 'lemon', 'lemonade', 'lettuce', 'license_plate', 'life_buoy', 'life_jacket', 'lightbulb', 'lightning_rod', 'lime', 'limousine', 'lion', 'lip_balm', 'liquor', 'lizard', 'log', 'lollipop', 'speaker_(stereo_equipment)', 'loveseat', 'machine_gun', 'magazine', 'magnet', 'mail_slot', 'mailbox_(at_home)', 'mallard', 'mallet', 'mammoth', 'manatee', 'mandarin_orange', 'manger', 'manhole', 'map', 'marker', 'martini', 'mascot', 'mashed_potato', 'masher', 'mask', 'mast', 'mat_(gym_equipment)', 'matchbox', 'mattress', 'measuring_cup', 'measuring_stick', 'meatball', 'medicine', 'melon', 'microphone', 'microscope', 'microwave_oven', 'milestone', 'milk', 'milk_can', 'milkshake', 'minivan', 'mint_candy', 'mirror', 'mitten', 'mixer_(kitchen_tool)', 'money', 'monitor_(computer_equipment) computer_monitor', 'monkey', 'motor', 'motor_scooter', 'motor_vehicle', 'motorcycle', 'mound_(baseball)', 'mouse_(computer_equipment)', 'mousepad', 'muffin', 'mug', 'mushroom', 'music_stool', 'musical_instrument', 'nailfile', 'napkin', 'neckerchief', 'necklace', 'necktie', 'needle', 'nest', 'newspaper', 'newsstand', 'nightshirt', 'nosebag_(for_animals)', 'noseband_(for_animals)', 'notebook', 'notepad', 'nut', 'nutcracker', 'oar', 'octopus_(food)', 'octopus_(animal)', 'oil_lamp', 'olive_oil', 'omelet', 'onion', 'orange_(fruit)', 'orange_juice', 'ostrich', 'ottoman', 'oven', 'overalls_(clothing)', 'owl', 'packet', 'inkpad', 'pad', 'paddle', 'padlock', 'paintbrush', 'painting', 'pajamas', 'palette', 'pan_(for_cooking)', 'pan_(metal_container)', 'pancake', 'pantyhose', 'papaya', 'paper_plate', 'paper_towel', 'paperback_book', 'paperweight', 'parachute', 'parakeet', 'parasail_(sports)', 'parasol', 'parchment', 'parka', 'parking_meter', 'parrot', 'passenger_car_(part_of_a_train)', 'passenger_ship', 'passport', 'pastry', 'patty_(food)', 'pea_(food)', 'peach', 'peanut_butter', 'pear', 'peeler_(tool_for_fruit_and_vegetables)', 'wooden_leg', 'pegboard', 'pelican', 'pen', 'pencil', 'pencil_box', 'pencil_sharpener', 'pendulum', 'penguin', 'pennant', 'penny_(coin)', 'pepper', 'pepper_mill', 'perfume', 'persimmon', 'person', 'pet', 'pew_(church_bench)', 'phonebook', 'phonograph_record', 'piano', 'pickle', 'pickup_truck', 'pie', 'pigeon', 'piggy_bank', 'pillow', 'pin_(non_jewelry)', 'pineapple', 'pinecone', 'ping-pong_ball', 'pinwheel', 'tobacco_pipe', 'pipe', 'pistol', 'pita_(bread)', 'pitcher_(vessel_for_liquid)', 'pitchfork', 'pizza', 'place_mat', 'plate', 'platter', 'playpen', 'pliers', 'plow_(farm_equipment)', 'plume', 'pocket_watch', 'pocketknife', 'poker_(fire_stirring_tool)', 'pole', 'polo_shirt', 'poncho', 'pony', 'pool_table', 'pop_(soda)', 'postbox_(public)', 'postcard', 'poster', 'pot', 'flowerpot', 'potato', 'potholder', 'pottery', 'pouch', 'power_shovel', 'prawn', 'pretzel', 'printer', 'projectile_(weapon)', 'projector', 'propeller', 'prune', 'pudding', 'puffer_(fish)', 'puffin', 'pug-dog', 'pumpkin', 'puncher', 'puppet', 'puppy', 'quesadilla', 'quiche', 'quilt', 'rabbit', 'race_car', 'racket', 'radar', 'radiator', 'radio_receiver', 'radish', 'raft', 'rag_doll', 'raincoat', 'ram_(animal)', 'raspberry', 'rat', 'razorblade', 'reamer_(juicer)', 'rearview_mirror', 'receipt', 'recliner', 'record_player', 'reflector', 'remote_control', 'rhinoceros', 'rib_(food)', 'rifle', 'ring', 'river_boat', 'road_map', 'robe', 'rocking_chair', 'rodent', 'roller_skate', 'Rollerblade', 'rolling_pin', 'root_beer', 'router_(computer_equipment)', 'rubber_band', 'runner_(carpet)', 'plastic_bag', 'saddle_(on_an_animal)', 'saddle_blanket', 'saddlebag', 'safety_pin', 'sail', 'salad', 'salad_plate', 'salami', 'salmon_(fish)', 'salmon_(food)', 'salsa', 'saltshaker', 'sandal_(type_of_shoe)', 'sandwich', 'satchel', 'saucepan', 'saucer', 'sausage', 'sawhorse', 'saxophone', 'scale_(measuring_instrument)', 'scarecrow', 'scarf', 'school_bus', 'scissors', 'scoreboard', 'scraper', 'screwdriver', 'scrubbing_brush', 'sculpture', 'seabird', 'seahorse', 'seaplane', 'seashell', 'sewing_machine', 'shaker', 'shampoo', 'shark', 'sharpener', 'Sharpie', 'shaver_(electric)', 'shaving_cream', 'shawl', 'shears', 'sheep', 'shepherd_dog', 'sherbert', 'shield', 'shirt', 'shoe', 'shopping_bag', 'shopping_cart', 'short_pants', 'shot_glass', 'shoulder_bag', 'shovel', 'shower_head', 'shower_cap', 'shower_curtain', 'shredder_(for_paper)', 'signboard', 'silo', 'sink', 'skateboard', 'skewer', 'ski', 'ski_boot', 'ski_parka', 'ski_pole', 'skirt', 'skullcap', 'sled', 'sleeping_bag', 'sling_(bandage)', 'slipper_(footwear)', 'smoothie', 'snake', 'snowboard', 'snowman', 'snowmobile', 'soap', 'soccer_ball', 'sock', 'sofa', 'softball', 'solar_array', 'sombrero', 'soup', 'soup_bowl', 'soupspoon', 'sour_cream', 'soya_milk', 'space_shuttle', 'sparkler_(fireworks)', 'spatula', 'spear', 'spectacles', 'spice_rack', 'spider', 'crawfish', 'sponge', 'spoon', 'sportswear', 'spotlight', 'squid_(food)', 'squirrel', 'stagecoach', 'stapler_(stapling_machine)', 'starfish', 'statue_(sculpture)', 'steak_(food)', 'steak_knife', 'steering_wheel', 'stepladder', 'step_stool', 'stereo_(sound_system)', 'stew', 'stirrer', 'stirrup', 'stool', 'stop_sign', 'brake_light', 'stove', 'strainer', 'strap', 'straw_(for_drinking)', 'strawberry', 'street_sign', 'streetlight', 'string_cheese', 'stylus', 'subwoofer', 'sugar_bowl', 'sugarcane_(plant)', 'suit_(clothing)', 'sunflower', 'sunglasses', 'sunhat', 'surfboard', 'sushi', 'mop', 'sweat_pants', 'sweatband', 'sweater', 'sweatshirt', 'sweet_potato', 'swimsuit', 'sword', 'syringe', 'Tabasco_sauce', 'table-tennis_table', 'table', 'table_lamp', 'tablecloth', 'tachometer', 'taco', 'tag', 'taillight', 'tambourine', 'army_tank', 'tank_(storage_vessel)', 'tank_top_(clothing)', 'tape_(sticky_cloth_or_paper)', 'tape_measure', 'tapestry', 'tarp', 'tartan', 'tassel', 'tea_bag', 'teacup', 'teakettle', 'teapot', 'teddy_bear', 'telephone', 'telephone_booth', 'telephone_pole', 'telephoto_lens', 'television_camera', 'television_set', 'tennis_ball', 'tennis_racket', 'tequila', 'thermometer', 'thermos_bottle', 'thermostat', 'thimble', 'thread', 'thumbtack', 'tiara', 'tiger', 'tights_(clothing)', 'timer', 'tinfoil', 'tinsel', 'tissue_paper', 'toast_(food)', 'toaster', 'toaster_oven', 'toilet', 'toilet_tissue', 'tomato', 'tongs', 'toolbox', 'toothbrush', 'toothpaste', 'toothpick', 'cover', 'tortilla', 'tow_truck', 'towel', 'towel_rack', 'toy', 'tractor_(farm_equipment)', 'traffic_light', 'dirt_bike', 'trailer_truck', 'train_(railroad_vehicle)', 'trampoline', 'tray', 'trench_coat', 'triangle_(musical_instrument)', 'tricycle', 'tripod', 'trousers', 'truck', 'truffle_(chocolate)', 'trunk', 'vat', 'turban', 'turkey_(food)', 'turnip', 'turtle', 'turtleneck_(clothing)', 'typewriter', 'umbrella', 'underwear', 'unicycle', 'urinal', 'urn', 'vacuum_cleaner', 'vase', 'vending_machine', 'vent', 'vest', 'videotape', 'vinegar', 'violin', 'vodka', 'volleyball', 'vulture', 'waffle', 'waffle_iron', 'wagon', 'wagon_wheel', 'walking_stick', 'wall_clock', 'wall_socket', 'wallet', 'walrus', 'wardrobe', 'washbasin', 'automatic_washer', 'watch', 'water_bottle', 'water_cooler', 'water_faucet', 'water_heater', 'water_jug', 'water_gun', 'water_scooter', 'water_ski', 'water_tower', 'watering_can', 'watermelon', 'weathervane', 'webcam', 'wedding_cake', 'wedding_ring', 'wet_suit', 'wheel', 'wheelchair', 'whipped_cream', 'whistle', 'wig', 'wind_chime', 'windmill', 'window_box_(for_plants)', 'windshield_wiper', 'windsock', 'wine_bottle', 'wine_bucket', 'wineglass', 'blinder_(for_horses)', 'wok', 'wolf', 'wooden_spoon', 'wreath', 'wrench', 'wristband', 'wristlet', 'yacht', 'yogurt', 'yoke_(animal_equipment)', 'zebra', 'zucchini'), 'palette': None } def load_data_list(self) -> List[dict]: """Load annotations from an annotation file named as ``self.ann_file`` Returns: List[dict]: A list of annotation. """ # noqa: E501 try: import lvis if getattr(lvis, '__version__', '0') >= '10.5.3': warnings.warn( 'mmlvis is deprecated, please install official lvis-api by "pip install git+https://github.com/lvis-dataset/lvis-api.git"', # noqa: E501 UserWarning) from lvis import LVIS except ImportError: raise ImportError( 'Package lvis is not installed. Please run "pip install git+https://github.com/lvis-dataset/lvis-api.git".' # noqa: E501 ) self.lvis = LVIS(self.ann_file) self.cat_ids = self.lvis.get_cat_ids() self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)} self.cat_img_map = copy.deepcopy(self.lvis.cat_img_map) img_ids = self.lvis.get_img_ids() data_list = [] total_ann_ids = [] for img_id in img_ids: raw_img_info = self.lvis.load_imgs([img_id])[0] raw_img_info['img_id'] = img_id # coco_url is used in LVISv1 instead of file_name # e.g. http://images.cocodataset.org/train2017/000000391895.jpg # train/val split in specified in url raw_img_info['file_name'] = raw_img_info['coco_url'].replace( 'http://images.cocodataset.org/', '') ann_ids = self.lvis.get_ann_ids(img_ids=[img_id]) raw_ann_info = self.lvis.load_anns(ann_ids) total_ann_ids.extend(ann_ids) parsed_data_info = self.parse_data_info({ 'raw_ann_info': raw_ann_info, 'raw_img_info': raw_img_info }) data_list.append(parsed_data_info) if self.ANN_ID_UNIQUE: assert len(set(total_ann_ids)) == len( total_ann_ids ), f"Annotation ids in '{self.ann_file}' are not unique!" del self.lvis return data_list ================================================ FILE: mmdet/datasets/objects365.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import os.path as osp from typing import List from mmdet.registry import DATASETS from .api_wrappers import COCO from .coco import CocoDataset # images exist in annotations but not in image folder. objv2_ignore_list = [ osp.join('patch16', 'objects365_v2_00908726.jpg'), osp.join('patch6', 'objects365_v1_00320532.jpg'), osp.join('patch6', 'objects365_v1_00320534.jpg'), ] @DATASETS.register_module() class Objects365V1Dataset(CocoDataset): """Objects365 v1 dataset for detection.""" METAINFO = { 'classes': ('person', 'sneakers', 'chair', 'hat', 'lamp', 'bottle', 'cabinet/shelf', 'cup', 'car', 'glasses', 'picture/frame', 'desk', 'handbag', 'street lights', 'book', 'plate', 'helmet', 'leather shoes', 'pillow', 'glove', 'potted plant', 'bracelet', 'flower', 'tv', 'storage box', 'vase', 'bench', 'wine glass', 'boots', 'bowl', 'dining table', 'umbrella', 'boat', 'flag', 'speaker', 'trash bin/can', 'stool', 'backpack', 'couch', 'belt', 'carpet', 'basket', 'towel/napkin', 'slippers', 'barrel/bucket', 'coffee table', 'suv', 'toy', 'tie', 'bed', 'traffic light', 'pen/pencil', 'microphone', 'sandals', 'canned', 'necklace', 'mirror', 'faucet', 'bicycle', 'bread', 'high heels', 'ring', 'van', 'watch', 'sink', 'horse', 'fish', 'apple', 'camera', 'candle', 'teddy bear', 'cake', 'motorcycle', 'wild bird', 'laptop', 'knife', 'traffic sign', 'cell phone', 'paddle', 'truck', 'cow', 'power outlet', 'clock', 'drum', 'fork', 'bus', 'hanger', 'nightstand', 'pot/pan', 'sheep', 'guitar', 'traffic cone', 'tea pot', 'keyboard', 'tripod', 'hockey', 'fan', 'dog', 'spoon', 'blackboard/whiteboard', 'balloon', 'air conditioner', 'cymbal', 'mouse', 'telephone', 'pickup truck', 'orange', 'banana', 'airplane', 'luggage', 'skis', 'soccer', 'trolley', 'oven', 'remote', 'baseball glove', 'paper towel', 'refrigerator', 'train', 'tomato', 'machinery vehicle', 'tent', 'shampoo/shower gel', 'head phone', 'lantern', 'donut', 'cleaning products', 'sailboat', 'tangerine', 'pizza', 'kite', 'computer box', 'elephant', 'toiletries', 'gas stove', 'broccoli', 'toilet', 'stroller', 'shovel', 'baseball bat', 'microwave', 'skateboard', 'surfboard', 'surveillance camera', 'gun', 'life saver', 'cat', 'lemon', 'liquid soap', 'zebra', 'duck', 'sports car', 'giraffe', 'pumpkin', 'piano', 'stop sign', 'radiator', 'converter', 'tissue ', 'carrot', 'washing machine', 'vent', 'cookies', 'cutting/chopping board', 'tennis racket', 'candy', 'skating and skiing shoes', 'scissors', 'folder', 'baseball', 'strawberry', 'bow tie', 'pigeon', 'pepper', 'coffee machine', 'bathtub', 'snowboard', 'suitcase', 'grapes', 'ladder', 'pear', 'american football', 'basketball', 'potato', 'paint brush', 'printer', 'billiards', 'fire hydrant', 'goose', 'projector', 'sausage', 'fire extinguisher', 'extension cord', 'facial mask', 'tennis ball', 'chopsticks', 'electronic stove and gas stove', 'pie', 'frisbee', 'kettle', 'hamburger', 'golf club', 'cucumber', 'clutch', 'blender', 'tong', 'slide', 'hot dog', 'toothbrush', 'facial cleanser', 'mango', 'deer', 'egg', 'violin', 'marker', 'ship', 'chicken', 'onion', 'ice cream', 'tape', 'wheelchair', 'plum', 'bar soap', 'scale', 'watermelon', 'cabbage', 'router/modem', 'golf ball', 'pine apple', 'crane', 'fire truck', 'peach', 'cello', 'notepaper', 'tricycle', 'toaster', 'helicopter', 'green beans', 'brush', 'carriage', 'cigar', 'earphone', 'penguin', 'hurdle', 'swing', 'radio', 'CD', 'parking meter', 'swan', 'garlic', 'french fries', 'horn', 'avocado', 'saxophone', 'trumpet', 'sandwich', 'cue', 'kiwi fruit', 'bear', 'fishing rod', 'cherry', 'tablet', 'green vegetables', 'nuts', 'corn', 'key', 'screwdriver', 'globe', 'broom', 'pliers', 'volleyball', 'hammer', 'eggplant', 'trophy', 'dates', 'board eraser', 'rice', 'tape measure/ruler', 'dumbbell', 'hamimelon', 'stapler', 'camel', 'lettuce', 'goldfish', 'meat balls', 'medal', 'toothpaste', 'antelope', 'shrimp', 'rickshaw', 'trombone', 'pomegranate', 'coconut', 'jellyfish', 'mushroom', 'calculator', 'treadmill', 'butterfly', 'egg tart', 'cheese', 'pig', 'pomelo', 'race car', 'rice cooker', 'tuba', 'crosswalk sign', 'papaya', 'hair drier', 'green onion', 'chips', 'dolphin', 'sushi', 'urinal', 'donkey', 'electric drill', 'spring rolls', 'tortoise/turtle', 'parrot', 'flute', 'measuring cup', 'shark', 'steak', 'poker card', 'binoculars', 'llama', 'radish', 'noodles', 'yak', 'mop', 'crab', 'microscope', 'barbell', 'bread/bun', 'baozi', 'lion', 'red cabbage', 'polar bear', 'lighter', 'seal', 'mangosteen', 'comb', 'eraser', 'pitaya', 'scallop', 'pencil case', 'saw', 'table tennis paddle', 'okra', 'starfish', 'eagle', 'monkey', 'durian', 'game board', 'rabbit', 'french horn', 'ambulance', 'asparagus', 'hoverboard', 'pasta', 'target', 'hotair balloon', 'chainsaw', 'lobster', 'iron', 'flashlight'), 'palette': None } COCOAPI = COCO # ann_id is unique in coco dataset. ANN_ID_UNIQUE = True def load_data_list(self) -> List[dict]: """Load annotations from an annotation file named as ``self.ann_file`` Returns: List[dict]: A list of annotation. """ # noqa: E501 with self.file_client.get_local_path(self.ann_file) as local_path: self.coco = self.COCOAPI(local_path) # 'categories' list in objects365_train.json and objects365_val.json # is inconsistent, need sort list(or dict) before get cat_ids. cats = self.coco.cats sorted_cats = {i: cats[i] for i in sorted(cats)} self.coco.cats = sorted_cats categories = self.coco.dataset['categories'] sorted_categories = sorted(categories, key=lambda i: i['id']) self.coco.dataset['categories'] = sorted_categories # The order of returned `cat_ids` will not # change with the order of the `classes` self.cat_ids = self.coco.get_cat_ids( cat_names=self.metainfo['classes']) self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)} self.cat_img_map = copy.deepcopy(self.coco.cat_img_map) img_ids = self.coco.get_img_ids() data_list = [] total_ann_ids = [] for img_id in img_ids: raw_img_info = self.coco.load_imgs([img_id])[0] raw_img_info['img_id'] = img_id ann_ids = self.coco.get_ann_ids(img_ids=[img_id]) raw_ann_info = self.coco.load_anns(ann_ids) total_ann_ids.extend(ann_ids) parsed_data_info = self.parse_data_info({ 'raw_ann_info': raw_ann_info, 'raw_img_info': raw_img_info }) data_list.append(parsed_data_info) if self.ANN_ID_UNIQUE: assert len(set(total_ann_ids)) == len( total_ann_ids ), f"Annotation ids in '{self.ann_file}' are not unique!" del self.coco return data_list @DATASETS.register_module() class Objects365V2Dataset(CocoDataset): """Objects365 v2 dataset for detection.""" METAINFO = { 'classes': ('Person', 'Sneakers', 'Chair', 'Other Shoes', 'Hat', 'Car', 'Lamp', 'Glasses', 'Bottle', 'Desk', 'Cup', 'Street Lights', 'Cabinet/shelf', 'Handbag/Satchel', 'Bracelet', 'Plate', 'Picture/Frame', 'Helmet', 'Book', 'Gloves', 'Storage box', 'Boat', 'Leather Shoes', 'Flower', 'Bench', 'Potted Plant', 'Bowl/Basin', 'Flag', 'Pillow', 'Boots', 'Vase', 'Microphone', 'Necklace', 'Ring', 'SUV', 'Wine Glass', 'Belt', 'Moniter/TV', 'Backpack', 'Umbrella', 'Traffic Light', 'Speaker', 'Watch', 'Tie', 'Trash bin Can', 'Slippers', 'Bicycle', 'Stool', 'Barrel/bucket', 'Van', 'Couch', 'Sandals', 'Bakset', 'Drum', 'Pen/Pencil', 'Bus', 'Wild Bird', 'High Heels', 'Motorcycle', 'Guitar', 'Carpet', 'Cell Phone', 'Bread', 'Camera', 'Canned', 'Truck', 'Traffic cone', 'Cymbal', 'Lifesaver', 'Towel', 'Stuffed Toy', 'Candle', 'Sailboat', 'Laptop', 'Awning', 'Bed', 'Faucet', 'Tent', 'Horse', 'Mirror', 'Power outlet', 'Sink', 'Apple', 'Air Conditioner', 'Knife', 'Hockey Stick', 'Paddle', 'Pickup Truck', 'Fork', 'Traffic Sign', 'Ballon', 'Tripod', 'Dog', 'Spoon', 'Clock', 'Pot', 'Cow', 'Cake', 'Dinning Table', 'Sheep', 'Hanger', 'Blackboard/Whiteboard', 'Napkin', 'Other Fish', 'Orange/Tangerine', 'Toiletry', 'Keyboard', 'Tomato', 'Lantern', 'Machinery Vehicle', 'Fan', 'Green Vegetables', 'Banana', 'Baseball Glove', 'Airplane', 'Mouse', 'Train', 'Pumpkin', 'Soccer', 'Skiboard', 'Luggage', 'Nightstand', 'Tea pot', 'Telephone', 'Trolley', 'Head Phone', 'Sports Car', 'Stop Sign', 'Dessert', 'Scooter', 'Stroller', 'Crane', 'Remote', 'Refrigerator', 'Oven', 'Lemon', 'Duck', 'Baseball Bat', 'Surveillance Camera', 'Cat', 'Jug', 'Broccoli', 'Piano', 'Pizza', 'Elephant', 'Skateboard', 'Surfboard', 'Gun', 'Skating and Skiing shoes', 'Gas stove', 'Donut', 'Bow Tie', 'Carrot', 'Toilet', 'Kite', 'Strawberry', 'Other Balls', 'Shovel', 'Pepper', 'Computer Box', 'Toilet Paper', 'Cleaning Products', 'Chopsticks', 'Microwave', 'Pigeon', 'Baseball', 'Cutting/chopping Board', 'Coffee Table', 'Side Table', 'Scissors', 'Marker', 'Pie', 'Ladder', 'Snowboard', 'Cookies', 'Radiator', 'Fire Hydrant', 'Basketball', 'Zebra', 'Grape', 'Giraffe', 'Potato', 'Sausage', 'Tricycle', 'Violin', 'Egg', 'Fire Extinguisher', 'Candy', 'Fire Truck', 'Billards', 'Converter', 'Bathtub', 'Wheelchair', 'Golf Club', 'Briefcase', 'Cucumber', 'Cigar/Cigarette ', 'Paint Brush', 'Pear', 'Heavy Truck', 'Hamburger', 'Extractor', 'Extention Cord', 'Tong', 'Tennis Racket', 'Folder', 'American Football', 'earphone', 'Mask', 'Kettle', 'Tennis', 'Ship', 'Swing', 'Coffee Machine', 'Slide', 'Carriage', 'Onion', 'Green beans', 'Projector', 'Frisbee', 'Washing Machine/Drying Machine', 'Chicken', 'Printer', 'Watermelon', 'Saxophone', 'Tissue', 'Toothbrush', 'Ice cream', 'Hotair ballon', 'Cello', 'French Fries', 'Scale', 'Trophy', 'Cabbage', 'Hot dog', 'Blender', 'Peach', 'Rice', 'Wallet/Purse', 'Volleyball', 'Deer', 'Goose', 'Tape', 'Tablet', 'Cosmetics', 'Trumpet', 'Pineapple', 'Golf Ball', 'Ambulance', 'Parking meter', 'Mango', 'Key', 'Hurdle', 'Fishing Rod', 'Medal', 'Flute', 'Brush', 'Penguin', 'Megaphone', 'Corn', 'Lettuce', 'Garlic', 'Swan', 'Helicopter', 'Green Onion', 'Sandwich', 'Nuts', 'Speed Limit Sign', 'Induction Cooker', 'Broom', 'Trombone', 'Plum', 'Rickshaw', 'Goldfish', 'Kiwi fruit', 'Router/modem', 'Poker Card', 'Toaster', 'Shrimp', 'Sushi', 'Cheese', 'Notepaper', 'Cherry', 'Pliers', 'CD', 'Pasta', 'Hammer', 'Cue', 'Avocado', 'Hamimelon', 'Flask', 'Mushroon', 'Screwdriver', 'Soap', 'Recorder', 'Bear', 'Eggplant', 'Board Eraser', 'Coconut', 'Tape Measur/ Ruler', 'Pig', 'Showerhead', 'Globe', 'Chips', 'Steak', 'Crosswalk Sign', 'Stapler', 'Campel', 'Formula 1 ', 'Pomegranate', 'Dishwasher', 'Crab', 'Hoverboard', 'Meat ball', 'Rice Cooker', 'Tuba', 'Calculator', 'Papaya', 'Antelope', 'Parrot', 'Seal', 'Buttefly', 'Dumbbell', 'Donkey', 'Lion', 'Urinal', 'Dolphin', 'Electric Drill', 'Hair Dryer', 'Egg tart', 'Jellyfish', 'Treadmill', 'Lighter', 'Grapefruit', 'Game board', 'Mop', 'Radish', 'Baozi', 'Target', 'French', 'Spring Rolls', 'Monkey', 'Rabbit', 'Pencil Case', 'Yak', 'Red Cabbage', 'Binoculars', 'Asparagus', 'Barbell', 'Scallop', 'Noddles', 'Comb', 'Dumpling', 'Oyster', 'Table Teniis paddle', 'Cosmetics Brush/Eyeliner Pencil', 'Chainsaw', 'Eraser', 'Lobster', 'Durian', 'Okra', 'Lipstick', 'Cosmetics Mirror', 'Curling', 'Table Tennis '), 'palette': None } COCOAPI = COCO # ann_id is unique in coco dataset. ANN_ID_UNIQUE = True def load_data_list(self) -> List[dict]: """Load annotations from an annotation file named as ``self.ann_file`` Returns: List[dict]: A list of annotation. """ # noqa: E501 with self.file_client.get_local_path(self.ann_file) as local_path: self.coco = self.COCOAPI(local_path) # The order of returned `cat_ids` will not # change with the order of the `classes` self.cat_ids = self.coco.get_cat_ids( cat_names=self.metainfo['classes']) self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)} self.cat_img_map = copy.deepcopy(self.coco.cat_img_map) img_ids = self.coco.get_img_ids() data_list = [] total_ann_ids = [] for img_id in img_ids: raw_img_info = self.coco.load_imgs([img_id])[0] raw_img_info['img_id'] = img_id ann_ids = self.coco.get_ann_ids(img_ids=[img_id]) raw_ann_info = self.coco.load_anns(ann_ids) total_ann_ids.extend(ann_ids) # file_name should be `patchX/xxx.jpg` file_name = osp.join( osp.split(osp.split(raw_img_info['file_name'])[0])[-1], osp.split(raw_img_info['file_name'])[-1]) if file_name in objv2_ignore_list: continue raw_img_info['file_name'] = file_name parsed_data_info = self.parse_data_info({ 'raw_ann_info': raw_ann_info, 'raw_img_info': raw_img_info }) data_list.append(parsed_data_info) if self.ANN_ID_UNIQUE: assert len(set(total_ann_ids)) == len( total_ann_ids ), f"Annotation ids in '{self.ann_file}' are not unique!" del self.coco return data_list ================================================ FILE: mmdet/datasets/openimages.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import csv import os.path as osp from collections import defaultdict from typing import Dict, List, Optional import numpy as np from mmengine.fileio import load from mmengine.utils import is_abs from mmdet.registry import DATASETS from .base_det_dataset import BaseDetDataset @DATASETS.register_module() class OpenImagesDataset(BaseDetDataset): """Open Images dataset for detection. Args: ann_file (str): Annotation file path. label_file (str): File path of the label description file that maps the classes names in MID format to their short descriptions. meta_file (str): File path to get image metas. hierarchy_file (str): The file path of the class hierarchy. image_level_ann_file (str): Human-verified image level annotation, which is used in evaluation. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. """ METAINFO: dict = dict(dataset_type='oid_v6') def __init__(self, label_file: str, meta_file: str, hierarchy_file: str, image_level_ann_file: Optional[str] = None, **kwargs) -> None: self.label_file = label_file self.meta_file = meta_file self.hierarchy_file = hierarchy_file self.image_level_ann_file = image_level_ann_file super().__init__(**kwargs) def load_data_list(self) -> List[dict]: """Load annotations from an annotation file named as ``self.ann_file`` Returns: List[dict]: A list of annotation. """ classes_names, label_id_mapping = self._parse_label_file( self.label_file) self._metainfo['classes'] = classes_names self.label_id_mapping = label_id_mapping if self.image_level_ann_file is not None: img_level_anns = self._parse_img_level_ann( self.image_level_ann_file) else: img_level_anns = None # OpenImagesMetric can get the relation matrix from the dataset meta relation_matrix = self._get_relation_matrix(self.hierarchy_file) self._metainfo['RELATION_MATRIX'] = relation_matrix data_list = [] with self.file_client.get_local_path(self.ann_file) as local_path: with open(local_path, 'r') as f: reader = csv.reader(f) last_img_id = None instances = [] for i, line in enumerate(reader): if i == 0: continue img_id = line[0] if last_img_id is None: last_img_id = img_id label_id = line[2] assert label_id in self.label_id_mapping label = int(self.label_id_mapping[label_id]) bbox = [ float(line[4]), # xmin float(line[6]), # ymin float(line[5]), # xmax float(line[7]) # ymax ] is_occluded = True if int(line[8]) == 1 else False is_truncated = True if int(line[9]) == 1 else False is_group_of = True if int(line[10]) == 1 else False is_depiction = True if int(line[11]) == 1 else False is_inside = True if int(line[12]) == 1 else False instance = dict( bbox=bbox, bbox_label=label, ignore_flag=0, is_occluded=is_occluded, is_truncated=is_truncated, is_group_of=is_group_of, is_depiction=is_depiction, is_inside=is_inside) last_img_path = osp.join(self.data_prefix['img'], f'{last_img_id}.jpg') if img_id != last_img_id: # switch to a new image, record previous image's data. data_info = dict( img_path=last_img_path, img_id=last_img_id, instances=instances, ) data_list.append(data_info) instances = [] instances.append(instance) last_img_id = img_id data_list.append( dict( img_path=last_img_path, img_id=last_img_id, instances=instances, )) # add image metas to data list img_metas = load( self.meta_file, file_format='pkl', file_client_args=self.file_client_args) assert len(img_metas) == len(data_list) for i, meta in enumerate(img_metas): img_id = data_list[i]['img_id'] assert f'{img_id}.jpg' == osp.split(meta['filename'])[-1] h, w = meta['ori_shape'][:2] data_list[i]['height'] = h data_list[i]['width'] = w # denormalize bboxes for j in range(len(data_list[i]['instances'])): data_list[i]['instances'][j]['bbox'][0] *= w data_list[i]['instances'][j]['bbox'][2] *= w data_list[i]['instances'][j]['bbox'][1] *= h data_list[i]['instances'][j]['bbox'][3] *= h # add image-level annotation if img_level_anns is not None: img_labels = [] confidences = [] img_ann_list = img_level_anns.get(img_id, []) for ann in img_ann_list: img_labels.append(int(ann['image_level_label'])) confidences.append(float(ann['confidence'])) data_list[i]['image_level_labels'] = np.array( img_labels, dtype=np.int64) data_list[i]['confidences'] = np.array( confidences, dtype=np.float32) return data_list def _parse_label_file(self, label_file: str) -> tuple: """Get classes name and index mapping from cls-label-description file. Args: label_file (str): File path of the label description file that maps the classes names in MID format to their short descriptions. Returns: tuple: Class name of OpenImages. """ index_list = [] classes_names = [] with self.file_client.get_local_path(label_file) as local_path: with open(local_path, 'r') as f: reader = csv.reader(f) for line in reader: # self.cat2label[line[0]] = line[1] classes_names.append(line[1]) index_list.append(line[0]) index_mapping = {index: i for i, index in enumerate(index_list)} return classes_names, index_mapping def _parse_img_level_ann(self, img_level_ann_file: str) -> Dict[str, List[dict]]: """Parse image level annotations from csv style ann_file. Args: img_level_ann_file (str): CSV style image level annotation file path. Returns: Dict[str, List[dict]]: Annotations where item of the defaultdict indicates an image, each of which has (n) dicts. Keys of dicts are: - `image_level_label` (int): Label id. - `confidence` (float): Labels that are human-verified to be present in an image have confidence = 1 (positive labels). Labels that are human-verified to be absent from an image have confidence = 0 (negative labels). Machine-generated labels have fractional confidences, generally >= 0.5. The higher the confidence, the smaller the chance for the label to be a false positive. """ item_lists = defaultdict(list) with self.file_client.get_local_path(img_level_ann_file) as local_path: with open(local_path, 'r') as f: reader = csv.reader(f) for i, line in enumerate(reader): if i == 0: continue img_id = line[0] item_lists[img_id].append( dict( image_level_label=int( self.label_id_mapping[line[2]]), confidence=float(line[3]))) return item_lists def _get_relation_matrix(self, hierarchy_file: str) -> np.ndarray: """Get the matrix of class hierarchy from the hierarchy file. Hierarchy for 600 classes can be found at https://storage.googleapis.com/openimag es/2018_04/bbox_labels_600_hierarchy_visualizer/circle.html. Args: hierarchy_file (str): File path to the hierarchy for classes. Returns: np.ndarray: The matrix of the corresponding relationship between the parent class and the child class, of shape (class_num, class_num). """ # noqa hierarchy = load( hierarchy_file, file_format='json', file_client_args=self.file_client_args) class_num = len(self._metainfo['classes']) relation_matrix = np.eye(class_num, class_num) relation_matrix = self._convert_hierarchy_tree(hierarchy, relation_matrix) return relation_matrix def _convert_hierarchy_tree(self, hierarchy_map: dict, relation_matrix: np.ndarray, parents: list = [], get_all_parents: bool = True) -> np.ndarray: """Get matrix of the corresponding relationship between the parent class and the child class. Args: hierarchy_map (dict): Including label name and corresponding subcategory. Keys of dicts are: - `LabeName` (str): Name of the label. - `Subcategory` (dict | list): Corresponding subcategory(ies). relation_matrix (ndarray): The matrix of the corresponding relationship between the parent class and the child class, of shape (class_num, class_num). parents (list): Corresponding parent class. get_all_parents (bool): Whether get all parent names. Default: True Returns: ndarray: The matrix of the corresponding relationship between the parent class and the child class, of shape (class_num, class_num). """ if 'Subcategory' in hierarchy_map: for node in hierarchy_map['Subcategory']: if 'LabelName' in node: children_name = node['LabelName'] children_index = self.label_id_mapping[children_name] children = [children_index] else: continue if len(parents) > 0: for parent_index in parents: if get_all_parents: children.append(parent_index) relation_matrix[children_index, parent_index] = 1 relation_matrix = self._convert_hierarchy_tree( node, relation_matrix, parents=children) return relation_matrix def _join_prefix(self): """Join ``self.data_root`` with annotation path.""" super()._join_prefix() if not is_abs(self.label_file) and self.label_file: self.label_file = osp.join(self.data_root, self.label_file) if not is_abs(self.meta_file) and self.meta_file: self.meta_file = osp.join(self.data_root, self.meta_file) if not is_abs(self.hierarchy_file) and self.hierarchy_file: self.hierarchy_file = osp.join(self.data_root, self.hierarchy_file) if self.image_level_ann_file and not is_abs(self.image_level_ann_file): self.image_level_ann_file = osp.join(self.data_root, self.image_level_ann_file) @DATASETS.register_module() class OpenImagesChallengeDataset(OpenImagesDataset): """Open Images Challenge dataset for detection. Args: ann_file (str): Open Images Challenge box annotation in txt format. """ METAINFO: dict = dict(dataset_type='oid_challenge') def __init__(self, ann_file: str, **kwargs) -> None: if not ann_file.endswith('txt'): raise TypeError('The annotation file of Open Images Challenge ' 'should be a txt file.') super().__init__(ann_file=ann_file, **kwargs) def load_data_list(self) -> List[dict]: """Load annotations from an annotation file named as ``self.ann_file`` Returns: List[dict]: A list of annotation. """ classes_names, label_id_mapping = self._parse_label_file( self.label_file) self._metainfo['classes'] = classes_names self.label_id_mapping = label_id_mapping if self.image_level_ann_file is not None: img_level_anns = self._parse_img_level_ann( self.image_level_ann_file) else: img_level_anns = None # OpenImagesMetric can get the relation matrix from the dataset meta relation_matrix = self._get_relation_matrix(self.hierarchy_file) self._metainfo['RELATION_MATRIX'] = relation_matrix data_list = [] with self.file_client.get_local_path(self.ann_file) as local_path: with open(local_path, 'r') as f: lines = f.readlines() i = 0 while i < len(lines): instances = [] filename = lines[i].rstrip() i += 2 img_gt_size = int(lines[i]) i += 1 for j in range(img_gt_size): sp = lines[i + j].split() instances.append( dict( bbox=[ float(sp[1]), float(sp[2]), float(sp[3]), float(sp[4]) ], bbox_label=int(sp[0]) - 1, # labels begin from 1 ignore_flag=0, is_group_ofs=True if int(sp[5]) == 1 else False)) i += img_gt_size data_list.append( dict( img_path=osp.join(self.data_prefix['img'], filename), instances=instances, )) # add image metas to data list img_metas = load( self.meta_file, file_format='pkl', file_client_args=self.file_client_args) assert len(img_metas) == len(data_list) for i, meta in enumerate(img_metas): img_id = osp.split(data_list[i]['img_path'])[-1][:-4] assert img_id == osp.split(meta['filename'])[-1][:-4] h, w = meta['ori_shape'][:2] data_list[i]['height'] = h data_list[i]['width'] = w data_list[i]['img_id'] = img_id # denormalize bboxes for j in range(len(data_list[i]['instances'])): data_list[i]['instances'][j]['bbox'][0] *= w data_list[i]['instances'][j]['bbox'][2] *= w data_list[i]['instances'][j]['bbox'][1] *= h data_list[i]['instances'][j]['bbox'][3] *= h # add image-level annotation if img_level_anns is not None: img_labels = [] confidences = [] img_ann_list = img_level_anns.get(img_id, []) for ann in img_ann_list: img_labels.append(int(ann['image_level_label'])) confidences.append(float(ann['confidence'])) data_list[i]['image_level_labels'] = np.array( img_labels, dtype=np.int64) data_list[i]['confidences'] = np.array( confidences, dtype=np.float32) return data_list def _parse_label_file(self, label_file: str) -> tuple: """Get classes name and index mapping from cls-label-description file. Args: label_file (str): File path of the label description file that maps the classes names in MID format to their short descriptions. Returns: tuple: Class name of OpenImages. """ label_list = [] id_list = [] index_mapping = {} with self.file_client.get_local_path(label_file) as local_path: with open(local_path, 'r') as f: reader = csv.reader(f) for line in reader: label_name = line[0] label_id = int(line[2]) label_list.append(line[1]) id_list.append(label_id) index_mapping[label_name] = label_id - 1 indexes = np.argsort(id_list) classes_names = [] for index in indexes: classes_names.append(label_list[index]) return classes_names, index_mapping def _parse_img_level_ann(self, image_level_ann_file): """Parse image level annotations from csv style ann_file. Args: image_level_ann_file (str): CSV style image level annotation file path. Returns: defaultdict[list[dict]]: Annotations where item of the defaultdict indicates an image, each of which has (n) dicts. Keys of dicts are: - `image_level_label` (int): of shape 1. - `confidence` (float): of shape 1. """ item_lists = defaultdict(list) with self.file_client.get_local_path( image_level_ann_file) as local_path: with open(local_path, 'r') as f: reader = csv.reader(f) i = -1 for line in reader: i += 1 if i == 0: continue else: img_id = line[0] label_id = line[1] assert label_id in self.label_id_mapping image_level_label = int( self.label_id_mapping[label_id]) confidence = float(line[2]) item_lists[img_id].append( dict( image_level_label=image_level_label, confidence=confidence)) return item_lists def _get_relation_matrix(self, hierarchy_file: str) -> np.ndarray: """Get the matrix of class hierarchy from the hierarchy file. Args: hierarchy_file (str): File path to the hierarchy for classes. Returns: np.ndarray: The matrix of the corresponding relationship between the parent class and the child class, of shape (class_num, class_num). """ with self.file_client.get_local_path(hierarchy_file) as local_path: class_label_tree = np.load(local_path, allow_pickle=True) return class_label_tree[1:, 1:] ================================================ FILE: mmdet/datasets/samplers/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .batch_sampler import AspectRatioBatchSampler from .class_aware_sampler import ClassAwareSampler from .multi_source_sampler import GroupMultiSourceSampler, MultiSourceSampler __all__ = [ 'ClassAwareSampler', 'AspectRatioBatchSampler', 'MultiSourceSampler', 'GroupMultiSourceSampler' ] ================================================ FILE: mmdet/datasets/samplers/batch_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Sequence from torch.utils.data import BatchSampler, Sampler from mmdet.registry import DATA_SAMPLERS # TODO: maybe replace with a data_loader wrapper @DATA_SAMPLERS.register_module() class AspectRatioBatchSampler(BatchSampler): """A sampler wrapper for grouping images with similar aspect ratio (< 1 or. >= 1) into a same batch. Args: sampler (Sampler): Base sampler. batch_size (int): Size of mini-batch. drop_last (bool): If ``True``, the sampler will drop the last batch if its size would be less than ``batch_size``. """ def __init__(self, sampler: Sampler, batch_size: int, drop_last: bool = False) -> None: if not isinstance(sampler, Sampler): raise TypeError('sampler should be an instance of ``Sampler``, ' f'but got {sampler}') if not isinstance(batch_size, int) or batch_size <= 0: raise ValueError('batch_size should be a positive integer value, ' f'but got batch_size={batch_size}') self.sampler = sampler self.batch_size = batch_size self.drop_last = drop_last # two groups for w < h and w >= h self._aspect_ratio_buckets = [[] for _ in range(2)] def __iter__(self) -> Sequence[int]: for idx in self.sampler: data_info = self.sampler.dataset.get_data_info(idx) width, height = data_info['width'], data_info['height'] bucket_id = 0 if width < height else 1 bucket = self._aspect_ratio_buckets[bucket_id] bucket.append(idx) # yield a batch of indices in the same aspect ratio group if len(bucket) == self.batch_size: yield bucket[:] del bucket[:] # yield the rest data and reset the bucket left_data = self._aspect_ratio_buckets[0] + self._aspect_ratio_buckets[ 1] self._aspect_ratio_buckets = [[] for _ in range(2)] while len(left_data) > 0: if len(left_data) <= self.batch_size: if not self.drop_last: yield left_data[:] left_data = [] else: yield left_data[:self.batch_size] left_data = left_data[self.batch_size:] def __len__(self) -> int: if self.drop_last: return len(self.sampler) // self.batch_size else: return (len(self.sampler) + self.batch_size - 1) // self.batch_size ================================================ FILE: mmdet/datasets/samplers/class_aware_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from typing import Dict, Iterator, Optional, Union import numpy as np import torch from mmengine.dataset import BaseDataset from mmengine.dist import get_dist_info, sync_random_seed from torch.utils.data import Sampler from mmdet.registry import DATA_SAMPLERS @DATA_SAMPLERS.register_module() class ClassAwareSampler(Sampler): r"""Sampler that restricts data loading to the label of the dataset. A class-aware sampling strategy to effectively tackle the non-uniform class distribution. The length of the training data is consistent with source data. Simple improvements based on `Relay Backpropagation for Effective Learning of Deep Convolutional Neural Networks `_ The implementation logic is referred to https://github.com/Sense-X/TSD/blob/master/mmdet/datasets/samplers/distributed_classaware_sampler.py Args: dataset: Dataset used for sampling. seed (int, optional): random seed used to shuffle the sampler. This number should be identical across all processes in the distributed group. Defaults to None. num_sample_class (int): The number of samples taken from each per-label list. Defaults to 1. """ def __init__(self, dataset: BaseDataset, seed: Optional[int] = None, num_sample_class: int = 1) -> None: rank, world_size = get_dist_info() self.rank = rank self.world_size = world_size self.dataset = dataset self.epoch = 0 # Must be the same across all workers. If None, will use a # random seed shared among workers # (require synchronization among all workers) if seed is None: seed = sync_random_seed() self.seed = seed # The number of samples taken from each per-label list assert num_sample_class > 0 and isinstance(num_sample_class, int) self.num_sample_class = num_sample_class # Get per-label image list from dataset self.cat_dict = self.get_cat2imgs() self.num_samples = int(math.ceil(len(self.dataset) * 1.0 / world_size)) self.total_size = self.num_samples * self.world_size # get number of images containing each category self.num_cat_imgs = [len(x) for x in self.cat_dict.values()] # filter labels without images self.valid_cat_inds = [ i for i, length in enumerate(self.num_cat_imgs) if length != 0 ] self.num_classes = len(self.valid_cat_inds) def get_cat2imgs(self) -> Dict[int, list]: """Get a dict with class as key and img_ids as values. Returns: dict[int, list]: A dict of per-label image list, the item of the dict indicates a label index, corresponds to the image index that contains the label. """ classes = self.dataset.metainfo.get('classes', None) if classes is None: raise ValueError('dataset metainfo must contain `classes`') # sort the label index cat2imgs = {i: [] for i in range(len(classes))} for i in range(len(self.dataset)): cat_ids = set(self.dataset.get_cat_ids(i)) for cat in cat_ids: cat2imgs[cat].append(i) return cat2imgs def __iter__(self) -> Iterator[int]: # deterministically shuffle based on epoch g = torch.Generator() g.manual_seed(self.epoch + self.seed) # initialize label list label_iter_list = RandomCycleIter(self.valid_cat_inds, generator=g) # initialize each per-label image list data_iter_dict = dict() for i in self.valid_cat_inds: data_iter_dict[i] = RandomCycleIter(self.cat_dict[i], generator=g) def gen_cat_img_inds(cls_list, data_dict, num_sample_cls): """Traverse the categories and extract `num_sample_cls` image indexes of the corresponding categories one by one.""" id_indices = [] for _ in range(len(cls_list)): cls_idx = next(cls_list) for _ in range(num_sample_cls): id = next(data_dict[cls_idx]) id_indices.append(id) return id_indices # deterministically shuffle based on epoch num_bins = int( math.ceil(self.total_size * 1.0 / self.num_classes / self.num_sample_class)) indices = [] for i in range(num_bins): indices += gen_cat_img_inds(label_iter_list, data_iter_dict, self.num_sample_class) # fix extra samples to make it evenly divisible if len(indices) >= self.total_size: indices = indices[:self.total_size] else: indices += indices[:(self.total_size - len(indices))] assert len(indices) == self.total_size # subsample offset = self.num_samples * self.rank indices = indices[offset:offset + self.num_samples] assert len(indices) == self.num_samples return iter(indices) def __len__(self) -> int: """The number of samples in this rank.""" return self.num_samples def set_epoch(self, epoch: int) -> None: """Sets the epoch for this sampler. When :attr:`shuffle=True`, this ensures all replicas use a different random ordering for each epoch. Otherwise, the next iteration of this sampler will yield the same ordering. Args: epoch (int): Epoch number. """ self.epoch = epoch class RandomCycleIter: """Shuffle the list and do it again after the list have traversed. The implementation logic is referred to https://github.com/wutong16/DistributionBalancedLoss/blob/master/mllt/datasets/loader/sampler.py Example: >>> label_list = [0, 1, 2, 4, 5] >>> g = torch.Generator() >>> g.manual_seed(0) >>> label_iter_list = RandomCycleIter(label_list, generator=g) >>> index = next(label_iter_list) Args: data (list or ndarray): The data that needs to be shuffled. generator: An torch.Generator object, which is used in setting the seed for generating random numbers. """ # noqa: W605 def __init__(self, data: Union[list, np.ndarray], generator: torch.Generator = None) -> None: self.data = data self.length = len(data) self.index = torch.randperm(self.length, generator=generator).numpy() self.i = 0 self.generator = generator def __iter__(self) -> Iterator: return self def __len__(self) -> int: return len(self.data) def __next__(self): if self.i == self.length: self.index = torch.randperm( self.length, generator=self.generator).numpy() self.i = 0 idx = self.data[self.index[self.i]] self.i += 1 return idx ================================================ FILE: mmdet/datasets/samplers/multi_source_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import itertools from typing import Iterator, List, Optional, Sized, Union import numpy as np import torch from mmengine.dataset import BaseDataset from mmengine.dist import get_dist_info, sync_random_seed from torch.utils.data import Sampler from mmdet.registry import DATA_SAMPLERS @DATA_SAMPLERS.register_module() class MultiSourceSampler(Sampler): r"""Multi-Source Infinite Sampler. According to the sampling ratio, sample data from different datasets to form batches. Args: dataset (Sized): The dataset. batch_size (int): Size of mini-batch. source_ratio (list[int | float]): The sampling ratio of different source datasets in a mini-batch. shuffle (bool): Whether shuffle the dataset or not. Defaults to True. seed (int, optional): Random seed. If None, set a random seed. Defaults to None. Examples: >>> dataset_type = 'ConcatDataset' >>> sub_dataset_type = 'CocoDataset' >>> data_root = 'data/coco/' >>> sup_ann = '../coco_semi_annos/instances_train2017.1@10.json' >>> unsup_ann = '../coco_semi_annos/' \ >>> 'instances_train2017.1@10-unlabeled.json' >>> dataset = dict(type=dataset_type, >>> datasets=[ >>> dict( >>> type=sub_dataset_type, >>> data_root=data_root, >>> ann_file=sup_ann, >>> data_prefix=dict(img='train2017/'), >>> filter_cfg=dict(filter_empty_gt=True, min_size=32), >>> pipeline=sup_pipeline), >>> dict( >>> type=sub_dataset_type, >>> data_root=data_root, >>> ann_file=unsup_ann, >>> data_prefix=dict(img='train2017/'), >>> filter_cfg=dict(filter_empty_gt=True, min_size=32), >>> pipeline=unsup_pipeline), >>> ]) >>> train_dataloader = dict( >>> batch_size=5, >>> num_workers=5, >>> persistent_workers=True, >>> sampler=dict(type='MultiSourceSampler', >>> batch_size=5, source_ratio=[1, 4]), >>> batch_sampler=None, >>> dataset=dataset) """ def __init__(self, dataset: Sized, batch_size: int, source_ratio: List[Union[int, float]], shuffle: bool = True, seed: Optional[int] = None) -> None: assert hasattr(dataset, 'cumulative_sizes'),\ f'The dataset must be ConcatDataset, but get {dataset}' assert isinstance(batch_size, int) and batch_size > 0, \ 'batch_size must be a positive integer value, ' \ f'but got batch_size={batch_size}' assert isinstance(source_ratio, list), \ f'source_ratio must be a list, but got source_ratio={source_ratio}' assert len(source_ratio) == len(dataset.cumulative_sizes), \ 'The length of source_ratio must be equal to ' \ f'the number of datasets, but got source_ratio={source_ratio}' rank, world_size = get_dist_info() self.rank = rank self.world_size = world_size self.dataset = dataset self.cumulative_sizes = [0] + dataset.cumulative_sizes self.batch_size = batch_size self.source_ratio = source_ratio self.num_per_source = [ int(batch_size * sr / sum(source_ratio)) for sr in source_ratio ] self.num_per_source[0] = batch_size - sum(self.num_per_source[1:]) assert sum(self.num_per_source) == batch_size, \ 'The sum of num_per_source must be equal to ' \ f'batch_size, but get {self.num_per_source}' self.seed = sync_random_seed() if seed is None else seed self.shuffle = shuffle self.source2inds = { source: self._indices_of_rank(len(ds)) for source, ds in enumerate(dataset.datasets) } def _infinite_indices(self, sample_size: int) -> Iterator[int]: """Infinitely yield a sequence of indices.""" g = torch.Generator() g.manual_seed(self.seed) while True: if self.shuffle: yield from torch.randperm(sample_size, generator=g).tolist() else: yield from torch.arange(sample_size).tolist() def _indices_of_rank(self, sample_size: int) -> Iterator[int]: """Slice the infinite indices by rank.""" yield from itertools.islice( self._infinite_indices(sample_size), self.rank, None, self.world_size) def __iter__(self) -> Iterator[int]: batch_buffer = [] while True: for source, num in enumerate(self.num_per_source): batch_buffer_per_source = [] for idx in self.source2inds[source]: idx += self.cumulative_sizes[source] batch_buffer_per_source.append(idx) if len(batch_buffer_per_source) == num: batch_buffer += batch_buffer_per_source break yield from batch_buffer batch_buffer = [] def __len__(self) -> int: return len(self.dataset) def set_epoch(self, epoch: int) -> None: """Not supported in `epoch-based runner.""" pass @DATA_SAMPLERS.register_module() class GroupMultiSourceSampler(MultiSourceSampler): r"""Group Multi-Source Infinite Sampler. According to the sampling ratio, sample data from different datasets but the same group to form batches. Args: dataset (Sized): The dataset. batch_size (int): Size of mini-batch. source_ratio (list[int | float]): The sampling ratio of different source datasets in a mini-batch. shuffle (bool): Whether shuffle the dataset or not. Defaults to True. seed (int, optional): Random seed. If None, set a random seed. Defaults to None. """ def __init__(self, dataset: BaseDataset, batch_size: int, source_ratio: List[Union[int, float]], shuffle: bool = True, seed: Optional[int] = None) -> None: super().__init__( dataset=dataset, batch_size=batch_size, source_ratio=source_ratio, shuffle=shuffle, seed=seed) self._get_source_group_info() self.group_source2inds = [{ source: self._indices_of_rank(self.group2size_per_source[source][group]) for source in range(len(dataset.datasets)) } for group in range(len(self.group_ratio))] def _get_source_group_info(self) -> None: self.group2size_per_source = [{0: 0, 1: 0}, {0: 0, 1: 0}] self.group2inds_per_source = [{0: [], 1: []}, {0: [], 1: []}] for source, dataset in enumerate(self.dataset.datasets): for idx in range(len(dataset)): data_info = dataset.get_data_info(idx) width, height = data_info['width'], data_info['height'] group = 0 if width < height else 1 self.group2size_per_source[source][group] += 1 self.group2inds_per_source[source][group].append(idx) self.group_sizes = np.zeros(2, dtype=np.int64) for group2size in self.group2size_per_source: for group, size in group2size.items(): self.group_sizes[group] += size self.group_ratio = self.group_sizes / sum(self.group_sizes) def __iter__(self) -> Iterator[int]: batch_buffer = [] while True: group = np.random.choice( list(range(len(self.group_ratio))), p=self.group_ratio) for source, num in enumerate(self.num_per_source): batch_buffer_per_source = [] for idx in self.group_source2inds[group][source]: idx = self.group2inds_per_source[source][group][ idx] + self.cumulative_sizes[source] batch_buffer_per_source.append(idx) if len(batch_buffer_per_source) == num: batch_buffer += batch_buffer_per_source break yield from batch_buffer batch_buffer = [] ================================================ FILE: mmdet/datasets/transforms/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .augment_wrappers import AutoAugment, RandAugment from .colorspace import (AutoContrast, Brightness, Color, ColorTransform, Contrast, Equalize, Invert, Posterize, Sharpness, Solarize, SolarizeAdd) from .formatting import ImageToTensor, PackDetInputs, ToTensor, Transpose from .geometric import (GeomTransform, Rotate, ShearX, ShearY, TranslateX, TranslateY) from .instaboost import InstaBoost from .loading import (FilterAnnotations, InferencerLoader, LoadAnnotations, LoadEmptyAnnotations, LoadImageFromNDArray, LoadMultiChannelImageFromFiles, LoadPanopticAnnotations, LoadProposals) from .transforms import (Albu, CachedMixUp, CachedMosaic, CopyPaste, CutOut, Expand, FixShapeResize, MinIoURandomCrop, MixUp, Mosaic, Pad, PhotoMetricDistortion, RandomAffine, RandomCenterCropPad, RandomCrop, RandomErasing, RandomFlip, RandomShift, Resize, SegRescale, YOLOXHSVRandomAug) from .wrappers import MultiBranch, ProposalBroadcaster, RandomOrder __all__ = [ 'PackDetInputs', 'ToTensor', 'ImageToTensor', 'Transpose', 'LoadImageFromNDArray', 'LoadAnnotations', 'LoadPanopticAnnotations', 'LoadMultiChannelImageFromFiles', 'LoadProposals', 'Resize', 'RandomFlip', 'RandomCrop', 'SegRescale', 'MinIoURandomCrop', 'Expand', 'PhotoMetricDistortion', 'Albu', 'InstaBoost', 'RandomCenterCropPad', 'AutoAugment', 'CutOut', 'ShearX', 'ShearY', 'Rotate', 'Color', 'Equalize', 'Brightness', 'Contrast', 'TranslateX', 'TranslateY', 'RandomShift', 'Mosaic', 'MixUp', 'RandomAffine', 'YOLOXHSVRandomAug', 'CopyPaste', 'FilterAnnotations', 'Pad', 'GeomTransform', 'ColorTransform', 'RandAugment', 'Sharpness', 'Solarize', 'SolarizeAdd', 'Posterize', 'AutoContrast', 'Invert', 'MultiBranch', 'RandomErasing', 'LoadEmptyAnnotations', 'RandomOrder', 'CachedMosaic', 'CachedMixUp', 'FixShapeResize', 'ProposalBroadcaster', 'InferencerLoader' ] ================================================ FILE: mmdet/datasets/transforms/augment_wrappers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Union import numpy as np from mmcv.transforms import RandomChoice from mmcv.transforms.utils import cache_randomness from mmengine.config import ConfigDict from mmdet.registry import TRANSFORMS # AutoAugment uses reinforcement learning to search for # some widely useful data augmentation strategies, # here we provide AUTOAUG_POLICIES_V0. # For AUTOAUG_POLICIES_V0, each tuple is an augmentation # operation of the form (operation, probability, magnitude). # Each element in policies is a policy that will be applied # sequentially on the image. # RandAugment defines a data augmentation search space, RANDAUG_SPACE, # sampling 1~3 data augmentations each time, and # setting the magnitude of each data augmentation randomly, # which will be applied sequentially on the image. _MAX_LEVEL = 10 AUTOAUG_POLICIES_V0 = [ [('Equalize', 0.8, 1), ('ShearY', 0.8, 4)], [('Color', 0.4, 9), ('Equalize', 0.6, 3)], [('Color', 0.4, 1), ('Rotate', 0.6, 8)], [('Solarize', 0.8, 3), ('Equalize', 0.4, 7)], [('Solarize', 0.4, 2), ('Solarize', 0.6, 2)], [('Color', 0.2, 0), ('Equalize', 0.8, 8)], [('Equalize', 0.4, 8), ('SolarizeAdd', 0.8, 3)], [('ShearX', 0.2, 9), ('Rotate', 0.6, 8)], [('Color', 0.6, 1), ('Equalize', 1.0, 2)], [('Invert', 0.4, 9), ('Rotate', 0.6, 0)], [('Equalize', 1.0, 9), ('ShearY', 0.6, 3)], [('Color', 0.4, 7), ('Equalize', 0.6, 0)], [('Posterize', 0.4, 6), ('AutoContrast', 0.4, 7)], [('Solarize', 0.6, 8), ('Color', 0.6, 9)], [('Solarize', 0.2, 4), ('Rotate', 0.8, 9)], [('Rotate', 1.0, 7), ('TranslateY', 0.8, 9)], [('ShearX', 0.0, 0), ('Solarize', 0.8, 4)], [('ShearY', 0.8, 0), ('Color', 0.6, 4)], [('Color', 1.0, 0), ('Rotate', 0.6, 2)], [('Equalize', 0.8, 4), ('Equalize', 0.0, 8)], [('Equalize', 1.0, 4), ('AutoContrast', 0.6, 2)], [('ShearY', 0.4, 7), ('SolarizeAdd', 0.6, 7)], [('Posterize', 0.8, 2), ('Solarize', 0.6, 10)], [('Solarize', 0.6, 8), ('Equalize', 0.6, 1)], [('Color', 0.8, 6), ('Rotate', 0.4, 5)], ] def policies_v0(): """Autoaugment policies that was used in AutoAugment Paper.""" policies = list() for policy_args in AUTOAUG_POLICIES_V0: policy = list() for args in policy_args: policy.append(dict(type=args[0], prob=args[1], level=args[2])) policies.append(policy) return policies RANDAUG_SPACE = [[dict(type='AutoContrast')], [dict(type='Equalize')], [dict(type='Invert')], [dict(type='Rotate')], [dict(type='Posterize')], [dict(type='Solarize')], [dict(type='SolarizeAdd')], [dict(type='Color')], [dict(type='Contrast')], [dict(type='Brightness')], [dict(type='Sharpness')], [dict(type='ShearX')], [dict(type='ShearY')], [dict(type='TranslateX')], [dict(type='TranslateY')]] def level_to_mag(level: Optional[int], min_mag: float, max_mag: float) -> float: """Map from level to magnitude.""" if level is None: return round(np.random.rand() * (max_mag - min_mag) + min_mag, 1) else: return round(level / _MAX_LEVEL * (max_mag - min_mag) + min_mag, 1) @TRANSFORMS.register_module() class AutoAugment(RandomChoice): """Auto augmentation. This data augmentation is proposed in `AutoAugment: Learning Augmentation Policies from Data `_ and in `Learning Data Augmentation Strategies for Object Detection `_. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_ignore_flags (bool) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - img_shape - gt_bboxes - gt_bboxes_labels - gt_masks - gt_ignore_flags - gt_seg_map Added Keys: - homography_matrix Args: policies (List[List[Union[dict, ConfigDict]]]): The policies of auto augmentation.Each policy in ``policies`` is a specific augmentation policy, and is composed by several augmentations. When AutoAugment is called, a random policy in ``policies`` will be selected to augment images. Defaults to policy_v0(). prob (list[float], optional): The probabilities associated with each policy. The length should be equal to the policy number and the sum should be 1. If not given, a uniform distribution will be assumed. Defaults to None. Examples: >>> policies = [ >>> [ >>> dict(type='Sharpness', prob=0.0, level=8), >>> dict(type='ShearX', prob=0.4, level=0,) >>> ], >>> [ >>> dict(type='Rotate', prob=0.6, level=10), >>> dict(type='Color', prob=1.0, level=6) >>> ] >>> ] >>> augmentation = AutoAugment(policies) >>> img = np.ones(100, 100, 3) >>> gt_bboxes = np.ones(10, 4) >>> results = dict(img=img, gt_bboxes=gt_bboxes) >>> results = augmentation(results) """ def __init__(self, policies: List[List[Union[dict, ConfigDict]]] = policies_v0(), prob: Optional[List[float]] = None) -> None: assert isinstance(policies, list) and len(policies) > 0, \ 'Policies must be a non-empty list.' for policy in policies: assert isinstance(policy, list) and len(policy) > 0, \ 'Each policy in policies must be a non-empty list.' for augment in policy: assert isinstance(augment, dict) and 'type' in augment, \ 'Each specific augmentation must be a dict with key' \ ' "type".' super().__init__(transforms=policies, prob=prob) self.policies = policies def __repr__(self) -> str: return f'{self.__class__.__name__}(policies={self.policies}, ' \ f'prob={self.prob})' @TRANSFORMS.register_module() class RandAugment(RandomChoice): """Rand augmentation. This data augmentation is proposed in `RandAugment: Practical automated data augmentation with a reduced search space `_. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_ignore_flags (bool) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - img_shape - gt_bboxes - gt_bboxes_labels - gt_masks - gt_ignore_flags - gt_seg_map Added Keys: - homography_matrix Args: aug_space (List[List[Union[dict, ConfigDict]]]): The augmentation space of rand augmentation. Each augmentation transform in ``aug_space`` is a specific transform, and is composed by several augmentations. When RandAugment is called, a random transform in ``aug_space`` will be selected to augment images. Defaults to aug_space. aug_num (int): Number of augmentation to apply equentially. Defaults to 2. prob (list[float], optional): The probabilities associated with each augmentation. The length should be equal to the augmentation space and the sum should be 1. If not given, a uniform distribution will be assumed. Defaults to None. Examples: >>> aug_space = [ >>> dict(type='Sharpness'), >>> dict(type='ShearX'), >>> dict(type='Color'), >>> ], >>> augmentation = RandAugment(aug_space) >>> img = np.ones(100, 100, 3) >>> gt_bboxes = np.ones(10, 4) >>> results = dict(img=img, gt_bboxes=gt_bboxes) >>> results = augmentation(results) """ def __init__(self, aug_space: List[Union[dict, ConfigDict]] = RANDAUG_SPACE, aug_num: int = 2, prob: Optional[List[float]] = None) -> None: assert isinstance(aug_space, list) and len(aug_space) > 0, \ 'Augmentation space must be a non-empty list.' for aug in aug_space: assert isinstance(aug, list) and len(aug) == 1, \ 'Each augmentation in aug_space must be a list.' for transform in aug: assert isinstance(transform, dict) and 'type' in transform, \ 'Each specific transform must be a dict with key' \ ' "type".' super().__init__(transforms=aug_space, prob=prob) self.aug_space = aug_space self.aug_num = aug_num @cache_randomness def random_pipeline_index(self): indices = np.arange(len(self.transforms)) return np.random.choice( indices, self.aug_num, p=self.prob, replace=False) def transform(self, results: dict) -> dict: """Transform function to use RandAugment. Args: results (dict): Result dict from loading pipeline. Returns: dict: Result dict with RandAugment. """ for idx in self.random_pipeline_index(): results = self.transforms[idx](results) return results def __repr__(self) -> str: return f'{self.__class__.__name__}(' \ f'aug_space={self.aug_space}, '\ f'aug_num={self.aug_num}, ' \ f'prob={self.prob})' ================================================ FILE: mmdet/datasets/transforms/colorspace.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from typing import Optional import mmcv import numpy as np from mmcv.transforms import BaseTransform from mmcv.transforms.utils import cache_randomness from mmdet.registry import TRANSFORMS from .augment_wrappers import _MAX_LEVEL, level_to_mag @TRANSFORMS.register_module() class ColorTransform(BaseTransform): """Base class for color transformations. All color transformations need to inherit from this base class. ``ColorTransform`` unifies the class attributes and class functions of color transformations (Color, Brightness, Contrast, Sharpness, Solarize, SolarizeAdd, Equalize, AutoContrast, Invert, and Posterize), and only distort color channels, without impacting the locations of the instances. Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing the geometric transformation and should be in range [0, 1]. Defaults to 1.0. level (int, optional): The level should be in range [0, _MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum magnitude for color transformation. Defaults to 0.1. max_mag (float): The maximum magnitude for color transformation. Defaults to 1.9. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.1, max_mag: float = 1.9) -> None: assert 0 <= prob <= 1.0, f'The probability of the transformation ' \ f'should be in range [0,1], got {prob}.' assert level is None or isinstance(level, int), \ f'The level should be None or type int, got {type(level)}.' assert level is None or 0 <= level <= _MAX_LEVEL, \ f'The level should be in range [0,{_MAX_LEVEL}], got {level}.' assert isinstance(min_mag, float), \ f'min_mag should be type float, got {type(min_mag)}.' assert isinstance(max_mag, float), \ f'max_mag should be type float, got {type(max_mag)}.' assert min_mag <= max_mag, \ f'min_mag should smaller than max_mag, ' \ f'got min_mag={min_mag} and max_mag={max_mag}' self.prob = prob self.level = level self.min_mag = min_mag self.max_mag = max_mag def _transform_img(self, results: dict, mag: float) -> None: """Transform the image.""" pass @cache_randomness def _random_disable(self): """Randomly disable the transform.""" return np.random.rand() > self.prob @cache_randomness def _get_mag(self): """Get the magnitude of the transform.""" return level_to_mag(self.level, self.min_mag, self.max_mag) def transform(self, results: dict) -> dict: """Transform function for images. Args: results (dict): Result dict from loading pipeline. Returns: dict: Transformed results. """ if self._random_disable(): return results mag = self._get_mag() self._transform_img(results, mag) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(prob={self.prob}, ' repr_str += f'level={self.level}, ' repr_str += f'min_mag={self.min_mag}, ' repr_str += f'max_mag={self.max_mag})' return repr_str @TRANSFORMS.register_module() class Color(ColorTransform): """Adjust the color balance of the image, in a manner similar to the controls on a colour TV set. A magnitude=0 gives a black & white image, whereas magnitude=1 gives the original image. The bboxes, masks and segmentations are not modified. Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing Color transformation. Defaults to 1.0. level (int, optional): Should be in range [0,_MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum magnitude for Color transformation. Defaults to 0.1. max_mag (float): The maximum magnitude for Color transformation. Defaults to 1.9. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.1, max_mag: float = 1.9) -> None: assert 0. <= min_mag <= 2.0, \ f'min_mag for Color should be in range [0,2], got {min_mag}.' assert 0. <= max_mag <= 2.0, \ f'max_mag for Color should be in range [0,2], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag) def _transform_img(self, results: dict, mag: float) -> None: """Apply Color transformation to image.""" # NOTE defaultly the image should be BGR format img = results['img'] results['img'] = mmcv.adjust_color(img, mag).astype(img.dtype) @TRANSFORMS.register_module() class Brightness(ColorTransform): """Adjust the brightness of the image. A magnitude=0 gives a black image, whereas magnitude=1 gives the original image. The bboxes, masks and segmentations are not modified. Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing Brightness transformation. Defaults to 1.0. level (int, optional): Should be in range [0,_MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum magnitude for Brightness transformation. Defaults to 0.1. max_mag (float): The maximum magnitude for Brightness transformation. Defaults to 1.9. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.1, max_mag: float = 1.9) -> None: assert 0. <= min_mag <= 2.0, \ f'min_mag for Brightness should be in range [0,2], got {min_mag}.' assert 0. <= max_mag <= 2.0, \ f'max_mag for Brightness should be in range [0,2], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag) def _transform_img(self, results: dict, mag: float) -> None: """Adjust the brightness of image.""" img = results['img'] results['img'] = mmcv.adjust_brightness(img, mag).astype(img.dtype) @TRANSFORMS.register_module() class Contrast(ColorTransform): """Control the contrast of the image. A magnitude=0 gives a gray image, whereas magnitude=1 gives the original imageThe bboxes, masks and segmentations are not modified. Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing Contrast transformation. Defaults to 1.0. level (int, optional): Should be in range [0,_MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum magnitude for Contrast transformation. Defaults to 0.1. max_mag (float): The maximum magnitude for Contrast transformation. Defaults to 1.9. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.1, max_mag: float = 1.9) -> None: assert 0. <= min_mag <= 2.0, \ f'min_mag for Contrast should be in range [0,2], got {min_mag}.' assert 0. <= max_mag <= 2.0, \ f'max_mag for Contrast should be in range [0,2], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag) def _transform_img(self, results: dict, mag: float) -> None: """Adjust the image contrast.""" img = results['img'] results['img'] = mmcv.adjust_contrast(img, mag).astype(img.dtype) @TRANSFORMS.register_module() class Sharpness(ColorTransform): """Adjust images sharpness. A positive magnitude would enhance the sharpness and a negative magnitude would make the image blurry. A magnitude=0 gives the origin img. Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing Sharpness transformation. Defaults to 1.0. level (int, optional): Should be in range [0,_MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum magnitude for Sharpness transformation. Defaults to 0.1. max_mag (float): The maximum magnitude for Sharpness transformation. Defaults to 1.9. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.1, max_mag: float = 1.9) -> None: assert 0. <= min_mag <= 2.0, \ f'min_mag for Sharpness should be in range [0,2], got {min_mag}.' assert 0. <= max_mag <= 2.0, \ f'max_mag for Sharpness should be in range [0,2], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag) def _transform_img(self, results: dict, mag: float) -> None: """Adjust the image sharpness.""" img = results['img'] results['img'] = mmcv.adjust_sharpness(img, mag).astype(img.dtype) @TRANSFORMS.register_module() class Solarize(ColorTransform): """Solarize images (Invert all pixels above a threshold value of magnitude.). Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing Solarize transformation. Defaults to 1.0. level (int, optional): Should be in range [0,_MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum magnitude for Solarize transformation. Defaults to 0.0. max_mag (float): The maximum magnitude for Solarize transformation. Defaults to 256.0. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.0, max_mag: float = 256.0) -> None: assert 0. <= min_mag <= 256.0, f'min_mag for Solarize should be ' \ f'in range [0, 256], got {min_mag}.' assert 0. <= max_mag <= 256.0, f'max_mag for Solarize should be ' \ f'in range [0, 256], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag) def _transform_img(self, results: dict, mag: float) -> None: """Invert all pixel values above magnitude.""" img = results['img'] results['img'] = mmcv.solarize(img, mag).astype(img.dtype) @TRANSFORMS.register_module() class SolarizeAdd(ColorTransform): """SolarizeAdd images. For each pixel in the image that is less than 128, add an additional amount to it decided by the magnitude. Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing SolarizeAdd transformation. Defaults to 1.0. level (int, optional): Should be in range [0,_MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum magnitude for SolarizeAdd transformation. Defaults to 0.0. max_mag (float): The maximum magnitude for SolarizeAdd transformation. Defaults to 110.0. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.0, max_mag: float = 110.0) -> None: assert 0. <= min_mag <= 110.0, f'min_mag for SolarizeAdd should be ' \ f'in range [0, 110], got {min_mag}.' assert 0. <= max_mag <= 110.0, f'max_mag for SolarizeAdd should be ' \ f'in range [0, 110], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag) def _transform_img(self, results: dict, mag: float) -> None: """SolarizeAdd the image.""" img = results['img'] img_solarized = np.where(img < 128, np.minimum(img + mag, 255), img) results['img'] = img_solarized.astype(img.dtype) @TRANSFORMS.register_module() class Posterize(ColorTransform): """Posterize images (reduce the number of bits for each color channel). Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing Posterize transformation. Defaults to 1.0. level (int, optional): Should be in range [0,_MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum magnitude for Posterize transformation. Defaults to 0.0. max_mag (float): The maximum magnitude for Posterize transformation. Defaults to 4.0. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.0, max_mag: float = 4.0) -> None: assert 0. <= min_mag <= 8.0, f'min_mag for Posterize should be ' \ f'in range [0, 8], got {min_mag}.' assert 0. <= max_mag <= 8.0, f'max_mag for Posterize should be ' \ f'in range [0, 8], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag) def _transform_img(self, results: dict, mag: float) -> None: """Posterize the image.""" img = results['img'] results['img'] = mmcv.posterize(img, math.ceil(mag)).astype(img.dtype) @TRANSFORMS.register_module() class Equalize(ColorTransform): """Equalize the image histogram. The bboxes, masks and segmentations are not modified. Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing Equalize transformation. Defaults to 1.0. level (int, optional): No use for Equalize transformation. Defaults to None. min_mag (float): No use for Equalize transformation. Defaults to 0.1. max_mag (float): No use for Equalize transformation. Defaults to 1.9. """ def _transform_img(self, results: dict, mag: float) -> None: """Equalizes the histogram of one image.""" img = results['img'] results['img'] = mmcv.imequalize(img).astype(img.dtype) @TRANSFORMS.register_module() class AutoContrast(ColorTransform): """Auto adjust image contrast. Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing AutoContrast should be in range [0, 1]. Defaults to 1.0. level (int, optional): No use for AutoContrast transformation. Defaults to None. min_mag (float): No use for AutoContrast transformation. Defaults to 0.1. max_mag (float): No use for AutoContrast transformation. Defaults to 1.9. """ def _transform_img(self, results: dict, mag: float) -> None: """Auto adjust image contrast.""" img = results['img'] results['img'] = mmcv.auto_contrast(img).astype(img.dtype) @TRANSFORMS.register_module() class Invert(ColorTransform): """Invert images. Required Keys: - img Modified Keys: - img Args: prob (float): The probability for performing invert therefore should be in range [0, 1]. Defaults to 1.0. level (int, optional): No use for Invert transformation. Defaults to None. min_mag (float): No use for Invert transformation. Defaults to 0.1. max_mag (float): No use for Invert transformation. Defaults to 1.9. """ def _transform_img(self, results: dict, mag: float) -> None: """Invert the image.""" img = results['img'] results['img'] = mmcv.iminvert(img).astype(img.dtype) ================================================ FILE: mmdet/datasets/transforms/formatting.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np from mmcv.transforms import to_tensor from mmcv.transforms.base import BaseTransform from mmengine.structures import InstanceData, PixelData from mmdet.registry import TRANSFORMS from mmdet.structures import DetDataSample from mmdet.structures.bbox import BaseBoxes @TRANSFORMS.register_module() class PackDetInputs(BaseTransform): """Pack the inputs data for the detection / semantic segmentation / panoptic segmentation. The ``img_meta`` item is always populated. The contents of the ``img_meta`` dictionary depends on ``meta_keys``. By default this includes: - ``img_id``: id of the image - ``img_path``: path to the image file - ``ori_shape``: original shape of the image as a tuple (h, w) - ``img_shape``: shape of the image input to the network as a tuple \ (h, w). Note that images may be zero padded on the \ bottom/right if the batch tensor is larger than this shape. - ``scale_factor``: a float indicating the preprocessing scale - ``flip``: a boolean indicating if image flip transform was used - ``flip_direction``: the flipping direction Args: meta_keys (Sequence[str], optional): Meta keys to be converted to ``mmcv.DataContainer`` and collected in ``data[img_metas]``. Default: ``('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction')`` """ mapping_table = { 'gt_bboxes': 'bboxes', 'gt_bboxes_labels': 'labels', 'gt_masks': 'masks' } def __init__(self, meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction')): self.meta_keys = meta_keys def transform(self, results: dict) -> dict: """Method to pack the input data. Args: results (dict): Result dict from the data pipeline. Returns: dict: - 'inputs' (obj:`torch.Tensor`): The forward data of models. - 'data_sample' (obj:`DetDataSample`): The annotation info of the sample. """ packed_results = dict() if 'img' in results: img = results['img'] if len(img.shape) < 3: img = np.expand_dims(img, -1) # To improve the computational speed by by 3-5 times, apply: # If image is not contiguous, use # `numpy.transpose()` followed by `numpy.ascontiguousarray()` # If image is already contiguous, use # `torch.permute()` followed by `torch.contiguous()` # Refer to https://github.com/open-mmlab/mmdetection/pull/9533 # for more details if not img.flags.c_contiguous: img = np.ascontiguousarray(img.transpose(2, 0, 1)) img = to_tensor(img) else: img = to_tensor(img).permute(2, 0, 1).contiguous() packed_results['inputs'] = img if 'gt_ignore_flags' in results: valid_idx = np.where(results['gt_ignore_flags'] == 0)[0] ignore_idx = np.where(results['gt_ignore_flags'] == 1)[0] data_sample = DetDataSample() instance_data = InstanceData() ignore_instance_data = InstanceData() for key in self.mapping_table.keys(): if key not in results: continue if key == 'gt_masks' or isinstance(results[key], BaseBoxes): if 'gt_ignore_flags' in results: instance_data[ self.mapping_table[key]] = results[key][valid_idx] ignore_instance_data[ self.mapping_table[key]] = results[key][ignore_idx] else: instance_data[self.mapping_table[key]] = results[key] else: if 'gt_ignore_flags' in results: instance_data[self.mapping_table[key]] = to_tensor( results[key][valid_idx]) ignore_instance_data[self.mapping_table[key]] = to_tensor( results[key][ignore_idx]) else: instance_data[self.mapping_table[key]] = to_tensor( results[key]) data_sample.gt_instances = instance_data data_sample.ignored_instances = ignore_instance_data if 'proposals' in results: proposals = InstanceData( bboxes=to_tensor(results['proposals']), scores=to_tensor(results['proposals_scores'])) data_sample.proposals = proposals if 'gt_seg_map' in results: gt_sem_seg_data = dict( sem_seg=to_tensor(results['gt_seg_map'][None, ...].copy())) data_sample.gt_sem_seg = PixelData(**gt_sem_seg_data) img_meta = {} for key in self.meta_keys: assert key in results, f'`{key}` is not found in `results`, ' \ f'the valid keys are {list(results)}.' img_meta[key] = results[key] data_sample.set_metainfo(img_meta) packed_results['data_samples'] = data_sample return packed_results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(meta_keys={self.meta_keys})' return repr_str @TRANSFORMS.register_module() class ToTensor: """Convert some results to :obj:`torch.Tensor` by given keys. Args: keys (Sequence[str]): Keys that need to be converted to Tensor. """ def __init__(self, keys): self.keys = keys def __call__(self, results): """Call function to convert data in results to :obj:`torch.Tensor`. Args: results (dict): Result dict contains the data to convert. Returns: dict: The result dict contains the data converted to :obj:`torch.Tensor`. """ for key in self.keys: results[key] = to_tensor(results[key]) return results def __repr__(self): return self.__class__.__name__ + f'(keys={self.keys})' @TRANSFORMS.register_module() class ImageToTensor: """Convert image to :obj:`torch.Tensor` by given keys. The dimension order of input image is (H, W, C). The pipeline will convert it to (C, H, W). If only 2 dimension (H, W) is given, the output would be (1, H, W). Args: keys (Sequence[str]): Key of images to be converted to Tensor. """ def __init__(self, keys): self.keys = keys def __call__(self, results): """Call function to convert image in results to :obj:`torch.Tensor` and transpose the channel order. Args: results (dict): Result dict contains the image data to convert. Returns: dict: The result dict contains the image converted to :obj:`torch.Tensor` and permuted to (C, H, W) order. """ for key in self.keys: img = results[key] if len(img.shape) < 3: img = np.expand_dims(img, -1) results[key] = to_tensor(img).permute(2, 0, 1).contiguous() return results def __repr__(self): return self.__class__.__name__ + f'(keys={self.keys})' @TRANSFORMS.register_module() class Transpose: """Transpose some results by given keys. Args: keys (Sequence[str]): Keys of results to be transposed. order (Sequence[int]): Order of transpose. """ def __init__(self, keys, order): self.keys = keys self.order = order def __call__(self, results): """Call function to transpose the channel order of data in results. Args: results (dict): Result dict contains the data to transpose. Returns: dict: The result dict contains the data transposed to \ ``self.order``. """ for key in self.keys: results[key] = results[key].transpose(self.order) return results def __repr__(self): return self.__class__.__name__ + \ f'(keys={self.keys}, order={self.order})' @TRANSFORMS.register_module() class WrapFieldsToLists: """Wrap fields of the data dictionary into lists for evaluation. This class can be used as a last step of a test or validation pipeline for single image evaluation or inference. Example: >>> test_pipeline = [ >>> dict(type='LoadImageFromFile'), >>> dict(type='Normalize', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True), >>> dict(type='Pad', size_divisor=32), >>> dict(type='ImageToTensor', keys=['img']), >>> dict(type='Collect', keys=['img']), >>> dict(type='WrapFieldsToLists') >>> ] """ def __call__(self, results): """Call function to wrap fields into lists. Args: results (dict): Result dict contains the data to wrap. Returns: dict: The result dict where value of ``self.keys`` are wrapped \ into list. """ # Wrap dict fields into lists for key, val in results.items(): results[key] = [val] return results def __repr__(self): return f'{self.__class__.__name__}()' ================================================ FILE: mmdet/datasets/transforms/geometric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Union import cv2 import mmcv import numpy as np from mmcv.transforms import BaseTransform from mmcv.transforms.utils import cache_randomness from mmdet.registry import TRANSFORMS from mmdet.structures.bbox import autocast_box_type from .augment_wrappers import _MAX_LEVEL, level_to_mag @TRANSFORMS.register_module() class GeomTransform(BaseTransform): """Base class for geometric transformations. All geometric transformations need to inherit from this base class. ``GeomTransform`` unifies the class attributes and class functions of geometric transformations (ShearX, ShearY, Rotate, TranslateX, and TranslateY), and records the homography matrix. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - gt_bboxes - gt_masks - gt_seg_map Added Keys: - homography_matrix Args: prob (float): The probability for performing the geometric transformation and should be in range [0, 1]. Defaults to 1.0. level (int, optional): The level should be in range [0, _MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum magnitude for geometric transformation. Defaults to 0.0. max_mag (float): The maximum magnitude for geometric transformation. Defaults to 1.0. reversal_prob (float): The probability that reverses the geometric transformation magnitude. Should be in range [0,1]. Defaults to 0.5. img_border_value (int | float | tuple): The filled values for image border. If float, the same fill value will be used for all the three channels of image. If tuple, it should be 3 elements. Defaults to 128. mask_border_value (int): The fill value used for masks. Defaults to 0. seg_ignore_label (int): The fill value used for segmentation map. Note this value must equals ``ignore_label`` in ``semantic_head`` of the corresponding config. Defaults to 255. interpolation (str): Interpolation method, accepted values are "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' backend, "nearest", "bilinear" for 'pillow' backend. Defaults to 'bilinear'. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.0, max_mag: float = 1.0, reversal_prob: float = 0.5, img_border_value: Union[int, float, tuple] = 128, mask_border_value: int = 0, seg_ignore_label: int = 255, interpolation: str = 'bilinear') -> None: assert 0 <= prob <= 1.0, f'The probability of the transformation ' \ f'should be in range [0,1], got {prob}.' assert level is None or isinstance(level, int), \ f'The level should be None or type int, got {type(level)}.' assert level is None or 0 <= level <= _MAX_LEVEL, \ f'The level should be in range [0,{_MAX_LEVEL}], got {level}.' assert isinstance(min_mag, float), \ f'min_mag should be type float, got {type(min_mag)}.' assert isinstance(max_mag, float), \ f'max_mag should be type float, got {type(max_mag)}.' assert min_mag <= max_mag, \ f'min_mag should smaller than max_mag, ' \ f'got min_mag={min_mag} and max_mag={max_mag}' assert isinstance(reversal_prob, float), \ f'reversal_prob should be type float, got {type(max_mag)}.' assert 0 <= reversal_prob <= 1.0, \ f'The reversal probability of the transformation magnitude ' \ f'should be type float, got {type(reversal_prob)}.' if isinstance(img_border_value, (float, int)): img_border_value = tuple([float(img_border_value)] * 3) elif isinstance(img_border_value, tuple): assert len(img_border_value) == 3, \ f'img_border_value as tuple must have 3 elements, ' \ f'got {len(img_border_value)}.' img_border_value = tuple([float(val) for val in img_border_value]) else: raise ValueError( 'img_border_value must be float or tuple with 3 elements.') assert np.all([0 <= val <= 255 for val in img_border_value]), 'all ' \ 'elements of img_border_value should between range [0,255].' \ f'got {img_border_value}.' self.prob = prob self.level = level self.min_mag = min_mag self.max_mag = max_mag self.reversal_prob = reversal_prob self.img_border_value = img_border_value self.mask_border_value = mask_border_value self.seg_ignore_label = seg_ignore_label self.interpolation = interpolation def _transform_img(self, results: dict, mag: float) -> None: """Transform the image.""" pass def _transform_masks(self, results: dict, mag: float) -> None: """Transform the masks.""" pass def _transform_seg(self, results: dict, mag: float) -> None: """Transform the segmentation map.""" pass def _get_homography_matrix(self, results: dict, mag: float) -> np.ndarray: """Get the homography matrix for the geometric transformation.""" return np.eye(3, dtype=np.float32) def _transform_bboxes(self, results: dict, mag: float) -> None: """Transform the bboxes.""" results['gt_bboxes'].project_(self.homography_matrix) results['gt_bboxes'].clip_(results['img_shape']) def _record_homography_matrix(self, results: dict) -> None: """Record the homography matrix for the geometric transformation.""" if results.get('homography_matrix', None) is None: results['homography_matrix'] = self.homography_matrix else: results['homography_matrix'] = self.homography_matrix @ results[ 'homography_matrix'] @cache_randomness def _random_disable(self): """Randomly disable the transform.""" return np.random.rand() > self.prob @cache_randomness def _get_mag(self): """Get the magnitude of the transform.""" mag = level_to_mag(self.level, self.min_mag, self.max_mag) return -mag if np.random.rand() > self.reversal_prob else mag @autocast_box_type() def transform(self, results: dict) -> dict: """Transform function for images, bounding boxes, masks and semantic segmentation map. Args: results (dict): Result dict from loading pipeline. Returns: dict: Transformed results. """ if self._random_disable(): return results mag = self._get_mag() self.homography_matrix = self._get_homography_matrix(results, mag) self._record_homography_matrix(results) self._transform_img(results, mag) if results.get('gt_bboxes', None) is not None: self._transform_bboxes(results, mag) if results.get('gt_masks', None) is not None: self._transform_masks(results, mag) if results.get('gt_seg_map', None) is not None: self._transform_seg(results, mag) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(prob={self.prob}, ' repr_str += f'level={self.level}, ' repr_str += f'min_mag={self.min_mag}, ' repr_str += f'max_mag={self.max_mag}, ' repr_str += f'reversal_prob={self.reversal_prob}, ' repr_str += f'img_border_value={self.img_border_value}, ' repr_str += f'mask_border_value={self.mask_border_value}, ' repr_str += f'seg_ignore_label={self.seg_ignore_label}, ' repr_str += f'interpolation={self.interpolation})' return repr_str @TRANSFORMS.register_module() class ShearX(GeomTransform): """Shear the images, bboxes, masks and segmentation map horizontally. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - gt_bboxes - gt_masks - gt_seg_map Added Keys: - homography_matrix Args: prob (float): The probability for performing Shear and should be in range [0, 1]. Defaults to 1.0. level (int, optional): The level should be in range [0, _MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum angle for the horizontal shear. Defaults to 0.0. max_mag (float): The maximum angle for the horizontal shear. Defaults to 30.0. reversal_prob (float): The probability that reverses the horizontal shear magnitude. Should be in range [0,1]. Defaults to 0.5. img_border_value (int | float | tuple): The filled values for image border. If float, the same fill value will be used for all the three channels of image. If tuple, it should be 3 elements. Defaults to 128. mask_border_value (int): The fill value used for masks. Defaults to 0. seg_ignore_label (int): The fill value used for segmentation map. Note this value must equals ``ignore_label`` in ``semantic_head`` of the corresponding config. Defaults to 255. interpolation (str): Interpolation method, accepted values are "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' backend, "nearest", "bilinear" for 'pillow' backend. Defaults to 'bilinear'. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.0, max_mag: float = 30.0, reversal_prob: float = 0.5, img_border_value: Union[int, float, tuple] = 128, mask_border_value: int = 0, seg_ignore_label: int = 255, interpolation: str = 'bilinear') -> None: assert 0. <= min_mag <= 90., \ f'min_mag angle for ShearX should be ' \ f'in range [0, 90], got {min_mag}.' assert 0. <= max_mag <= 90., \ f'max_mag angle for ShearX should be ' \ f'in range [0, 90], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag, reversal_prob=reversal_prob, img_border_value=img_border_value, mask_border_value=mask_border_value, seg_ignore_label=seg_ignore_label, interpolation=interpolation) @cache_randomness def _get_mag(self): """Get the magnitude of the transform.""" mag = level_to_mag(self.level, self.min_mag, self.max_mag) mag = np.tan(mag * np.pi / 180) return -mag if np.random.rand() > self.reversal_prob else mag def _get_homography_matrix(self, results: dict, mag: float) -> np.ndarray: """Get the homography matrix for ShearX.""" return np.array([[1, mag, 0], [0, 1, 0], [0, 0, 1]], dtype=np.float32) def _transform_img(self, results: dict, mag: float) -> None: """Shear the image horizontally.""" results['img'] = mmcv.imshear( results['img'], mag, direction='horizontal', border_value=self.img_border_value, interpolation=self.interpolation) def _transform_masks(self, results: dict, mag: float) -> None: """Shear the masks horizontally.""" results['gt_masks'] = results['gt_masks'].shear( results['img_shape'], mag, direction='horizontal', border_value=self.mask_border_value, interpolation=self.interpolation) def _transform_seg(self, results: dict, mag: float) -> None: """Shear the segmentation map horizontally.""" results['gt_seg_map'] = mmcv.imshear( results['gt_seg_map'], mag, direction='horizontal', border_value=self.seg_ignore_label, interpolation='nearest') @TRANSFORMS.register_module() class ShearY(GeomTransform): """Shear the images, bboxes, masks and segmentation map vertically. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - gt_bboxes - gt_masks - gt_seg_map Added Keys: - homography_matrix Args: prob (float): The probability for performing ShearY and should be in range [0, 1]. Defaults to 1.0. level (int, optional): The level should be in range [0,_MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum angle for the vertical shear. Defaults to 0.0. max_mag (float): The maximum angle for the vertical shear. Defaults to 30.0. reversal_prob (float): The probability that reverses the vertical shear magnitude. Should be in range [0,1]. Defaults to 0.5. img_border_value (int | float | tuple): The filled values for image border. If float, the same fill value will be used for all the three channels of image. If tuple, it should be 3 elements. Defaults to 128. mask_border_value (int): The fill value used for masks. Defaults to 0. seg_ignore_label (int): The fill value used for segmentation map. Note this value must equals ``ignore_label`` in ``semantic_head`` of the corresponding config. Defaults to 255. interpolation (str): Interpolation method, accepted values are "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' backend, "nearest", "bilinear" for 'pillow' backend. Defaults to 'bilinear'. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.0, max_mag: float = 30., reversal_prob: float = 0.5, img_border_value: Union[int, float, tuple] = 128, mask_border_value: int = 0, seg_ignore_label: int = 255, interpolation: str = 'bilinear') -> None: assert 0. <= min_mag <= 90., \ f'min_mag angle for ShearY should be ' \ f'in range [0, 90], got {min_mag}.' assert 0. <= max_mag <= 90., \ f'max_mag angle for ShearY should be ' \ f'in range [0, 90], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag, reversal_prob=reversal_prob, img_border_value=img_border_value, mask_border_value=mask_border_value, seg_ignore_label=seg_ignore_label, interpolation=interpolation) @cache_randomness def _get_mag(self): """Get the magnitude of the transform.""" mag = level_to_mag(self.level, self.min_mag, self.max_mag) mag = np.tan(mag * np.pi / 180) return -mag if np.random.rand() > self.reversal_prob else mag def _get_homography_matrix(self, results: dict, mag: float) -> np.ndarray: """Get the homography matrix for ShearY.""" return np.array([[1, 0, 0], [mag, 1, 0], [0, 0, 1]], dtype=np.float32) def _transform_img(self, results: dict, mag: float) -> None: """Shear the image vertically.""" results['img'] = mmcv.imshear( results['img'], mag, direction='vertical', border_value=self.img_border_value, interpolation=self.interpolation) def _transform_masks(self, results: dict, mag: float) -> None: """Shear the masks vertically.""" results['gt_masks'] = results['gt_masks'].shear( results['img_shape'], mag, direction='vertical', border_value=self.mask_border_value, interpolation=self.interpolation) def _transform_seg(self, results: dict, mag: float) -> None: """Shear the segmentation map vertically.""" results['gt_seg_map'] = mmcv.imshear( results['gt_seg_map'], mag, direction='vertical', border_value=self.seg_ignore_label, interpolation='nearest') @TRANSFORMS.register_module() class Rotate(GeomTransform): """Rotate the images, bboxes, masks and segmentation map. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - gt_bboxes - gt_masks - gt_seg_map Added Keys: - homography_matrix Args: prob (float): The probability for perform transformation and should be in range 0 to 1. Defaults to 1.0. level (int, optional): The level should be in range [0, _MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The maximum angle for rotation. Defaults to 0.0. max_mag (float): The maximum angle for rotation. Defaults to 30.0. reversal_prob (float): The probability that reverses the rotation magnitude. Should be in range [0,1]. Defaults to 0.5. img_border_value (int | float | tuple): The filled values for image border. If float, the same fill value will be used for all the three channels of image. If tuple, it should be 3 elements. Defaults to 128. mask_border_value (int): The fill value used for masks. Defaults to 0. seg_ignore_label (int): The fill value used for segmentation map. Note this value must equals ``ignore_label`` in ``semantic_head`` of the corresponding config. Defaults to 255. interpolation (str): Interpolation method, accepted values are "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' backend, "nearest", "bilinear" for 'pillow' backend. Defaults to 'bilinear'. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.0, max_mag: float = 30.0, reversal_prob: float = 0.5, img_border_value: Union[int, float, tuple] = 128, mask_border_value: int = 0, seg_ignore_label: int = 255, interpolation: str = 'bilinear') -> None: assert 0. <= min_mag <= 180., \ f'min_mag for Rotate should be in range [0,180], got {min_mag}.' assert 0. <= max_mag <= 180., \ f'max_mag for Rotate should be in range [0,180], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag, reversal_prob=reversal_prob, img_border_value=img_border_value, mask_border_value=mask_border_value, seg_ignore_label=seg_ignore_label, interpolation=interpolation) def _get_homography_matrix(self, results: dict, mag: float) -> np.ndarray: """Get the homography matrix for Rotate.""" img_shape = results['img_shape'] center = ((img_shape[1] - 1) * 0.5, (img_shape[0] - 1) * 0.5) cv2_rotation_matrix = cv2.getRotationMatrix2D(center, -mag, 1.0) return np.concatenate( [cv2_rotation_matrix, np.array([0, 0, 1]).reshape((1, 3))]).astype(np.float32) def _transform_img(self, results: dict, mag: float) -> None: """Rotate the image.""" results['img'] = mmcv.imrotate( results['img'], mag, border_value=self.img_border_value, interpolation=self.interpolation) def _transform_masks(self, results: dict, mag: float) -> None: """Rotate the masks.""" results['gt_masks'] = results['gt_masks'].rotate( results['img_shape'], mag, border_value=self.mask_border_value, interpolation=self.interpolation) def _transform_seg(self, results: dict, mag: float) -> None: """Rotate the segmentation map.""" results['gt_seg_map'] = mmcv.imrotate( results['gt_seg_map'], mag, border_value=self.seg_ignore_label, interpolation='nearest') @TRANSFORMS.register_module() class TranslateX(GeomTransform): """Translate the images, bboxes, masks and segmentation map horizontally. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - gt_bboxes - gt_masks - gt_seg_map Added Keys: - homography_matrix Args: prob (float): The probability for perform transformation and should be in range 0 to 1. Defaults to 1.0. level (int, optional): The level should be in range [0, _MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum pixel's offset ratio for horizontal translation. Defaults to 0.0. max_mag (float): The maximum pixel's offset ratio for horizontal translation. Defaults to 0.1. reversal_prob (float): The probability that reverses the horizontal translation magnitude. Should be in range [0,1]. Defaults to 0.5. img_border_value (int | float | tuple): The filled values for image border. If float, the same fill value will be used for all the three channels of image. If tuple, it should be 3 elements. Defaults to 128. mask_border_value (int): The fill value used for masks. Defaults to 0. seg_ignore_label (int): The fill value used for segmentation map. Note this value must equals ``ignore_label`` in ``semantic_head`` of the corresponding config. Defaults to 255. interpolation (str): Interpolation method, accepted values are "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' backend, "nearest", "bilinear" for 'pillow' backend. Defaults to 'bilinear'. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.0, max_mag: float = 0.1, reversal_prob: float = 0.5, img_border_value: Union[int, float, tuple] = 128, mask_border_value: int = 0, seg_ignore_label: int = 255, interpolation: str = 'bilinear') -> None: assert 0. <= min_mag <= 1., \ f'min_mag ratio for TranslateX should be ' \ f'in range [0, 1], got {min_mag}.' assert 0. <= max_mag <= 1., \ f'max_mag ratio for TranslateX should be ' \ f'in range [0, 1], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag, reversal_prob=reversal_prob, img_border_value=img_border_value, mask_border_value=mask_border_value, seg_ignore_label=seg_ignore_label, interpolation=interpolation) def _get_homography_matrix(self, results: dict, mag: float) -> np.ndarray: """Get the homography matrix for TranslateX.""" mag = int(results['img_shape'][1] * mag) return np.array([[1, 0, mag], [0, 1, 0], [0, 0, 1]], dtype=np.float32) def _transform_img(self, results: dict, mag: float) -> None: """Translate the image horizontally.""" mag = int(results['img_shape'][1] * mag) results['img'] = mmcv.imtranslate( results['img'], mag, direction='horizontal', border_value=self.img_border_value, interpolation=self.interpolation) def _transform_masks(self, results: dict, mag: float) -> None: """Translate the masks horizontally.""" mag = int(results['img_shape'][1] * mag) results['gt_masks'] = results['gt_masks'].translate( results['img_shape'], mag, direction='horizontal', border_value=self.mask_border_value, interpolation=self.interpolation) def _transform_seg(self, results: dict, mag: float) -> None: """Translate the segmentation map horizontally.""" mag = int(results['img_shape'][1] * mag) results['gt_seg_map'] = mmcv.imtranslate( results['gt_seg_map'], mag, direction='horizontal', border_value=self.seg_ignore_label, interpolation='nearest') @TRANSFORMS.register_module() class TranslateY(GeomTransform): """Translate the images, bboxes, masks and segmentation map vertically. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - gt_bboxes - gt_masks - gt_seg_map Added Keys: - homography_matrix Args: prob (float): The probability for perform transformation and should be in range 0 to 1. Defaults to 1.0. level (int, optional): The level should be in range [0, _MAX_LEVEL]. If level is None, it will generate from [0, _MAX_LEVEL] randomly. Defaults to None. min_mag (float): The minimum pixel's offset ratio for vertical translation. Defaults to 0.0. max_mag (float): The maximum pixel's offset ratio for vertical translation. Defaults to 0.1. reversal_prob (float): The probability that reverses the vertical translation magnitude. Should be in range [0,1]. Defaults to 0.5. img_border_value (int | float | tuple): The filled values for image border. If float, the same fill value will be used for all the three channels of image. If tuple, it should be 3 elements. Defaults to 128. mask_border_value (int): The fill value used for masks. Defaults to 0. seg_ignore_label (int): The fill value used for segmentation map. Note this value must equals ``ignore_label`` in ``semantic_head`` of the corresponding config. Defaults to 255. interpolation (str): Interpolation method, accepted values are "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' backend, "nearest", "bilinear" for 'pillow' backend. Defaults to 'bilinear'. """ def __init__(self, prob: float = 1.0, level: Optional[int] = None, min_mag: float = 0.0, max_mag: float = 0.1, reversal_prob: float = 0.5, img_border_value: Union[int, float, tuple] = 128, mask_border_value: int = 0, seg_ignore_label: int = 255, interpolation: str = 'bilinear') -> None: assert 0. <= min_mag <= 1., \ f'min_mag ratio for TranslateY should be ' \ f'in range [0,1], got {min_mag}.' assert 0. <= max_mag <= 1., \ f'max_mag ratio for TranslateY should be ' \ f'in range [0,1], got {max_mag}.' super().__init__( prob=prob, level=level, min_mag=min_mag, max_mag=max_mag, reversal_prob=reversal_prob, img_border_value=img_border_value, mask_border_value=mask_border_value, seg_ignore_label=seg_ignore_label, interpolation=interpolation) def _get_homography_matrix(self, results: dict, mag: float) -> np.ndarray: """Get the homography matrix for TranslateY.""" mag = int(results['img_shape'][0] * mag) return np.array([[1, 0, 0], [0, 1, mag], [0, 0, 1]], dtype=np.float32) def _transform_img(self, results: dict, mag: float) -> None: """Translate the image vertically.""" mag = int(results['img_shape'][0] * mag) results['img'] = mmcv.imtranslate( results['img'], mag, direction='vertical', border_value=self.img_border_value, interpolation=self.interpolation) def _transform_masks(self, results: dict, mag: float) -> None: """Translate masks vertically.""" mag = int(results['img_shape'][0] * mag) results['gt_masks'] = results['gt_masks'].translate( results['img_shape'], mag, direction='vertical', border_value=self.mask_border_value, interpolation=self.interpolation) def _transform_seg(self, results: dict, mag: float) -> None: """Translate segmentation map vertically.""" mag = int(results['img_shape'][0] * mag) results['gt_seg_map'] = mmcv.imtranslate( results['gt_seg_map'], mag, direction='vertical', border_value=self.seg_ignore_label, interpolation='nearest') ================================================ FILE: mmdet/datasets/transforms/instaboost.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple import numpy as np from mmcv.transforms import BaseTransform from mmdet.registry import TRANSFORMS @TRANSFORMS.register_module() class InstaBoost(BaseTransform): r"""Data augmentation method in `InstaBoost: Boosting Instance Segmentation Via Probability Map Guided Copy-Pasting `_. Refer to https://github.com/GothicAi/Instaboost for implementation details. Required Keys: - img (np.uint8) - instances Modified Keys: - img (np.uint8) - instances Args: action_candidate (tuple): Action candidates. "normal", "horizontal", \ "vertical", "skip" are supported. Defaults to ('normal', \ 'horizontal', 'skip'). action_prob (tuple): Corresponding action probabilities. Should be \ the same length as action_candidate. Defaults to (1, 0, 0). scale (tuple): (min scale, max scale). Defaults to (0.8, 1.2). dx (int): The maximum x-axis shift will be (instance width) / dx. Defaults to 15. dy (int): The maximum y-axis shift will be (instance height) / dy. Defaults to 15. theta (tuple): (min rotation degree, max rotation degree). \ Defaults to (-1, 1). color_prob (float): Probability of images for color augmentation. Defaults to 0.5. hflag (bool): Whether to use heatmap guided. Defaults to False. aug_ratio (float): Probability of applying this transformation. \ Defaults to 0.5. """ def __init__(self, action_candidate: tuple = ('normal', 'horizontal', 'skip'), action_prob: tuple = (1, 0, 0), scale: tuple = (0.8, 1.2), dx: int = 15, dy: int = 15, theta: tuple = (-1, 1), color_prob: float = 0.5, hflag: bool = False, aug_ratio: float = 0.5) -> None: import matplotlib import matplotlib.pyplot as plt default_backend = plt.get_backend() try: import instaboostfast as instaboost except ImportError: raise ImportError( 'Please run "pip install instaboostfast" ' 'to install instaboostfast first for instaboost augmentation.') # instaboost will modify the default backend # and cause visualization to fail. matplotlib.use(default_backend) self.cfg = instaboost.InstaBoostConfig(action_candidate, action_prob, scale, dx, dy, theta, color_prob, hflag) self.aug_ratio = aug_ratio def _load_anns(self, results: dict) -> Tuple[list, list]: """Convert raw anns to instaboost expected input format.""" anns = [] ignore_anns = [] for instance in results['instances']: label = instance['bbox_label'] bbox = instance['bbox'] mask = instance['mask'] x1, y1, x2, y2 = bbox # assert (x2 - x1) >= 1 and (y2 - y1) >= 1 bbox = [x1, y1, x2 - x1, y2 - y1] if instance['ignore_flag'] == 0: anns.append({ 'category_id': label, 'segmentation': mask, 'bbox': bbox }) else: # Ignore instances without data augmentation ignore_anns.append(instance) return anns, ignore_anns def _parse_anns(self, results: dict, anns: list, ignore_anns: list, img: np.ndarray) -> dict: """Restore the result of instaboost processing to the original anns format.""" instances = [] for ann in anns: x1, y1, w, h = ann['bbox'] # TODO: more essential bug need to be fixed in instaboost if w <= 0 or h <= 0: continue bbox = [x1, y1, x1 + w, y1 + h] instances.append( dict( bbox=bbox, bbox_label=ann['category_id'], mask=ann['segmentation'], ignore_flag=0)) instances.extend(ignore_anns) results['img'] = img results['instances'] = instances return results def transform(self, results) -> dict: """The transform function.""" img = results['img'] ori_type = img.dtype if 'instances' not in results or len(results['instances']) == 0: return results anns, ignore_anns = self._load_anns(results) if np.random.choice([0, 1], p=[1 - self.aug_ratio, self.aug_ratio]): try: import instaboostfast as instaboost except ImportError: raise ImportError('Please run "pip install instaboostfast" ' 'to install instaboostfast first.') anns, img = instaboost.get_new_data( anns, img.astype(np.uint8), self.cfg, background=None) results = self._parse_anns(results, anns, ignore_anns, img.astype(ori_type)) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(aug_ratio={self.aug_ratio})' return repr_str ================================================ FILE: mmdet/datasets/transforms/loading.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple, Union import mmcv import numpy as np import pycocotools.mask as maskUtils import torch from mmcv.transforms import BaseTransform from mmcv.transforms import LoadAnnotations as MMCV_LoadAnnotations from mmcv.transforms import LoadImageFromFile from mmengine.fileio import FileClient from mmengine.structures import BaseDataElement from mmdet.registry import TRANSFORMS from mmdet.structures.bbox import get_box_type from mmdet.structures.bbox.box_type import autocast_box_type from mmdet.structures.mask import BitmapMasks, PolygonMasks @TRANSFORMS.register_module() class LoadImageFromNDArray(LoadImageFromFile): """Load an image from ``results['img']``. Similar with :obj:`LoadImageFromFile`, but the image has been loaded as :obj:`np.ndarray` in ``results['img']``. Can be used when loading image from webcam. Required Keys: - img Modified Keys: - img - img_path - img_shape - ori_shape Args: to_float32 (bool): Whether to convert the loaded image to a float32 numpy array. If set to False, the loaded image is an uint8 array. Defaults to False. """ def transform(self, results: dict) -> dict: """Transform function to add image meta information. Args: results (dict): Result dict with Webcam read image in ``results['img']``. Returns: dict: The dict contains loaded image and meta information. """ img = results['img'] if self.to_float32: img = img.astype(np.float32) results['img_path'] = None results['img'] = img results['img_shape'] = img.shape[:2] results['ori_shape'] = img.shape[:2] return results @TRANSFORMS.register_module() class LoadMultiChannelImageFromFiles(BaseTransform): """Load multi-channel images from a list of separate channel files. Required Keys: - img_path Modified Keys: - img - img_shape - ori_shape Args: to_float32 (bool): Whether to convert the loaded image to a float32 numpy array. If set to False, the loaded image is an uint8 array. Defaults to False. color_type (str): The flag argument for :func:``mmcv.imfrombytes``. Defaults to 'unchanged'. imdecode_backend (str): The image decoding backend type. The backend argument for :func:``mmcv.imfrombytes``. See :func:``mmcv.imfrombytes`` for details. Defaults to 'cv2'. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. """ def __init__( self, to_float32: bool = False, color_type: str = 'unchanged', imdecode_backend: str = 'cv2', file_client_args: dict = dict(backend='disk') ) -> None: self.to_float32 = to_float32 self.color_type = color_type self.imdecode_backend = imdecode_backend self.file_client_args = file_client_args.copy() self.file_client = FileClient(**self.file_client_args) def transform(self, results: dict) -> dict: """Transform functions to load multiple images and get images meta information. Args: results (dict): Result dict from :obj:`mmdet.CustomDataset`. Returns: dict: The dict contains loaded images and meta information. """ assert isinstance(results['img_path'], list) img = [] for name in results['img_path']: img_bytes = self.file_client.get(name) img.append( mmcv.imfrombytes( img_bytes, flag=self.color_type, backend=self.imdecode_backend)) img = np.stack(img, axis=-1) if self.to_float32: img = img.astype(np.float32) results['img'] = img results['img_shape'] = img.shape[:2] results['ori_shape'] = img.shape[:2] return results def __repr__(self): repr_str = (f'{self.__class__.__name__}(' f'to_float32={self.to_float32}, ' f"color_type='{self.color_type}', " f"imdecode_backend='{self.imdecode_backend}', " f'file_client_args={self.file_client_args})') return repr_str @TRANSFORMS.register_module() class LoadAnnotations(MMCV_LoadAnnotations): """Load and process the ``instances`` and ``seg_map`` annotation provided by dataset. The annotation format is as the following: .. code-block:: python { 'instances': [ { # List of 4 numbers representing the bounding box of the # instance, in (x1, y1, x2, y2) order. 'bbox': [x1, y1, x2, y2], # Label of image classification. 'bbox_label': 1, # Used in instance/panoptic segmentation. The segmentation mask # of the instance or the information of segments. # 1. If list[list[float]], it represents a list of polygons, # one for each connected component of the object. Each # list[float] is one simple polygon in the format of # [x1, y1, ..., xn, yn] (n≥3). The Xs and Ys are absolute # coordinates in unit of pixels. # 2. If dict, it represents the per-pixel segmentation mask in # COCO’s compressed RLE format. The dict should have keys # “size” and “counts”. Can be loaded by pycocotools 'mask': list[list[float]] or dict, } ] # Filename of semantic or panoptic segmentation ground truth file. 'seg_map_path': 'a/b/c' } After this module, the annotation has been changed to the format below: .. code-block:: python { # In (x1, y1, x2, y2) order, float type. N is the number of bboxes # in an image 'gt_bboxes': BaseBoxes(N, 4) # In int type. 'gt_bboxes_labels': np.ndarray(N, ) # In built-in class 'gt_masks': PolygonMasks (H, W) or BitmapMasks (H, W) # In uint8 type. 'gt_seg_map': np.ndarray (H, W) # in (x, y, v) order, float type. } Required Keys: - height - width - instances - bbox (optional) - bbox_label - mask (optional) - ignore_flag - seg_map_path (optional) Added Keys: - gt_bboxes (BaseBoxes[torch.float32]) - gt_bboxes_labels (np.int64) - gt_masks (BitmapMasks | PolygonMasks) - gt_seg_map (np.uint8) - gt_ignore_flags (bool) Args: with_bbox (bool): Whether to parse and load the bbox annotation. Defaults to True. with_label (bool): Whether to parse and load the label annotation. Defaults to True. with_mask (bool): Whether to parse and load the mask annotation. Default: False. with_seg (bool): Whether to parse and load the semantic segmentation annotation. Defaults to False. poly2mask (bool): Whether to convert mask to bitmap. Default: True. box_type (str): The box type used to wrap the bboxes. If ``box_type`` is None, gt_bboxes will keep being np.ndarray. Defaults to 'hbox'. imdecode_backend (str): The image decoding backend type. The backend argument for :func:``mmcv.imfrombytes``. See :fun:``mmcv.imfrombytes`` for details. Defaults to 'cv2'. file_client_args (dict): Arguments to instantiate a FileClient. See :class:``mmengine.fileio.FileClient`` for details. Defaults to ``dict(backend='disk')``. """ def __init__(self, with_mask: bool = False, poly2mask: bool = True, box_type: str = 'hbox', **kwargs) -> None: super(LoadAnnotations, self).__init__(**kwargs) self.with_mask = with_mask self.poly2mask = poly2mask self.box_type = box_type def _load_bboxes(self, results: dict) -> None: """Private function to load bounding box annotations. Args: results (dict): Result dict from :obj:``mmengine.BaseDataset``. Returns: dict: The dict contains loaded bounding box annotations. """ gt_bboxes = [] gt_ignore_flags = [] for instance in results.get('instances', []): gt_bboxes.append(instance['bbox']) gt_ignore_flags.append(instance['ignore_flag']) if self.box_type is None: results['gt_bboxes'] = np.array( gt_bboxes, dtype=np.float32).reshape((-1, 4)) else: _, box_type_cls = get_box_type(self.box_type) results['gt_bboxes'] = box_type_cls(gt_bboxes, dtype=torch.float32) results['gt_ignore_flags'] = np.array(gt_ignore_flags, dtype=bool) def _load_labels(self, results: dict) -> None: """Private function to load label annotations. Args: results (dict): Result dict from :obj:``mmengine.BaseDataset``. Returns: dict: The dict contains loaded label annotations. """ gt_bboxes_labels = [] for instance in results.get('instances', []): gt_bboxes_labels.append(instance['bbox_label']) # TODO: Inconsistent with mmcv, consider how to deal with it later. results['gt_bboxes_labels'] = np.array( gt_bboxes_labels, dtype=np.int64) def _poly2mask(self, mask_ann: Union[list, dict], img_h: int, img_w: int) -> np.ndarray: """Private function to convert masks represented with polygon to bitmaps. Args: mask_ann (list | dict): Polygon mask annotation input. img_h (int): The height of output mask. img_w (int): The width of output mask. Returns: np.ndarray: The decode bitmap mask of shape (img_h, img_w). """ if isinstance(mask_ann, list): # polygon -- a single object might consist of multiple parts # we merge all parts into one mask rle code rles = maskUtils.frPyObjects(mask_ann, img_h, img_w) rle = maskUtils.merge(rles) elif isinstance(mask_ann['counts'], list): # uncompressed RLE rle = maskUtils.frPyObjects(mask_ann, img_h, img_w) else: # rle rle = mask_ann mask = maskUtils.decode(rle) return mask def _process_masks(self, results: dict) -> list: """Process gt_masks and filter invalid polygons. Args: results (dict): Result dict from :obj:``mmengine.BaseDataset``. Returns: list: Processed gt_masks. """ gt_masks = [] gt_ignore_flags = [] for instance in results.get('instances', []): gt_mask = instance['mask'] # If the annotation of segmentation mask is invalid, # ignore the whole instance. if isinstance(gt_mask, list): gt_mask = [ np.array(polygon) for polygon in gt_mask if len(polygon) % 2 == 0 and len(polygon) >= 6 ] if len(gt_mask) == 0: # ignore this instance and set gt_mask to a fake mask instance['ignore_flag'] = 1 gt_mask = [np.zeros(6)] elif not self.poly2mask: # `PolygonMasks` requires a ploygon of format List[np.array], # other formats are invalid. instance['ignore_flag'] = 1 gt_mask = [np.zeros(6)] elif isinstance(gt_mask, dict) and \ not (gt_mask.get('counts') is not None and gt_mask.get('size') is not None and isinstance(gt_mask['counts'], (list, str))): # if gt_mask is a dict, it should include `counts` and `size`, # so that `BitmapMasks` can uncompressed RLE instance['ignore_flag'] = 1 gt_mask = [np.zeros(6)] gt_masks.append(gt_mask) # re-process gt_ignore_flags gt_ignore_flags.append(instance['ignore_flag']) results['gt_ignore_flags'] = np.array(gt_ignore_flags, dtype=bool) return gt_masks def _load_masks(self, results: dict) -> None: """Private function to load mask annotations. Args: results (dict): Result dict from :obj:``mmengine.BaseDataset``. """ h, w = results['ori_shape'] gt_masks = self._process_masks(results) if self.poly2mask: gt_masks = BitmapMasks( [self._poly2mask(mask, h, w) for mask in gt_masks], h, w) else: # fake polygon masks will be ignored in `PackDetInputs` gt_masks = PolygonMasks([mask for mask in gt_masks], h, w) results['gt_masks'] = gt_masks def transform(self, results: dict) -> dict: """Function to load multiple types annotations. Args: results (dict): Result dict from :obj:``mmengine.BaseDataset``. Returns: dict: The dict contains loaded bounding box, label and semantic segmentation. """ if self.with_bbox: self._load_bboxes(results) if self.with_label: self._load_labels(results) if self.with_mask: self._load_masks(results) if self.with_seg: self._load_seg_map(results) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(with_bbox={self.with_bbox}, ' repr_str += f'with_label={self.with_label}, ' repr_str += f'with_mask={self.with_mask}, ' repr_str += f'with_seg={self.with_seg}, ' repr_str += f'poly2mask={self.poly2mask}, ' repr_str += f"imdecode_backend='{self.imdecode_backend}', " repr_str += f'file_client_args={self.file_client_args})' return repr_str @TRANSFORMS.register_module() class LoadPanopticAnnotations(LoadAnnotations): """Load multiple types of panoptic annotations. The annotation format is as the following: .. code-block:: python { 'instances': [ { # List of 4 numbers representing the bounding box of the # instance, in (x1, y1, x2, y2) order. 'bbox': [x1, y1, x2, y2], # Label of image classification. 'bbox_label': 1, }, ... ] 'segments_info': [ { # id = cls_id + instance_id * INSTANCE_OFFSET 'id': int, # Contiguous category id defined in dataset. 'category': int # Thing flag. 'is_thing': bool }, ... ] # Filename of semantic or panoptic segmentation ground truth file. 'seg_map_path': 'a/b/c' } After this module, the annotation has been changed to the format below: .. code-block:: python { # In (x1, y1, x2, y2) order, float type. N is the number of bboxes # in an image 'gt_bboxes': BaseBoxes(N, 4) # In int type. 'gt_bboxes_labels': np.ndarray(N, ) # In built-in class 'gt_masks': PolygonMasks (H, W) or BitmapMasks (H, W) # In uint8 type. 'gt_seg_map': np.ndarray (H, W) # in (x, y, v) order, float type. } Required Keys: - height - width - instances - bbox - bbox_label - ignore_flag - segments_info - id - category - is_thing - seg_map_path Added Keys: - gt_bboxes (BaseBoxes[torch.float32]) - gt_bboxes_labels (np.int64) - gt_masks (BitmapMasks | PolygonMasks) - gt_seg_map (np.uint8) - gt_ignore_flags (bool) Args: with_bbox (bool): Whether to parse and load the bbox annotation. Defaults to True. with_label (bool): Whether to parse and load the label annotation. Defaults to True. with_mask (bool): Whether to parse and load the mask annotation. Defaults to True. with_seg (bool): Whether to parse and load the semantic segmentation annotation. Defaults to False. box_type (str): The box mode used to wrap the bboxes. imdecode_backend (str): The image decoding backend type. The backend argument for :func:``mmcv.imfrombytes``. See :fun:``mmcv.imfrombytes`` for details. Defaults to 'cv2'. file_client_args (dict): Arguments to instantiate a FileClient. See :class:``mmengine.fileio.FileClient`` for details. Defaults to ``dict(backend='disk')``. """ def __init__( self, with_bbox: bool = True, with_label: bool = True, with_mask: bool = True, with_seg: bool = True, box_type: str = 'hbox', imdecode_backend: str = 'cv2', file_client_args: dict = dict(backend='disk') ) -> None: try: from panopticapi import utils except ImportError: raise ImportError( 'panopticapi is not installed, please install it by: ' 'pip install git+https://github.com/cocodataset/' 'panopticapi.git.') self.rgb2id = utils.rgb2id self.file_client = FileClient(**file_client_args) super(LoadPanopticAnnotations, self).__init__( with_bbox=with_bbox, with_label=with_label, with_mask=with_mask, with_seg=with_seg, with_keypoints=False, box_type=box_type, imdecode_backend=imdecode_backend, file_client_args=file_client_args) def _load_masks_and_semantic_segs(self, results: dict) -> None: """Private function to load mask and semantic segmentation annotations. In gt_semantic_seg, the foreground label is from ``0`` to ``num_things - 1``, the background label is from ``num_things`` to ``num_things + num_stuff - 1``, 255 means the ignored label (``VOID``). Args: results (dict): Result dict from :obj:``mmdet.CustomDataset``. """ # seg_map_path is None, when inference on the dataset without gts. if results.get('seg_map_path', None) is None: return img_bytes = self.file_client.get(results['seg_map_path']) pan_png = mmcv.imfrombytes( img_bytes, flag='color', channel_order='rgb').squeeze() pan_png = self.rgb2id(pan_png) gt_masks = [] gt_seg = np.zeros_like(pan_png) + 255 # 255 as ignore for segment_info in results['segments_info']: mask = (pan_png == segment_info['id']) gt_seg = np.where(mask, segment_info['category'], gt_seg) # The legal thing masks if segment_info.get('is_thing'): gt_masks.append(mask.astype(np.uint8)) if self.with_mask: h, w = results['ori_shape'] gt_masks = BitmapMasks(gt_masks, h, w) results['gt_masks'] = gt_masks if self.with_seg: results['gt_seg_map'] = gt_seg def transform(self, results: dict) -> dict: """Function to load multiple types panoptic annotations. Args: results (dict): Result dict from :obj:``mmdet.CustomDataset``. Returns: dict: The dict contains loaded bounding box, label, mask and semantic segmentation annotations. """ if self.with_bbox: self._load_bboxes(results) if self.with_label: self._load_labels(results) if self.with_mask or self.with_seg: # The tasks completed by '_load_masks' and '_load_semantic_segs' # in LoadAnnotations are merged to one function. self._load_masks_and_semantic_segs(results) return results @TRANSFORMS.register_module() class LoadProposals(BaseTransform): """Load proposal pipeline. Required Keys: - proposals Modified Keys: - proposals Args: num_max_proposals (int, optional): Maximum number of proposals to load. If not specified, all proposals will be loaded. """ def __init__(self, num_max_proposals: Optional[int] = None) -> None: self.num_max_proposals = num_max_proposals def transform(self, results: dict) -> dict: """Transform function to load proposals from file. Args: results (dict): Result dict from :obj:`mmdet.CustomDataset`. Returns: dict: The dict contains loaded proposal annotations. """ proposals = results['proposals'] # the type of proposals should be `dict` or `InstanceData` assert isinstance(proposals, dict) \ or isinstance(proposals, BaseDataElement) bboxes = proposals['bboxes'].astype(np.float32) assert bboxes.shape[1] == 4, \ f'Proposals should have shapes (n, 4), but found {bboxes.shape}' if 'scores' in proposals: scores = proposals['scores'].astype(np.float32) assert bboxes.shape[0] == scores.shape[0] else: scores = np.zeros(bboxes.shape[0], dtype=np.float32) if self.num_max_proposals is not None: # proposals should sort by scores during dumping the proposals bboxes = bboxes[:self.num_max_proposals] scores = scores[:self.num_max_proposals] if len(bboxes) == 0: bboxes = np.zeros((0, 4), dtype=np.float32) scores = np.zeros(0, dtype=np.float32) results['proposals'] = bboxes results['proposals_scores'] = scores return results def __repr__(self): return self.__class__.__name__ + \ f'(num_max_proposals={self.num_max_proposals})' @TRANSFORMS.register_module() class FilterAnnotations(BaseTransform): """Filter invalid annotations. Required Keys: - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_ignore_flags (bool) (optional) Modified Keys: - gt_bboxes (optional) - gt_bboxes_labels (optional) - gt_masks (optional) - gt_ignore_flags (optional) Args: min_gt_bbox_wh (tuple[float]): Minimum width and height of ground truth boxes. Default: (1., 1.) min_gt_mask_area (int): Minimum foreground area of ground truth masks. Default: 1 by_box (bool): Filter instances with bounding boxes not meeting the min_gt_bbox_wh threshold. Default: True by_mask (bool): Filter instances with masks not meeting min_gt_mask_area threshold. Default: False keep_empty (bool): Whether to return None when it becomes an empty bbox after filtering. Defaults to True. """ def __init__(self, min_gt_bbox_wh: Tuple[int, int] = (1, 1), min_gt_mask_area: int = 1, by_box: bool = True, by_mask: bool = False, keep_empty: bool = True) -> None: # TODO: add more filter options assert by_box or by_mask self.min_gt_bbox_wh = min_gt_bbox_wh self.min_gt_mask_area = min_gt_mask_area self.by_box = by_box self.by_mask = by_mask self.keep_empty = keep_empty @autocast_box_type() def transform(self, results: dict) -> Union[dict, None]: """Transform function to filter annotations. Args: results (dict): Result dict. Returns: dict: Updated result dict. """ assert 'gt_bboxes' in results gt_bboxes = results['gt_bboxes'] if gt_bboxes.shape[0] == 0: return results tests = [] if self.by_box: tests.append( ((gt_bboxes.widths > self.min_gt_bbox_wh[0]) & (gt_bboxes.heights > self.min_gt_bbox_wh[1])).numpy()) if self.by_mask: assert 'gt_masks' in results gt_masks = results['gt_masks'] tests.append(gt_masks.areas >= self.min_gt_mask_area) keep = tests[0] for t in tests[1:]: keep = keep & t if not keep.any(): if self.keep_empty: return None keys = ('gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags') for key in keys: if key in results: results[key] = results[key][keep] return results def __repr__(self): return self.__class__.__name__ + \ f'(min_gt_bbox_wh={self.min_gt_bbox_wh}, ' \ f'keep_empty={self.keep_empty})' @TRANSFORMS.register_module() class LoadEmptyAnnotations(BaseTransform): """Load Empty Annotations for unlabeled images. Added Keys: - gt_bboxes (np.float32) - gt_bboxes_labels (np.int64) - gt_masks (BitmapMasks | PolygonMasks) - gt_seg_map (np.uint8) - gt_ignore_flags (bool) Args: with_bbox (bool): Whether to load the pseudo bbox annotation. Defaults to True. with_label (bool): Whether to load the pseudo label annotation. Defaults to True. with_mask (bool): Whether to load the pseudo mask annotation. Default: False. with_seg (bool): Whether to load the pseudo semantic segmentation annotation. Defaults to False. seg_ignore_label (int): The fill value used for segmentation map. Note this value must equals ``ignore_label`` in ``semantic_head`` of the corresponding config. Defaults to 255. """ def __init__(self, with_bbox: bool = True, with_label: bool = True, with_mask: bool = False, with_seg: bool = False, seg_ignore_label: int = 255) -> None: self.with_bbox = with_bbox self.with_label = with_label self.with_mask = with_mask self.with_seg = with_seg self.seg_ignore_label = seg_ignore_label def transform(self, results: dict) -> dict: """Transform function to load empty annotations. Args: results (dict): Result dict. Returns: dict: Updated result dict. """ if self.with_bbox: results['gt_bboxes'] = np.zeros((0, 4), dtype=np.float32) results['gt_ignore_flags'] = np.zeros((0, ), dtype=bool) if self.with_label: results['gt_bboxes_labels'] = np.zeros((0, ), dtype=np.int64) if self.with_mask: # TODO: support PolygonMasks h, w = results['img_shape'] gt_masks = np.zeros((0, h, w), dtype=np.uint8) results['gt_masks'] = BitmapMasks(gt_masks, h, w) if self.with_seg: h, w = results['img_shape'] results['gt_seg_map'] = self.seg_ignore_label * np.ones( (h, w), dtype=np.uint8) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(with_bbox={self.with_bbox}, ' repr_str += f'with_label={self.with_label}, ' repr_str += f'with_mask={self.with_mask}, ' repr_str += f'with_seg={self.with_seg}, ' repr_str += f'seg_ignore_label={self.seg_ignore_label})' return repr_str @TRANSFORMS.register_module() class InferencerLoader(BaseTransform): """Load an image from ``results['img']``. Similar with :obj:`LoadImageFromFile`, but the image has been loaded as :obj:`np.ndarray` in ``results['img']``. Can be used when loading image from webcam. Required Keys: - img Modified Keys: - img - img_path - img_shape - ori_shape Args: to_float32 (bool): Whether to convert the loaded image to a float32 numpy array. If set to False, the loaded image is an uint8 array. Defaults to False. """ def __init__(self, **kwargs) -> None: super().__init__() self.from_file = TRANSFORMS.build( dict(type='LoadImageFromFile', **kwargs)) self.from_ndarray = TRANSFORMS.build( dict(type='mmdet.LoadImageFromNDArray', **kwargs)) def transform(self, results: Union[str, np.ndarray, dict]) -> dict: """Transform function to add image meta information. Args: results (str, np.ndarray or dict): The result. Returns: dict: The dict contains loaded image and meta information. """ if isinstance(results, str): inputs = dict(img_path=results) elif isinstance(results, np.ndarray): inputs = dict(img=results) elif isinstance(results, dict): inputs = results else: raise NotImplementedError if 'img' in inputs: return self.from_ndarray(inputs) return self.from_file(inputs) ================================================ FILE: mmdet/datasets/transforms/transforms.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import inspect import math from typing import List, Optional, Sequence, Tuple, Union import cv2 import mmcv import numpy as np from mmcv.image.geometric import _scale_size from mmcv.transforms import BaseTransform from mmcv.transforms import Pad as MMCV_Pad from mmcv.transforms import RandomFlip as MMCV_RandomFlip from mmcv.transforms import Resize as MMCV_Resize from mmcv.transforms.utils import avoid_cache_randomness, cache_randomness from mmengine.dataset import BaseDataset from mmengine.utils import is_str from numpy import random from mmdet.registry import TRANSFORMS from mmdet.structures.bbox import HorizontalBoxes, autocast_box_type from mmdet.structures.mask import BitmapMasks, PolygonMasks from mmdet.utils import log_img_scale try: from imagecorruptions import corrupt except ImportError: corrupt = None try: import albumentations from albumentations import Compose except ImportError: albumentations = None Compose = None Number = Union[int, float] @TRANSFORMS.register_module() class Resize(MMCV_Resize): """Resize images & bbox & seg. This transform resizes the input image according to ``scale`` or ``scale_factor``. Bboxes, masks, and seg map are then resized with the same scale factor. if ``scale`` and ``scale_factor`` are both set, it will use ``scale`` to resize. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - img_shape - gt_bboxes - gt_masks - gt_seg_map Added Keys: - scale - scale_factor - keep_ratio - homography_matrix Args: scale (int or tuple): Images scales for resizing. Defaults to None scale_factor (float or tuple[float]): Scale factors for resizing. Defaults to None. keep_ratio (bool): Whether to keep the aspect ratio when resizing the image. Defaults to False. clip_object_border (bool): Whether to clip the objects outside the border of the image. In some dataset like MOT17, the gt bboxes are allowed to cross the border of images. Therefore, we don't need to clip the gt bboxes in these cases. Defaults to True. backend (str): Image resize backend, choices are 'cv2' and 'pillow'. These two backends generates slightly different results. Defaults to 'cv2'. interpolation (str): Interpolation method, accepted values are "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' backend, "nearest", "bilinear" for 'pillow' backend. Defaults to 'bilinear'. """ def _resize_masks(self, results: dict) -> None: """Resize masks with ``results['scale']``""" if results.get('gt_masks', None) is not None: if self.keep_ratio: results['gt_masks'] = results['gt_masks'].rescale( results['scale']) else: results['gt_masks'] = results['gt_masks'].resize( results['img_shape']) def _resize_bboxes(self, results: dict) -> None: """Resize bounding boxes with ``results['scale_factor']``.""" if results.get('gt_bboxes', None) is not None: results['gt_bboxes'].rescale_(results['scale_factor']) if self.clip_object_border: results['gt_bboxes'].clip_(results['img_shape']) def _resize_seg(self, results: dict) -> None: """Resize semantic segmentation map with ``results['scale']``.""" if results.get('gt_seg_map', None) is not None: if self.keep_ratio: gt_seg = mmcv.imrescale( results['gt_seg_map'], results['scale'], interpolation='nearest', backend=self.backend) else: gt_seg = mmcv.imresize( results['gt_seg_map'], results['scale'], interpolation='nearest', backend=self.backend) results['gt_seg_map'] = gt_seg def _record_homography_matrix(self, results: dict) -> None: """Record the homography matrix for the Resize.""" w_scale, h_scale = results['scale_factor'] homography_matrix = np.array( [[w_scale, 0, 0], [0, h_scale, 0], [0, 0, 1]], dtype=np.float32) if results.get('homography_matrix', None) is None: results['homography_matrix'] = homography_matrix else: results['homography_matrix'] = homography_matrix @ results[ 'homography_matrix'] @autocast_box_type() def transform(self, results: dict) -> dict: """Transform function to resize images, bounding boxes and semantic segmentation map. Args: results (dict): Result dict from loading pipeline. Returns: dict: Resized results, 'img', 'gt_bboxes', 'gt_seg_map', 'scale', 'scale_factor', 'height', 'width', and 'keep_ratio' keys are updated in result dict. """ if self.scale: results['scale'] = self.scale else: img_shape = results['img'].shape[:2] results['scale'] = _scale_size(img_shape[::-1], self.scale_factor) self._resize_img(results) self._resize_bboxes(results) self._resize_masks(results) self._resize_seg(results) self._record_homography_matrix(results) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(scale={self.scale}, ' repr_str += f'scale_factor={self.scale_factor}, ' repr_str += f'keep_ratio={self.keep_ratio}, ' repr_str += f'clip_object_border={self.clip_object_border}), ' repr_str += f'backend={self.backend}), ' repr_str += f'interpolation={self.interpolation})' return repr_str @TRANSFORMS.register_module() class FixShapeResize(Resize): """Resize images & bbox & seg to the specified size. This transform resizes the input image according to ``width`` and ``height``. Bboxes, masks, and seg map are then resized with the same parameters. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - img_shape - gt_bboxes - gt_masks - gt_seg_map Added Keys: - scale - scale_factor - keep_ratio - homography_matrix Args: width (int): width for resizing. height (int): height for resizing. Defaults to None. pad_val (Number | dict[str, Number], optional): Padding value for if the pad_mode is "constant". If it is a single number, the value to pad the image is the number and to pad the semantic segmentation map is 255. If it is a dict, it should have the following keys: - img: The value to pad the image. - seg: The value to pad the semantic segmentation map. Defaults to dict(img=0, seg=255). keep_ratio (bool): Whether to keep the aspect ratio when resizing the image. Defaults to False. clip_object_border (bool): Whether to clip the objects outside the border of the image. In some dataset like MOT17, the gt bboxes are allowed to cross the border of images. Therefore, we don't need to clip the gt bboxes in these cases. Defaults to True. backend (str): Image resize backend, choices are 'cv2' and 'pillow'. These two backends generates slightly different results. Defaults to 'cv2'. interpolation (str): Interpolation method, accepted values are "nearest", "bilinear", "bicubic", "area", "lanczos" for 'cv2' backend, "nearest", "bilinear" for 'pillow' backend. Defaults to 'bilinear'. """ def __init__(self, width: int, height: int, pad_val: Union[Number, dict] = dict(img=0, seg=255), keep_ratio: bool = False, clip_object_border: bool = True, backend: str = 'cv2', interpolation: str = 'bilinear') -> None: assert width is not None and height is not None, ( '`width` and' '`height` can not be `None`') self.width = width self.height = height self.scale = (width, height) self.backend = backend self.interpolation = interpolation self.keep_ratio = keep_ratio self.clip_object_border = clip_object_border if keep_ratio is True: # padding to the fixed size when keep_ratio=True self.pad_transform = Pad(size=self.scale, pad_val=pad_val) @autocast_box_type() def transform(self, results: dict) -> dict: """Transform function to resize images, bounding boxes and semantic segmentation map. Args: results (dict): Result dict from loading pipeline. Returns: dict: Resized results, 'img', 'gt_bboxes', 'gt_seg_map', 'scale', 'scale_factor', 'height', 'width', and 'keep_ratio' keys are updated in result dict. """ img = results['img'] h, w = img.shape[:2] if self.keep_ratio: scale_factor = min(self.width / w, self.height / h) results['scale_factor'] = (scale_factor, scale_factor) real_w, real_h = int(w * float(scale_factor) + 0.5), int(h * float(scale_factor) + 0.5) img, scale_factor = mmcv.imrescale( results['img'], (real_w, real_h), interpolation=self.interpolation, return_scale=True, backend=self.backend) # the w_scale and h_scale has minor difference # a real fix should be done in the mmcv.imrescale in the future results['img'] = img results['img_shape'] = img.shape[:2] results['keep_ratio'] = self.keep_ratio results['scale'] = (real_w, real_h) else: results['scale'] = (self.width, self.height) results['scale_factor'] = (self.width / w, self.height / h) super()._resize_img(results) self._resize_bboxes(results) self._resize_masks(results) self._resize_seg(results) self._record_homography_matrix(results) if self.keep_ratio: self.pad_transform(results) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(width={self.width}, height={self.height}, ' repr_str += f'keep_ratio={self.keep_ratio}, ' repr_str += f'clip_object_border={self.clip_object_border}), ' repr_str += f'backend={self.backend}), ' repr_str += f'interpolation={self.interpolation})' return repr_str @TRANSFORMS.register_module() class RandomFlip(MMCV_RandomFlip): """Flip the image & bbox & mask & segmentation map. Added or Updated keys: flip, flip_direction, img, gt_bboxes, and gt_seg_map. There are 3 flip modes: - ``prob`` is float, ``direction`` is string: the image will be ``direction``ly flipped with probability of ``prob`` . E.g., ``prob=0.5``, ``direction='horizontal'``, then image will be horizontally flipped with probability of 0.5. - ``prob`` is float, ``direction`` is list of string: the image will be ``direction[i]``ly flipped with probability of ``prob/len(direction)``. E.g., ``prob=0.5``, ``direction=['horizontal', 'vertical']``, then image will be horizontally flipped with probability of 0.25, vertically with probability of 0.25. - ``prob`` is list of float, ``direction`` is list of string: given ``len(prob) == len(direction)``, the image will be ``direction[i]``ly flipped with probability of ``prob[i]``. E.g., ``prob=[0.3, 0.5]``, ``direction=['horizontal', 'vertical']``, then image will be horizontally flipped with probability of 0.3, vertically with probability of 0.5. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - gt_bboxes - gt_masks - gt_seg_map Added Keys: - flip - flip_direction - homography_matrix Args: prob (float | list[float], optional): The flipping probability. Defaults to None. direction(str | list[str]): The flipping direction. Options If input is a list, the length must equal ``prob``. Each element in ``prob`` indicates the flip probability of corresponding direction. Defaults to 'horizontal'. """ def _record_homography_matrix(self, results: dict) -> None: """Record the homography matrix for the RandomFlip.""" cur_dir = results['flip_direction'] h, w = results['img'].shape[:2] if cur_dir == 'horizontal': homography_matrix = np.array([[-1, 0, w], [0, 1, 0], [0, 0, 1]], dtype=np.float32) elif cur_dir == 'vertical': homography_matrix = np.array([[1, 0, 0], [0, -1, h], [0, 0, 1]], dtype=np.float32) elif cur_dir == 'diagonal': homography_matrix = np.array([[-1, 0, w], [0, -1, h], [0, 0, 1]], dtype=np.float32) else: homography_matrix = np.eye(3, dtype=np.float32) if results.get('homography_matrix', None) is None: results['homography_matrix'] = homography_matrix else: results['homography_matrix'] = homography_matrix @ results[ 'homography_matrix'] @autocast_box_type() def _flip(self, results: dict) -> None: """Flip images, bounding boxes, and semantic segmentation map.""" # flip image results['img'] = mmcv.imflip( results['img'], direction=results['flip_direction']) img_shape = results['img'].shape[:2] # flip bboxes if results.get('gt_bboxes', None) is not None: results['gt_bboxes'].flip_(img_shape, results['flip_direction']) # flip masks if results.get('gt_masks', None) is not None: results['gt_masks'] = results['gt_masks'].flip( results['flip_direction']) # flip segs if results.get('gt_seg_map', None) is not None: results['gt_seg_map'] = mmcv.imflip( results['gt_seg_map'], direction=results['flip_direction']) # record homography matrix for flip self._record_homography_matrix(results) @TRANSFORMS.register_module() class RandomShift(BaseTransform): """Shift the image and box given shift pixels and probability. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) - gt_bboxes_labels (np.int64) - gt_ignore_flags (bool) (optional) Modified Keys: - img - gt_bboxes - gt_bboxes_labels - gt_ignore_flags (bool) (optional) Args: prob (float): Probability of shifts. Defaults to 0.5. max_shift_px (int): The max pixels for shifting. Defaults to 32. filter_thr_px (int): The width and height threshold for filtering. The bbox and the rest of the targets below the width and height threshold will be filtered. Defaults to 1. """ def __init__(self, prob: float = 0.5, max_shift_px: int = 32, filter_thr_px: int = 1) -> None: assert 0 <= prob <= 1 assert max_shift_px >= 0 self.prob = prob self.max_shift_px = max_shift_px self.filter_thr_px = int(filter_thr_px) @cache_randomness def _random_prob(self) -> float: return random.uniform(0, 1) @autocast_box_type() def transform(self, results: dict) -> dict: """Transform function to random shift images, bounding boxes. Args: results (dict): Result dict from loading pipeline. Returns: dict: Shift results. """ if self._random_prob() < self.prob: img_shape = results['img'].shape[:2] random_shift_x = random.randint(-self.max_shift_px, self.max_shift_px) random_shift_y = random.randint(-self.max_shift_px, self.max_shift_px) new_x = max(0, random_shift_x) ori_x = max(0, -random_shift_x) new_y = max(0, random_shift_y) ori_y = max(0, -random_shift_y) # TODO: support mask and semantic segmentation maps. bboxes = results['gt_bboxes'].clone() bboxes.translate_([random_shift_x, random_shift_y]) # clip border bboxes.clip_(img_shape) # remove invalid bboxes valid_inds = (bboxes.widths > self.filter_thr_px).numpy() & ( bboxes.heights > self.filter_thr_px).numpy() # If the shift does not contain any gt-bbox area, skip this # image. if not valid_inds.any(): return results bboxes = bboxes[valid_inds] results['gt_bboxes'] = bboxes results['gt_bboxes_labels'] = results['gt_bboxes_labels'][ valid_inds] if results.get('gt_ignore_flags', None) is not None: results['gt_ignore_flags'] = \ results['gt_ignore_flags'][valid_inds] # shift img img = results['img'] new_img = np.zeros_like(img) img_h, img_w = img.shape[:2] new_h = img_h - np.abs(random_shift_y) new_w = img_w - np.abs(random_shift_x) new_img[new_y:new_y + new_h, new_x:new_x + new_w] \ = img[ori_y:ori_y + new_h, ori_x:ori_x + new_w] results['img'] = new_img return results def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(prob={self.prob}, ' repr_str += f'max_shift_px={self.max_shift_px}, ' repr_str += f'filter_thr_px={self.filter_thr_px})' return repr_str @TRANSFORMS.register_module() class Pad(MMCV_Pad): """Pad the image & segmentation map. There are three padding modes: (1) pad to a fixed size and (2) pad to the minimum size that is divisible by some number. and (3)pad to square. Also, pad to square and pad to the minimum size can be used as the same time. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - img_shape - gt_masks - gt_seg_map Added Keys: - pad_shape - pad_fixed_size - pad_size_divisor Args: size (tuple, optional): Fixed padding size. Expected padding shape (width, height). Defaults to None. size_divisor (int, optional): The divisor of padded size. Defaults to None. pad_to_square (bool): Whether to pad the image into a square. Currently only used for YOLOX. Defaults to False. pad_val (Number | dict[str, Number], optional) - Padding value for if the pad_mode is "constant". If it is a single number, the value to pad the image is the number and to pad the semantic segmentation map is 255. If it is a dict, it should have the following keys: - img: The value to pad the image. - seg: The value to pad the semantic segmentation map. Defaults to dict(img=0, seg=255). padding_mode (str): Type of padding. Should be: constant, edge, reflect or symmetric. Defaults to 'constant'. - constant: pads with a constant value, this value is specified with pad_val. - edge: pads with the last value at the edge of the image. - reflect: pads with reflection of image without repeating the last value on the edge. For example, padding [1, 2, 3, 4] with 2 elements on both sides in reflect mode will result in [3, 2, 1, 2, 3, 4, 3, 2]. - symmetric: pads with reflection of image repeating the last value on the edge. For example, padding [1, 2, 3, 4] with 2 elements on both sides in symmetric mode will result in [2, 1, 1, 2, 3, 4, 4, 3] """ def _pad_masks(self, results: dict) -> None: """Pad masks according to ``results['pad_shape']``.""" if results.get('gt_masks', None) is not None: pad_val = self.pad_val.get('masks', 0) pad_shape = results['pad_shape'][:2] results['gt_masks'] = results['gt_masks'].pad( pad_shape, pad_val=pad_val) def transform(self, results: dict) -> dict: """Call function to pad images, masks, semantic segmentation maps. Args: results (dict): Result dict from loading pipeline. Returns: dict: Updated result dict. """ self._pad_img(results) self._pad_seg(results) self._pad_masks(results) return results @TRANSFORMS.register_module() class RandomCrop(BaseTransform): """Random crop the image & bboxes & masks. The absolute ``crop_size`` is sampled based on ``crop_type`` and ``image_size``, then the cropped results are generated. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_ignore_flags (bool) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - img_shape - gt_bboxes (optional) - gt_bboxes_labels (optional) - gt_masks (optional) - gt_ignore_flags (optional) - gt_seg_map (optional) Added Keys: - homography_matrix Args: crop_size (tuple): The relative ratio or absolute pixels of (width, height). crop_type (str, optional): One of "relative_range", "relative", "absolute", "absolute_range". "relative" randomly crops (h * crop_size[0], w * crop_size[1]) part from an input of size (h, w). "relative_range" uniformly samples relative crop size from range [crop_size[0], 1] and [crop_size[1], 1] for height and width respectively. "absolute" crops from an input with absolute size (crop_size[0], crop_size[1]). "absolute_range" uniformly samples crop_h in range [crop_size[0], min(h, crop_size[1])] and crop_w in range [crop_size[0], min(w, crop_size[1])]. Defaults to "absolute". allow_negative_crop (bool, optional): Whether to allow a crop that does not contain any bbox area. Defaults to False. recompute_bbox (bool, optional): Whether to re-compute the boxes based on cropped instance masks. Defaults to False. bbox_clip_border (bool, optional): Whether clip the objects outside the border of the image. Defaults to True. Note: - If the image is smaller than the absolute crop size, return the original image. - The keys for bboxes, labels and masks must be aligned. That is, ``gt_bboxes`` corresponds to ``gt_labels`` and ``gt_masks``, and ``gt_bboxes_ignore`` corresponds to ``gt_labels_ignore`` and ``gt_masks_ignore``. - If the crop does not contain any gt-bbox region and ``allow_negative_crop`` is set to False, skip this image. """ def __init__(self, crop_size: tuple, crop_type: str = 'absolute', allow_negative_crop: bool = False, recompute_bbox: bool = False, bbox_clip_border: bool = True) -> None: if crop_type not in [ 'relative_range', 'relative', 'absolute', 'absolute_range' ]: raise ValueError(f'Invalid crop_type {crop_type}.') if crop_type in ['absolute', 'absolute_range']: assert crop_size[0] > 0 and crop_size[1] > 0 assert isinstance(crop_size[0], int) and isinstance( crop_size[1], int) if crop_type == 'absolute_range': assert crop_size[0] <= crop_size[1] else: assert 0 < crop_size[0] <= 1 and 0 < crop_size[1] <= 1 self.crop_size = crop_size self.crop_type = crop_type self.allow_negative_crop = allow_negative_crop self.bbox_clip_border = bbox_clip_border self.recompute_bbox = recompute_bbox def _crop_data(self, results: dict, crop_size: Tuple[int, int], allow_negative_crop: bool) -> Union[dict, None]: """Function to randomly crop images, bounding boxes, masks, semantic segmentation maps. Args: results (dict): Result dict from loading pipeline. crop_size (Tuple[int, int]): Expected absolute size after cropping, (h, w). allow_negative_crop (bool): Whether to allow a crop that does not contain any bbox area. Returns: results (Union[dict, None]): Randomly cropped results, 'img_shape' key in result dict is updated according to crop size. None will be returned when there is no valid bbox after cropping. """ assert crop_size[0] > 0 and crop_size[1] > 0 img = results['img'] margin_h = max(img.shape[0] - crop_size[0], 0) margin_w = max(img.shape[1] - crop_size[1], 0) offset_h, offset_w = self._rand_offset((margin_h, margin_w)) crop_y1, crop_y2 = offset_h, offset_h + crop_size[0] crop_x1, crop_x2 = offset_w, offset_w + crop_size[1] # Record the homography matrix for the RandomCrop homography_matrix = np.array( [[1, 0, -offset_w], [0, 1, -offset_h], [0, 0, 1]], dtype=np.float32) if results.get('homography_matrix', None) is None: results['homography_matrix'] = homography_matrix else: results['homography_matrix'] = homography_matrix @ results[ 'homography_matrix'] # crop the image img = img[crop_y1:crop_y2, crop_x1:crop_x2, ...] img_shape = img.shape results['img'] = img results['img_shape'] = img_shape # crop bboxes accordingly and clip to the image boundary if results.get('gt_bboxes', None) is not None: bboxes = results['gt_bboxes'] bboxes.translate_([-offset_w, -offset_h]) if self.bbox_clip_border: bboxes.clip_(img_shape[:2]) valid_inds = bboxes.is_inside(img_shape[:2]).numpy() # If the crop does not contain any gt-bbox area and # allow_negative_crop is False, skip this image. if (not valid_inds.any() and not allow_negative_crop): return None results['gt_bboxes'] = bboxes[valid_inds] if results.get('gt_ignore_flags', None) is not None: results['gt_ignore_flags'] = \ results['gt_ignore_flags'][valid_inds] if results.get('gt_bboxes_labels', None) is not None: results['gt_bboxes_labels'] = \ results['gt_bboxes_labels'][valid_inds] if results.get('gt_masks', None) is not None: results['gt_masks'] = results['gt_masks'][ valid_inds.nonzero()[0]].crop( np.asarray([crop_x1, crop_y1, crop_x2, crop_y2])) if self.recompute_bbox: results['gt_bboxes'] = results['gt_masks'].get_bboxes( type(results['gt_bboxes'])) # crop semantic seg if results.get('gt_seg_map', None) is not None: results['gt_seg_map'] = results['gt_seg_map'][crop_y1:crop_y2, crop_x1:crop_x2] return results @cache_randomness def _rand_offset(self, margin: Tuple[int, int]) -> Tuple[int, int]: """Randomly generate crop offset. Args: margin (Tuple[int, int]): The upper bound for the offset generated randomly. Returns: Tuple[int, int]: The random offset for the crop. """ margin_h, margin_w = margin offset_h = np.random.randint(0, margin_h + 1) offset_w = np.random.randint(0, margin_w + 1) return offset_h, offset_w @cache_randomness def _get_crop_size(self, image_size: Tuple[int, int]) -> Tuple[int, int]: """Randomly generates the absolute crop size based on `crop_type` and `image_size`. Args: image_size (Tuple[int, int]): (h, w). Returns: crop_size (Tuple[int, int]): (crop_h, crop_w) in absolute pixels. """ h, w = image_size if self.crop_type == 'absolute': return min(self.crop_size[1], h), min(self.crop_size[0], w) elif self.crop_type == 'absolute_range': crop_h = np.random.randint( min(h, self.crop_size[0]), min(h, self.crop_size[1]) + 1) crop_w = np.random.randint( min(w, self.crop_size[0]), min(w, self.crop_size[1]) + 1) return crop_h, crop_w elif self.crop_type == 'relative': crop_w, crop_h = self.crop_size return int(h * crop_h + 0.5), int(w * crop_w + 0.5) else: # 'relative_range' crop_size = np.asarray(self.crop_size, dtype=np.float32) crop_h, crop_w = crop_size + np.random.rand(2) * (1 - crop_size) return int(h * crop_h + 0.5), int(w * crop_w + 0.5) @autocast_box_type() def transform(self, results: dict) -> Union[dict, None]: """Transform function to randomly crop images, bounding boxes, masks, semantic segmentation maps. Args: results (dict): Result dict from loading pipeline. Returns: results (Union[dict, None]): Randomly cropped results, 'img_shape' key in result dict is updated according to crop size. None will be returned when there is no valid bbox after cropping. """ image_size = results['img'].shape[:2] crop_size = self._get_crop_size(image_size) results = self._crop_data(results, crop_size, self.allow_negative_crop) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(crop_size={self.crop_size}, ' repr_str += f'crop_type={self.crop_type}, ' repr_str += f'allow_negative_crop={self.allow_negative_crop}, ' repr_str += f'recompute_bbox={self.recompute_bbox}, ' repr_str += f'bbox_clip_border={self.bbox_clip_border})' return repr_str @TRANSFORMS.register_module() class SegRescale(BaseTransform): """Rescale semantic segmentation maps. This transform rescale the ``gt_seg_map`` according to ``scale_factor``. Required Keys: - gt_seg_map Modified Keys: - gt_seg_map Args: scale_factor (float): The scale factor of the final output. Defaults to 1. backend (str): Image rescale backend, choices are 'cv2' and 'pillow'. These two backends generates slightly different results. Defaults to 'cv2'. """ def __init__(self, scale_factor: float = 1, backend: str = 'cv2') -> None: self.scale_factor = scale_factor self.backend = backend def transform(self, results: dict) -> dict: """Transform function to scale the semantic segmentation map. Args: results (dict): Result dict from loading pipeline. Returns: dict: Result dict with semantic segmentation map scaled. """ if self.scale_factor != 1: results['gt_seg_map'] = mmcv.imrescale( results['gt_seg_map'], self.scale_factor, interpolation='nearest', backend=self.backend) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(scale_factor={self.scale_factor}, ' repr_str += f'backend={self.backend})' return repr_str @TRANSFORMS.register_module() class PhotoMetricDistortion(BaseTransform): """Apply photometric distortion to image sequentially, every transformation is applied with a probability of 0.5. The position of random contrast is in second or second to last. 1. random brightness 2. random contrast (mode 0) 3. convert color from BGR to HSV 4. random saturation 5. random hue 6. convert color from HSV to BGR 7. random contrast (mode 1) 8. randomly swap channels Required Keys: - img (np.uint8) Modified Keys: - img (np.float32) Args: brightness_delta (int): delta of brightness. contrast_range (sequence): range of contrast. saturation_range (sequence): range of saturation. hue_delta (int): delta of hue. """ def __init__(self, brightness_delta: int = 32, contrast_range: Sequence[Number] = (0.5, 1.5), saturation_range: Sequence[Number] = (0.5, 1.5), hue_delta: int = 18) -> None: self.brightness_delta = brightness_delta self.contrast_lower, self.contrast_upper = contrast_range self.saturation_lower, self.saturation_upper = saturation_range self.hue_delta = hue_delta @cache_randomness def _random_flags(self) -> Sequence[Number]: mode = random.randint(2) brightness_flag = random.randint(2) contrast_flag = random.randint(2) saturation_flag = random.randint(2) hue_flag = random.randint(2) swap_flag = random.randint(2) delta_value = random.uniform(-self.brightness_delta, self.brightness_delta) alpha_value = random.uniform(self.contrast_lower, self.contrast_upper) saturation_value = random.uniform(self.saturation_lower, self.saturation_upper) hue_value = random.uniform(-self.hue_delta, self.hue_delta) swap_value = random.permutation(3) return (mode, brightness_flag, contrast_flag, saturation_flag, hue_flag, swap_flag, delta_value, alpha_value, saturation_value, hue_value, swap_value) def transform(self, results: dict) -> dict: """Transform function to perform photometric distortion on images. Args: results (dict): Result dict from loading pipeline. Returns: dict: Result dict with images distorted. """ assert 'img' in results, '`img` is not found in results' img = results['img'] img = img.astype(np.float32) (mode, brightness_flag, contrast_flag, saturation_flag, hue_flag, swap_flag, delta_value, alpha_value, saturation_value, hue_value, swap_value) = self._random_flags() # random brightness if brightness_flag: img += delta_value # mode == 0 --> do random contrast first # mode == 1 --> do random contrast last if mode == 1: if contrast_flag: img *= alpha_value # convert color from BGR to HSV img = mmcv.bgr2hsv(img) # random saturation if saturation_flag: img[..., 1] *= saturation_value # For image(type=float32), after convert bgr to hsv by opencv, # valid saturation value range is [0, 1] if saturation_value > 1: img[..., 1] = img[..., 1].clip(0, 1) # random hue if hue_flag: img[..., 0] += hue_value img[..., 0][img[..., 0] > 360] -= 360 img[..., 0][img[..., 0] < 0] += 360 # convert color from HSV to BGR img = mmcv.hsv2bgr(img) # random contrast if mode == 0: if contrast_flag: img *= alpha_value # randomly swap channels if swap_flag: img = img[..., swap_value] results['img'] = img return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(brightness_delta={self.brightness_delta}, ' repr_str += 'contrast_range=' repr_str += f'{(self.contrast_lower, self.contrast_upper)}, ' repr_str += 'saturation_range=' repr_str += f'{(self.saturation_lower, self.saturation_upper)}, ' repr_str += f'hue_delta={self.hue_delta})' return repr_str @TRANSFORMS.register_module() class Expand(BaseTransform): """Random expand the image & bboxes & masks & segmentation map. Randomly place the original image on a canvas of ``ratio`` x original image size filled with mean values. The ratio is in the range of ratio_range. Required Keys: - img - img_shape - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - img_shape - gt_bboxes - gt_masks - gt_seg_map Args: mean (sequence): mean value of dataset. to_rgb (bool): if need to convert the order of mean to align with RGB. ratio_range (sequence)): range of expand ratio. seg_ignore_label (int): label of ignore segmentation map. prob (float): probability of applying this transformation """ def __init__(self, mean: Sequence[Number] = (0, 0, 0), to_rgb: bool = True, ratio_range: Sequence[Number] = (1, 4), seg_ignore_label: int = None, prob: float = 0.5) -> None: self.to_rgb = to_rgb self.ratio_range = ratio_range if to_rgb: self.mean = mean[::-1] else: self.mean = mean self.min_ratio, self.max_ratio = ratio_range self.seg_ignore_label = seg_ignore_label self.prob = prob @cache_randomness def _random_prob(self) -> float: return random.uniform(0, 1) @cache_randomness def _random_ratio(self) -> float: return random.uniform(self.min_ratio, self.max_ratio) @cache_randomness def _random_left_top(self, ratio: float, h: int, w: int) -> Tuple[int, int]: left = int(random.uniform(0, w * ratio - w)) top = int(random.uniform(0, h * ratio - h)) return left, top @autocast_box_type() def transform(self, results: dict) -> dict: """Transform function to expand images, bounding boxes, masks, segmentation map. Args: results (dict): Result dict from loading pipeline. Returns: dict: Result dict with images, bounding boxes, masks, segmentation map expanded. """ if self._random_prob() > self.prob: return results assert 'img' in results, '`img` is not found in results' img = results['img'] h, w, c = img.shape ratio = self._random_ratio() # speedup expand when meets large image if np.all(self.mean == self.mean[0]): expand_img = np.empty((int(h * ratio), int(w * ratio), c), img.dtype) expand_img.fill(self.mean[0]) else: expand_img = np.full((int(h * ratio), int(w * ratio), c), self.mean, dtype=img.dtype) left, top = self._random_left_top(ratio, h, w) expand_img[top:top + h, left:left + w] = img results['img'] = expand_img results['img_shape'] = expand_img.shape[:2] # expand bboxes if results.get('gt_bboxes', None) is not None: results['gt_bboxes'].translate_([left, top]) # expand masks if results.get('gt_masks', None) is not None: results['gt_masks'] = results['gt_masks'].expand( int(h * ratio), int(w * ratio), top, left) # expand segmentation map if results.get('gt_seg_map', None) is not None: gt_seg = results['gt_seg_map'] expand_gt_seg = np.full((int(h * ratio), int(w * ratio)), self.seg_ignore_label, dtype=gt_seg.dtype) expand_gt_seg[top:top + h, left:left + w] = gt_seg results['gt_seg_map'] = expand_gt_seg return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(mean={self.mean}, to_rgb={self.to_rgb}, ' repr_str += f'ratio_range={self.ratio_range}, ' repr_str += f'seg_ignore_label={self.seg_ignore_label}, ' repr_str += f'prob={self.prob})' return repr_str @TRANSFORMS.register_module() class MinIoURandomCrop(BaseTransform): """Random crop the image & bboxes & masks & segmentation map, the cropped patches have minimum IoU requirement with original image & bboxes & masks. & segmentation map, the IoU threshold is randomly selected from min_ious. Required Keys: - img - img_shape - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - gt_ignore_flags (bool) (optional) - gt_seg_map (np.uint8) (optional) Modified Keys: - img - img_shape - gt_bboxes - gt_bboxes_labels - gt_masks - gt_ignore_flags - gt_seg_map Args: min_ious (Sequence[float]): minimum IoU threshold for all intersections with bounding boxes. min_crop_size (float): minimum crop's size (i.e. h,w := a*h, a*w, where a >= min_crop_size). bbox_clip_border (bool, optional): Whether clip the objects outside the border of the image. Defaults to True. """ def __init__(self, min_ious: Sequence[float] = (0.1, 0.3, 0.5, 0.7, 0.9), min_crop_size: float = 0.3, bbox_clip_border: bool = True) -> None: self.min_ious = min_ious self.sample_mode = (1, *min_ious, 0) self.min_crop_size = min_crop_size self.bbox_clip_border = bbox_clip_border @cache_randomness def _random_mode(self) -> Number: return random.choice(self.sample_mode) @autocast_box_type() def transform(self, results: dict) -> dict: """Transform function to crop images and bounding boxes with minimum IoU constraint. Args: results (dict): Result dict from loading pipeline. Returns: dict: Result dict with images and bounding boxes cropped, \ 'img_shape' key is updated. """ assert 'img' in results, '`img` is not found in results' assert 'gt_bboxes' in results, '`gt_bboxes` is not found in results' img = results['img'] boxes = results['gt_bboxes'] h, w, c = img.shape while True: mode = self._random_mode() self.mode = mode if mode == 1: return results min_iou = self.mode for i in range(50): new_w = random.uniform(self.min_crop_size * w, w) new_h = random.uniform(self.min_crop_size * h, h) # h / w in [0.5, 2] if new_h / new_w < 0.5 or new_h / new_w > 2: continue left = random.uniform(w - new_w) top = random.uniform(h - new_h) patch = np.array( (int(left), int(top), int(left + new_w), int(top + new_h))) # Line or point crop is not allowed if patch[2] == patch[0] or patch[3] == patch[1]: continue overlaps = boxes.overlaps( HorizontalBoxes(patch.reshape(-1, 4).astype(np.float32)), boxes).numpy().reshape(-1) if len(overlaps) > 0 and overlaps.min() < min_iou: continue # center of boxes should inside the crop img # only adjust boxes and instance masks when the gt is not empty if len(overlaps) > 0: # adjust boxes def is_center_of_bboxes_in_patch(boxes, patch): centers = boxes.centers.numpy() mask = ((centers[:, 0] > patch[0]) * (centers[:, 1] > patch[1]) * (centers[:, 0] < patch[2]) * (centers[:, 1] < patch[3])) return mask mask = is_center_of_bboxes_in_patch(boxes, patch) if not mask.any(): continue if results.get('gt_bboxes', None) is not None: boxes = results['gt_bboxes'] mask = is_center_of_bboxes_in_patch(boxes, patch) boxes = boxes[mask] boxes.translate_([-patch[0], -patch[1]]) if self.bbox_clip_border: boxes.clip_( [patch[3] - patch[1], patch[2] - patch[0]]) results['gt_bboxes'] = boxes # ignore_flags if results.get('gt_ignore_flags', None) is not None: results['gt_ignore_flags'] = \ results['gt_ignore_flags'][mask] # labels if results.get('gt_bboxes_labels', None) is not None: results['gt_bboxes_labels'] = results[ 'gt_bboxes_labels'][mask] # mask fields if results.get('gt_masks', None) is not None: results['gt_masks'] = results['gt_masks'][ mask.nonzero()[0]].crop(patch) # adjust the img no matter whether the gt is empty before crop img = img[patch[1]:patch[3], patch[0]:patch[2]] results['img'] = img results['img_shape'] = img.shape[:2] # seg fields if results.get('gt_seg_map', None) is not None: results['gt_seg_map'] = results['gt_seg_map'][ patch[1]:patch[3], patch[0]:patch[2]] return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(min_ious={self.min_ious}, ' repr_str += f'min_crop_size={self.min_crop_size}, ' repr_str += f'bbox_clip_border={self.bbox_clip_border})' return repr_str @TRANSFORMS.register_module() class Corrupt(BaseTransform): """Corruption augmentation. Corruption transforms implemented based on `imagecorruptions `_. Required Keys: - img (np.uint8) Modified Keys: - img (np.uint8) Args: corruption (str): Corruption name. severity (int): The severity of corruption. Defaults to 1. """ def __init__(self, corruption: str, severity: int = 1) -> None: self.corruption = corruption self.severity = severity def transform(self, results: dict) -> dict: """Call function to corrupt image. Args: results (dict): Result dict from loading pipeline. Returns: dict: Result dict with images corrupted. """ if corrupt is None: raise RuntimeError('imagecorruptions is not installed') results['img'] = corrupt( results['img'].astype(np.uint8), corruption_name=self.corruption, severity=self.severity) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(corruption={self.corruption}, ' repr_str += f'severity={self.severity})' return repr_str @TRANSFORMS.register_module() @avoid_cache_randomness class Albu(BaseTransform): """Albumentation augmentation. Adds custom transformations from Albumentations library. Please, visit `https://albumentations.readthedocs.io` to get more information. Required Keys: - img (np.uint8) - gt_bboxes (HorizontalBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) Modified Keys: - img (np.uint8) - gt_bboxes (HorizontalBoxes[torch.float32]) (optional) - gt_masks (BitmapMasks | PolygonMasks) (optional) - img_shape (tuple) An example of ``transforms`` is as followed: .. code-block:: [ dict( type='ShiftScaleRotate', shift_limit=0.0625, scale_limit=0.0, rotate_limit=0, interpolation=1, p=0.5), dict( type='RandomBrightnessContrast', brightness_limit=[0.1, 0.3], contrast_limit=[0.1, 0.3], p=0.2), dict(type='ChannelShuffle', p=0.1), dict( type='OneOf', transforms=[ dict(type='Blur', blur_limit=3, p=1.0), dict(type='MedianBlur', blur_limit=3, p=1.0) ], p=0.1), ] Args: transforms (list[dict]): A list of albu transformations bbox_params (dict, optional): Bbox_params for albumentation `Compose` keymap (dict, optional): Contains {'input key':'albumentation-style key'} skip_img_without_anno (bool): Whether to skip the image if no ann left after aug. Defaults to False. """ def __init__(self, transforms: List[dict], bbox_params: Optional[dict] = None, keymap: Optional[dict] = None, skip_img_without_anno: bool = False) -> None: if Compose is None: raise RuntimeError('albumentations is not installed') # Args will be modified later, copying it will be safer transforms = copy.deepcopy(transforms) if bbox_params is not None: bbox_params = copy.deepcopy(bbox_params) if keymap is not None: keymap = copy.deepcopy(keymap) self.transforms = transforms self.filter_lost_elements = False self.skip_img_without_anno = skip_img_without_anno # A simple workaround to remove masks without boxes if (isinstance(bbox_params, dict) and 'label_fields' in bbox_params and 'filter_lost_elements' in bbox_params): self.filter_lost_elements = True self.origin_label_fields = bbox_params['label_fields'] bbox_params['label_fields'] = ['idx_mapper'] del bbox_params['filter_lost_elements'] self.bbox_params = ( self.albu_builder(bbox_params) if bbox_params else None) self.aug = Compose([self.albu_builder(t) for t in self.transforms], bbox_params=self.bbox_params) if not keymap: self.keymap_to_albu = { 'img': 'image', 'gt_masks': 'masks', 'gt_bboxes': 'bboxes' } else: self.keymap_to_albu = keymap self.keymap_back = {v: k for k, v in self.keymap_to_albu.items()} def albu_builder(self, cfg: dict) -> albumentations: """Import a module from albumentations. It inherits some of :func:`build_from_cfg` logic. Args: cfg (dict): Config dict. It should at least contain the key "type". Returns: obj: The constructed object. """ assert isinstance(cfg, dict) and 'type' in cfg args = cfg.copy() obj_type = args.pop('type') if is_str(obj_type): if albumentations is None: raise RuntimeError('albumentations is not installed') obj_cls = getattr(albumentations, obj_type) elif inspect.isclass(obj_type): obj_cls = obj_type else: raise TypeError( f'type must be a str or valid type, but got {type(obj_type)}') if 'transforms' in args: args['transforms'] = [ self.albu_builder(transform) for transform in args['transforms'] ] return obj_cls(**args) @staticmethod def mapper(d: dict, keymap: dict) -> dict: """Dictionary mapper. Renames keys according to keymap provided. Args: d (dict): old dict keymap (dict): {'old_key':'new_key'} Returns: dict: new dict. """ updated_dict = {} for k, v in zip(d.keys(), d.values()): new_k = keymap.get(k, k) updated_dict[new_k] = d[k] return updated_dict @autocast_box_type() def transform(self, results: dict) -> Union[dict, None]: """Transform function of Albu.""" # TODO: gt_seg_map is not currently supported # dict to albumentations format results = self.mapper(results, self.keymap_to_albu) results, ori_masks = self._preprocess_results(results) results = self.aug(**results) results = self._postprocess_results(results, ori_masks) if results is None: return None # back to the original format results = self.mapper(results, self.keymap_back) results['img_shape'] = results['img'].shape return results def _preprocess_results(self, results: dict) -> tuple: """Pre-processing results to facilitate the use of Albu.""" if 'bboxes' in results: # to list of boxes if not isinstance(results['bboxes'], HorizontalBoxes): raise NotImplementedError( 'Albu only supports horizontal boxes now') bboxes = results['bboxes'].numpy() results['bboxes'] = [x for x in bboxes] # add pseudo-field for filtration if self.filter_lost_elements: results['idx_mapper'] = np.arange(len(results['bboxes'])) # TODO: Support mask structure in albu ori_masks = None if 'masks' in results: if isinstance(results['masks'], PolygonMasks): raise NotImplementedError( 'Albu only supports BitMap masks now') ori_masks = results['masks'] if albumentations.__version__ < '0.5': results['masks'] = results['masks'].masks else: results['masks'] = [mask for mask in results['masks'].masks] return results, ori_masks def _postprocess_results( self, results: dict, ori_masks: Optional[Union[BitmapMasks, PolygonMasks]] = None) -> dict: """Post-processing Albu output.""" # albumentations may return np.array or list on different versions if 'gt_bboxes_labels' in results and isinstance( results['gt_bboxes_labels'], list): results['gt_bboxes_labels'] = np.array( results['gt_bboxes_labels'], dtype=np.int64) if 'gt_ignore_flags' in results and isinstance( results['gt_ignore_flags'], list): results['gt_ignore_flags'] = np.array( results['gt_ignore_flags'], dtype=bool) if 'bboxes' in results: if isinstance(results['bboxes'], list): results['bboxes'] = np.array( results['bboxes'], dtype=np.float32) results['bboxes'] = results['bboxes'].reshape(-1, 4) results['bboxes'] = HorizontalBoxes(results['bboxes']) # filter label_fields if self.filter_lost_elements: for label in self.origin_label_fields: results[label] = np.array( [results[label][i] for i in results['idx_mapper']]) if 'masks' in results: assert ori_masks is not None results['masks'] = np.array( [results['masks'][i] for i in results['idx_mapper']]) results['masks'] = ori_masks.__class__( results['masks'], results['image'].shape[0], results['image'].shape[1]) if (not len(results['idx_mapper']) and self.skip_img_without_anno): return None elif 'masks' in results: results['masks'] = ori_masks.__class__( results['masks'], results['image'].shape[0], results['image'].shape[1]) return results def __repr__(self) -> str: repr_str = self.__class__.__name__ + f'(transforms={self.transforms})' return repr_str @TRANSFORMS.register_module() @avoid_cache_randomness class RandomCenterCropPad(BaseTransform): """Random center crop and random around padding for CornerNet. This operation generates randomly cropped image from the original image and pads it simultaneously. Different from :class:`RandomCrop`, the output shape may not equal to ``crop_size`` strictly. We choose a random value from ``ratios`` and the output shape could be larger or smaller than ``crop_size``. The padding operation is also different from :class:`Pad`, here we use around padding instead of right-bottom padding. The relation between output image (padding image) and original image: .. code:: text output image +----------------------------+ | padded area | +------|----------------------------|----------+ | | cropped area | | | | +---------------+ | | | | | . center | | | original image | | | range | | | | | +---------------+ | | +------|----------------------------|----------+ | padded area | +----------------------------+ There are 5 main areas in the figure: - output image: output image of this operation, also called padding image in following instruction. - original image: input image of this operation. - padded area: non-intersect area of output image and original image. - cropped area: the overlap of output image and original image. - center range: a smaller area where random center chosen from. center range is computed by ``border`` and original image's shape to avoid our random center is too close to original image's border. Also this operation act differently in train and test mode, the summary pipeline is listed below. Train pipeline: 1. Choose a ``random_ratio`` from ``ratios``, the shape of padding image will be ``random_ratio * crop_size``. 2. Choose a ``random_center`` in center range. 3. Generate padding image with center matches the ``random_center``. 4. Initialize the padding image with pixel value equals to ``mean``. 5. Copy the cropped area to padding image. 6. Refine annotations. Test pipeline: 1. Compute output shape according to ``test_pad_mode``. 2. Generate padding image with center matches the original image center. 3. Initialize the padding image with pixel value equals to ``mean``. 4. Copy the ``cropped area`` to padding image. Required Keys: - img (np.float32) - img_shape (tuple) - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_ignore_flags (bool) (optional) Modified Keys: - img (np.float32) - img_shape (tuple) - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_ignore_flags (bool) (optional) Args: crop_size (tuple, optional): expected size after crop, final size will computed according to ratio. Requires (width, height) in train mode, and None in test mode. ratios (tuple, optional): random select a ratio from tuple and crop image to (crop_size[0] * ratio) * (crop_size[1] * ratio). Only available in train mode. Defaults to (0.9, 1.0, 1.1). border (int, optional): max distance from center select area to image border. Only available in train mode. Defaults to 128. mean (sequence, optional): Mean values of 3 channels. std (sequence, optional): Std values of 3 channels. to_rgb (bool, optional): Whether to convert the image from BGR to RGB. test_mode (bool): whether involve random variables in transform. In train mode, crop_size is fixed, center coords and ratio is random selected from predefined lists. In test mode, crop_size is image's original shape, center coords and ratio is fixed. Defaults to False. test_pad_mode (tuple, optional): padding method and padding shape value, only available in test mode. Default is using 'logical_or' with 127 as padding shape value. - 'logical_or': final_shape = input_shape | padding_shape_value - 'size_divisor': final_shape = int( ceil(input_shape / padding_shape_value) * padding_shape_value) Defaults to ('logical_or', 127). test_pad_add_pix (int): Extra padding pixel in test mode. Defaults to 0. bbox_clip_border (bool): Whether clip the objects outside the border of the image. Defaults to True. """ def __init__(self, crop_size: Optional[tuple] = None, ratios: Optional[tuple] = (0.9, 1.0, 1.1), border: Optional[int] = 128, mean: Optional[Sequence] = None, std: Optional[Sequence] = None, to_rgb: Optional[bool] = None, test_mode: bool = False, test_pad_mode: Optional[tuple] = ('logical_or', 127), test_pad_add_pix: int = 0, bbox_clip_border: bool = True) -> None: if test_mode: assert crop_size is None, 'crop_size must be None in test mode' assert ratios is None, 'ratios must be None in test mode' assert border is None, 'border must be None in test mode' assert isinstance(test_pad_mode, (list, tuple)) assert test_pad_mode[0] in ['logical_or', 'size_divisor'] else: assert isinstance(crop_size, (list, tuple)) assert crop_size[0] > 0 and crop_size[1] > 0, ( 'crop_size must > 0 in train mode') assert isinstance(ratios, (list, tuple)) assert test_pad_mode is None, ( 'test_pad_mode must be None in train mode') self.crop_size = crop_size self.ratios = ratios self.border = border # We do not set default value to mean, std and to_rgb because these # hyper-parameters are easy to forget but could affect the performance. # Please use the same setting as Normalize for performance assurance. assert mean is not None and std is not None and to_rgb is not None self.to_rgb = to_rgb self.input_mean = mean self.input_std = std if to_rgb: self.mean = mean[::-1] self.std = std[::-1] else: self.mean = mean self.std = std self.test_mode = test_mode self.test_pad_mode = test_pad_mode self.test_pad_add_pix = test_pad_add_pix self.bbox_clip_border = bbox_clip_border def _get_border(self, border, size): """Get final border for the target size. This function generates a ``final_border`` according to image's shape. The area between ``final_border`` and ``size - final_border`` is the ``center range``. We randomly choose center from the ``center range`` to avoid our random center is too close to original image's border. Also ``center range`` should be larger than 0. Args: border (int): The initial border, default is 128. size (int): The width or height of original image. Returns: int: The final border. """ k = 2 * border / size i = pow(2, np.ceil(np.log2(np.ceil(k))) + (k == int(k))) return border // i def _filter_boxes(self, patch, boxes): """Check whether the center of each box is in the patch. Args: patch (list[int]): The cropped area, [left, top, right, bottom]. boxes (numpy array, (N x 4)): Ground truth boxes. Returns: mask (numpy array, (N,)): Each box is inside or outside the patch. """ center = boxes.centers.numpy() mask = (center[:, 0] > patch[0]) * (center[:, 1] > patch[1]) * ( center[:, 0] < patch[2]) * ( center[:, 1] < patch[3]) return mask def _crop_image_and_paste(self, image, center, size): """Crop image with a given center and size, then paste the cropped image to a blank image with two centers align. This function is equivalent to generating a blank image with ``size`` as its shape. Then cover it on the original image with two centers ( the center of blank image and the random center of original image) aligned. The overlap area is paste from the original image and the outside area is filled with ``mean pixel``. Args: image (np array, H x W x C): Original image. center (list[int]): Target crop center coord. size (list[int]): Target crop size. [target_h, target_w] Returns: cropped_img (np array, target_h x target_w x C): Cropped image. border (np array, 4): The distance of four border of ``cropped_img`` to the original image area, [top, bottom, left, right] patch (list[int]): The cropped area, [left, top, right, bottom]. """ center_y, center_x = center target_h, target_w = size img_h, img_w, img_c = image.shape x0 = max(0, center_x - target_w // 2) x1 = min(center_x + target_w // 2, img_w) y0 = max(0, center_y - target_h // 2) y1 = min(center_y + target_h // 2, img_h) patch = np.array((int(x0), int(y0), int(x1), int(y1))) left, right = center_x - x0, x1 - center_x top, bottom = center_y - y0, y1 - center_y cropped_center_y, cropped_center_x = target_h // 2, target_w // 2 cropped_img = np.zeros((target_h, target_w, img_c), dtype=image.dtype) for i in range(img_c): cropped_img[:, :, i] += self.mean[i] y_slice = slice(cropped_center_y - top, cropped_center_y + bottom) x_slice = slice(cropped_center_x - left, cropped_center_x + right) cropped_img[y_slice, x_slice, :] = image[y0:y1, x0:x1, :] border = np.array([ cropped_center_y - top, cropped_center_y + bottom, cropped_center_x - left, cropped_center_x + right ], dtype=np.float32) return cropped_img, border, patch def _train_aug(self, results): """Random crop and around padding the original image. Args: results (dict): Image infomations in the augment pipeline. Returns: results (dict): The updated dict. """ img = results['img'] h, w, c = img.shape gt_bboxes = results['gt_bboxes'] while True: scale = random.choice(self.ratios) new_h = int(self.crop_size[1] * scale) new_w = int(self.crop_size[0] * scale) h_border = self._get_border(self.border, h) w_border = self._get_border(self.border, w) for i in range(50): center_x = random.randint(low=w_border, high=w - w_border) center_y = random.randint(low=h_border, high=h - h_border) cropped_img, border, patch = self._crop_image_and_paste( img, [center_y, center_x], [new_h, new_w]) if len(gt_bboxes) == 0: results['img'] = cropped_img results['img_shape'] = cropped_img.shape return results # if image do not have valid bbox, any crop patch is valid. mask = self._filter_boxes(patch, gt_bboxes) if not mask.any(): continue results['img'] = cropped_img results['img_shape'] = cropped_img.shape x0, y0, x1, y1 = patch left_w, top_h = center_x - x0, center_y - y0 cropped_center_x, cropped_center_y = new_w // 2, new_h // 2 # crop bboxes accordingly and clip to the image boundary gt_bboxes = gt_bboxes[mask] gt_bboxes.translate_([ cropped_center_x - left_w - x0, cropped_center_y - top_h - y0 ]) if self.bbox_clip_border: gt_bboxes.clip_([new_h, new_w]) keep = gt_bboxes.is_inside([new_h, new_w]).numpy() gt_bboxes = gt_bboxes[keep] results['gt_bboxes'] = gt_bboxes # ignore_flags if results.get('gt_ignore_flags', None) is not None: gt_ignore_flags = results['gt_ignore_flags'][mask] results['gt_ignore_flags'] = \ gt_ignore_flags[keep] # labels if results.get('gt_bboxes_labels', None) is not None: gt_labels = results['gt_bboxes_labels'][mask] results['gt_bboxes_labels'] = gt_labels[keep] if 'gt_masks' in results or 'gt_seg_map' in results: raise NotImplementedError( 'RandomCenterCropPad only supports bbox.') return results def _test_aug(self, results): """Around padding the original image without cropping. The padding mode and value are from ``test_pad_mode``. Args: results (dict): Image infomations in the augment pipeline. Returns: results (dict): The updated dict. """ img = results['img'] h, w, c = img.shape if self.test_pad_mode[0] in ['logical_or']: # self.test_pad_add_pix is only used for centernet target_h = (h | self.test_pad_mode[1]) + self.test_pad_add_pix target_w = (w | self.test_pad_mode[1]) + self.test_pad_add_pix elif self.test_pad_mode[0] in ['size_divisor']: divisor = self.test_pad_mode[1] target_h = int(np.ceil(h / divisor)) * divisor target_w = int(np.ceil(w / divisor)) * divisor else: raise NotImplementedError( 'RandomCenterCropPad only support two testing pad mode:' 'logical-or and size_divisor.') cropped_img, border, _ = self._crop_image_and_paste( img, [h // 2, w // 2], [target_h, target_w]) results['img'] = cropped_img results['img_shape'] = cropped_img.shape results['border'] = border return results @autocast_box_type() def transform(self, results: dict) -> dict: img = results['img'] assert img.dtype == np.float32, ( 'RandomCenterCropPad needs the input image of dtype np.float32,' ' please set "to_float32=True" in "LoadImageFromFile" pipeline') h, w, c = img.shape assert c == len(self.mean) if self.test_mode: return self._test_aug(results) else: return self._train_aug(results) def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(crop_size={self.crop_size}, ' repr_str += f'ratios={self.ratios}, ' repr_str += f'border={self.border}, ' repr_str += f'mean={self.input_mean}, ' repr_str += f'std={self.input_std}, ' repr_str += f'to_rgb={self.to_rgb}, ' repr_str += f'test_mode={self.test_mode}, ' repr_str += f'test_pad_mode={self.test_pad_mode}, ' repr_str += f'bbox_clip_border={self.bbox_clip_border})' return repr_str @TRANSFORMS.register_module() class CutOut(BaseTransform): """CutOut operation. Randomly drop some regions of image used in `Cutout `_. Required Keys: - img Modified Keys: - img Args: n_holes (int or tuple[int, int]): Number of regions to be dropped. If it is given as a list, number of holes will be randomly selected from the closed interval [``n_holes[0]``, ``n_holes[1]``]. cutout_shape (tuple[int, int] or list[tuple[int, int]], optional): The candidate shape of dropped regions. It can be ``tuple[int, int]`` to use a fixed cutout shape, or ``list[tuple[int, int]]`` to randomly choose shape from the list. Defaults to None. cutout_ratio (tuple[float, float] or list[tuple[float, float]], optional): The candidate ratio of dropped regions. It can be ``tuple[float, float]`` to use a fixed ratio or ``list[tuple[float, float]]`` to randomly choose ratio from the list. Please note that ``cutout_shape`` and ``cutout_ratio`` cannot be both given at the same time. Defaults to None. fill_in (tuple[float, float, float] or tuple[int, int, int]): The value of pixel to fill in the dropped regions. Defaults to (0, 0, 0). """ def __init__( self, n_holes: Union[int, Tuple[int, int]], cutout_shape: Optional[Union[Tuple[int, int], List[Tuple[int, int]]]] = None, cutout_ratio: Optional[Union[Tuple[float, float], List[Tuple[float, float]]]] = None, fill_in: Union[Tuple[float, float, float], Tuple[int, int, int]] = (0, 0, 0) ) -> None: assert (cutout_shape is None) ^ (cutout_ratio is None), \ 'Either cutout_shape or cutout_ratio should be specified.' assert (isinstance(cutout_shape, (list, tuple)) or isinstance(cutout_ratio, (list, tuple))) if isinstance(n_holes, tuple): assert len(n_holes) == 2 and 0 <= n_holes[0] < n_holes[1] else: n_holes = (n_holes, n_holes) self.n_holes = n_holes self.fill_in = fill_in self.with_ratio = cutout_ratio is not None self.candidates = cutout_ratio if self.with_ratio else cutout_shape if not isinstance(self.candidates, list): self.candidates = [self.candidates] @autocast_box_type() def transform(self, results: dict) -> dict: """Call function to drop some regions of image.""" h, w, c = results['img'].shape n_holes = np.random.randint(self.n_holes[0], self.n_holes[1] + 1) for _ in range(n_holes): x1 = np.random.randint(0, w) y1 = np.random.randint(0, h) index = np.random.randint(0, len(self.candidates)) if not self.with_ratio: cutout_w, cutout_h = self.candidates[index] else: cutout_w = int(self.candidates[index][0] * w) cutout_h = int(self.candidates[index][1] * h) x2 = np.clip(x1 + cutout_w, 0, w) y2 = np.clip(y1 + cutout_h, 0, h) results['img'][y1:y2, x1:x2, :] = self.fill_in return results def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(n_holes={self.n_holes}, ' repr_str += (f'cutout_ratio={self.candidates}, ' if self.with_ratio else f'cutout_shape={self.candidates}, ') repr_str += f'fill_in={self.fill_in})' return repr_str @TRANSFORMS.register_module() class Mosaic(BaseTransform): """Mosaic augmentation. Given 4 images, mosaic transform combines them into one output image. The output image is composed of the parts from each sub- image. .. code:: text mosaic transform center_x +------------------------------+ | pad | pad | | +-----------+ | | | | | | | image1 |--------+ | | | | | | | | | image2 | | center_y |----+-------------+-----------| | | cropped | | |pad | image3 | image4 | | | | | +----|-------------+-----------+ | | +-------------+ The mosaic transform steps are as follows: 1. Choose the mosaic center as the intersections of 4 images 2. Get the left top image according to the index, and randomly sample another 3 images from the custom dataset. 3. Sub image will be cropped if image is larger than mosaic patch Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_ignore_flags (bool) (optional) - mix_results (List[dict]) Modified Keys: - img - img_shape - gt_bboxes (optional) - gt_bboxes_labels (optional) - gt_ignore_flags (optional) Args: img_scale (Sequence[int]): Image size after mosaic pipeline of single image. The shape order should be (width, height). Defaults to (640, 640). center_ratio_range (Sequence[float]): Center ratio range of mosaic output. Defaults to (0.5, 1.5). bbox_clip_border (bool, optional): Whether to clip the objects outside the border of the image. In some dataset like MOT17, the gt bboxes are allowed to cross the border of images. Therefore, we don't need to clip the gt bboxes in these cases. Defaults to True. pad_val (int): Pad value. Defaults to 114. prob (float): Probability of applying this transformation. Defaults to 1.0. """ def __init__(self, img_scale: Tuple[int, int] = (640, 640), center_ratio_range: Tuple[float, float] = (0.5, 1.5), bbox_clip_border: bool = True, pad_val: float = 114.0, prob: float = 1.0) -> None: assert isinstance(img_scale, tuple) assert 0 <= prob <= 1.0, 'The probability should be in range [0,1]. ' \ f'got {prob}.' log_img_scale(img_scale, skip_square=True, shape_order='wh') self.img_scale = img_scale self.center_ratio_range = center_ratio_range self.bbox_clip_border = bbox_clip_border self.pad_val = pad_val self.prob = prob @cache_randomness def get_indexes(self, dataset: BaseDataset) -> int: """Call function to collect indexes. Args: dataset (:obj:`MultiImageMixDataset`): The dataset. Returns: list: indexes. """ indexes = [random.randint(0, len(dataset)) for _ in range(3)] return indexes @autocast_box_type() def transform(self, results: dict) -> dict: """Mosaic transform function. Args: results (dict): Result dict. Returns: dict: Updated result dict. """ if random.uniform(0, 1) > self.prob: return results assert 'mix_results' in results mosaic_bboxes = [] mosaic_bboxes_labels = [] mosaic_ignore_flags = [] if len(results['img'].shape) == 3: mosaic_img = np.full( (int(self.img_scale[1] * 2), int(self.img_scale[0] * 2), 3), self.pad_val, dtype=results['img'].dtype) else: mosaic_img = np.full( (int(self.img_scale[1] * 2), int(self.img_scale[0] * 2)), self.pad_val, dtype=results['img'].dtype) # mosaic center x, y center_x = int( random.uniform(*self.center_ratio_range) * self.img_scale[0]) center_y = int( random.uniform(*self.center_ratio_range) * self.img_scale[1]) center_position = (center_x, center_y) loc_strs = ('top_left', 'top_right', 'bottom_left', 'bottom_right') for i, loc in enumerate(loc_strs): if loc == 'top_left': results_patch = copy.deepcopy(results) else: results_patch = copy.deepcopy(results['mix_results'][i - 1]) img_i = results_patch['img'] h_i, w_i = img_i.shape[:2] # keep_ratio resize scale_ratio_i = min(self.img_scale[1] / h_i, self.img_scale[0] / w_i) img_i = mmcv.imresize( img_i, (int(w_i * scale_ratio_i), int(h_i * scale_ratio_i))) # compute the combine parameters paste_coord, crop_coord = self._mosaic_combine( loc, center_position, img_i.shape[:2][::-1]) x1_p, y1_p, x2_p, y2_p = paste_coord x1_c, y1_c, x2_c, y2_c = crop_coord # crop and paste image mosaic_img[y1_p:y2_p, x1_p:x2_p] = img_i[y1_c:y2_c, x1_c:x2_c] # adjust coordinate gt_bboxes_i = results_patch['gt_bboxes'] gt_bboxes_labels_i = results_patch['gt_bboxes_labels'] gt_ignore_flags_i = results_patch['gt_ignore_flags'] padw = x1_p - x1_c padh = y1_p - y1_c gt_bboxes_i.rescale_([scale_ratio_i, scale_ratio_i]) gt_bboxes_i.translate_([padw, padh]) mosaic_bboxes.append(gt_bboxes_i) mosaic_bboxes_labels.append(gt_bboxes_labels_i) mosaic_ignore_flags.append(gt_ignore_flags_i) mosaic_bboxes = mosaic_bboxes[0].cat(mosaic_bboxes, 0) mosaic_bboxes_labels = np.concatenate(mosaic_bboxes_labels, 0) mosaic_ignore_flags = np.concatenate(mosaic_ignore_flags, 0) if self.bbox_clip_border: mosaic_bboxes.clip_([2 * self.img_scale[1], 2 * self.img_scale[0]]) # remove outside bboxes inside_inds = mosaic_bboxes.is_inside( [2 * self.img_scale[1], 2 * self.img_scale[0]]).numpy() mosaic_bboxes = mosaic_bboxes[inside_inds] mosaic_bboxes_labels = mosaic_bboxes_labels[inside_inds] mosaic_ignore_flags = mosaic_ignore_flags[inside_inds] results['img'] = mosaic_img results['img_shape'] = mosaic_img.shape results['gt_bboxes'] = mosaic_bboxes results['gt_bboxes_labels'] = mosaic_bboxes_labels results['gt_ignore_flags'] = mosaic_ignore_flags return results def _mosaic_combine( self, loc: str, center_position_xy: Sequence[float], img_shape_wh: Sequence[int]) -> Tuple[Tuple[int], Tuple[int]]: """Calculate global coordinate of mosaic image and local coordinate of cropped sub-image. Args: loc (str): Index for the sub-image, loc in ('top_left', 'top_right', 'bottom_left', 'bottom_right'). center_position_xy (Sequence[float]): Mixing center for 4 images, (x, y). img_shape_wh (Sequence[int]): Width and height of sub-image Returns: tuple[tuple[float]]: Corresponding coordinate of pasting and cropping - paste_coord (tuple): paste corner coordinate in mosaic image. - crop_coord (tuple): crop corner coordinate in mosaic image. """ assert loc in ('top_left', 'top_right', 'bottom_left', 'bottom_right') if loc == 'top_left': # index0 to top left part of image x1, y1, x2, y2 = max(center_position_xy[0] - img_shape_wh[0], 0), \ max(center_position_xy[1] - img_shape_wh[1], 0), \ center_position_xy[0], \ center_position_xy[1] crop_coord = img_shape_wh[0] - (x2 - x1), img_shape_wh[1] - ( y2 - y1), img_shape_wh[0], img_shape_wh[1] elif loc == 'top_right': # index1 to top right part of image x1, y1, x2, y2 = center_position_xy[0], \ max(center_position_xy[1] - img_shape_wh[1], 0), \ min(center_position_xy[0] + img_shape_wh[0], self.img_scale[0] * 2), \ center_position_xy[1] crop_coord = 0, img_shape_wh[1] - (y2 - y1), min( img_shape_wh[0], x2 - x1), img_shape_wh[1] elif loc == 'bottom_left': # index2 to bottom left part of image x1, y1, x2, y2 = max(center_position_xy[0] - img_shape_wh[0], 0), \ center_position_xy[1], \ center_position_xy[0], \ min(self.img_scale[1] * 2, center_position_xy[1] + img_shape_wh[1]) crop_coord = img_shape_wh[0] - (x2 - x1), 0, img_shape_wh[0], min( y2 - y1, img_shape_wh[1]) else: # index3 to bottom right part of image x1, y1, x2, y2 = center_position_xy[0], \ center_position_xy[1], \ min(center_position_xy[0] + img_shape_wh[0], self.img_scale[0] * 2), \ min(self.img_scale[1] * 2, center_position_xy[1] + img_shape_wh[1]) crop_coord = 0, 0, min(img_shape_wh[0], x2 - x1), min(y2 - y1, img_shape_wh[1]) paste_coord = x1, y1, x2, y2 return paste_coord, crop_coord def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(img_scale={self.img_scale}, ' repr_str += f'center_ratio_range={self.center_ratio_range}, ' repr_str += f'pad_val={self.pad_val}, ' repr_str += f'prob={self.prob})' return repr_str @TRANSFORMS.register_module() class MixUp(BaseTransform): """MixUp data augmentation. .. code:: text mixup transform +------------------------------+ | mixup image | | | +--------|--------+ | | | | | | |---------------+ | | | | | | | | image | | | | | | | | | | | |-----------------+ | | pad | +------------------------------+ The mixup transform steps are as follows: 1. Another random image is picked by dataset and embedded in the top left patch(after padding and resizing) 2. The target of mixup transform is the weighted average of mixup image and origin image. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_ignore_flags (bool) (optional) - mix_results (List[dict]) Modified Keys: - img - img_shape - gt_bboxes (optional) - gt_bboxes_labels (optional) - gt_ignore_flags (optional) Args: img_scale (Sequence[int]): Image output size after mixup pipeline. The shape order should be (width, height). Defaults to (640, 640). ratio_range (Sequence[float]): Scale ratio of mixup image. Defaults to (0.5, 1.5). flip_ratio (float): Horizontal flip ratio of mixup image. Defaults to 0.5. pad_val (int): Pad value. Defaults to 114. max_iters (int): The maximum number of iterations. If the number of iterations is greater than `max_iters`, but gt_bbox is still empty, then the iteration is terminated. Defaults to 15. bbox_clip_border (bool, optional): Whether to clip the objects outside the border of the image. In some dataset like MOT17, the gt bboxes are allowed to cross the border of images. Therefore, we don't need to clip the gt bboxes in these cases. Defaults to True. """ def __init__(self, img_scale: Tuple[int, int] = (640, 640), ratio_range: Tuple[float, float] = (0.5, 1.5), flip_ratio: float = 0.5, pad_val: float = 114.0, max_iters: int = 15, bbox_clip_border: bool = True) -> None: assert isinstance(img_scale, tuple) log_img_scale(img_scale, skip_square=True, shape_order='wh') self.dynamic_scale = img_scale self.ratio_range = ratio_range self.flip_ratio = flip_ratio self.pad_val = pad_val self.max_iters = max_iters self.bbox_clip_border = bbox_clip_border @cache_randomness def get_indexes(self, dataset: BaseDataset) -> int: """Call function to collect indexes. Args: dataset (:obj:`MultiImageMixDataset`): The dataset. Returns: list: indexes. """ for i in range(self.max_iters): index = random.randint(0, len(dataset)) gt_bboxes_i = dataset[index]['gt_bboxes'] if len(gt_bboxes_i) != 0: break return index @autocast_box_type() def transform(self, results: dict) -> dict: """MixUp transform function. Args: results (dict): Result dict. Returns: dict: Updated result dict. """ assert 'mix_results' in results assert len( results['mix_results']) == 1, 'MixUp only support 2 images now !' if results['mix_results'][0]['gt_bboxes'].shape[0] == 0: # empty bbox return results retrieve_results = results['mix_results'][0] retrieve_img = retrieve_results['img'] jit_factor = random.uniform(*self.ratio_range) is_filp = random.uniform(0, 1) > self.flip_ratio if len(retrieve_img.shape) == 3: out_img = np.ones( (self.dynamic_scale[1], self.dynamic_scale[0], 3), dtype=retrieve_img.dtype) * self.pad_val else: out_img = np.ones( self.dynamic_scale[::-1], dtype=retrieve_img.dtype) * self.pad_val # 1. keep_ratio resize scale_ratio = min(self.dynamic_scale[1] / retrieve_img.shape[0], self.dynamic_scale[0] / retrieve_img.shape[1]) retrieve_img = mmcv.imresize( retrieve_img, (int(retrieve_img.shape[1] * scale_ratio), int(retrieve_img.shape[0] * scale_ratio))) # 2. paste out_img[:retrieve_img.shape[0], :retrieve_img.shape[1]] = retrieve_img # 3. scale jit scale_ratio *= jit_factor out_img = mmcv.imresize(out_img, (int(out_img.shape[1] * jit_factor), int(out_img.shape[0] * jit_factor))) # 4. flip if is_filp: out_img = out_img[:, ::-1, :] # 5. random crop ori_img = results['img'] origin_h, origin_w = out_img.shape[:2] target_h, target_w = ori_img.shape[:2] padded_img = np.ones((max(origin_h, target_h), max( origin_w, target_w), 3)) * self.pad_val padded_img = padded_img.astype(np.uint8) padded_img[:origin_h, :origin_w] = out_img x_offset, y_offset = 0, 0 if padded_img.shape[0] > target_h: y_offset = random.randint(0, padded_img.shape[0] - target_h) if padded_img.shape[1] > target_w: x_offset = random.randint(0, padded_img.shape[1] - target_w) padded_cropped_img = padded_img[y_offset:y_offset + target_h, x_offset:x_offset + target_w] # 6. adjust bbox retrieve_gt_bboxes = retrieve_results['gt_bboxes'] retrieve_gt_bboxes.rescale_([scale_ratio, scale_ratio]) if self.bbox_clip_border: retrieve_gt_bboxes.clip_([origin_h, origin_w]) if is_filp: retrieve_gt_bboxes.flip_([origin_h, origin_w], direction='horizontal') # 7. filter cp_retrieve_gt_bboxes = retrieve_gt_bboxes.clone() cp_retrieve_gt_bboxes.translate_([-x_offset, -y_offset]) if self.bbox_clip_border: cp_retrieve_gt_bboxes.clip_([target_h, target_w]) # 8. mix up ori_img = ori_img.astype(np.float32) mixup_img = 0.5 * ori_img + 0.5 * padded_cropped_img.astype(np.float32) retrieve_gt_bboxes_labels = retrieve_results['gt_bboxes_labels'] retrieve_gt_ignore_flags = retrieve_results['gt_ignore_flags'] mixup_gt_bboxes = cp_retrieve_gt_bboxes.cat( (results['gt_bboxes'], cp_retrieve_gt_bboxes), dim=0) mixup_gt_bboxes_labels = np.concatenate( (results['gt_bboxes_labels'], retrieve_gt_bboxes_labels), axis=0) mixup_gt_ignore_flags = np.concatenate( (results['gt_ignore_flags'], retrieve_gt_ignore_flags), axis=0) # remove outside bbox inside_inds = mixup_gt_bboxes.is_inside([target_h, target_w]).numpy() mixup_gt_bboxes = mixup_gt_bboxes[inside_inds] mixup_gt_bboxes_labels = mixup_gt_bboxes_labels[inside_inds] mixup_gt_ignore_flags = mixup_gt_ignore_flags[inside_inds] results['img'] = mixup_img.astype(np.uint8) results['img_shape'] = mixup_img.shape results['gt_bboxes'] = mixup_gt_bboxes results['gt_bboxes_labels'] = mixup_gt_bboxes_labels results['gt_ignore_flags'] = mixup_gt_ignore_flags return results def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(dynamic_scale={self.dynamic_scale}, ' repr_str += f'ratio_range={self.ratio_range}, ' repr_str += f'flip_ratio={self.flip_ratio}, ' repr_str += f'pad_val={self.pad_val}, ' repr_str += f'max_iters={self.max_iters}, ' repr_str += f'bbox_clip_border={self.bbox_clip_border})' return repr_str @TRANSFORMS.register_module() class RandomAffine(BaseTransform): """Random affine transform data augmentation. This operation randomly generates affine transform matrix which including rotation, translation, shear and scaling transforms. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_ignore_flags (bool) (optional) Modified Keys: - img - img_shape - gt_bboxes (optional) - gt_bboxes_labels (optional) - gt_ignore_flags (optional) Args: max_rotate_degree (float): Maximum degrees of rotation transform. Defaults to 10. max_translate_ratio (float): Maximum ratio of translation. Defaults to 0.1. scaling_ratio_range (tuple[float]): Min and max ratio of scaling transform. Defaults to (0.5, 1.5). max_shear_degree (float): Maximum degrees of shear transform. Defaults to 2. border (tuple[int]): Distance from width and height sides of input image to adjust output shape. Only used in mosaic dataset. Defaults to (0, 0). border_val (tuple[int]): Border padding values of 3 channels. Defaults to (114, 114, 114). bbox_clip_border (bool, optional): Whether to clip the objects outside the border of the image. In some dataset like MOT17, the gt bboxes are allowed to cross the border of images. Therefore, we don't need to clip the gt bboxes in these cases. Defaults to True. """ def __init__(self, max_rotate_degree: float = 10.0, max_translate_ratio: float = 0.1, scaling_ratio_range: Tuple[float, float] = (0.5, 1.5), max_shear_degree: float = 2.0, border: Tuple[int, int] = (0, 0), border_val: Tuple[int, int, int] = (114, 114, 114), bbox_clip_border: bool = True) -> None: assert 0 <= max_translate_ratio <= 1 assert scaling_ratio_range[0] <= scaling_ratio_range[1] assert scaling_ratio_range[0] > 0 self.max_rotate_degree = max_rotate_degree self.max_translate_ratio = max_translate_ratio self.scaling_ratio_range = scaling_ratio_range self.max_shear_degree = max_shear_degree self.border = border self.border_val = border_val self.bbox_clip_border = bbox_clip_border @cache_randomness def _get_random_homography_matrix(self, height, width): # Rotation rotation_degree = random.uniform(-self.max_rotate_degree, self.max_rotate_degree) rotation_matrix = self._get_rotation_matrix(rotation_degree) # Scaling scaling_ratio = random.uniform(self.scaling_ratio_range[0], self.scaling_ratio_range[1]) scaling_matrix = self._get_scaling_matrix(scaling_ratio) # Shear x_degree = random.uniform(-self.max_shear_degree, self.max_shear_degree) y_degree = random.uniform(-self.max_shear_degree, self.max_shear_degree) shear_matrix = self._get_shear_matrix(x_degree, y_degree) # Translation trans_x = random.uniform(-self.max_translate_ratio, self.max_translate_ratio) * width trans_y = random.uniform(-self.max_translate_ratio, self.max_translate_ratio) * height translate_matrix = self._get_translation_matrix(trans_x, trans_y) warp_matrix = ( translate_matrix @ shear_matrix @ rotation_matrix @ scaling_matrix) return warp_matrix @autocast_box_type() def transform(self, results: dict) -> dict: img = results['img'] height = img.shape[0] + self.border[1] * 2 width = img.shape[1] + self.border[0] * 2 warp_matrix = self._get_random_homography_matrix(height, width) img = cv2.warpPerspective( img, warp_matrix, dsize=(width, height), borderValue=self.border_val) results['img'] = img results['img_shape'] = img.shape bboxes = results['gt_bboxes'] num_bboxes = len(bboxes) if num_bboxes: bboxes.project_(warp_matrix) if self.bbox_clip_border: bboxes.clip_([height, width]) # remove outside bbox valid_index = bboxes.is_inside([height, width]).numpy() results['gt_bboxes'] = bboxes[valid_index] results['gt_bboxes_labels'] = results['gt_bboxes_labels'][ valid_index] results['gt_ignore_flags'] = results['gt_ignore_flags'][ valid_index] if 'gt_masks' in results: raise NotImplementedError('RandomAffine only supports bbox.') return results def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(max_rotate_degree={self.max_rotate_degree}, ' repr_str += f'max_translate_ratio={self.max_translate_ratio}, ' repr_str += f'scaling_ratio_range={self.scaling_ratio_range}, ' repr_str += f'max_shear_degree={self.max_shear_degree}, ' repr_str += f'border={self.border}, ' repr_str += f'border_val={self.border_val}, ' repr_str += f'bbox_clip_border={self.bbox_clip_border})' return repr_str @staticmethod def _get_rotation_matrix(rotate_degrees: float) -> np.ndarray: radian = math.radians(rotate_degrees) rotation_matrix = np.array( [[np.cos(radian), -np.sin(radian), 0.], [np.sin(radian), np.cos(radian), 0.], [0., 0., 1.]], dtype=np.float32) return rotation_matrix @staticmethod def _get_scaling_matrix(scale_ratio: float) -> np.ndarray: scaling_matrix = np.array( [[scale_ratio, 0., 0.], [0., scale_ratio, 0.], [0., 0., 1.]], dtype=np.float32) return scaling_matrix @staticmethod def _get_shear_matrix(x_shear_degrees: float, y_shear_degrees: float) -> np.ndarray: x_radian = math.radians(x_shear_degrees) y_radian = math.radians(y_shear_degrees) shear_matrix = np.array([[1, np.tan(x_radian), 0.], [np.tan(y_radian), 1, 0.], [0., 0., 1.]], dtype=np.float32) return shear_matrix @staticmethod def _get_translation_matrix(x: float, y: float) -> np.ndarray: translation_matrix = np.array([[1, 0., x], [0., 1, y], [0., 0., 1.]], dtype=np.float32) return translation_matrix @TRANSFORMS.register_module() class YOLOXHSVRandomAug(BaseTransform): """Apply HSV augmentation to image sequentially. It is referenced from https://github.com/Megvii- BaseDetection/YOLOX/blob/main/yolox/data/data_augment.py#L21. Required Keys: - img Modified Keys: - img Args: hue_delta (int): delta of hue. Defaults to 5. saturation_delta (int): delta of saturation. Defaults to 30. value_delta (int): delat of value. Defaults to 30. """ def __init__(self, hue_delta: int = 5, saturation_delta: int = 30, value_delta: int = 30) -> None: self.hue_delta = hue_delta self.saturation_delta = saturation_delta self.value_delta = value_delta @cache_randomness def _get_hsv_gains(self): hsv_gains = np.random.uniform(-1, 1, 3) * [ self.hue_delta, self.saturation_delta, self.value_delta ] # random selection of h, s, v hsv_gains *= np.random.randint(0, 2, 3) # prevent overflow hsv_gains = hsv_gains.astype(np.int16) return hsv_gains def transform(self, results: dict) -> dict: img = results['img'] hsv_gains = self._get_hsv_gains() img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV).astype(np.int16) img_hsv[..., 0] = (img_hsv[..., 0] + hsv_gains[0]) % 180 img_hsv[..., 1] = np.clip(img_hsv[..., 1] + hsv_gains[1], 0, 255) img_hsv[..., 2] = np.clip(img_hsv[..., 2] + hsv_gains[2], 0, 255) cv2.cvtColor(img_hsv.astype(img.dtype), cv2.COLOR_HSV2BGR, dst=img) results['img'] = img return results def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(hue_delta={self.hue_delta}, ' repr_str += f'saturation_delta={self.saturation_delta}, ' repr_str += f'value_delta={self.value_delta})' return repr_str @TRANSFORMS.register_module() class CopyPaste(BaseTransform): """Simple Copy-Paste is a Strong Data Augmentation Method for Instance Segmentation The simple copy-paste transform steps are as follows: 1. The destination image is already resized with aspect ratio kept, cropped and padded. 2. Randomly select a source image, which is also already resized with aspect ratio kept, cropped and padded in a similar way as the destination image. 3. Randomly select some objects from the source image. 4. Paste these source objects to the destination image directly, due to the source and destination image have the same size. 5. Update object masks of the destination image, for some origin objects may be occluded. 6. Generate bboxes from the updated destination masks and filter some objects which are totally occluded, and adjust bboxes which are partly occluded. 7. Append selected source bboxes, masks, and labels. Required Keys: - img - gt_bboxes (BaseBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_ignore_flags (bool) (optional) - gt_masks (BitmapMasks) (optional) Modified Keys: - img - gt_bboxes (optional) - gt_bboxes_labels (optional) - gt_ignore_flags (optional) - gt_masks (optional) Args: max_num_pasted (int): The maximum number of pasted objects. Defaults to 100. bbox_occluded_thr (int): The threshold of occluded bbox. Defaults to 10. mask_occluded_thr (int): The threshold of occluded mask. Defaults to 300. selected (bool): Whether select objects or not. If select is False, all objects of the source image will be pasted to the destination image. Defaults to True. """ def __init__( self, max_num_pasted: int = 100, bbox_occluded_thr: int = 10, mask_occluded_thr: int = 300, selected: bool = True, ) -> None: self.max_num_pasted = max_num_pasted self.bbox_occluded_thr = bbox_occluded_thr self.mask_occluded_thr = mask_occluded_thr self.selected = selected @cache_randomness def get_indexes(self, dataset: BaseDataset) -> int: """Call function to collect indexes.s. Args: dataset (:obj:`MultiImageMixDataset`): The dataset. Returns: list: Indexes. """ return random.randint(0, len(dataset)) @autocast_box_type() def transform(self, results: dict) -> dict: """Transform function to make a copy-paste of image. Args: results (dict): Result dict. Returns: dict: Result dict with copy-paste transformed. """ assert 'mix_results' in results num_images = len(results['mix_results']) assert num_images == 1, \ f'CopyPaste only supports processing 2 images, got {num_images}' if self.selected: selected_results = self._select_object(results['mix_results'][0]) else: selected_results = results['mix_results'][0] return self._copy_paste(results, selected_results) @cache_randomness def _get_selected_inds(self, num_bboxes: int) -> np.ndarray: max_num_pasted = min(num_bboxes + 1, self.max_num_pasted) num_pasted = np.random.randint(0, max_num_pasted) return np.random.choice(num_bboxes, size=num_pasted, replace=False) def _select_object(self, results: dict) -> dict: """Select some objects from the source results.""" bboxes = results['gt_bboxes'] labels = results['gt_bboxes_labels'] masks = results['gt_masks'] ignore_flags = results['gt_ignore_flags'] selected_inds = self._get_selected_inds(bboxes.shape[0]) selected_bboxes = bboxes[selected_inds] selected_labels = labels[selected_inds] selected_masks = masks[selected_inds] selected_ignore_flags = ignore_flags[selected_inds] results['gt_bboxes'] = selected_bboxes results['gt_bboxes_labels'] = selected_labels results['gt_masks'] = selected_masks results['gt_ignore_flags'] = selected_ignore_flags return results def _copy_paste(self, dst_results: dict, src_results: dict) -> dict: """CopyPaste transform function. Args: dst_results (dict): Result dict of the destination image. src_results (dict): Result dict of the source image. Returns: dict: Updated result dict. """ dst_img = dst_results['img'] dst_bboxes = dst_results['gt_bboxes'] dst_labels = dst_results['gt_bboxes_labels'] dst_masks = dst_results['gt_masks'] dst_ignore_flags = dst_results['gt_ignore_flags'] src_img = src_results['img'] src_bboxes = src_results['gt_bboxes'] src_labels = src_results['gt_bboxes_labels'] src_masks = src_results['gt_masks'] src_ignore_flags = src_results['gt_ignore_flags'] if len(src_bboxes) == 0: return dst_results # update masks and generate bboxes from updated masks composed_mask = np.where(np.any(src_masks.masks, axis=0), 1, 0) updated_dst_masks = self._get_updated_masks(dst_masks, composed_mask) updated_dst_bboxes = updated_dst_masks.get_bboxes(type(dst_bboxes)) assert len(updated_dst_bboxes) == len(updated_dst_masks) # filter totally occluded objects l1_distance = (updated_dst_bboxes.tensor - dst_bboxes.tensor).abs() bboxes_inds = (l1_distance <= self.bbox_occluded_thr).all( dim=-1).numpy() masks_inds = updated_dst_masks.masks.sum( axis=(1, 2)) > self.mask_occluded_thr valid_inds = bboxes_inds | masks_inds # Paste source objects to destination image directly img = dst_img * (1 - composed_mask[..., np.newaxis] ) + src_img * composed_mask[..., np.newaxis] bboxes = src_bboxes.cat([updated_dst_bboxes[valid_inds], src_bboxes]) labels = np.concatenate([dst_labels[valid_inds], src_labels]) masks = np.concatenate( [updated_dst_masks.masks[valid_inds], src_masks.masks]) ignore_flags = np.concatenate( [dst_ignore_flags[valid_inds], src_ignore_flags]) dst_results['img'] = img dst_results['gt_bboxes'] = bboxes dst_results['gt_bboxes_labels'] = labels dst_results['gt_masks'] = BitmapMasks(masks, masks.shape[1], masks.shape[2]) dst_results['gt_ignore_flags'] = ignore_flags return dst_results def _get_updated_masks(self, masks: BitmapMasks, composed_mask: np.ndarray) -> BitmapMasks: """Update masks with composed mask.""" assert masks.masks.shape[-2:] == composed_mask.shape[-2:], \ 'Cannot compare two arrays of different size' masks.masks = np.where(composed_mask, 0, masks.masks) return masks def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(max_num_pasted={self.max_num_pasted}, ' repr_str += f'bbox_occluded_thr={self.bbox_occluded_thr}, ' repr_str += f'mask_occluded_thr={self.mask_occluded_thr}, ' repr_str += f'selected={self.selected})' return repr_str @TRANSFORMS.register_module() class RandomErasing(BaseTransform): """RandomErasing operation. Random Erasing randomly selects a rectangle region in an image and erases its pixels with random values. `RandomErasing `_. Required Keys: - img - gt_bboxes (HorizontalBoxes[torch.float32]) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_ignore_flags (bool) (optional) - gt_masks (BitmapMasks) (optional) Modified Keys: - img - gt_bboxes (optional) - gt_bboxes_labels (optional) - gt_ignore_flags (optional) - gt_masks (optional) Args: n_patches (int or tuple[int, int]): Number of regions to be dropped. If it is given as a tuple, number of patches will be randomly selected from the closed interval [``n_patches[0]``, ``n_patches[1]``]. ratio (float or tuple[float, float]): The ratio of erased regions. It can be ``float`` to use a fixed ratio or ``tuple[float, float]`` to randomly choose ratio from the interval. squared (bool): Whether to erase square region. Defaults to True. bbox_erased_thr (float): The threshold for the maximum area proportion of the bbox to be erased. When the proportion of the area where the bbox is erased is greater than the threshold, the bbox will be removed. Defaults to 0.9. img_border_value (int or float or tuple): The filled values for image border. If float, the same fill value will be used for all the three channels of image. If tuple, it should be 3 elements. Defaults to 128. mask_border_value (int): The fill value used for masks. Defaults to 0. seg_ignore_label (int): The fill value used for segmentation map. Note this value must equals ``ignore_label`` in ``semantic_head`` of the corresponding config. Defaults to 255. """ def __init__( self, n_patches: Union[int, Tuple[int, int]], ratio: Union[float, Tuple[float, float]], squared: bool = True, bbox_erased_thr: float = 0.9, img_border_value: Union[int, float, tuple] = 128, mask_border_value: int = 0, seg_ignore_label: int = 255, ) -> None: if isinstance(n_patches, tuple): assert len(n_patches) == 2 and 0 <= n_patches[0] < n_patches[1] else: n_patches = (n_patches, n_patches) if isinstance(ratio, tuple): assert len(ratio) == 2 and 0 <= ratio[0] < ratio[1] <= 1 else: ratio = (ratio, ratio) self.n_patches = n_patches self.ratio = ratio self.squared = squared self.bbox_erased_thr = bbox_erased_thr self.img_border_value = img_border_value self.mask_border_value = mask_border_value self.seg_ignore_label = seg_ignore_label @cache_randomness def _get_patches(self, img_shape: Tuple[int, int]) -> List[list]: """Get patches for random erasing.""" patches = [] n_patches = np.random.randint(self.n_patches[0], self.n_patches[1] + 1) for _ in range(n_patches): if self.squared: ratio = np.random.random() * (self.ratio[1] - self.ratio[0]) + self.ratio[0] ratio = (ratio, ratio) else: ratio = (np.random.random() * (self.ratio[1] - self.ratio[0]) + self.ratio[0], np.random.random() * (self.ratio[1] - self.ratio[0]) + self.ratio[0]) ph, pw = int(img_shape[0] * ratio[0]), int(img_shape[1] * ratio[1]) px1, py1 = np.random.randint(0, img_shape[1] - pw), np.random.randint( 0, img_shape[0] - ph) px2, py2 = px1 + pw, py1 + ph patches.append([px1, py1, px2, py2]) return np.array(patches) def _transform_img(self, results: dict, patches: List[list]) -> None: """Random erasing the image.""" for patch in patches: px1, py1, px2, py2 = patch results['img'][py1:py2, px1:px2, :] = self.img_border_value def _transform_bboxes(self, results: dict, patches: List[list]) -> None: """Random erasing the bboxes.""" bboxes = results['gt_bboxes'] # TODO: unify the logic by using operators in BaseBoxes. assert isinstance(bboxes, HorizontalBoxes) bboxes = bboxes.numpy() left_top = np.maximum(bboxes[:, None, :2], patches[:, :2]) right_bottom = np.minimum(bboxes[:, None, 2:], patches[:, 2:]) wh = np.maximum(right_bottom - left_top, 0) inter_areas = wh[:, :, 0] * wh[:, :, 1] bbox_areas = (bboxes[:, 2] - bboxes[:, 0]) * ( bboxes[:, 3] - bboxes[:, 1]) bboxes_erased_ratio = inter_areas.sum(-1) / (bbox_areas + 1e-7) valid_inds = bboxes_erased_ratio < self.bbox_erased_thr results['gt_bboxes'] = HorizontalBoxes(bboxes[valid_inds]) results['gt_bboxes_labels'] = results['gt_bboxes_labels'][valid_inds] results['gt_ignore_flags'] = results['gt_ignore_flags'][valid_inds] if results.get('gt_masks', None) is not None: results['gt_masks'] = results['gt_masks'][valid_inds] def _transform_masks(self, results: dict, patches: List[list]) -> None: """Random erasing the masks.""" for patch in patches: px1, py1, px2, py2 = patch results['gt_masks'].masks[:, py1:py2, px1:px2] = self.mask_border_value def _transform_seg(self, results: dict, patches: List[list]) -> None: """Random erasing the segmentation map.""" for patch in patches: px1, py1, px2, py2 = patch results['gt_seg_map'][py1:py2, px1:px2] = self.seg_ignore_label @autocast_box_type() def transform(self, results: dict) -> dict: """Transform function to erase some regions of image.""" patches = self._get_patches(results['img_shape']) self._transform_img(results, patches) if results.get('gt_bboxes', None) is not None: self._transform_bboxes(results, patches) if results.get('gt_masks', None) is not None: self._transform_masks(results, patches) if results.get('gt_seg_map', None) is not None: self._transform_seg(results, patches) return results def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(n_patches={self.n_patches}, ' repr_str += f'ratio={self.ratio}, ' repr_str += f'squared={self.squared}, ' repr_str += f'bbox_erased_thr={self.bbox_erased_thr}, ' repr_str += f'img_border_value={self.img_border_value}, ' repr_str += f'mask_border_value={self.mask_border_value}, ' repr_str += f'seg_ignore_label={self.seg_ignore_label})' return repr_str @TRANSFORMS.register_module() class CachedMosaic(Mosaic): """Cached mosaic augmentation. Cached mosaic transform will random select images from the cache and combine them into one output image. .. code:: text mosaic transform center_x +------------------------------+ | pad | pad | | +-----------+ | | | | | | | image1 |--------+ | | | | | | | | | image2 | | center_y |----+-------------+-----------| | | cropped | | |pad | image3 | image4 | | | | | +----|-------------+-----------+ | | +-------------+ The cached mosaic transform steps are as follows: 1. Append the results from the last transform into the cache. 2. Choose the mosaic center as the intersections of 4 images 3. Get the left top image according to the index, and randomly sample another 3 images from the result cache. 4. Sub image will be cropped if image is larger than mosaic patch Required Keys: - img - gt_bboxes (np.float32) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_ignore_flags (bool) (optional) Modified Keys: - img - img_shape - gt_bboxes (optional) - gt_bboxes_labels (optional) - gt_ignore_flags (optional) Args: img_scale (Sequence[int]): Image size after mosaic pipeline of single image. The shape order should be (width, height). Defaults to (640, 640). center_ratio_range (Sequence[float]): Center ratio range of mosaic output. Defaults to (0.5, 1.5). bbox_clip_border (bool, optional): Whether to clip the objects outside the border of the image. In some dataset like MOT17, the gt bboxes are allowed to cross the border of images. Therefore, we don't need to clip the gt bboxes in these cases. Defaults to True. pad_val (int): Pad value. Defaults to 114. prob (float): Probability of applying this transformation. Defaults to 1.0. max_cached_images (int): The maximum length of the cache. The larger the cache, the stronger the randomness of this transform. As a rule of thumb, providing 10 caches for each image suffices for randomness. Defaults to 40. random_pop (bool): Whether to randomly pop a result from the cache when the cache is full. If set to False, use FIFO popping method. Defaults to True. """ def __init__(self, *args, max_cached_images: int = 40, random_pop: bool = True, **kwargs) -> None: super().__init__(*args, **kwargs) self.results_cache = [] self.random_pop = random_pop assert max_cached_images >= 4, 'The length of cache must >= 4, ' \ f'but got {max_cached_images}.' self.max_cached_images = max_cached_images @cache_randomness def get_indexes(self, cache: list) -> list: """Call function to collect indexes. Args: cache (list): The results cache. Returns: list: indexes. """ indexes = [random.randint(0, len(cache) - 1) for _ in range(3)] return indexes @autocast_box_type() def transform(self, results: dict) -> dict: """Mosaic transform function. Args: results (dict): Result dict. Returns: dict: Updated result dict. """ # cache and pop images self.results_cache.append(copy.deepcopy(results)) if len(self.results_cache) > self.max_cached_images: if self.random_pop: index = random.randint(0, len(self.results_cache) - 1) else: index = 0 self.results_cache.pop(index) if len(self.results_cache) <= 4: return results if random.uniform(0, 1) > self.prob: return results indices = self.get_indexes(self.results_cache) mix_results = [copy.deepcopy(self.results_cache[i]) for i in indices] # TODO: refactor mosaic to reuse these code. mosaic_bboxes = [] mosaic_bboxes_labels = [] mosaic_ignore_flags = [] mosaic_masks = [] with_mask = True if 'gt_masks' in results else False if len(results['img'].shape) == 3: mosaic_img = np.full( (int(self.img_scale[1] * 2), int(self.img_scale[0] * 2), 3), self.pad_val, dtype=results['img'].dtype) else: mosaic_img = np.full( (int(self.img_scale[1] * 2), int(self.img_scale[0] * 2)), self.pad_val, dtype=results['img'].dtype) # mosaic center x, y center_x = int( random.uniform(*self.center_ratio_range) * self.img_scale[0]) center_y = int( random.uniform(*self.center_ratio_range) * self.img_scale[1]) center_position = (center_x, center_y) loc_strs = ('top_left', 'top_right', 'bottom_left', 'bottom_right') for i, loc in enumerate(loc_strs): if loc == 'top_left': results_patch = copy.deepcopy(results) else: results_patch = copy.deepcopy(mix_results[i - 1]) img_i = results_patch['img'] h_i, w_i = img_i.shape[:2] # keep_ratio resize scale_ratio_i = min(self.img_scale[1] / h_i, self.img_scale[0] / w_i) img_i = mmcv.imresize( img_i, (int(w_i * scale_ratio_i), int(h_i * scale_ratio_i))) # compute the combine parameters paste_coord, crop_coord = self._mosaic_combine( loc, center_position, img_i.shape[:2][::-1]) x1_p, y1_p, x2_p, y2_p = paste_coord x1_c, y1_c, x2_c, y2_c = crop_coord # crop and paste image mosaic_img[y1_p:y2_p, x1_p:x2_p] = img_i[y1_c:y2_c, x1_c:x2_c] # adjust coordinate gt_bboxes_i = results_patch['gt_bboxes'] gt_bboxes_labels_i = results_patch['gt_bboxes_labels'] gt_ignore_flags_i = results_patch['gt_ignore_flags'] padw = x1_p - x1_c padh = y1_p - y1_c gt_bboxes_i.rescale_([scale_ratio_i, scale_ratio_i]) gt_bboxes_i.translate_([padw, padh]) mosaic_bboxes.append(gt_bboxes_i) mosaic_bboxes_labels.append(gt_bboxes_labels_i) mosaic_ignore_flags.append(gt_ignore_flags_i) if with_mask and results_patch.get('gt_masks', None) is not None: gt_masks_i = results_patch['gt_masks'] gt_masks_i = gt_masks_i.rescale(float(scale_ratio_i)) gt_masks_i = gt_masks_i.translate( out_shape=(int(self.img_scale[0] * 2), int(self.img_scale[1] * 2)), offset=padw, direction='horizontal') gt_masks_i = gt_masks_i.translate( out_shape=(int(self.img_scale[0] * 2), int(self.img_scale[1] * 2)), offset=padh, direction='vertical') mosaic_masks.append(gt_masks_i) mosaic_bboxes = mosaic_bboxes[0].cat(mosaic_bboxes, 0) mosaic_bboxes_labels = np.concatenate(mosaic_bboxes_labels, 0) mosaic_ignore_flags = np.concatenate(mosaic_ignore_flags, 0) if self.bbox_clip_border: mosaic_bboxes.clip_([2 * self.img_scale[1], 2 * self.img_scale[0]]) # remove outside bboxes inside_inds = mosaic_bboxes.is_inside( [2 * self.img_scale[1], 2 * self.img_scale[0]]).numpy() mosaic_bboxes = mosaic_bboxes[inside_inds] mosaic_bboxes_labels = mosaic_bboxes_labels[inside_inds] mosaic_ignore_flags = mosaic_ignore_flags[inside_inds] results['img'] = mosaic_img results['img_shape'] = mosaic_img.shape results['gt_bboxes'] = mosaic_bboxes results['gt_bboxes_labels'] = mosaic_bboxes_labels results['gt_ignore_flags'] = mosaic_ignore_flags if with_mask: mosaic_masks = mosaic_masks[0].cat(mosaic_masks) results['gt_masks'] = mosaic_masks[inside_inds] return results def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(img_scale={self.img_scale}, ' repr_str += f'center_ratio_range={self.center_ratio_range}, ' repr_str += f'pad_val={self.pad_val}, ' repr_str += f'prob={self.prob}, ' repr_str += f'max_cached_images={self.max_cached_images}, ' repr_str += f'random_pop={self.random_pop})' return repr_str @TRANSFORMS.register_module() class CachedMixUp(BaseTransform): """Cached mixup data augmentation. .. code:: text mixup transform +------------------------------+ | mixup image | | | +--------|--------+ | | | | | | |---------------+ | | | | | | | | image | | | | | | | | | | | |-----------------+ | | pad | +------------------------------+ The cached mixup transform steps are as follows: 1. Append the results from the last transform into the cache. 2. Another random image is picked from the cache and embedded in the top left patch(after padding and resizing) 3. The target of mixup transform is the weighted average of mixup image and origin image. Required Keys: - img - gt_bboxes (np.float32) (optional) - gt_bboxes_labels (np.int64) (optional) - gt_ignore_flags (bool) (optional) - mix_results (List[dict]) Modified Keys: - img - img_shape - gt_bboxes (optional) - gt_bboxes_labels (optional) - gt_ignore_flags (optional) Args: img_scale (Sequence[int]): Image output size after mixup pipeline. The shape order should be (width, height). Defaults to (640, 640). ratio_range (Sequence[float]): Scale ratio of mixup image. Defaults to (0.5, 1.5). flip_ratio (float): Horizontal flip ratio of mixup image. Defaults to 0.5. pad_val (int): Pad value. Defaults to 114. max_iters (int): The maximum number of iterations. If the number of iterations is greater than `max_iters`, but gt_bbox is still empty, then the iteration is terminated. Defaults to 15. bbox_clip_border (bool, optional): Whether to clip the objects outside the border of the image. In some dataset like MOT17, the gt bboxes are allowed to cross the border of images. Therefore, we don't need to clip the gt bboxes in these cases. Defaults to True. max_cached_images (int): The maximum length of the cache. The larger the cache, the stronger the randomness of this transform. As a rule of thumb, providing 10 caches for each image suffices for randomness. Defaults to 20. random_pop (bool): Whether to randomly pop a result from the cache when the cache is full. If set to False, use FIFO popping method. Defaults to True. prob (float): Probability of applying this transformation. Defaults to 1.0. """ def __init__(self, img_scale: Tuple[int, int] = (640, 640), ratio_range: Tuple[float, float] = (0.5, 1.5), flip_ratio: float = 0.5, pad_val: float = 114.0, max_iters: int = 15, bbox_clip_border: bool = True, max_cached_images: int = 20, random_pop: bool = True, prob: float = 1.0) -> None: assert isinstance(img_scale, tuple) assert max_cached_images >= 2, 'The length of cache must >= 2, ' \ f'but got {max_cached_images}.' assert 0 <= prob <= 1.0, 'The probability should be in range [0,1]. ' \ f'got {prob}.' self.dynamic_scale = img_scale self.ratio_range = ratio_range self.flip_ratio = flip_ratio self.pad_val = pad_val self.max_iters = max_iters self.bbox_clip_border = bbox_clip_border self.results_cache = [] self.max_cached_images = max_cached_images self.random_pop = random_pop self.prob = prob @cache_randomness def get_indexes(self, cache: list) -> int: """Call function to collect indexes. Args: cache (list): The result cache. Returns: int: index. """ for i in range(self.max_iters): index = random.randint(0, len(cache) - 1) gt_bboxes_i = cache[index]['gt_bboxes'] if len(gt_bboxes_i) != 0: break return index @autocast_box_type() def transform(self, results: dict) -> dict: """MixUp transform function. Args: results (dict): Result dict. Returns: dict: Updated result dict. """ # cache and pop images self.results_cache.append(copy.deepcopy(results)) if len(self.results_cache) > self.max_cached_images: if self.random_pop: index = random.randint(0, len(self.results_cache) - 1) else: index = 0 self.results_cache.pop(index) if len(self.results_cache) <= 1: return results if random.uniform(0, 1) > self.prob: return results index = self.get_indexes(self.results_cache) retrieve_results = copy.deepcopy(self.results_cache[index]) # TODO: refactor mixup to reuse these code. if retrieve_results['gt_bboxes'].shape[0] == 0: # empty bbox return results retrieve_img = retrieve_results['img'] with_mask = True if 'gt_masks' in results else False jit_factor = random.uniform(*self.ratio_range) is_filp = random.uniform(0, 1) > self.flip_ratio if len(retrieve_img.shape) == 3: out_img = np.ones( (self.dynamic_scale[1], self.dynamic_scale[0], 3), dtype=retrieve_img.dtype) * self.pad_val else: out_img = np.ones( self.dynamic_scale[::-1], dtype=retrieve_img.dtype) * self.pad_val # 1. keep_ratio resize scale_ratio = min(self.dynamic_scale[1] / retrieve_img.shape[0], self.dynamic_scale[0] / retrieve_img.shape[1]) retrieve_img = mmcv.imresize( retrieve_img, (int(retrieve_img.shape[1] * scale_ratio), int(retrieve_img.shape[0] * scale_ratio))) # 2. paste out_img[:retrieve_img.shape[0], :retrieve_img.shape[1]] = retrieve_img # 3. scale jit scale_ratio *= jit_factor out_img = mmcv.imresize(out_img, (int(out_img.shape[1] * jit_factor), int(out_img.shape[0] * jit_factor))) # 4. flip if is_filp: out_img = out_img[:, ::-1, :] # 5. random crop ori_img = results['img'] origin_h, origin_w = out_img.shape[:2] target_h, target_w = ori_img.shape[:2] padded_img = np.ones((max(origin_h, target_h), max( origin_w, target_w), 3)) * self.pad_val padded_img = padded_img.astype(np.uint8) padded_img[:origin_h, :origin_w] = out_img x_offset, y_offset = 0, 0 if padded_img.shape[0] > target_h: y_offset = random.randint(0, padded_img.shape[0] - target_h) if padded_img.shape[1] > target_w: x_offset = random.randint(0, padded_img.shape[1] - target_w) padded_cropped_img = padded_img[y_offset:y_offset + target_h, x_offset:x_offset + target_w] # 6. adjust bbox retrieve_gt_bboxes = retrieve_results['gt_bboxes'] retrieve_gt_bboxes.rescale_([scale_ratio, scale_ratio]) if with_mask: retrieve_gt_masks = retrieve_results['gt_masks'].rescale( scale_ratio) if self.bbox_clip_border: retrieve_gt_bboxes.clip_([origin_h, origin_w]) if is_filp: retrieve_gt_bboxes.flip_([origin_h, origin_w], direction='horizontal') if with_mask: retrieve_gt_masks = retrieve_gt_masks.flip() # 7. filter cp_retrieve_gt_bboxes = retrieve_gt_bboxes.clone() cp_retrieve_gt_bboxes.translate_([-x_offset, -y_offset]) if with_mask: retrieve_gt_masks = retrieve_gt_masks.translate( out_shape=(target_h, target_w), offset=-x_offset, direction='horizontal') retrieve_gt_masks = retrieve_gt_masks.translate( out_shape=(target_h, target_w), offset=-y_offset, direction='vertical') if self.bbox_clip_border: cp_retrieve_gt_bboxes.clip_([target_h, target_w]) # 8. mix up ori_img = ori_img.astype(np.float32) mixup_img = 0.5 * ori_img + 0.5 * padded_cropped_img.astype(np.float32) retrieve_gt_bboxes_labels = retrieve_results['gt_bboxes_labels'] retrieve_gt_ignore_flags = retrieve_results['gt_ignore_flags'] mixup_gt_bboxes = cp_retrieve_gt_bboxes.cat( (results['gt_bboxes'], cp_retrieve_gt_bboxes), dim=0) mixup_gt_bboxes_labels = np.concatenate( (results['gt_bboxes_labels'], retrieve_gt_bboxes_labels), axis=0) mixup_gt_ignore_flags = np.concatenate( (results['gt_ignore_flags'], retrieve_gt_ignore_flags), axis=0) if with_mask: mixup_gt_masks = retrieve_gt_masks.cat( [results['gt_masks'], retrieve_gt_masks]) # remove outside bbox inside_inds = mixup_gt_bboxes.is_inside([target_h, target_w]).numpy() mixup_gt_bboxes = mixup_gt_bboxes[inside_inds] mixup_gt_bboxes_labels = mixup_gt_bboxes_labels[inside_inds] mixup_gt_ignore_flags = mixup_gt_ignore_flags[inside_inds] if with_mask: mixup_gt_masks = mixup_gt_masks[inside_inds] results['img'] = mixup_img.astype(np.uint8) results['img_shape'] = mixup_img.shape results['gt_bboxes'] = mixup_gt_bboxes results['gt_bboxes_labels'] = mixup_gt_bboxes_labels results['gt_ignore_flags'] = mixup_gt_ignore_flags if with_mask: results['gt_masks'] = mixup_gt_masks return results def __repr__(self): repr_str = self.__class__.__name__ repr_str += f'(dynamic_scale={self.dynamic_scale}, ' repr_str += f'ratio_range={self.ratio_range}, ' repr_str += f'flip_ratio={self.flip_ratio}, ' repr_str += f'pad_val={self.pad_val}, ' repr_str += f'max_iters={self.max_iters}, ' repr_str += f'bbox_clip_border={self.bbox_clip_border}, ' repr_str += f'max_cached_images={self.max_cached_images}, ' repr_str += f'random_pop={self.random_pop}, ' repr_str += f'prob={self.prob})' return repr_str ================================================ FILE: mmdet/datasets/transforms/wrappers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import Callable, Dict, List, Optional, Union import numpy as np from mmcv.transforms import BaseTransform, Compose from mmcv.transforms.utils import cache_random_params, cache_randomness from mmdet.registry import TRANSFORMS @TRANSFORMS.register_module() class MultiBranch(BaseTransform): r"""Multiple branch pipeline wrapper. Generate multiple data-augmented versions of the same image. `MultiBranch` needs to specify the branch names of all pipelines of the dataset, perform corresponding data augmentation for the current branch, and return None for other branches, which ensures the consistency of return format across different samples. Args: branch_field (list): List of branch names. branch_pipelines (dict): Dict of different pipeline configs to be composed. Examples: >>> branch_field = ['sup', 'unsup_teacher', 'unsup_student'] >>> sup_pipeline = [ >>> dict(type='LoadImageFromFile', >>> file_client_args=dict(backend='disk')), >>> dict(type='LoadAnnotations', with_bbox=True), >>> dict(type='Resize', scale=(1333, 800), keep_ratio=True), >>> dict(type='RandomFlip', prob=0.5), >>> dict( >>> type='MultiBranch', >>> branch_field=branch_field, >>> sup=dict(type='PackDetInputs')) >>> ] >>> weak_pipeline = [ >>> dict(type='LoadImageFromFile', >>> file_client_args=dict(backend='disk')), >>> dict(type='LoadAnnotations', with_bbox=True), >>> dict(type='Resize', scale=(1333, 800), keep_ratio=True), >>> dict(type='RandomFlip', prob=0.0), >>> dict( >>> type='MultiBranch', >>> branch_field=branch_field, >>> sup=dict(type='PackDetInputs')) >>> ] >>> strong_pipeline = [ >>> dict(type='LoadImageFromFile', >>> file_client_args=dict(backend='disk')), >>> dict(type='LoadAnnotations', with_bbox=True), >>> dict(type='Resize', scale=(1333, 800), keep_ratio=True), >>> dict(type='RandomFlip', prob=1.0), >>> dict( >>> type='MultiBranch', >>> branch_field=branch_field, >>> sup=dict(type='PackDetInputs')) >>> ] >>> unsup_pipeline = [ >>> dict(type='LoadImageFromFile', >>> file_client_args=file_client_args), >>> dict(type='LoadEmptyAnnotations'), >>> dict( >>> type='MultiBranch', >>> branch_field=branch_field, >>> unsup_teacher=weak_pipeline, >>> unsup_student=strong_pipeline) >>> ] >>> from mmcv.transforms import Compose >>> sup_branch = Compose(sup_pipeline) >>> unsup_branch = Compose(unsup_pipeline) >>> print(sup_branch) >>> Compose( >>> LoadImageFromFile(ignore_empty=False, to_float32=False, color_type='color', imdecode_backend='cv2', file_client_args={'backend': 'disk'}) # noqa >>> LoadAnnotations(with_bbox=True, with_label=True, with_mask=False, with_seg=False, poly2mask=True, imdecode_backend='cv2', file_client_args={'backend': 'disk'}) # noqa >>> Resize(scale=(1333, 800), scale_factor=None, keep_ratio=True, clip_object_border=True), backend=cv2), interpolation=bilinear) # noqa >>> RandomFlip(prob=0.5, direction=horizontal) >>> MultiBranch(branch_pipelines=['sup']) >>> ) >>> print(unsup_branch) >>> Compose( >>> LoadImageFromFile(ignore_empty=False, to_float32=False, color_type='color', imdecode_backend='cv2', file_client_args={'backend': 'disk'}) # noqa >>> LoadEmptyAnnotations(with_bbox=True, with_label=True, with_mask=False, with_seg=False, seg_ignore_label=255) # noqa >>> MultiBranch(branch_pipelines=['unsup_teacher', 'unsup_student']) >>> ) """ def __init__(self, branch_field: List[str], **branch_pipelines: dict) -> None: self.branch_field = branch_field self.branch_pipelines = { branch: Compose(pipeline) for branch, pipeline in branch_pipelines.items() } def transform(self, results: dict) -> dict: """Transform function to apply transforms sequentially. Args: results (dict): Result dict from loading pipeline. Returns: dict: - 'inputs' (Dict[str, obj:`torch.Tensor`]): The forward data of models from different branches. - 'data_sample' (Dict[str,obj:`DetDataSample`]): The annotation info of the sample from different branches. """ multi_results = {} for branch in self.branch_field: multi_results[branch] = {'inputs': None, 'data_samples': None} for branch, pipeline in self.branch_pipelines.items(): branch_results = pipeline(copy.deepcopy(results)) # If one branch pipeline returns None, # it will sample another data from dataset. if branch_results is None: return None multi_results[branch] = branch_results format_results = {} for branch, results in multi_results.items(): for key in results.keys(): if format_results.get(key, None) is None: format_results[key] = {branch: results[key]} else: format_results[key][branch] = results[key] return format_results def __repr__(self) -> str: repr_str = self.__class__.__name__ repr_str += f'(branch_pipelines={list(self.branch_pipelines.keys())})' return repr_str @TRANSFORMS.register_module() class RandomOrder(Compose): """Shuffle the transform Sequence.""" @cache_randomness def _random_permutation(self): return np.random.permutation(len(self.transforms)) def transform(self, results: Dict) -> Optional[Dict]: """Transform function to apply transforms in random order. Args: results (dict): A result dict contains the results to transform. Returns: dict or None: Transformed results. """ inds = self._random_permutation() for idx in inds: t = self.transforms[idx] results = t(results) if results is None: return None return results def __repr__(self): """Compute the string representation.""" format_string = self.__class__.__name__ + '(' for t in self.transforms: format_string += f'{t.__class__.__name__}, ' format_string += ')' return format_string @TRANSFORMS.register_module() class ProposalBroadcaster(BaseTransform): """A transform wrapper to apply the wrapped transforms to process both `gt_bboxes` and `proposals` without adding any codes. It will do the following steps: 1. Scatter the broadcasting targets to a list of inputs of the wrapped transforms. The type of the list should be list[dict, dict], which the first is the original inputs, the second is the processing results that `gt_bboxes` being rewritten by the `proposals`. 2. Apply ``self.transforms``, with same random parameters, which is sharing with a context manager. The type of the outputs is a list[dict, dict]. 3. Gather the outputs, update the `proposals` in the first item of the outputs with the `gt_bboxes` in the second . Args: transforms (list, optional): Sequence of transform object or config dict to be wrapped. Defaults to []. Note: The `TransformBroadcaster` in MMCV can achieve the same operation as `ProposalBroadcaster`, but need to set more complex parameters. Examples: >>> pipeline = [ >>> dict(type='LoadImageFromFile'), >>> dict(type='LoadProposals', num_max_proposals=2000), >>> dict(type='LoadAnnotations', with_bbox=True), >>> dict( >>> type='ProposalBroadcaster', >>> transforms=[ >>> dict(type='Resize', scale=(1333, 800), >>> keep_ratio=True), >>> dict(type='RandomFlip', prob=0.5), >>> ]), >>> dict(type='PackDetInputs')] """ def __init__(self, transforms: List[Union[dict, Callable]] = []) -> None: self.transforms = Compose(transforms) def transform(self, results: dict) -> dict: """Apply wrapped transform functions to process both `gt_bboxes` and `proposals`. Args: results (dict): Result dict from loading pipeline. Returns: dict: Updated result dict. """ assert results.get('proposals', None) is not None, \ '`proposals` should be in the results, please delete ' \ '`ProposalBroadcaster` in your configs, or check whether ' \ 'you have load proposals successfully.' inputs = self._process_input(results) outputs = self._apply_transforms(inputs) outputs = self._process_output(outputs) return outputs def _process_input(self, data: dict) -> list: """Scatter the broadcasting targets to a list of inputs of the wrapped transforms. Args: data (dict): The original input data. Returns: list[dict]: A list of input data. """ cp_data = copy.deepcopy(data) cp_data['gt_bboxes'] = cp_data['proposals'] scatters = [data, cp_data] return scatters def _apply_transforms(self, inputs: list) -> list: """Apply ``self.transforms``. Args: inputs (list[dict, dict]): list of input data. Returns: list[dict]: The output of the wrapped pipeline. """ assert len(inputs) == 2 ctx = cache_random_params with ctx(self.transforms): output_scatters = [self.transforms(_input) for _input in inputs] return output_scatters def _process_output(self, output_scatters: list) -> dict: """Gathering and renaming data items. Args: output_scatters (list[dict, dict]): The output of the wrapped pipeline. Returns: dict: Updated result dict. """ assert isinstance(output_scatters, list) and \ isinstance(output_scatters[0], dict) and \ len(output_scatters) == 2 outputs = output_scatters[0] outputs['proposals'] = output_scatters[1]['gt_bboxes'] return outputs ================================================ FILE: mmdet/datasets/utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmcv.transforms import LoadImageFromFile from mmdet.datasets.transforms import LoadAnnotations, LoadPanopticAnnotations from mmdet.registry import TRANSFORMS def get_loading_pipeline(pipeline): """Only keep loading image and annotations related configuration. Args: pipeline (list[dict]): Data pipeline configs. Returns: list[dict]: The new pipeline list with only keep loading image and annotations related configuration. Examples: >>> pipelines = [ ... dict(type='LoadImageFromFile'), ... dict(type='LoadAnnotations', with_bbox=True), ... dict(type='Resize', img_scale=(1333, 800), keep_ratio=True), ... dict(type='RandomFlip', flip_ratio=0.5), ... dict(type='Normalize', **img_norm_cfg), ... dict(type='Pad', size_divisor=32), ... dict(type='DefaultFormatBundle'), ... dict(type='Collect', keys=['img', 'gt_bboxes', 'gt_labels']) ... ] >>> expected_pipelines = [ ... dict(type='LoadImageFromFile'), ... dict(type='LoadAnnotations', with_bbox=True) ... ] >>> assert expected_pipelines ==\ ... get_loading_pipeline(pipelines) """ loading_pipeline_cfg = [] for cfg in pipeline: obj_cls = TRANSFORMS.get(cfg['type']) # TODO：use more elegant way to distinguish loading modules if obj_cls is not None and obj_cls in (LoadImageFromFile, LoadAnnotations, LoadPanopticAnnotations): loading_pipeline_cfg.append(cfg) assert len(loading_pipeline_cfg) == 2, \ 'The data pipeline in your config file must include ' \ 'loading image and annotations related pipeline.' return loading_pipeline_cfg ================================================ FILE: mmdet/datasets/voc.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import DATASETS from .xml_style import XMLDataset @DATASETS.register_module() class VOCDataset(XMLDataset): """Dataset for PASCAL VOC.""" METAINFO = { 'classes': ('aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor'), # palette is a list of color tuples, which is used for visualization. 'palette': [(106, 0, 228), (119, 11, 32), (165, 42, 42), (0, 0, 192), (197, 226, 255), (0, 60, 100), (0, 0, 142), (255, 77, 255), (153, 69, 1), (120, 166, 157), (0, 182, 199), (0, 226, 252), (182, 182, 255), (0, 0, 230), (220, 20, 60), (163, 255, 0), (0, 82, 0), (3, 95, 161), (0, 80, 100), (183, 130, 88)] } def __init__(self, **kwargs): super().__init__(**kwargs) if 'VOC2007' in self.sub_data_root: self._metainfo['dataset_type'] = 'VOC2007' elif 'VOC2012' in self.sub_data_root: self._metainfo['dataset_type'] = 'VOC2012' else: self._metainfo['dataset_type'] = None ================================================ FILE: mmdet/datasets/wider_face.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp import xml.etree.ElementTree as ET from mmengine.fileio import list_from_file from mmdet.registry import DATASETS from .xml_style import XMLDataset @DATASETS.register_module() class WIDERFaceDataset(XMLDataset): """Reader for the WIDER Face dataset in PASCAL VOC format. Conversion scripts can be found in https://github.com/sovrasov/wider-face-pascal-voc-annotations """ METAINFO = {'classes': ('face', ), 'palette': [(0, 255, 0)]} def __init__(self, **kwargs): super(WIDERFaceDataset, self).__init__(**kwargs) def load_annotations(self, ann_file): """Load annotation from WIDERFace XML style annotation file. Args: ann_file (str): Path of XML file. Returns: list[dict]: Annotation info from XML file. """ data_infos = [] img_ids = list_from_file(ann_file) for img_id in img_ids: filename = f'{img_id}.jpg' xml_path = osp.join(self.img_prefix, 'Annotations', f'{img_id}.xml') tree = ET.parse(xml_path) root = tree.getroot() size = root.find('size') width = int(size.find('width').text) height = int(size.find('height').text) folder = root.find('folder').text data_infos.append( dict( id=img_id, filename=osp.join(folder, filename), width=width, height=height)) return data_infos ================================================ FILE: mmdet/datasets/xml_style.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp import xml.etree.ElementTree as ET from typing import List, Optional, Union import mmcv from mmengine.fileio import list_from_file from mmdet.registry import DATASETS from .base_det_dataset import BaseDetDataset @DATASETS.register_module() class XMLDataset(BaseDetDataset): """XML dataset for detection. Args: img_subdir (str): Subdir where images are stored. Default: JPEGImages. ann_subdir (str): Subdir where annotations are. Default: Annotations. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. """ def __init__(self, img_subdir: str = 'JPEGImages', ann_subdir: str = 'Annotations', **kwargs) -> None: self.img_subdir = img_subdir self.ann_subdir = ann_subdir super().__init__(**kwargs) @property def sub_data_root(self) -> str: """Return the sub data root.""" return self.data_prefix.get('sub_data_root', '') def load_data_list(self) -> List[dict]: """Load annotation from XML style ann_file. Returns: list[dict]: Annotation info from XML file. """ assert self._metainfo.get('classes', None) is not None, \ '`classes` in `XMLDataset` can not be None.' self.cat2label = { cat: i for i, cat in enumerate(self._metainfo['classes']) } data_list = [] img_ids = list_from_file( self.ann_file, file_client_args=self.file_client_args) for img_id in img_ids: file_name = osp.join(self.img_subdir, f'{img_id}.jpg') xml_path = osp.join(self.sub_data_root, self.ann_subdir, f'{img_id}.xml') raw_img_info = {} raw_img_info['img_id'] = img_id raw_img_info['file_name'] = file_name raw_img_info['xml_path'] = xml_path parsed_data_info = self.parse_data_info(raw_img_info) data_list.append(parsed_data_info) return data_list @property def bbox_min_size(self) -> Optional[str]: """Return the minimum size of bounding boxes in the images.""" if self.filter_cfg is not None: return self.filter_cfg.get('bbox_min_size', None) else: return None def parse_data_info(self, img_info: dict) -> Union[dict, List[dict]]: """Parse raw annotation to target format. Args: img_info (dict): Raw image information, usually it includes `img_id`, `file_name`, and `xml_path`. Returns: Union[dict, List[dict]]: Parsed annotation. """ data_info = {} img_path = osp.join(self.sub_data_root, img_info['file_name']) data_info['img_path'] = img_path data_info['img_id'] = img_info['img_id'] data_info['xml_path'] = img_info['xml_path'] # deal with xml file with self.file_client.get_local_path( img_info['xml_path']) as local_path: raw_ann_info = ET.parse(local_path) root = raw_ann_info.getroot() size = root.find('size') if size is not None: width = int(size.find('width').text) height = int(size.find('height').text) else: img_bytes = self.file_client.get(img_path) img = mmcv.imfrombytes(img_bytes, backend='cv2') height, width = img.shape[:2] del img, img_bytes data_info['height'] = height data_info['width'] = width instances = [] for obj in raw_ann_info.findall('object'): instance = {} name = obj.find('name').text if name not in self._metainfo['classes']: continue difficult = obj.find('difficult') difficult = 0 if difficult is None else int(difficult.text) bnd_box = obj.find('bndbox') bbox = [ int(float(bnd_box.find('xmin').text)) - 1, int(float(bnd_box.find('ymin').text)) - 1, int(float(bnd_box.find('xmax').text)) - 1, int(float(bnd_box.find('ymax').text)) - 1 ] ignore = False if self.bbox_min_size is not None: assert not self.test_mode w = bbox[2] - bbox[0] h = bbox[3] - bbox[1] if w < self.bbox_min_size or h < self.bbox_min_size: ignore = True if difficult or ignore: instance['ignore_flag'] = 1 else: instance['ignore_flag'] = 0 instance['bbox'] = bbox instance['bbox_label'] = self.cat2label[name] instances.append(instance) data_info['instances'] = instances return data_info def filter_data(self) -> List[dict]: """Filter annotations according to filter_cfg. Returns: List[dict]: Filtered results. """ if self.test_mode: return self.data_list filter_empty_gt = self.filter_cfg.get('filter_empty_gt', False) \ if self.filter_cfg is not None else False min_size = self.filter_cfg.get('min_size', 0) \ if self.filter_cfg is not None else 0 valid_data_infos = [] for i, data_info in enumerate(self.data_list): width = data_info['width'] height = data_info['height'] if filter_empty_gt and len(data_info['instances']) == 0: continue if min(width, height) >= min_size: valid_data_infos.append(data_info) return valid_data_infos ================================================ FILE: mmdet/engine/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .hooks import * # noqa: F401, F403 from .optimizers import * # noqa: F401, F403 from .runner import * # noqa: F401, F403 from .schedulers import * # noqa: F401, F403 ================================================ FILE: mmdet/engine/hooks/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .checkloss_hook import CheckInvalidLossHook from .mean_teacher_hook import MeanTeacherHook from .memory_profiler_hook import MemoryProfilerHook from .num_class_check_hook import NumClassCheckHook from .pipeline_switch_hook import PipelineSwitchHook from .set_epoch_info_hook import SetEpochInfoHook from .sync_norm_hook import SyncNormHook from .utils import trigger_visualization_hook from .visualization_hook import DetVisualizationHook from .yolox_mode_switch_hook import YOLOXModeSwitchHook __all__ = [ 'YOLOXModeSwitchHook', 'SyncNormHook', 'CheckInvalidLossHook', 'SetEpochInfoHook', 'MemoryProfilerHook', 'DetVisualizationHook', 'NumClassCheckHook', 'MeanTeacherHook', 'trigger_visualization_hook', 'PipelineSwitchHook' ] ================================================ FILE: mmdet/engine/hooks/checkloss_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch from mmengine.hooks import Hook from mmengine.runner import Runner from mmdet.registry import HOOKS @HOOKS.register_module() class CheckInvalidLossHook(Hook): """Check invalid loss hook. This hook will regularly check whether the loss is valid during training. Args: interval (int): Checking interval (every k iterations). Default: 50. """ def __init__(self, interval: int = 50) -> None: self.interval = interval def after_train_iter(self, runner: Runner, batch_idx: int, data_batch: Optional[dict] = None, outputs: Optional[dict] = None) -> None: """Regularly check whether the loss is valid every n iterations. Args: runner (:obj:`Runner`): The runner of the training process. batch_idx (int): The index of the current batch in the train loop. data_batch (dict, Optional): Data from dataloader. Defaults to None. outputs (dict, Optional): Outputs from model. Defaults to None. """ if self.every_n_train_iters(runner, self.interval): assert torch.isfinite(outputs['loss']), \ runner.logger.info('loss become infinite or NaN!') ================================================ FILE: mmdet/engine/hooks/mean_teacher_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch.nn as nn from mmengine.hooks import Hook from mmengine.model import is_model_wrapper from mmengine.runner import Runner from mmdet.registry import HOOKS @HOOKS.register_module() class MeanTeacherHook(Hook): """Mean Teacher Hook. Mean Teacher is an efficient semi-supervised learning method in `Mean Teacher `_. This method requires two models with exactly the same structure, as the student model and the teacher model, respectively. The student model updates the parameters through gradient descent, and the teacher model updates the parameters through exponential moving average of the student model. Compared with the student model, the teacher model is smoother and accumulates more knowledge. Args: momentum (float): The momentum used for updating teacher's parameter. Teacher's parameter are updated with the formula: `teacher = (1-momentum) * teacher + momentum * student`. Defaults to 0.001. interval (int): Update teacher's parameter every interval iteration. Defaults to 1. skip_buffers (bool): Whether to skip the model buffers, such as batchnorm running stats (running_mean, running_var), it does not perform the ema operation. Default to True. """ def __init__(self, momentum: float = 0.001, interval: int = 1, skip_buffer=True) -> None: assert 0 < momentum < 1 self.momentum = momentum self.interval = interval self.skip_buffers = skip_buffer def before_train(self, runner: Runner) -> None: """To check that teacher model and student model exist.""" model = runner.model if is_model_wrapper(model): model = model.module assert hasattr(model, 'teacher') assert hasattr(model, 'student') # only do it at initial stage if runner.iter == 0: self.momentum_update(model, 1) def after_train_iter(self, runner: Runner, batch_idx: int, data_batch: Optional[dict] = None, outputs: Optional[dict] = None) -> None: """Update teacher's parameter every self.interval iterations.""" if (runner.iter + 1) % self.interval != 0: return model = runner.model if is_model_wrapper(model): model = model.module self.momentum_update(model, self.momentum) def momentum_update(self, model: nn.Module, momentum: float) -> None: """Compute the moving average of the parameters using exponential moving average.""" if self.skip_buffers: for (src_name, src_parm), (dst_name, dst_parm) in zip( model.student.named_parameters(), model.teacher.named_parameters()): dst_parm.data.mul_(1 - momentum).add_( src_parm.data, alpha=momentum) else: for (src_parm, dst_parm) in zip(model.student.state_dict().values(), model.teacher.state_dict().values()): # exclude num_tracking if dst_parm.dtype.is_floating_point: dst_parm.data.mul_(1 - momentum).add_( src_parm.data, alpha=momentum) ================================================ FILE: mmdet/engine/hooks/memory_profiler_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Sequence from mmengine.hooks import Hook from mmengine.runner import Runner from mmdet.registry import HOOKS from mmdet.structures import DetDataSample @HOOKS.register_module() class MemoryProfilerHook(Hook): """Memory profiler hook recording memory information including virtual memory, swap memory, and the memory of the current process. Args: interval (int): Checking interval (every k iterations). Default: 50. """ def __init__(self, interval: int = 50) -> None: try: from psutil import swap_memory, virtual_memory self._swap_memory = swap_memory self._virtual_memory = virtual_memory except ImportError: raise ImportError('psutil is not installed, please install it by: ' 'pip install psutil') try: from memory_profiler import memory_usage self._memory_usage = memory_usage except ImportError: raise ImportError( 'memory_profiler is not installed, please install it by: ' 'pip install memory_profiler') self.interval = interval def _record_memory_information(self, runner: Runner) -> None: """Regularly record memory information. Args: runner (:obj:`Runner`): The runner of the training or evaluation process. """ # in Byte virtual_memory = self._virtual_memory() swap_memory = self._swap_memory() # in MB process_memory = self._memory_usage()[0] factor = 1024 * 1024 runner.logger.info( 'Memory information ' 'available_memory: ' f'{round(virtual_memory.available / factor)} MB, ' 'used_memory: ' f'{round(virtual_memory.used / factor)} MB, ' f'memory_utilization: {virtual_memory.percent} %, ' 'available_swap_memory: ' f'{round((swap_memory.total - swap_memory.used) / factor)}' ' MB, ' f'used_swap_memory: {round(swap_memory.used / factor)} MB, ' f'swap_memory_utilization: {swap_memory.percent} %, ' 'current_process_memory: ' f'{round(process_memory)} MB') def after_train_iter(self, runner: Runner, batch_idx: int, data_batch: Optional[dict] = None, outputs: Optional[dict] = None) -> None: """Regularly record memory information. Args: runner (:obj:`Runner`): The runner of the training process. batch_idx (int): The index of the current batch in the train loop. data_batch (dict, optional): Data from dataloader. Defaults to None. outputs (dict, optional): Outputs from model. Defaults to None. """ if self.every_n_inner_iters(batch_idx, self.interval): self._record_memory_information(runner) def after_val_iter( self, runner: Runner, batch_idx: int, data_batch: Optional[dict] = None, outputs: Optional[Sequence[DetDataSample]] = None) -> None: """Regularly record memory information. Args: runner (:obj:`Runner`): The runner of the validation process. batch_idx (int): The index of the current batch in the val loop. data_batch (dict, optional): Data from dataloader. Defaults to None. outputs (Sequence[:obj:`DetDataSample`], optional): Outputs from model. Defaults to None. """ if self.every_n_inner_iters(batch_idx, self.interval): self._record_memory_information(runner) def after_test_iter( self, runner: Runner, batch_idx: int, data_batch: Optional[dict] = None, outputs: Optional[Sequence[DetDataSample]] = None) -> None: """Regularly record memory information. Args: runner (:obj:`Runner`): The runner of the testing process. batch_idx (int): The index of the current batch in the test loop. data_batch (dict, optional): Data from dataloader. Defaults to None. outputs (Sequence[:obj:`DetDataSample`], optional): Outputs from model. Defaults to None. """ if self.every_n_inner_iters(batch_idx, self.interval): self._record_memory_information(runner) ================================================ FILE: mmdet/engine/hooks/num_class_check_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmcv.cnn import VGG from mmengine.hooks import Hook from mmengine.runner import Runner from mmdet.registry import HOOKS @HOOKS.register_module() class NumClassCheckHook(Hook): """Check whether the `num_classes` in head matches the length of `classes` in `dataset.metainfo`.""" def _check_head(self, runner: Runner, mode: str) -> None: """Check whether the `num_classes` in head matches the length of `classes` in `dataset.metainfo`. Args: runner (:obj:`Runner`): The runner of the training or evaluation process. """ assert mode in ['train', 'val'] model = runner.model dataset = runner.train_dataloader.dataset if mode == 'train' else \ runner.val_dataloader.dataset if dataset.metainfo.get('classes', None) is None: runner.logger.warning( f'Please set `classes` ' f'in the {dataset.__class__.__name__} `metainfo` and' f'check if it is consistent with the `num_classes` ' f'of head') else: classes = dataset.metainfo['classes'] assert type(classes) is not str, \ (f'`classes` in {dataset.__class__.__name__}' f'should be a tuple of str.' f'Add comma if number of classes is 1 as ' f'classes = ({classes},)') from mmdet.models.roi_heads.mask_heads import FusedSemanticHead for name, module in model.named_modules(): if hasattr(module, 'num_classes') and not name.endswith( 'rpn_head') and not isinstance( module, (VGG, FusedSemanticHead)): assert module.num_classes == len(classes), \ (f'The `num_classes` ({module.num_classes}) in ' f'{module.__class__.__name__} of ' f'{model.__class__.__name__} does not matches ' f'the length of `classes` ' f'{len(classes)}) in ' f'{dataset.__class__.__name__}') def before_train_epoch(self, runner: Runner) -> None: """Check whether the training dataset is compatible with head. Args: runner (:obj:`Runner`): The runner of the training or evaluation process. """ self._check_head(runner, 'train') def before_val_epoch(self, runner: Runner) -> None: """Check whether the dataset in val epoch is compatible with head. Args: runner (:obj:`Runner`): The runner of the training or evaluation process. """ self._check_head(runner, 'val') ================================================ FILE: mmdet/engine/hooks/pipeline_switch_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmcv.transforms import Compose from mmengine.hooks import Hook from mmdet.registry import HOOKS @HOOKS.register_module() class PipelineSwitchHook(Hook): """Switch data pipeline at switch_epoch. Args: switch_epoch (int): switch pipeline at this epoch. switch_pipeline (list[dict]): the pipeline to switch to. """ def __init__(self, switch_epoch, switch_pipeline): self.switch_epoch = switch_epoch self.switch_pipeline = switch_pipeline self._restart_dataloader = False def before_train_epoch(self, runner): """switch pipeline.""" epoch = runner.epoch train_loader = runner.train_dataloader if epoch == self.switch_epoch: runner.logger.info('Switch pipeline now!') # The dataset pipeline cannot be updated when persistent_workers # is True, so we need to force the dataloader's multi-process # restart. This is a very hacky approach. train_loader.dataset.pipeline = Compose(self.switch_pipeline) if hasattr(train_loader, 'persistent_workers' ) and train_loader.persistent_workers is True: train_loader._DataLoader__initialized = False train_loader._iterator = None self._restart_dataloader = True else: # Once the restart is complete, we need to restore # the initialization flag. if self._restart_dataloader: train_loader._DataLoader__initialized = True ================================================ FILE: mmdet/engine/hooks/set_epoch_info_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmengine.hooks import Hook from mmengine.model.wrappers import is_model_wrapper from mmdet.registry import HOOKS @HOOKS.register_module() class SetEpochInfoHook(Hook): """Set runner's epoch information to the model.""" def before_train_epoch(self, runner): epoch = runner.epoch model = runner.model if is_model_wrapper(model): model = model.module model.set_epoch(epoch) ================================================ FILE: mmdet/engine/hooks/sync_norm_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from collections import OrderedDict from mmengine.dist import get_dist_info from mmengine.hooks import Hook from torch import nn from mmdet.registry import HOOKS from mmdet.utils import all_reduce_dict def get_norm_states(module: nn.Module) -> OrderedDict: """Get the state_dict of batch norms in the module.""" async_norm_states = OrderedDict() for name, child in module.named_modules(): if isinstance(child, nn.modules.batchnorm._NormBase): for k, v in child.state_dict().items(): async_norm_states['.'.join([name, k])] = v return async_norm_states @HOOKS.register_module() class SyncNormHook(Hook): """Synchronize Norm states before validation, currently used in YOLOX.""" def before_val_epoch(self, runner): """Synchronizing norm.""" module = runner.model _, world_size = get_dist_info() if world_size == 1: return norm_states = get_norm_states(module) if len(norm_states) == 0: return # TODO: use `all_reduce_dict` in mmengine norm_states = all_reduce_dict(norm_states, op='mean') module.load_state_dict(norm_states, strict=False) ================================================ FILE: mmdet/engine/hooks/utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. def trigger_visualization_hook(cfg, args): default_hooks = cfg.default_hooks if 'visualization' in default_hooks: visualization_hook = default_hooks['visualization'] # Turn on visualization visualization_hook['draw'] = True if args.show: visualization_hook['show'] = True visualization_hook['wait_time'] = args.wait_time if args.show_dir: visualization_hook['test_out_dir'] = args.show_dir else: raise RuntimeError( 'VisualizationHook must be included in default_hooks.' 'refer to usage ' '"visualization=dict(type=\'VisualizationHook\')"') return cfg ================================================ FILE: mmdet/engine/hooks/visualization_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp import warnings from typing import Optional, Sequence import mmcv from mmengine.fileio import FileClient from mmengine.hooks import Hook from mmengine.runner import Runner from mmengine.utils import mkdir_or_exist from mmengine.visualization import Visualizer from mmdet.registry import HOOKS from mmdet.structures import DetDataSample @HOOKS.register_module() class DetVisualizationHook(Hook): """Detection Visualization Hook. Used to visualize validation and testing process prediction results. In the testing phase: 1. If ``show`` is True, it means that only the prediction results are visualized without storing data, so ``vis_backends`` needs to be excluded. 2. If ``test_out_dir`` is specified, it means that the prediction results need to be saved to ``test_out_dir``. In order to avoid vis_backends also storing data, so ``vis_backends`` needs to be excluded. 3. ``vis_backends`` takes effect if the user does not specify ``show`` and `test_out_dir``. You can set ``vis_backends`` to WandbVisBackend or TensorboardVisBackend to store the prediction result in Wandb or Tensorboard. Args: draw (bool): whether to draw prediction results. If it is False, it means that no drawing will be done. Defaults to False. interval (int): The interval of visualization. Defaults to 50. score_thr (float): The threshold to visualize the bboxes and masks. Defaults to 0.3. show (bool): Whether to display the drawn image. Default to False. wait_time (float): The interval of show (s). Defaults to 0. test_out_dir (str, optional): directory where painted images will be saved in testing process. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. """ def __init__(self, draw: bool = False, interval: int = 50, score_thr: float = 0.3, show: bool = False, wait_time: float = 0., test_out_dir: Optional[str] = None, file_client_args: dict = dict(backend='disk')): self._visualizer: Visualizer = Visualizer.get_current_instance() self.interval = interval self.score_thr = score_thr self.show = show if self.show: # No need to think about vis backends. self._visualizer._vis_backends = {} warnings.warn('The show is True, it means that only ' 'the prediction results are visualized ' 'without storing data, so vis_backends ' 'needs to be excluded.') self.wait_time = wait_time self.file_client_args = file_client_args.copy() self.file_client = None self.draw = draw self.test_out_dir = test_out_dir self._test_index = 0 def after_val_iter(self, runner: Runner, batch_idx: int, data_batch: dict, outputs: Sequence[DetDataSample]) -> None: """Run after every ``self.interval`` validation iterations. Args: runner (:obj:`Runner`): The runner of the validation process. batch_idx (int): The index of the current batch in the val loop. data_batch (dict): Data from dataloader. outputs (Sequence[:obj:`DetDataSample`]]): A batch of data samples that contain annotations and predictions. """ if self.draw is False: return if self.file_client is None: self.file_client = FileClient(**self.file_client_args) # There is no guarantee that the same batch of images # is visualized for each evaluation. total_curr_iter = runner.iter + batch_idx # Visualize only the first data img_path = outputs[0].img_path img_bytes = self.file_client.get(img_path) img = mmcv.imfrombytes(img_bytes, channel_order='rgb') if total_curr_iter % self.interval == 0: self._visualizer.add_datasample( osp.basename(img_path) if self.show else 'val_img', img, data_sample=outputs[0], show=self.show, wait_time=self.wait_time, pred_score_thr=self.score_thr, step=total_curr_iter) def after_test_iter(self, runner: Runner, batch_idx: int, data_batch: dict, outputs: Sequence[DetDataSample]) -> None: """Run after every testing iterations. Args: runner (:obj:`Runner`): The runner of the testing process. batch_idx (int): The index of the current batch in the val loop. data_batch (dict): Data from dataloader. outputs (Sequence[:obj:`DetDataSample`]): A batch of data samples that contain annotations and predictions. """ if self.draw is False: return if self.test_out_dir is not None: self.test_out_dir = osp.join(runner.work_dir, runner.timestamp, self.test_out_dir) mkdir_or_exist(self.test_out_dir) if self.file_client is None: self.file_client = FileClient(**self.file_client_args) for data_sample in outputs: self._test_index += 1 img_path = data_sample.img_path img_bytes = self.file_client.get(img_path) img = mmcv.imfrombytes(img_bytes, channel_order='rgb') out_file = None if self.test_out_dir is not None: out_file = osp.basename(img_path) out_file = osp.join(self.test_out_dir, out_file) self._visualizer.add_datasample( osp.basename(img_path) if self.show else 'test_img', img, data_sample=data_sample, show=self.show, wait_time=self.wait_time, pred_score_thr=self.score_thr, out_file=out_file, step=self._test_index) ================================================ FILE: mmdet/engine/hooks/yolox_mode_switch_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Sequence from mmengine.hooks import Hook from mmengine.model import is_model_wrapper from mmdet.registry import HOOKS @HOOKS.register_module() class YOLOXModeSwitchHook(Hook): """Switch the mode of YOLOX during training. This hook turns off the mosaic and mixup data augmentation and switches to use L1 loss in bbox_head. Args: num_last_epochs (int): The number of latter epochs in the end of the training to close the data augmentation and switch to L1 loss. Defaults to 15. skip_type_keys (Sequence[str], optional): Sequence of type string to be skip pipeline. Defaults to ('Mosaic', 'RandomAffine', 'MixUp'). """ def __init__( self, num_last_epochs: int = 15, skip_type_keys: Sequence[str] = ('Mosaic', 'RandomAffine', 'MixUp') ) -> None: self.num_last_epochs = num_last_epochs self.skip_type_keys = skip_type_keys self._restart_dataloader = False def before_train_epoch(self, runner) -> None: """Close mosaic and mixup augmentation and switches to use L1 loss.""" epoch = runner.epoch train_loader = runner.train_dataloader model = runner.model # TODO: refactor after mmengine using model wrapper if is_model_wrapper(model): model = model.module if (epoch + 1) == runner.max_epochs - self.num_last_epochs: runner.logger.info('No mosaic and mixup aug now!') # The dataset pipeline cannot be updated when persistent_workers # is True, so we need to force the dataloader's multi-process # restart. This is a very hacky approach. train_loader.dataset.update_skip_type_keys(self.skip_type_keys) if hasattr(train_loader, 'persistent_workers' ) and train_loader.persistent_workers is True: train_loader._DataLoader__initialized = False train_loader._iterator = None self._restart_dataloader = True runner.logger.info('Add additional L1 loss now!') model.bbox_head.use_l1 = True else: # Once the restart is complete, we need to restore # the initialization flag. if self._restart_dataloader: train_loader._DataLoader__initialized = True ================================================ FILE: mmdet/engine/optimizers/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .layer_decay_optimizer_constructor import \ LearningRateDecayOptimizerConstructor __all__ = ['LearningRateDecayOptimizerConstructor'] ================================================ FILE: mmdet/engine/optimizers/layer_decay_optimizer_constructor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import json from typing import List import torch.nn as nn from mmengine.dist import get_dist_info from mmengine.logging import MMLogger from mmengine.optim import DefaultOptimWrapperConstructor from mmdet.registry import OPTIM_WRAPPER_CONSTRUCTORS def get_layer_id_for_convnext(var_name, max_layer_id): """Get the layer id to set the different learning rates in ``layer_wise`` decay_type. Args: var_name (str): The key of the model. max_layer_id (int): Maximum layer id. Returns: int: The id number corresponding to different learning rate in ``LearningRateDecayOptimizerConstructor``. """ if var_name in ('backbone.cls_token', 'backbone.mask_token', 'backbone.pos_embed'): return 0 elif var_name.startswith('backbone.downsample_layers'): stage_id = int(var_name.split('.')[2]) if stage_id == 0: layer_id = 0 elif stage_id == 1: layer_id = 2 elif stage_id == 2: layer_id = 3 elif stage_id == 3: layer_id = max_layer_id return layer_id elif var_name.startswith('backbone.stages'): stage_id = int(var_name.split('.')[2]) block_id = int(var_name.split('.')[3]) if stage_id == 0: layer_id = 1 elif stage_id == 1: layer_id = 2 elif stage_id == 2: layer_id = 3 + block_id // 3 elif stage_id == 3: layer_id = max_layer_id return layer_id else: return max_layer_id + 1 def get_stage_id_for_convnext(var_name, max_stage_id): """Get the stage id to set the different learning rates in ``stage_wise`` decay_type. Args: var_name (str): The key of the model. max_stage_id (int): Maximum stage id. Returns: int: The id number corresponding to different learning rate in ``LearningRateDecayOptimizerConstructor``. """ if var_name in ('backbone.cls_token', 'backbone.mask_token', 'backbone.pos_embed'): return 0 elif var_name.startswith('backbone.downsample_layers'): return 0 elif var_name.startswith('backbone.stages'): stage_id = int(var_name.split('.')[2]) return stage_id + 1 else: return max_stage_id - 1 @OPTIM_WRAPPER_CONSTRUCTORS.register_module() class LearningRateDecayOptimizerConstructor(DefaultOptimWrapperConstructor): # Different learning rates are set for different layers of backbone. # Note: Currently, this optimizer constructor is built for ConvNeXt. def add_params(self, params: List[dict], module: nn.Module, **kwargs) -> None: """Add all parameters of module to the params list. The parameters of the given module will be added to the list of param groups, with specific rules defined by paramwise_cfg. Args: params (list[dict]): A list of param groups, it will be modified in place. module (nn.Module): The module to be added. """ logger = MMLogger.get_current_instance() parameter_groups = {} logger.info(f'self.paramwise_cfg is {self.paramwise_cfg}') num_layers = self.paramwise_cfg.get('num_layers') + 2 decay_rate = self.paramwise_cfg.get('decay_rate') decay_type = self.paramwise_cfg.get('decay_type', 'layer_wise') logger.info('Build LearningRateDecayOptimizerConstructor ' f'{decay_type} {decay_rate} - {num_layers}') weight_decay = self.base_wd for name, param in module.named_parameters(): if not param.requires_grad: continue # frozen weights if len(param.shape) == 1 or name.endswith('.bias') or name in ( 'pos_embed', 'cls_token'): group_name = 'no_decay' this_weight_decay = 0. else: group_name = 'decay' this_weight_decay = weight_decay if 'layer_wise' in decay_type: if 'ConvNeXt' in module.backbone.__class__.__name__: layer_id = get_layer_id_for_convnext( name, self.paramwise_cfg.get('num_layers')) logger.info(f'set param {name} as id {layer_id}') else: raise NotImplementedError() elif decay_type == 'stage_wise': if 'ConvNeXt' in module.backbone.__class__.__name__: layer_id = get_stage_id_for_convnext(name, num_layers) logger.info(f'set param {name} as id {layer_id}') else: raise NotImplementedError() group_name = f'layer_{layer_id}_{group_name}' if group_name not in parameter_groups: scale = decay_rate**(num_layers - layer_id - 1) parameter_groups[group_name] = { 'weight_decay': this_weight_decay, 'params': [], 'param_names': [], 'lr_scale': scale, 'group_name': group_name, 'lr': scale * self.base_lr, } parameter_groups[group_name]['params'].append(param) parameter_groups[group_name]['param_names'].append(name) rank, _ = get_dist_info() if rank == 0: to_display = {} for key in parameter_groups: to_display[key] = { 'param_names': parameter_groups[key]['param_names'], 'lr_scale': parameter_groups[key]['lr_scale'], 'lr': parameter_groups[key]['lr'], 'weight_decay': parameter_groups[key]['weight_decay'], } logger.info(f'Param groups = {json.dumps(to_display, indent=2)}') params.extend(parameter_groups.values()) ================================================ FILE: mmdet/engine/runner/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .loops import TeacherStudentValLoop __all__ = ['TeacherStudentValLoop'] ================================================ FILE: mmdet/engine/runner/loops.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmengine.model import is_model_wrapper from mmengine.runner import ValLoop from mmdet.registry import LOOPS @LOOPS.register_module() class TeacherStudentValLoop(ValLoop): """Loop for validation of model teacher and student.""" def run(self): """Launch validation for model teacher and student.""" self.runner.call_hook('before_val') self.runner.call_hook('before_val_epoch') self.runner.model.eval() model = self.runner.model if is_model_wrapper(model): model = model.module assert hasattr(model, 'teacher') assert hasattr(model, 'student') predict_on = model.semi_test_cfg.get('predict_on', None) multi_metrics = dict() for _predict_on in ['teacher', 'student']: model.semi_test_cfg['predict_on'] = _predict_on for idx, data_batch in enumerate(self.dataloader): self.run_iter(idx, data_batch) # compute metrics metrics = self.evaluator.evaluate(len(self.dataloader.dataset)) multi_metrics.update( {'/'.join((_predict_on, k)): v for k, v in metrics.items()}) model.semi_test_cfg['predict_on'] = predict_on self.runner.call_hook('after_val_epoch', metrics=multi_metrics) self.runner.call_hook('after_val') ================================================ FILE: mmdet/engine/schedulers/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .quadratic_warmup import (QuadraticWarmupLR, QuadraticWarmupMomentum, QuadraticWarmupParamScheduler) __all__ = [ 'QuadraticWarmupParamScheduler', 'QuadraticWarmupMomentum', 'QuadraticWarmupLR' ] ================================================ FILE: mmdet/engine/schedulers/quadratic_warmup.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmengine.optim.scheduler.lr_scheduler import LRSchedulerMixin from mmengine.optim.scheduler.momentum_scheduler import MomentumSchedulerMixin from mmengine.optim.scheduler.param_scheduler import INF, _ParamScheduler from torch.optim import Optimizer from mmdet.registry import PARAM_SCHEDULERS @PARAM_SCHEDULERS.register_module() class QuadraticWarmupParamScheduler(_ParamScheduler): r"""Warm up the parameter value of each parameter group by quadratic formula: .. math:: X_{t} = X_{t-1} + \frac{2t+1}{{(end-begin)}^{2}} \times X_{base} Args: optimizer (Optimizer): Wrapped optimizer. param_name (str): Name of the parameter to be adjusted, such as ``lr``, ``momentum``. begin (int): Step at which to start updating the parameters. Defaults to 0. end (int): Step at which to stop updating the parameters. Defaults to INF. last_step (int): The index of last step. Used for resume without state dict. Defaults to -1. by_epoch (bool): Whether the scheduled parameters are updated by epochs. Defaults to True. verbose (bool): Whether to print the value for each update. Defaults to False. """ def __init__(self, optimizer: Optimizer, param_name: str, begin: int = 0, end: int = INF, last_step: int = -1, by_epoch: bool = True, verbose: bool = False): if end >= INF: raise ValueError('``end`` must be less than infinity,' 'Please set ``end`` parameter of ' '``QuadraticWarmupScheduler`` as the ' 'number of warmup end.') self.total_iters = end - begin super().__init__( optimizer=optimizer, param_name=param_name, begin=begin, end=end, last_step=last_step, by_epoch=by_epoch, verbose=verbose) @classmethod def build_iter_from_epoch(cls, *args, begin=0, end=INF, by_epoch=True, epoch_length=None, **kwargs): """Build an iter-based instance of this scheduler from an epoch-based config.""" assert by_epoch, 'Only epoch-based kwargs whose `by_epoch=True` can ' \ 'be converted to iter-based.' assert epoch_length is not None and epoch_length > 0, \ f'`epoch_length` must be a positive integer, ' \ f'but got {epoch_length}.' by_epoch = False begin = begin * epoch_length if end != INF: end = end * epoch_length return cls(*args, begin=begin, end=end, by_epoch=by_epoch, **kwargs) def _get_value(self): """Compute value using chainable form of the scheduler.""" if self.last_step == 0: return [ base_value * (2 * self.last_step + 1) / self.total_iters**2 for base_value in self.base_values ] return [ group[self.param_name] + base_value * (2 * self.last_step + 1) / self.total_iters**2 for base_value, group in zip(self.base_values, self.optimizer.param_groups) ] @PARAM_SCHEDULERS.register_module() class QuadraticWarmupLR(LRSchedulerMixin, QuadraticWarmupParamScheduler): """Warm up the learning rate of each parameter group by quadratic formula. Args: optimizer (Optimizer): Wrapped optimizer. begin (int): Step at which to start updating the parameters. Defaults to 0. end (int): Step at which to stop updating the parameters. Defaults to INF. last_step (int): The index of last step. Used for resume without state dict. Defaults to -1. by_epoch (bool): Whether the scheduled parameters are updated by epochs. Defaults to True. verbose (bool): Whether to print the value for each update. Defaults to False. """ @PARAM_SCHEDULERS.register_module() class QuadraticWarmupMomentum(MomentumSchedulerMixin, QuadraticWarmupParamScheduler): """Warm up the momentum value of each parameter group by quadratic formula. Args: optimizer (Optimizer): Wrapped optimizer. begin (int): Step at which to start updating the parameters. Defaults to 0. end (int): Step at which to stop updating the parameters. Defaults to INF. last_step (int): The index of last step. Used for resume without state dict. Defaults to -1. by_epoch (bool): Whether the scheduled parameters are updated by epochs. Defaults to True. verbose (bool): Whether to print the value for each update. Defaults to False. """ ================================================ FILE: mmdet/evaluation/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .functional import * # noqa: F401,F403 from .metrics import * # noqa: F401,F403 ================================================ FILE: mmdet/evaluation/functional/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .bbox_overlaps import bbox_overlaps from .class_names import (cityscapes_classes, coco_classes, coco_panoptic_classes, dataset_aliases, get_classes, imagenet_det_classes, imagenet_vid_classes, objects365v1_classes, objects365v2_classes, oid_challenge_classes, oid_v6_classes, voc_classes) from .mean_ap import average_precision, eval_map, print_map_summary from .panoptic_utils import (INSTANCE_OFFSET, pq_compute_multi_core, pq_compute_single_core) from .recall import (eval_recalls, plot_iou_recall, plot_num_recall, print_recall_summary) __all__ = [ 'voc_classes', 'imagenet_det_classes', 'imagenet_vid_classes', 'coco_classes', 'cityscapes_classes', 'dataset_aliases', 'get_classes', 'average_precision', 'eval_map', 'print_map_summary', 'eval_recalls', 'print_recall_summary', 'plot_num_recall', 'plot_iou_recall', 'oid_v6_classes', 'oid_challenge_classes', 'INSTANCE_OFFSET', 'pq_compute_single_core', 'pq_compute_multi_core', 'bbox_overlaps', 'objects365v1_classes', 'objects365v2_classes', 'coco_panoptic_classes' ] ================================================ FILE: mmdet/evaluation/functional/bbox_overlaps.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np def bbox_overlaps(bboxes1, bboxes2, mode='iou', eps=1e-6, use_legacy_coordinate=False): """Calculate the ious between each bbox of bboxes1 and bboxes2. Args: bboxes1 (ndarray): Shape (n, 4) bboxes2 (ndarray): Shape (k, 4) mode (str): IOU (intersection over union) or IOF (intersection over foreground) use_legacy_coordinate (bool): Whether to use coordinate system in mmdet v1.x. which means width, height should be calculated as 'x2 - x1 + 1` and 'y2 - y1 + 1' respectively. Note when function is used in `VOCDataset`, it should be True to align with the official implementation `http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCdevkit_18-May-2011.tar` Default: False. Returns: ious (ndarray): Shape (n, k) """ assert mode in ['iou', 'iof'] if not use_legacy_coordinate: extra_length = 0. else: extra_length = 1. bboxes1 = bboxes1.astype(np.float32) bboxes2 = bboxes2.astype(np.float32) rows = bboxes1.shape[0] cols = bboxes2.shape[0] ious = np.zeros((rows, cols), dtype=np.float32) if rows * cols == 0: return ious exchange = False if bboxes1.shape[0] > bboxes2.shape[0]: bboxes1, bboxes2 = bboxes2, bboxes1 ious = np.zeros((cols, rows), dtype=np.float32) exchange = True area1 = (bboxes1[:, 2] - bboxes1[:, 0] + extra_length) * ( bboxes1[:, 3] - bboxes1[:, 1] + extra_length) area2 = (bboxes2[:, 2] - bboxes2[:, 0] + extra_length) * ( bboxes2[:, 3] - bboxes2[:, 1] + extra_length) for i in range(bboxes1.shape[0]): x_start = np.maximum(bboxes1[i, 0], bboxes2[:, 0]) y_start = np.maximum(bboxes1[i, 1], bboxes2[:, 1]) x_end = np.minimum(bboxes1[i, 2], bboxes2[:, 2]) y_end = np.minimum(bboxes1[i, 3], bboxes2[:, 3]) overlap = np.maximum(x_end - x_start + extra_length, 0) * np.maximum( y_end - y_start + extra_length, 0) if mode == 'iou': union = area1[i] + area2 - overlap else: union = area1[i] if not exchange else area2 union = np.maximum(union, eps) ious[i, :] = overlap / union if exchange: ious = ious.T return ious ================================================ FILE: mmdet/evaluation/functional/class_names.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmengine.utils import is_str def wider_face_classes() -> list: """Class names of WIDERFace.""" return ['face'] def voc_classes() -> list: """Class names of PASCAL VOC.""" return [ 'aeroplane', 'bicycle', 'bird', 'boat', 'bottle', 'bus', 'car', 'cat', 'chair', 'cow', 'diningtable', 'dog', 'horse', 'motorbike', 'person', 'pottedplant', 'sheep', 'sofa', 'train', 'tvmonitor' ] def imagenet_det_classes() -> list: """Class names of ImageNet Det.""" return [ 'accordion', 'airplane', 'ant', 'antelope', 'apple', 'armadillo', 'artichoke', 'axe', 'baby_bed', 'backpack', 'bagel', 'balance_beam', 'banana', 'band_aid', 'banjo', 'baseball', 'basketball', 'bathing_cap', 'beaker', 'bear', 'bee', 'bell_pepper', 'bench', 'bicycle', 'binder', 'bird', 'bookshelf', 'bow_tie', 'bow', 'bowl', 'brassiere', 'burrito', 'bus', 'butterfly', 'camel', 'can_opener', 'car', 'cart', 'cattle', 'cello', 'centipede', 'chain_saw', 'chair', 'chime', 'cocktail_shaker', 'coffee_maker', 'computer_keyboard', 'computer_mouse', 'corkscrew', 'cream', 'croquet_ball', 'crutch', 'cucumber', 'cup_or_mug', 'diaper', 'digital_clock', 'dishwasher', 'dog', 'domestic_cat', 'dragonfly', 'drum', 'dumbbell', 'electric_fan', 'elephant', 'face_powder', 'fig', 'filing_cabinet', 'flower_pot', 'flute', 'fox', 'french_horn', 'frog', 'frying_pan', 'giant_panda', 'goldfish', 'golf_ball', 'golfcart', 'guacamole', 'guitar', 'hair_dryer', 'hair_spray', 'hamburger', 'hammer', 'hamster', 'harmonica', 'harp', 'hat_with_a_wide_brim', 'head_cabbage', 'helmet', 'hippopotamus', 'horizontal_bar', 'horse', 'hotdog', 'iPod', 'isopod', 'jellyfish', 'koala_bear', 'ladle', 'ladybug', 'lamp', 'laptop', 'lemon', 'lion', 'lipstick', 'lizard', 'lobster', 'maillot', 'maraca', 'microphone', 'microwave', 'milk_can', 'miniskirt', 'monkey', 'motorcycle', 'mushroom', 'nail', 'neck_brace', 'oboe', 'orange', 'otter', 'pencil_box', 'pencil_sharpener', 'perfume', 'person', 'piano', 'pineapple', 'ping-pong_ball', 'pitcher', 'pizza', 'plastic_bag', 'plate_rack', 'pomegranate', 'popsicle', 'porcupine', 'power_drill', 'pretzel', 'printer', 'puck', 'punching_bag', 'purse', 'rabbit', 'racket', 'ray', 'red_panda', 'refrigerator', 'remote_control', 'rubber_eraser', 'rugby_ball', 'ruler', 'salt_or_pepper_shaker', 'saxophone', 'scorpion', 'screwdriver', 'seal', 'sheep', 'ski', 'skunk', 'snail', 'snake', 'snowmobile', 'snowplow', 'soap_dispenser', 'soccer_ball', 'sofa', 'spatula', 'squirrel', 'starfish', 'stethoscope', 'stove', 'strainer', 'strawberry', 'stretcher', 'sunglasses', 'swimming_trunks', 'swine', 'syringe', 'table', 'tape_player', 'tennis_ball', 'tick', 'tie', 'tiger', 'toaster', 'traffic_light', 'train', 'trombone', 'trumpet', 'turtle', 'tv_or_monitor', 'unicycle', 'vacuum', 'violin', 'volleyball', 'waffle_iron', 'washer', 'water_bottle', 'watercraft', 'whale', 'wine_bottle', 'zebra' ] def imagenet_vid_classes() -> list: """Class names of ImageNet VID.""" return [ 'airplane', 'antelope', 'bear', 'bicycle', 'bird', 'bus', 'car', 'cattle', 'dog', 'domestic_cat', 'elephant', 'fox', 'giant_panda', 'hamster', 'horse', 'lion', 'lizard', 'monkey', 'motorcycle', 'rabbit', 'red_panda', 'sheep', 'snake', 'squirrel', 'tiger', 'train', 'turtle', 'watercraft', 'whale', 'zebra' ] def coco_classes() -> list: """Class names of COCO.""" return [ 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic_light', 'fire_hydrant', 'stop_sign', 'parking_meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports_ball', 'kite', 'baseball_bat', 'baseball_glove', 'skateboard', 'surfboard', 'tennis_racket', 'bottle', 'wine_glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot_dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted_plant', 'bed', 'dining_table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell_phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy_bear', 'hair_drier', 'toothbrush' ] def coco_panoptic_classes() -> list: """Class names of COCO panoptic.""" return [ 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush', 'banner', 'blanket', 'bridge', 'cardboard', 'counter', 'curtain', 'door-stuff', 'floor-wood', 'flower', 'fruit', 'gravel', 'house', 'light', 'mirror-stuff', 'net', 'pillow', 'platform', 'playingfield', 'railroad', 'river', 'road', 'roof', 'sand', 'sea', 'shelf', 'snow', 'stairs', 'tent', 'towel', 'wall-brick', 'wall-stone', 'wall-tile', 'wall-wood', 'water-other', 'window-blind', 'window-other', 'tree-merged', 'fence-merged', 'ceiling-merged', 'sky-other-merged', 'cabinet-merged', 'table-merged', 'floor-other-merged', 'pavement-merged', 'mountain-merged', 'grass-merged', 'dirt-merged', 'paper-merged', 'food-other-merged', 'building-other-merged', 'rock-merged', 'wall-other-merged', 'rug-merged' ] def cityscapes_classes() -> list: """Class names of Cityscapes.""" return [ 'person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', 'bicycle' ] def oid_challenge_classes() -> list: """Class names of Open Images Challenge.""" return [ 'Footwear', 'Jeans', 'House', 'Tree', 'Woman', 'Man', 'Land vehicle', 'Person', 'Wheel', 'Bus', 'Human face', 'Bird', 'Dress', 'Girl', 'Vehicle', 'Building', 'Cat', 'Car', 'Belt', 'Elephant', 'Dessert', 'Butterfly', 'Train', 'Guitar', 'Poster', 'Book', 'Boy', 'Bee', 'Flower', 'Window', 'Hat', 'Human head', 'Dog', 'Human arm', 'Drink', 'Human mouth', 'Human hair', 'Human nose', 'Human hand', 'Table', 'Marine invertebrates', 'Fish', 'Sculpture', 'Rose', 'Street light', 'Glasses', 'Fountain', 'Skyscraper', 'Swimwear', 'Brassiere', 'Drum', 'Duck', 'Countertop', 'Furniture', 'Ball', 'Human leg', 'Boat', 'Balloon', 'Bicycle helmet', 'Goggles', 'Door', 'Human eye', 'Shirt', 'Toy', 'Teddy bear', 'Pasta', 'Tomato', 'Human ear', 'Vehicle registration plate', 'Microphone', 'Musical keyboard', 'Tower', 'Houseplant', 'Flowerpot', 'Fruit', 'Vegetable', 'Musical instrument', 'Suit', 'Motorcycle', 'Bagel', 'French fries', 'Hamburger', 'Chair', 'Salt and pepper shakers', 'Snail', 'Airplane', 'Horse', 'Laptop', 'Computer keyboard', 'Football helmet', 'Cocktail', 'Juice', 'Tie', 'Computer monitor', 'Human beard', 'Bottle', 'Saxophone', 'Lemon', 'Mouse', 'Sock', 'Cowboy hat', 'Sun hat', 'Football', 'Porch', 'Sunglasses', 'Lobster', 'Crab', 'Picture frame', 'Van', 'Crocodile', 'Surfboard', 'Shorts', 'Helicopter', 'Helmet', 'Sports uniform', 'Taxi', 'Swan', 'Goose', 'Coat', 'Jacket', 'Handbag', 'Flag', 'Skateboard', 'Television', 'Tire', 'Spoon', 'Palm tree', 'Stairs', 'Salad', 'Castle', 'Oven', 'Microwave oven', 'Wine', 'Ceiling fan', 'Mechanical fan', 'Cattle', 'Truck', 'Box', 'Ambulance', 'Desk', 'Wine glass', 'Reptile', 'Tank', 'Traffic light', 'Billboard', 'Tent', 'Insect', 'Spider', 'Treadmill', 'Cupboard', 'Shelf', 'Seat belt', 'Human foot', 'Bicycle', 'Bicycle wheel', 'Couch', 'Bookcase', 'Fedora', 'Backpack', 'Bench', 'Oyster', 'Moths and butterflies', 'Lavender', 'Waffle', 'Fork', 'Animal', 'Accordion', 'Mobile phone', 'Plate', 'Coffee cup', 'Saucer', 'Platter', 'Dagger', 'Knife', 'Bull', 'Tortoise', 'Sea turtle', 'Deer', 'Weapon', 'Apple', 'Ski', 'Taco', 'Traffic sign', 'Beer', 'Necklace', 'Sunflower', 'Piano', 'Organ', 'Harpsichord', 'Bed', 'Cabinetry', 'Nightstand', 'Curtain', 'Chest of drawers', 'Drawer', 'Parrot', 'Sandal', 'High heels', 'Tableware', 'Cart', 'Mushroom', 'Kite', 'Missile', 'Seafood', 'Camera', 'Paper towel', 'Toilet paper', 'Sombrero', 'Radish', 'Lighthouse', 'Segway', 'Pig', 'Watercraft', 'Golf cart', 'studio couch', 'Dolphin', 'Whale', 'Earrings', 'Otter', 'Sea lion', 'Whiteboard', 'Monkey', 'Gondola', 'Zebra', 'Baseball glove', 'Scarf', 'Adhesive tape', 'Trousers', 'Scoreboard', 'Lily', 'Carnivore', 'Power plugs and sockets', 'Office building', 'Sandwich', 'Swimming pool', 'Headphones', 'Tin can', 'Crown', 'Doll', 'Cake', 'Frog', 'Beetle', 'Ant', 'Gas stove', 'Canoe', 'Falcon', 'Blue jay', 'Egg', 'Fire hydrant', 'Raccoon', 'Muffin', 'Wall clock', 'Coffee', 'Mug', 'Tea', 'Bear', 'Waste container', 'Home appliance', 'Candle', 'Lion', 'Mirror', 'Starfish', 'Marine mammal', 'Wheelchair', 'Umbrella', 'Alpaca', 'Violin', 'Cello', 'Brown bear', 'Canary', 'Bat', 'Ruler', 'Plastic bag', 'Penguin', 'Watermelon', 'Harbor seal', 'Pen', 'Pumpkin', 'Harp', 'Kitchen appliance', 'Roller skates', 'Bust', 'Coffee table', 'Tennis ball', 'Tennis racket', 'Ladder', 'Boot', 'Bowl', 'Stop sign', 'Volleyball', 'Eagle', 'Paddle', 'Chicken', 'Skull', 'Lamp', 'Beehive', 'Maple', 'Sink', 'Goldfish', 'Tripod', 'Coconut', 'Bidet', 'Tap', 'Bathroom cabinet', 'Toilet', 'Filing cabinet', 'Pretzel', 'Table tennis racket', 'Bronze sculpture', 'Rocket', 'Mouse', 'Hamster', 'Lizard', 'Lifejacket', 'Goat', 'Washing machine', 'Trumpet', 'Horn', 'Trombone', 'Sheep', 'Tablet computer', 'Pillow', 'Kitchen & dining room table', 'Parachute', 'Raven', 'Glove', 'Loveseat', 'Christmas tree', 'Shellfish', 'Rifle', 'Shotgun', 'Sushi', 'Sparrow', 'Bread', 'Toaster', 'Watch', 'Asparagus', 'Artichoke', 'Suitcase', 'Antelope', 'Broccoli', 'Ice cream', 'Racket', 'Banana', 'Cookie', 'Cucumber', 'Dragonfly', 'Lynx', 'Caterpillar', 'Light bulb', 'Office supplies', 'Miniskirt', 'Skirt', 'Fireplace', 'Potato', 'Light switch', 'Croissant', 'Cabbage', 'Ladybug', 'Handgun', 'Luggage and bags', 'Window blind', 'Snowboard', 'Baseball bat', 'Digital clock', 'Serving tray', 'Infant bed', 'Sofa bed', 'Guacamole', 'Fox', 'Pizza', 'Snowplow', 'Jet ski', 'Refrigerator', 'Lantern', 'Convenience store', 'Sword', 'Rugby ball', 'Owl', 'Ostrich', 'Pancake', 'Strawberry', 'Carrot', 'Tart', 'Dice', 'Turkey', 'Rabbit', 'Invertebrate', 'Vase', 'Stool', 'Swim cap', 'Shower', 'Clock', 'Jellyfish', 'Aircraft', 'Chopsticks', 'Orange', 'Snake', 'Sewing machine', 'Kangaroo', 'Mixer', 'Food processor', 'Shrimp', 'Towel', 'Porcupine', 'Jaguar', 'Cannon', 'Limousine', 'Mule', 'Squirrel', 'Kitchen knife', 'Tiara', 'Tiger', 'Bow and arrow', 'Candy', 'Rhinoceros', 'Shark', 'Cricket ball', 'Doughnut', 'Plumbing fixture', 'Camel', 'Polar bear', 'Coin', 'Printer', 'Blender', 'Giraffe', 'Billiard table', 'Kettle', 'Dinosaur', 'Pineapple', 'Zucchini', 'Jug', 'Barge', 'Teapot', 'Golf ball', 'Binoculars', 'Scissors', 'Hot dog', 'Door handle', 'Seahorse', 'Bathtub', 'Leopard', 'Centipede', 'Grapefruit', 'Snowman', 'Cheetah', 'Alarm clock', 'Grape', 'Wrench', 'Wok', 'Bell pepper', 'Cake stand', 'Barrel', 'Woodpecker', 'Flute', 'Corded phone', 'Willow', 'Punching bag', 'Pomegranate', 'Telephone', 'Pear', 'Common fig', 'Bench', 'Wood-burning stove', 'Burrito', 'Nail', 'Turtle', 'Submarine sandwich', 'Drinking straw', 'Peach', 'Popcorn', 'Frying pan', 'Picnic basket', 'Honeycomb', 'Envelope', 'Mango', 'Cutting board', 'Pitcher', 'Stationary bicycle', 'Dumbbell', 'Personal care', 'Dog bed', 'Snowmobile', 'Oboe', 'Briefcase', 'Squash', 'Tick', 'Slow cooker', 'Coffeemaker', 'Measuring cup', 'Crutch', 'Stretcher', 'Screwdriver', 'Flashlight', 'Spatula', 'Pressure cooker', 'Ring binder', 'Beaker', 'Torch', 'Winter melon' ] def oid_v6_classes() -> list: """Class names of Open Images V6.""" return [ 'Tortoise', 'Container', 'Magpie', 'Sea turtle', 'Football', 'Ambulance', 'Ladder', 'Toothbrush', 'Syringe', 'Sink', 'Toy', 'Organ (Musical Instrument)', 'Cassette deck', 'Apple', 'Human eye', 'Cosmetics', 'Paddle', 'Snowman', 'Beer', 'Chopsticks', 'Human beard', 'Bird', 'Parking meter', 'Traffic light', 'Croissant', 'Cucumber', 'Radish', 'Towel', 'Doll', 'Skull', 'Washing machine', 'Glove', 'Tick', 'Belt', 'Sunglasses', 'Banjo', 'Cart', 'Ball', 'Backpack', 'Bicycle', 'Home appliance', 'Centipede', 'Boat', 'Surfboard', 'Boot', 'Headphones', 'Hot dog', 'Shorts', 'Fast food', 'Bus', 'Boy', 'Screwdriver', 'Bicycle wheel', 'Barge', 'Laptop', 'Miniskirt', 'Drill (Tool)', 'Dress', 'Bear', 'Waffle', 'Pancake', 'Brown bear', 'Woodpecker', 'Blue jay', 'Pretzel', 'Bagel', 'Tower', 'Teapot', 'Person', 'Bow and arrow', 'Swimwear', 'Beehive', 'Brassiere', 'Bee', 'Bat (Animal)', 'Starfish', 'Popcorn', 'Burrito', 'Chainsaw', 'Balloon', 'Wrench', 'Tent', 'Vehicle registration plate', 'Lantern', 'Toaster', 'Flashlight', 'Billboard', 'Tiara', 'Limousine', 'Necklace', 'Carnivore', 'Scissors', 'Stairs', 'Computer keyboard', 'Printer', 'Traffic sign', 'Chair', 'Shirt', 'Poster', 'Cheese', 'Sock', 'Fire hydrant', 'Land vehicle', 'Earrings', 'Tie', 'Watercraft', 'Cabinetry', 'Suitcase', 'Muffin', 'Bidet', 'Snack', 'Snowmobile', 'Clock', 'Medical equipment', 'Cattle', 'Cello', 'Jet ski', 'Camel', 'Coat', 'Suit', 'Desk', 'Cat', 'Bronze sculpture', 'Juice', 'Gondola', 'Beetle', 'Cannon', 'Computer mouse', 'Cookie', 'Office building', 'Fountain', 'Coin', 'Calculator', 'Cocktail', 'Computer monitor', 'Box', 'Stapler', 'Christmas tree', 'Cowboy hat', 'Hiking equipment', 'Studio couch', 'Drum', 'Dessert', 'Wine rack', 'Drink', 'Zucchini', 'Ladle', 'Human mouth', 'Dairy Product', 'Dice', 'Oven', 'Dinosaur', 'Ratchet (Device)', 'Couch', 'Cricket ball', 'Winter melon', 'Spatula', 'Whiteboard', 'Pencil sharpener', 'Door', 'Hat', 'Shower', 'Eraser', 'Fedora', 'Guacamole', 'Dagger', 'Scarf', 'Dolphin', 'Sombrero', 'Tin can', 'Mug', 'Tap', 'Harbor seal', 'Stretcher', 'Can opener', 'Goggles', 'Human body', 'Roller skates', 'Coffee cup', 'Cutting board', 'Blender', 'Plumbing fixture', 'Stop sign', 'Office supplies', 'Volleyball (Ball)', 'Vase', 'Slow cooker', 'Wardrobe', 'Coffee', 'Whisk', 'Paper towel', 'Personal care', 'Food', 'Sun hat', 'Tree house', 'Flying disc', 'Skirt', 'Gas stove', 'Salt and pepper shakers', 'Mechanical fan', 'Face powder', 'Fax', 'Fruit', 'French fries', 'Nightstand', 'Barrel', 'Kite', 'Tart', 'Treadmill', 'Fox', 'Flag', 'French horn', 'Window blind', 'Human foot', 'Golf cart', 'Jacket', 'Egg (Food)', 'Street light', 'Guitar', 'Pillow', 'Human leg', 'Isopod', 'Grape', 'Human ear', 'Power plugs and sockets', 'Panda', 'Giraffe', 'Woman', 'Door handle', 'Rhinoceros', 'Bathtub', 'Goldfish', 'Houseplant', 'Goat', 'Baseball bat', 'Baseball glove', 'Mixing bowl', 'Marine invertebrates', 'Kitchen utensil', 'Light switch', 'House', 'Horse', 'Stationary bicycle', 'Hammer', 'Ceiling fan', 'Sofa bed', 'Adhesive tape', 'Harp', 'Sandal', 'Bicycle helmet', 'Saucer', 'Harpsichord', 'Human hair', 'Heater', 'Harmonica', 'Hamster', 'Curtain', 'Bed', 'Kettle', 'Fireplace', 'Scale', 'Drinking straw', 'Insect', 'Hair dryer', 'Kitchenware', 'Indoor rower', 'Invertebrate', 'Food processor', 'Bookcase', 'Refrigerator', 'Wood-burning stove', 'Punching bag', 'Common fig', 'Cocktail shaker', 'Jaguar (Animal)', 'Golf ball', 'Fashion accessory', 'Alarm clock', 'Filing cabinet', 'Artichoke', 'Table', 'Tableware', 'Kangaroo', 'Koala', 'Knife', 'Bottle', 'Bottle opener', 'Lynx', 'Lavender (Plant)', 'Lighthouse', 'Dumbbell', 'Human head', 'Bowl', 'Humidifier', 'Porch', 'Lizard', 'Billiard table', 'Mammal', 'Mouse', 'Motorcycle', 'Musical instrument', 'Swim cap', 'Frying pan', 'Snowplow', 'Bathroom cabinet', 'Missile', 'Bust', 'Man', 'Waffle iron', 'Milk', 'Ring binder', 'Plate', 'Mobile phone', 'Baked goods', 'Mushroom', 'Crutch', 'Pitcher (Container)', 'Mirror', 'Personal flotation device', 'Table tennis racket', 'Pencil case', 'Musical keyboard', 'Scoreboard', 'Briefcase', 'Kitchen knife', 'Nail (Construction)', 'Tennis ball', 'Plastic bag', 'Oboe', 'Chest of drawers', 'Ostrich', 'Piano', 'Girl', 'Plant', 'Potato', 'Hair spray', 'Sports equipment', 'Pasta', 'Penguin', 'Pumpkin', 'Pear', 'Infant bed', 'Polar bear', 'Mixer', 'Cupboard', 'Jacuzzi', 'Pizza', 'Digital clock', 'Pig', 'Reptile', 'Rifle', 'Lipstick', 'Skateboard', 'Raven', 'High heels', 'Red panda', 'Rose', 'Rabbit', 'Sculpture', 'Saxophone', 'Shotgun', 'Seafood', 'Submarine sandwich', 'Snowboard', 'Sword', 'Picture frame', 'Sushi', 'Loveseat', 'Ski', 'Squirrel', 'Tripod', 'Stethoscope', 'Submarine', 'Scorpion', 'Segway', 'Training bench', 'Snake', 'Coffee table', 'Skyscraper', 'Sheep', 'Television', 'Trombone', 'Tea', 'Tank', 'Taco', 'Telephone', 'Torch', 'Tiger', 'Strawberry', 'Trumpet', 'Tree', 'Tomato', 'Train', 'Tool', 'Picnic basket', 'Cooking spray', 'Trousers', 'Bowling equipment', 'Football helmet', 'Truck', 'Measuring cup', 'Coffeemaker', 'Violin', 'Vehicle', 'Handbag', 'Paper cutter', 'Wine', 'Weapon', 'Wheel', 'Worm', 'Wok', 'Whale', 'Zebra', 'Auto part', 'Jug', 'Pizza cutter', 'Cream', 'Monkey', 'Lion', 'Bread', 'Platter', 'Chicken', 'Eagle', 'Helicopter', 'Owl', 'Duck', 'Turtle', 'Hippopotamus', 'Crocodile', 'Toilet', 'Toilet paper', 'Squid', 'Clothing', 'Footwear', 'Lemon', 'Spider', 'Deer', 'Frog', 'Banana', 'Rocket', 'Wine glass', 'Countertop', 'Tablet computer', 'Waste container', 'Swimming pool', 'Dog', 'Book', 'Elephant', 'Shark', 'Candle', 'Leopard', 'Axe', 'Hand dryer', 'Soap dispenser', 'Porcupine', 'Flower', 'Canary', 'Cheetah', 'Palm tree', 'Hamburger', 'Maple', 'Building', 'Fish', 'Lobster', 'Garden Asparagus', 'Furniture', 'Hedgehog', 'Airplane', 'Spoon', 'Otter', 'Bull', 'Oyster', 'Horizontal bar', 'Convenience store', 'Bomb', 'Bench', 'Ice cream', 'Caterpillar', 'Butterfly', 'Parachute', 'Orange', 'Antelope', 'Beaker', 'Moths and butterflies', 'Window', 'Closet', 'Castle', 'Jellyfish', 'Goose', 'Mule', 'Swan', 'Peach', 'Coconut', 'Seat belt', 'Raccoon', 'Chisel', 'Fork', 'Lamp', 'Camera', 'Squash (Plant)', 'Racket', 'Human face', 'Human arm', 'Vegetable', 'Diaper', 'Unicycle', 'Falcon', 'Chime', 'Snail', 'Shellfish', 'Cabbage', 'Carrot', 'Mango', 'Jeans', 'Flowerpot', 'Pineapple', 'Drawer', 'Stool', 'Envelope', 'Cake', 'Dragonfly', 'Common sunflower', 'Microwave oven', 'Honeycomb', 'Marine mammal', 'Sea lion', 'Ladybug', 'Shelf', 'Watch', 'Candy', 'Salad', 'Parrot', 'Handgun', 'Sparrow', 'Van', 'Grinder', 'Spice rack', 'Light bulb', 'Corded phone', 'Sports uniform', 'Tennis racket', 'Wall clock', 'Serving tray', 'Kitchen & dining room table', 'Dog bed', 'Cake stand', 'Cat furniture', 'Bathroom accessory', 'Facial tissue holder', 'Pressure cooker', 'Kitchen appliance', 'Tire', 'Ruler', 'Luggage and bags', 'Microphone', 'Broccoli', 'Umbrella', 'Pastry', 'Grapefruit', 'Band-aid', 'Animal', 'Bell pepper', 'Turkey', 'Lily', 'Pomegranate', 'Doughnut', 'Glasses', 'Human nose', 'Pen', 'Ant', 'Car', 'Aircraft', 'Human hand', 'Skunk', 'Teddy bear', 'Watermelon', 'Cantaloupe', 'Dishwasher', 'Flute', 'Balance beam', 'Sandwich', 'Shrimp', 'Sewing machine', 'Binoculars', 'Rays and skates', 'Ipod', 'Accordion', 'Willow', 'Crab', 'Crown', 'Seahorse', 'Perfume', 'Alpaca', 'Taxi', 'Canoe', 'Remote control', 'Wheelchair', 'Rugby ball', 'Armadillo', 'Maracas', 'Helmet' ] def objects365v1_classes() -> list: """Class names of Objects365 V1.""" return [ 'person', 'sneakers', 'chair', 'hat', 'lamp', 'bottle', 'cabinet/shelf', 'cup', 'car', 'glasses', 'picture/frame', 'desk', 'handbag', 'street lights', 'book', 'plate', 'helmet', 'leather shoes', 'pillow', 'glove', 'potted plant', 'bracelet', 'flower', 'tv', 'storage box', 'vase', 'bench', 'wine glass', 'boots', 'bowl', 'dining table', 'umbrella', 'boat', 'flag', 'speaker', 'trash bin/can', 'stool', 'backpack', 'couch', 'belt', 'carpet', 'basket', 'towel/napkin', 'slippers', 'barrel/bucket', 'coffee table', 'suv', 'toy', 'tie', 'bed', 'traffic light', 'pen/pencil', 'microphone', 'sandals', 'canned', 'necklace', 'mirror', 'faucet', 'bicycle', 'bread', 'high heels', 'ring', 'van', 'watch', 'sink', 'horse', 'fish', 'apple', 'camera', 'candle', 'teddy bear', 'cake', 'motorcycle', 'wild bird', 'laptop', 'knife', 'traffic sign', 'cell phone', 'paddle', 'truck', 'cow', 'power outlet', 'clock', 'drum', 'fork', 'bus', 'hanger', 'nightstand', 'pot/pan', 'sheep', 'guitar', 'traffic cone', 'tea pot', 'keyboard', 'tripod', 'hockey', 'fan', 'dog', 'spoon', 'blackboard/whiteboard', 'balloon', 'air conditioner', 'cymbal', 'mouse', 'telephone', 'pickup truck', 'orange', 'banana', 'airplane', 'luggage', 'skis', 'soccer', 'trolley', 'oven', 'remote', 'baseball glove', 'paper towel', 'refrigerator', 'train', 'tomato', 'machinery vehicle', 'tent', 'shampoo/shower gel', 'head phone', 'lantern', 'donut', 'cleaning products', 'sailboat', 'tangerine', 'pizza', 'kite', 'computer box', 'elephant', 'toiletries', 'gas stove', 'broccoli', 'toilet', 'stroller', 'shovel', 'baseball bat', 'microwave', 'skateboard', 'surfboard', 'surveillance camera', 'gun', 'life saver', 'cat', 'lemon', 'liquid soap', 'zebra', 'duck', 'sports car', 'giraffe', 'pumpkin', 'piano', 'stop sign', 'radiator', 'converter', 'tissue ', 'carrot', 'washing machine', 'vent', 'cookies', 'cutting/chopping board', 'tennis racket', 'candy', 'skating and skiing shoes', 'scissors', 'folder', 'baseball', 'strawberry', 'bow tie', 'pigeon', 'pepper', 'coffee machine', 'bathtub', 'snowboard', 'suitcase', 'grapes', 'ladder', 'pear', 'american football', 'basketball', 'potato', 'paint brush', 'printer', 'billiards', 'fire hydrant', 'goose', 'projector', 'sausage', 'fire extinguisher', 'extension cord', 'facial mask', 'tennis ball', 'chopsticks', 'electronic stove and gas stove', 'pie', 'frisbee', 'kettle', 'hamburger', 'golf club', 'cucumber', 'clutch', 'blender', 'tong', 'slide', 'hot dog', 'toothbrush', 'facial cleanser', 'mango', 'deer', 'egg', 'violin', 'marker', 'ship', 'chicken', 'onion', 'ice cream', 'tape', 'wheelchair', 'plum', 'bar soap', 'scale', 'watermelon', 'cabbage', 'router/modem', 'golf ball', 'pine apple', 'crane', 'fire truck', 'peach', 'cello', 'notepaper', 'tricycle', 'toaster', 'helicopter', 'green beans', 'brush', 'carriage', 'cigar', 'earphone', 'penguin', 'hurdle', 'swing', 'radio', 'CD', 'parking meter', 'swan', 'garlic', 'french fries', 'horn', 'avocado', 'saxophone', 'trumpet', 'sandwich', 'cue', 'kiwi fruit', 'bear', 'fishing rod', 'cherry', 'tablet', 'green vegetables', 'nuts', 'corn', 'key', 'screwdriver', 'globe', 'broom', 'pliers', 'volleyball', 'hammer', 'eggplant', 'trophy', 'dates', 'board eraser', 'rice', 'tape measure/ruler', 'dumbbell', 'hamimelon', 'stapler', 'camel', 'lettuce', 'goldfish', 'meat balls', 'medal', 'toothpaste', 'antelope', 'shrimp', 'rickshaw', 'trombone', 'pomegranate', 'coconut', 'jellyfish', 'mushroom', 'calculator', 'treadmill', 'butterfly', 'egg tart', 'cheese', 'pig', 'pomelo', 'race car', 'rice cooker', 'tuba', 'crosswalk sign', 'papaya', 'hair drier', 'green onion', 'chips', 'dolphin', 'sushi', 'urinal', 'donkey', 'electric drill', 'spring rolls', 'tortoise/turtle', 'parrot', 'flute', 'measuring cup', 'shark', 'steak', 'poker card', 'binoculars', 'llama', 'radish', 'noodles', 'yak', 'mop', 'crab', 'microscope', 'barbell', 'bread/bun', 'baozi', 'lion', 'red cabbage', 'polar bear', 'lighter', 'seal', 'mangosteen', 'comb', 'eraser', 'pitaya', 'scallop', 'pencil case', 'saw', 'table tennis paddle', 'okra', 'starfish', 'eagle', 'monkey', 'durian', 'game board', 'rabbit', 'french horn', 'ambulance', 'asparagus', 'hoverboard', 'pasta', 'target', 'hotair balloon', 'chainsaw', 'lobster', 'iron', 'flashlight' ] def objects365v2_classes() -> list: """Class names of Objects365 V2.""" return [ 'Person', 'Sneakers', 'Chair', 'Other Shoes', 'Hat', 'Car', 'Lamp', 'Glasses', 'Bottle', 'Desk', 'Cup', 'Street Lights', 'Cabinet/shelf', 'Handbag/Satchel', 'Bracelet', 'Plate', 'Picture/Frame', 'Helmet', 'Book', 'Gloves', 'Storage box', 'Boat', 'Leather Shoes', 'Flower', 'Bench', 'Potted Plant', 'Bowl/Basin', 'Flag', 'Pillow', 'Boots', 'Vase', 'Microphone', 'Necklace', 'Ring', 'SUV', 'Wine Glass', 'Belt', 'Moniter/TV', 'Backpack', 'Umbrella', 'Traffic Light', 'Speaker', 'Watch', 'Tie', 'Trash bin Can', 'Slippers', 'Bicycle', 'Stool', 'Barrel/bucket', 'Van', 'Couch', 'Sandals', 'Bakset', 'Drum', 'Pen/Pencil', 'Bus', 'Wild Bird', 'High Heels', 'Motorcycle', 'Guitar', 'Carpet', 'Cell Phone', 'Bread', 'Camera', 'Canned', 'Truck', 'Traffic cone', 'Cymbal', 'Lifesaver', 'Towel', 'Stuffed Toy', 'Candle', 'Sailboat', 'Laptop', 'Awning', 'Bed', 'Faucet', 'Tent', 'Horse', 'Mirror', 'Power outlet', 'Sink', 'Apple', 'Air Conditioner', 'Knife', 'Hockey Stick', 'Paddle', 'Pickup Truck', 'Fork', 'Traffic Sign', 'Ballon', 'Tripod', 'Dog', 'Spoon', 'Clock', 'Pot', 'Cow', 'Cake', 'Dinning Table', 'Sheep', 'Hanger', 'Blackboard/Whiteboard', 'Napkin', 'Other Fish', 'Orange/Tangerine', 'Toiletry', 'Keyboard', 'Tomato', 'Lantern', 'Machinery Vehicle', 'Fan', 'Green Vegetables', 'Banana', 'Baseball Glove', 'Airplane', 'Mouse', 'Train', 'Pumpkin', 'Soccer', 'Skiboard', 'Luggage', 'Nightstand', 'Tea pot', 'Telephone', 'Trolley', 'Head Phone', 'Sports Car', 'Stop Sign', 'Dessert', 'Scooter', 'Stroller', 'Crane', 'Remote', 'Refrigerator', 'Oven', 'Lemon', 'Duck', 'Baseball Bat', 'Surveillance Camera', 'Cat', 'Jug', 'Broccoli', 'Piano', 'Pizza', 'Elephant', 'Skateboard', 'Surfboard', 'Gun', 'Skating and Skiing shoes', 'Gas stove', 'Donut', 'Bow Tie', 'Carrot', 'Toilet', 'Kite', 'Strawberry', 'Other Balls', 'Shovel', 'Pepper', 'Computer Box', 'Toilet Paper', 'Cleaning Products', 'Chopsticks', 'Microwave', 'Pigeon', 'Baseball', 'Cutting/chopping Board', 'Coffee Table', 'Side Table', 'Scissors', 'Marker', 'Pie', 'Ladder', 'Snowboard', 'Cookies', 'Radiator', 'Fire Hydrant', 'Basketball', 'Zebra', 'Grape', 'Giraffe', 'Potato', 'Sausage', 'Tricycle', 'Violin', 'Egg', 'Fire Extinguisher', 'Candy', 'Fire Truck', 'Billards', 'Converter', 'Bathtub', 'Wheelchair', 'Golf Club', 'Briefcase', 'Cucumber', 'Cigar/Cigarette ', 'Paint Brush', 'Pear', 'Heavy Truck', 'Hamburger', 'Extractor', 'Extention Cord', 'Tong', 'Tennis Racket', 'Folder', 'American Football', 'earphone', 'Mask', 'Kettle', 'Tennis', 'Ship', 'Swing', 'Coffee Machine', 'Slide', 'Carriage', 'Onion', 'Green beans', 'Projector', 'Frisbee', 'Washing Machine/Drying Machine', 'Chicken', 'Printer', 'Watermelon', 'Saxophone', 'Tissue', 'Toothbrush', 'Ice cream', 'Hotair ballon', 'Cello', 'French Fries', 'Scale', 'Trophy', 'Cabbage', 'Hot dog', 'Blender', 'Peach', 'Rice', 'Wallet/Purse', 'Volleyball', 'Deer', 'Goose', 'Tape', 'Tablet', 'Cosmetics', 'Trumpet', 'Pineapple', 'Golf Ball', 'Ambulance', 'Parking meter', 'Mango', 'Key', 'Hurdle', 'Fishing Rod', 'Medal', 'Flute', 'Brush', 'Penguin', 'Megaphone', 'Corn', 'Lettuce', 'Garlic', 'Swan', 'Helicopter', 'Green Onion', 'Sandwich', 'Nuts', 'Speed Limit Sign', 'Induction Cooker', 'Broom', 'Trombone', 'Plum', 'Rickshaw', 'Goldfish', 'Kiwi fruit', 'Router/modem', 'Poker Card', 'Toaster', 'Shrimp', 'Sushi', 'Cheese', 'Notepaper', 'Cherry', 'Pliers', 'CD', 'Pasta', 'Hammer', 'Cue', 'Avocado', 'Hamimelon', 'Flask', 'Mushroon', 'Screwdriver', 'Soap', 'Recorder', 'Bear', 'Eggplant', 'Board Eraser', 'Coconut', 'Tape Measur/ Ruler', 'Pig', 'Showerhead', 'Globe', 'Chips', 'Steak', 'Crosswalk Sign', 'Stapler', 'Campel', 'Formula 1 ', 'Pomegranate', 'Dishwasher', 'Crab', 'Hoverboard', 'Meat ball', 'Rice Cooker', 'Tuba', 'Calculator', 'Papaya', 'Antelope', 'Parrot', 'Seal', 'Buttefly', 'Dumbbell', 'Donkey', 'Lion', 'Urinal', 'Dolphin', 'Electric Drill', 'Hair Dryer', 'Egg tart', 'Jellyfish', 'Treadmill', 'Lighter', 'Grapefruit', 'Game board', 'Mop', 'Radish', 'Baozi', 'Target', 'French', 'Spring Rolls', 'Monkey', 'Rabbit', 'Pencil Case', 'Yak', 'Red Cabbage', 'Binoculars', 'Asparagus', 'Barbell', 'Scallop', 'Noddles', 'Comb', 'Dumpling', 'Oyster', 'Table Teniis paddle', 'Cosmetics Brush/Eyeliner Pencil', 'Chainsaw', 'Eraser', 'Lobster', 'Durian', 'Okra', 'Lipstick', 'Cosmetics Mirror', 'Curling', 'Table Tennis ' ] dataset_aliases = { 'voc': ['voc', 'pascal_voc', 'voc07', 'voc12'], 'imagenet_det': ['det', 'imagenet_det', 'ilsvrc_det'], 'imagenet_vid': ['vid', 'imagenet_vid', 'ilsvrc_vid'], 'coco': ['coco', 'mscoco', 'ms_coco'], 'coco_panoptic': ['coco_panoptic', 'panoptic'], 'wider_face': ['WIDERFaceDataset', 'wider_face', 'WIDERFace'], 'cityscapes': ['cityscapes'], 'oid_challenge': ['oid_challenge', 'openimages_challenge'], 'oid_v6': ['oid_v6', 'openimages_v6'], 'objects365v1': ['objects365v1', 'obj365v1'], 'objects365v2': ['objects365v2', 'obj365v2'] } def get_classes(dataset) -> list: """Get class names of a dataset.""" alias2name = {} for name, aliases in dataset_aliases.items(): for alias in aliases: alias2name[alias] = name if is_str(dataset): if dataset in alias2name: labels = eval(alias2name[dataset] + '_classes()') else: raise ValueError(f'Unrecognized dataset: {dataset}') else: raise TypeError(f'dataset must a str, but got {type(dataset)}') return labels ================================================ FILE: mmdet/evaluation/functional/mean_ap.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from multiprocessing import Pool import numpy as np from mmengine.logging import print_log from mmengine.utils import is_str from terminaltables import AsciiTable from .bbox_overlaps import bbox_overlaps from .class_names import get_classes def average_precision(recalls, precisions, mode='area'): """Calculate average precision (for single or multiple scales). Args: recalls (ndarray): shape (num_scales, num_dets) or (num_dets, ) precisions (ndarray): shape (num_scales, num_dets) or (num_dets, ) mode (str): 'area' or '11points', 'area' means calculating the area under precision-recall curve, '11points' means calculating the average precision of recalls at [0, 0.1, ..., 1] Returns: float or ndarray: calculated average precision """ no_scale = False if recalls.ndim == 1: no_scale = True recalls = recalls[np.newaxis, :] precisions = precisions[np.newaxis, :] assert recalls.shape == precisions.shape and recalls.ndim == 2 num_scales = recalls.shape[0] ap = np.zeros(num_scales, dtype=np.float32) if mode == 'area': zeros = np.zeros((num_scales, 1), dtype=recalls.dtype) ones = np.ones((num_scales, 1), dtype=recalls.dtype) mrec = np.hstack((zeros, recalls, ones)) mpre = np.hstack((zeros, precisions, zeros)) for i in range(mpre.shape[1] - 1, 0, -1): mpre[:, i - 1] = np.maximum(mpre[:, i - 1], mpre[:, i]) for i in range(num_scales): ind = np.where(mrec[i, 1:] != mrec[i, :-1])[0] ap[i] = np.sum( (mrec[i, ind + 1] - mrec[i, ind]) * mpre[i, ind + 1]) elif mode == '11points': for i in range(num_scales): for thr in np.arange(0, 1 + 1e-3, 0.1): precs = precisions[i, recalls[i, :] >= thr] prec = precs.max() if precs.size > 0 else 0 ap[i] += prec ap /= 11 else: raise ValueError( 'Unrecognized mode, only "area" and "11points" are supported') if no_scale: ap = ap[0] return ap def tpfp_imagenet(det_bboxes, gt_bboxes, gt_bboxes_ignore=None, default_iou_thr=0.5, area_ranges=None, use_legacy_coordinate=False, **kwargs): """Check if detected bboxes are true positive or false positive. Args: det_bbox (ndarray): Detected bboxes of this image, of shape (m, 5). gt_bboxes (ndarray): GT bboxes of this image, of shape (n, 4). gt_bboxes_ignore (ndarray): Ignored gt bboxes of this image, of shape (k, 4). Defaults to None default_iou_thr (float): IoU threshold to be considered as matched for medium and large bboxes (small ones have special rules). Defaults to 0.5. area_ranges (list[tuple] | None): Range of bbox areas to be evaluated, in the format [(min1, max1), (min2, max2), ...]. Defaults to None. use_legacy_coordinate (bool): Whether to use coordinate system in mmdet v1.x. which means width, height should be calculated as 'x2 - x1 + 1` and 'y2 - y1 + 1' respectively. Defaults to False. Returns: tuple[np.ndarray]: (tp, fp) whose elements are 0 and 1. The shape of each array is (num_scales, m). """ if not use_legacy_coordinate: extra_length = 0. else: extra_length = 1. # an indicator of ignored gts gt_ignore_inds = np.concatenate( (np.zeros(gt_bboxes.shape[0], dtype=bool), np.ones(gt_bboxes_ignore.shape[0], dtype=bool))) # stack gt_bboxes and gt_bboxes_ignore for convenience gt_bboxes = np.vstack((gt_bboxes, gt_bboxes_ignore)) num_dets = det_bboxes.shape[0] num_gts = gt_bboxes.shape[0] if area_ranges is None: area_ranges = [(None, None)] num_scales = len(area_ranges) # tp and fp are of shape (num_scales, num_gts), each row is tp or fp # of a certain scale. tp = np.zeros((num_scales, num_dets), dtype=np.float32) fp = np.zeros((num_scales, num_dets), dtype=np.float32) if gt_bboxes.shape[0] == 0: if area_ranges == [(None, None)]: fp[...] = 1 else: det_areas = ( det_bboxes[:, 2] - det_bboxes[:, 0] + extra_length) * ( det_bboxes[:, 3] - det_bboxes[:, 1] + extra_length) for i, (min_area, max_area) in enumerate(area_ranges): fp[i, (det_areas >= min_area) & (det_areas < max_area)] = 1 return tp, fp ious = bbox_overlaps( det_bboxes, gt_bboxes - 1, use_legacy_coordinate=use_legacy_coordinate) gt_w = gt_bboxes[:, 2] - gt_bboxes[:, 0] + extra_length gt_h = gt_bboxes[:, 3] - gt_bboxes[:, 1] + extra_length iou_thrs = np.minimum((gt_w * gt_h) / ((gt_w + 10.0) * (gt_h + 10.0)), default_iou_thr) # sort all detections by scores in descending order sort_inds = np.argsort(-det_bboxes[:, -1]) for k, (min_area, max_area) in enumerate(area_ranges): gt_covered = np.zeros(num_gts, dtype=bool) # if no area range is specified, gt_area_ignore is all False if min_area is None: gt_area_ignore = np.zeros_like(gt_ignore_inds, dtype=bool) else: gt_areas = gt_w * gt_h gt_area_ignore = (gt_areas < min_area) | (gt_areas >= max_area) for i in sort_inds: max_iou = -1 matched_gt = -1 # find best overlapped available gt for j in range(num_gts): # different from PASCAL VOC: allow finding other gts if the # best overlapped ones are already matched by other det bboxes if gt_covered[j]: continue elif ious[i, j] >= iou_thrs[j] and ious[i, j] > max_iou: max_iou = ious[i, j] matched_gt = j # there are 4 cases for a det bbox: # 1. it matches a gt, tp = 1, fp = 0 # 2. it matches an ignored gt, tp = 0, fp = 0 # 3. it matches no gt and within area range, tp = 0, fp = 1 # 4. it matches no gt but is beyond area range, tp = 0, fp = 0 if matched_gt >= 0: gt_covered[matched_gt] = 1 if not (gt_ignore_inds[matched_gt] or gt_area_ignore[matched_gt]): tp[k, i] = 1 elif min_area is None: fp[k, i] = 1 else: bbox = det_bboxes[i, :4] area = (bbox[2] - bbox[0] + extra_length) * ( bbox[3] - bbox[1] + extra_length) if area >= min_area and area < max_area: fp[k, i] = 1 return tp, fp def tpfp_default(det_bboxes, gt_bboxes, gt_bboxes_ignore=None, iou_thr=0.5, area_ranges=None, use_legacy_coordinate=False, **kwargs): """Check if detected bboxes are true positive or false positive. Args: det_bbox (ndarray): Detected bboxes of this image, of shape (m, 5). gt_bboxes (ndarray): GT bboxes of this image, of shape (n, 4). gt_bboxes_ignore (ndarray): Ignored gt bboxes of this image, of shape (k, 4). Defaults to None iou_thr (float): IoU threshold to be considered as matched. Defaults to 0.5. area_ranges (list[tuple] | None): Range of bbox areas to be evaluated, in the format [(min1, max1), (min2, max2), ...]. Defaults to None. use_legacy_coordinate (bool): Whether to use coordinate system in mmdet v1.x. which means width, height should be calculated as 'x2 - x1 + 1` and 'y2 - y1 + 1' respectively. Defaults to False. Returns: tuple[np.ndarray]: (tp, fp) whose elements are 0 and 1. The shape of each array is (num_scales, m). """ if not use_legacy_coordinate: extra_length = 0. else: extra_length = 1. # an indicator of ignored gts gt_ignore_inds = np.concatenate( (np.zeros(gt_bboxes.shape[0], dtype=bool), np.ones(gt_bboxes_ignore.shape[0], dtype=bool))) # stack gt_bboxes and gt_bboxes_ignore for convenience gt_bboxes = np.vstack((gt_bboxes, gt_bboxes_ignore)) num_dets = det_bboxes.shape[0] num_gts = gt_bboxes.shape[0] if area_ranges is None: area_ranges = [(None, None)] num_scales = len(area_ranges) # tp and fp are of shape (num_scales, num_gts), each row is tp or fp of # a certain scale tp = np.zeros((num_scales, num_dets), dtype=np.float32) fp = np.zeros((num_scales, num_dets), dtype=np.float32) # if there is no gt bboxes in this image, then all det bboxes # within area range are false positives if gt_bboxes.shape[0] == 0: if area_ranges == [(None, None)]: fp[...] = 1 else: det_areas = ( det_bboxes[:, 2] - det_bboxes[:, 0] + extra_length) * ( det_bboxes[:, 3] - det_bboxes[:, 1] + extra_length) for i, (min_area, max_area) in enumerate(area_ranges): fp[i, (det_areas >= min_area) & (det_areas < max_area)] = 1 return tp, fp ious = bbox_overlaps( det_bboxes, gt_bboxes, use_legacy_coordinate=use_legacy_coordinate) # for each det, the max iou with all gts ious_max = ious.max(axis=1) # for each det, which gt overlaps most with it ious_argmax = ious.argmax(axis=1) # sort all dets in descending order by scores sort_inds = np.argsort(-det_bboxes[:, -1]) for k, (min_area, max_area) in enumerate(area_ranges): gt_covered = np.zeros(num_gts, dtype=bool) # if no area range is specified, gt_area_ignore is all False if min_area is None: gt_area_ignore = np.zeros_like(gt_ignore_inds, dtype=bool) else: gt_areas = (gt_bboxes[:, 2] - gt_bboxes[:, 0] + extra_length) * ( gt_bboxes[:, 3] - gt_bboxes[:, 1] + extra_length) gt_area_ignore = (gt_areas < min_area) | (gt_areas >= max_area) for i in sort_inds: if ious_max[i] >= iou_thr: matched_gt = ious_argmax[i] if not (gt_ignore_inds[matched_gt] or gt_area_ignore[matched_gt]): if not gt_covered[matched_gt]: gt_covered[matched_gt] = True tp[k, i] = 1 else: fp[k, i] = 1 # otherwise ignore this detected bbox, tp = 0, fp = 0 elif min_area is None: fp[k, i] = 1 else: bbox = det_bboxes[i, :4] area = (bbox[2] - bbox[0] + extra_length) * ( bbox[3] - bbox[1] + extra_length) if area >= min_area and area < max_area: fp[k, i] = 1 return tp, fp def tpfp_openimages(det_bboxes, gt_bboxes, gt_bboxes_ignore=None, iou_thr=0.5, area_ranges=None, use_legacy_coordinate=False, gt_bboxes_group_of=None, use_group_of=True, ioa_thr=0.5, **kwargs): """Check if detected bboxes are true positive or false positive. Args: det_bbox (ndarray): Detected bboxes of this image, of shape (m, 5). gt_bboxes (ndarray): GT bboxes of this image, of shape (n, 4). gt_bboxes_ignore (ndarray): Ignored gt bboxes of this image, of shape (k, 4). Defaults to None iou_thr (float): IoU threshold to be considered as matched. Defaults to 0.5. area_ranges (list[tuple] | None): Range of bbox areas to be evaluated, in the format [(min1, max1), (min2, max2), ...]. Defaults to None. use_legacy_coordinate (bool): Whether to use coordinate system in mmdet v1.x. which means width, height should be calculated as 'x2 - x1 + 1` and 'y2 - y1 + 1' respectively. Defaults to False. gt_bboxes_group_of (ndarray): GT group_of of this image, of shape (k, 1). Defaults to None use_group_of (bool): Whether to use group of when calculate TP and FP, which only used in OpenImages evaluation. Defaults to True. ioa_thr (float | None): IoA threshold to be considered as matched, which only used in OpenImages evaluation. Defaults to 0.5. Returns: tuple[np.ndarray]: Returns a tuple (tp, fp, det_bboxes), where (tp, fp) whose elements are 0 and 1. The shape of each array is (num_scales, m). (det_bboxes) whose will filter those are not matched by group of gts when processing Open Images evaluation. The shape is (num_scales, m). """ if not use_legacy_coordinate: extra_length = 0. else: extra_length = 1. # an indicator of ignored gts gt_ignore_inds = np.concatenate( (np.zeros(gt_bboxes.shape[0], dtype=bool), np.ones(gt_bboxes_ignore.shape[0], dtype=bool))) # stack gt_bboxes and gt_bboxes_ignore for convenience gt_bboxes = np.vstack((gt_bboxes, gt_bboxes_ignore)) num_dets = det_bboxes.shape[0] num_gts = gt_bboxes.shape[0] if area_ranges is None: area_ranges = [(None, None)] num_scales = len(area_ranges) # tp and fp are of shape (num_scales, num_gts), each row is tp or fp of # a certain scale tp = np.zeros((num_scales, num_dets), dtype=np.float32) fp = np.zeros((num_scales, num_dets), dtype=np.float32) # if there is no gt bboxes in this image, then all det bboxes # within area range are false positives if gt_bboxes.shape[0] == 0: if area_ranges == [(None, None)]: fp[...] = 1 else: det_areas = ( det_bboxes[:, 2] - det_bboxes[:, 0] + extra_length) * ( det_bboxes[:, 3] - det_bboxes[:, 1] + extra_length) for i, (min_area, max_area) in enumerate(area_ranges): fp[i, (det_areas >= min_area) & (det_areas < max_area)] = 1 return tp, fp, det_bboxes if gt_bboxes_group_of is not None and use_group_of: # if handle group-of boxes, divided gt boxes into two parts: # non-group-of and group-of.Then calculate ious and ioas through # non-group-of group-of gts respectively. This only used in # OpenImages evaluation. assert gt_bboxes_group_of.shape[0] == gt_bboxes.shape[0] non_group_gt_bboxes = gt_bboxes[~gt_bboxes_group_of] group_gt_bboxes = gt_bboxes[gt_bboxes_group_of] num_gts_group = group_gt_bboxes.shape[0] ious = bbox_overlaps(det_bboxes, non_group_gt_bboxes) ioas = bbox_overlaps(det_bboxes, group_gt_bboxes, mode='iof') else: # if not consider group-of boxes, only calculate ious through gt boxes ious = bbox_overlaps( det_bboxes, gt_bboxes, use_legacy_coordinate=use_legacy_coordinate) ioas = None if ious.shape[1] > 0: # for each det, the max iou with all gts ious_max = ious.max(axis=1) # for each det, which gt overlaps most with it ious_argmax = ious.argmax(axis=1) # sort all dets in descending order by scores sort_inds = np.argsort(-det_bboxes[:, -1]) for k, (min_area, max_area) in enumerate(area_ranges): gt_covered = np.zeros(num_gts, dtype=bool) # if no area range is specified, gt_area_ignore is all False if min_area is None: gt_area_ignore = np.zeros_like(gt_ignore_inds, dtype=bool) else: gt_areas = ( gt_bboxes[:, 2] - gt_bboxes[:, 0] + extra_length) * ( gt_bboxes[:, 3] - gt_bboxes[:, 1] + extra_length) gt_area_ignore = (gt_areas < min_area) | (gt_areas >= max_area) for i in sort_inds: if ious_max[i] >= iou_thr: matched_gt = ious_argmax[i] if not (gt_ignore_inds[matched_gt] or gt_area_ignore[matched_gt]): if not gt_covered[matched_gt]: gt_covered[matched_gt] = True tp[k, i] = 1 else: fp[k, i] = 1 # otherwise ignore this detected bbox, tp = 0, fp = 0 elif min_area is None: fp[k, i] = 1 else: bbox = det_bboxes[i, :4] area = (bbox[2] - bbox[0] + extra_length) * ( bbox[3] - bbox[1] + extra_length) if area >= min_area and area < max_area: fp[k, i] = 1 else: # if there is no no-group-of gt bboxes in this image, # then all det bboxes within area range are false positives. # Only used in OpenImages evaluation. if area_ranges == [(None, None)]: fp[...] = 1 else: det_areas = ( det_bboxes[:, 2] - det_bboxes[:, 0] + extra_length) * ( det_bboxes[:, 3] - det_bboxes[:, 1] + extra_length) for i, (min_area, max_area) in enumerate(area_ranges): fp[i, (det_areas >= min_area) & (det_areas < max_area)] = 1 if ioas is None or ioas.shape[1] <= 0: return tp, fp, det_bboxes else: # The evaluation of group-of TP and FP are done in two stages: # 1. All detections are first matched to non group-of boxes; true # positives are determined. # 2. Detections that are determined as false positives are matched # against group-of boxes and calculated group-of TP and FP. # Only used in OpenImages evaluation. det_bboxes_group = np.zeros( (num_scales, ioas.shape[1], det_bboxes.shape[1]), dtype=float) match_group_of = np.zeros((num_scales, num_dets), dtype=bool) tp_group = np.zeros((num_scales, num_gts_group), dtype=np.float32) ioas_max = ioas.max(axis=1) # for each det, which gt overlaps most with it ioas_argmax = ioas.argmax(axis=1) # sort all dets in descending order by scores sort_inds = np.argsort(-det_bboxes[:, -1]) for k, (min_area, max_area) in enumerate(area_ranges): box_is_covered = tp[k] # if no area range is specified, gt_area_ignore is all False if min_area is None: gt_area_ignore = np.zeros_like(gt_ignore_inds, dtype=bool) else: gt_areas = (gt_bboxes[:, 2] - gt_bboxes[:, 0]) * ( gt_bboxes[:, 3] - gt_bboxes[:, 1]) gt_area_ignore = (gt_areas < min_area) | (gt_areas >= max_area) for i in sort_inds: matched_gt = ioas_argmax[i] if not box_is_covered[i]: if ioas_max[i] >= ioa_thr: if not (gt_ignore_inds[matched_gt] or gt_area_ignore[matched_gt]): if not tp_group[k, matched_gt]: tp_group[k, matched_gt] = 1 match_group_of[k, i] = True else: match_group_of[k, i] = True if det_bboxes_group[k, matched_gt, -1] < \ det_bboxes[i, -1]: det_bboxes_group[k, matched_gt] = \ det_bboxes[i] fp_group = (tp_group <= 0).astype(float) tps = [] fps = [] # concatenate tp, fp, and det-boxes which not matched group of # gt boxes and tp_group, fp_group, and det_bboxes_group which # matched group of boxes respectively. for i in range(num_scales): tps.append( np.concatenate((tp[i][~match_group_of[i]], tp_group[i]))) fps.append( np.concatenate((fp[i][~match_group_of[i]], fp_group[i]))) det_bboxes = np.concatenate( (det_bboxes[~match_group_of[i]], det_bboxes_group[i])) tp = np.vstack(tps) fp = np.vstack(fps) return tp, fp, det_bboxes def get_cls_results(det_results, annotations, class_id): """Get det results and gt information of a certain class. Args: det_results (list[list]): Same as `eval_map()`. annotations (list[dict]): Same as `eval_map()`. class_id (int): ID of a specific class. Returns: tuple[list[np.ndarray]]: detected bboxes, gt bboxes, ignored gt bboxes """ cls_dets = [img_res[class_id] for img_res in det_results] cls_gts = [] cls_gts_ignore = [] for ann in annotations: gt_inds = ann['labels'] == class_id cls_gts.append(ann['bboxes'][gt_inds, :]) if ann.get('labels_ignore', None) is not None: ignore_inds = ann['labels_ignore'] == class_id cls_gts_ignore.append(ann['bboxes_ignore'][ignore_inds, :]) else: cls_gts_ignore.append(np.empty((0, 4), dtype=np.float32)) return cls_dets, cls_gts, cls_gts_ignore def get_cls_group_ofs(annotations, class_id): """Get `gt_group_of` of a certain class, which is used in Open Images. Args: annotations (list[dict]): Same as `eval_map()`. class_id (int): ID of a specific class. Returns: list[np.ndarray]: `gt_group_of` of a certain class. """ gt_group_ofs = [] for ann in annotations: gt_inds = ann['labels'] == class_id if ann.get('gt_is_group_ofs', None) is not None: gt_group_ofs.append(ann['gt_is_group_ofs'][gt_inds]) else: gt_group_ofs.append(np.empty((0, 1), dtype=bool)) return gt_group_ofs def eval_map(det_results, annotations, scale_ranges=None, iou_thr=0.5, ioa_thr=None, dataset=None, logger=None, tpfp_fn=None, nproc=4, use_legacy_coordinate=False, use_group_of=False, eval_mode='area'): """Evaluate mAP of a dataset. Args: det_results (list[list]): [[cls1_det, cls2_det, ...], ...]. The outer list indicates images, and the inner list indicates per-class detected bboxes. annotations (list[dict]): Ground truth annotations where each item of the list indicates an image. Keys of annotations are: - `bboxes`: numpy array of shape (n, 4) - `labels`: numpy array of shape (n, ) - `bboxes_ignore` (optional): numpy array of shape (k, 4) - `labels_ignore` (optional): numpy array of shape (k, ) scale_ranges (list[tuple] | None): Range of scales to be evaluated, in the format [(min1, max1), (min2, max2), ...]. A range of (32, 64) means the area range between (32**2, 64**2). Defaults to None. iou_thr (float): IoU threshold to be considered as matched. Defaults to 0.5. ioa_thr (float | None): IoA threshold to be considered as matched, which only used in OpenImages evaluation. Defaults to None. dataset (list[str] | str | None): Dataset name or dataset classes, there are minor differences in metrics for different datasets, e.g. "voc", "imagenet_det", etc. Defaults to None. logger (logging.Logger | str | None): The way to print the mAP summary. See `mmengine.logging.print_log()` for details. Defaults to None. tpfp_fn (callable | None): The function used to determine true/ false positives. If None, :func:`tpfp_default` is used as default unless dataset is 'det' or 'vid' (:func:`tpfp_imagenet` in this case). If it is given as a function, then this function is used to evaluate tp & fp. Default None. nproc (int): Processes used for computing TP and FP. Defaults to 4. use_legacy_coordinate (bool): Whether to use coordinate system in mmdet v1.x. which means width, height should be calculated as 'x2 - x1 + 1` and 'y2 - y1 + 1' respectively. Defaults to False. use_group_of (bool): Whether to use group of when calculate TP and FP, which only used in OpenImages evaluation. Defaults to False. eval_mode (str): 'area' or '11points', 'area' means calculating the area under precision-recall curve, '11points' means calculating the average precision of recalls at [0, 0.1, ..., 1], PASCAL VOC2007 uses `11points` as default evaluate mode, while others are 'area'. Defaults to 'area'. Returns: tuple: (mAP, [dict, dict, ...]) """ assert len(det_results) == len(annotations) assert eval_mode in ['area', '11points'], \ f'Unrecognized {eval_mode} mode, only "area" and "11points" ' \ 'are supported' if not use_legacy_coordinate: extra_length = 0. else: extra_length = 1. num_imgs = len(det_results) num_scales = len(scale_ranges) if scale_ranges is not None else 1 num_classes = len(det_results[0]) # positive class num area_ranges = ([(rg[0]**2, rg[1]**2) for rg in scale_ranges] if scale_ranges is not None else None) # There is no need to use multi processes to process # when num_imgs = 1 . if num_imgs > 1: assert nproc > 0, 'nproc must be at least one.' nproc = min(nproc, num_imgs) pool = Pool(nproc) eval_results = [] for i in range(num_classes): # get gt and det bboxes of this class cls_dets, cls_gts, cls_gts_ignore = get_cls_results( det_results, annotations, i) # choose proper function according to datasets to compute tp and fp if tpfp_fn is None: if dataset in ['det', 'vid']: tpfp_fn = tpfp_imagenet elif dataset in ['oid_challenge', 'oid_v6'] \ or use_group_of is True: tpfp_fn = tpfp_openimages else: tpfp_fn = tpfp_default if not callable(tpfp_fn): raise ValueError( f'tpfp_fn has to be a function or None, but got {tpfp_fn}') if num_imgs > 1: # compute tp and fp for each image with multiple processes args = [] if use_group_of: # used in Open Images Dataset evaluation gt_group_ofs = get_cls_group_ofs(annotations, i) args.append(gt_group_ofs) args.append([use_group_of for _ in range(num_imgs)]) if ioa_thr is not None: args.append([ioa_thr for _ in range(num_imgs)]) tpfp = pool.starmap( tpfp_fn, zip(cls_dets, cls_gts, cls_gts_ignore, [iou_thr for _ in range(num_imgs)], [area_ranges for _ in range(num_imgs)], [use_legacy_coordinate for _ in range(num_imgs)], *args)) else: tpfp = tpfp_fn( cls_dets[0], cls_gts[0], cls_gts_ignore[0], iou_thr, area_ranges, use_legacy_coordinate, gt_bboxes_group_of=(get_cls_group_ofs(annotations, i)[0] if use_group_of else None), use_group_of=use_group_of, ioa_thr=ioa_thr) tpfp = [tpfp] if use_group_of: tp, fp, cls_dets = tuple(zip(*tpfp)) else: tp, fp = tuple(zip(*tpfp)) # calculate gt number of each scale # ignored gts or gts beyond the specific scale are not counted num_gts = np.zeros(num_scales, dtype=int) for j, bbox in enumerate(cls_gts): if area_ranges is None: num_gts[0] += bbox.shape[0] else: gt_areas = (bbox[:, 2] - bbox[:, 0] + extra_length) * ( bbox[:, 3] - bbox[:, 1] + extra_length) for k, (min_area, max_area) in enumerate(area_ranges): num_gts[k] += np.sum((gt_areas >= min_area) & (gt_areas < max_area)) # sort all det bboxes by score, also sort tp and fp cls_dets = np.vstack(cls_dets) num_dets = cls_dets.shape[0] sort_inds = np.argsort(-cls_dets[:, -1]) tp = np.hstack(tp)[:, sort_inds] fp = np.hstack(fp)[:, sort_inds] # calculate recall and precision with tp and fp tp = np.cumsum(tp, axis=1) fp = np.cumsum(fp, axis=1) eps = np.finfo(np.float32).eps recalls = tp / np.maximum(num_gts[:, np.newaxis], eps) precisions = tp / np.maximum((tp + fp), eps) # calculate AP if scale_ranges is None: recalls = recalls[0, :] precisions = precisions[0, :] num_gts = num_gts.item() ap = average_precision(recalls, precisions, eval_mode) eval_results.append({ 'num_gts': num_gts, 'num_dets': num_dets, 'recall': recalls, 'precision': precisions, 'ap': ap }) if num_imgs > 1: pool.close() if scale_ranges is not None: # shape (num_classes, num_scales) all_ap = np.vstack([cls_result['ap'] for cls_result in eval_results]) all_num_gts = np.vstack( [cls_result['num_gts'] for cls_result in eval_results]) mean_ap = [] for i in range(num_scales): if np.any(all_num_gts[:, i] > 0): mean_ap.append(all_ap[all_num_gts[:, i] > 0, i].mean()) else: mean_ap.append(0.0) else: aps = [] for cls_result in eval_results: if cls_result['num_gts'] > 0: aps.append(cls_result['ap']) mean_ap = np.array(aps).mean().item() if aps else 0.0 print_map_summary( mean_ap, eval_results, dataset, area_ranges, logger=logger) return mean_ap, eval_results def print_map_summary(mean_ap, results, dataset=None, scale_ranges=None, logger=None): """Print mAP and results of each class. A table will be printed to show the gts/dets/recall/AP of each class and the mAP. Args: mean_ap (float): Calculated from `eval_map()`. results (list[dict]): Calculated from `eval_map()`. dataset (list[str] | str | None): Dataset name or dataset classes. scale_ranges (list[tuple] | None): Range of scales to be evaluated. logger (logging.Logger | str | None): The way to print the mAP summary. See `mmengine.logging.print_log()` for details. Defaults to None. """ if logger == 'silent': return if isinstance(results[0]['ap'], np.ndarray): num_scales = len(results[0]['ap']) else: num_scales = 1 if scale_ranges is not None: assert len(scale_ranges) == num_scales num_classes = len(results) recalls = np.zeros((num_scales, num_classes), dtype=np.float32) aps = np.zeros((num_scales, num_classes), dtype=np.float32) num_gts = np.zeros((num_scales, num_classes), dtype=int) for i, cls_result in enumerate(results): if cls_result['recall'].size > 0: recalls[:, i] = np.array(cls_result['recall'], ndmin=2)[:, -1] aps[:, i] = cls_result['ap'] num_gts[:, i] = cls_result['num_gts'] if dataset is None: label_names = [str(i) for i in range(num_classes)] elif is_str(dataset): label_names = get_classes(dataset) else: label_names = dataset if not isinstance(mean_ap, list): mean_ap = [mean_ap] header = ['class', 'gts', 'dets', 'recall', 'ap'] for i in range(num_scales): if scale_ranges is not None: print_log(f'Scale range {scale_ranges[i]}', logger=logger) table_data = [header] for j in range(num_classes): row_data = [ label_names[j], num_gts[i, j], results[j]['num_dets'], f'{recalls[i, j]:.3f}', f'{aps[i, j]:.3f}' ] table_data.append(row_data) table_data.append(['mAP', '', '', '', f'{mean_ap[i]:.3f}']) table = AsciiTable(table_data) table.inner_footing_row_border = True print_log('\n' + table.table, logger=logger) ================================================ FILE: mmdet/evaluation/functional/panoptic_utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Copyright (c) 2018, Alexander Kirillov # This file supports `file_client` for `panopticapi`, # the source code is copied from `panopticapi`, # only the way to load the gt images is modified. import multiprocessing import os import mmcv import numpy as np from mmengine.fileio import FileClient # A custom value to distinguish instance ID and category ID; need to # be greater than the number of categories. # For a pixel in the panoptic result map: # pan_id = ins_id * INSTANCE_OFFSET + cat_id INSTANCE_OFFSET = 1000 try: from panopticapi.evaluation import OFFSET, VOID, PQStat from panopticapi.utils import rgb2id except ImportError: PQStat = None rgb2id = None VOID = 0 OFFSET = 256 * 256 * 256 def pq_compute_single_core(proc_id, annotation_set, gt_folder, pred_folder, categories, file_client=None, print_log=False): """The single core function to evaluate the metric of Panoptic Segmentation. Same as the function with the same name in `panopticapi`. Only the function to load the images is changed to use the file client. Args: proc_id (int): The id of the mini process. gt_folder (str): The path of the ground truth images. pred_folder (str): The path of the prediction images. categories (str): The categories of the dataset. file_client (object): The file client of the dataset. If None, the backend will be set to `disk`. print_log (bool): Whether to print the log. Defaults to False. """ if PQStat is None: raise RuntimeError( 'panopticapi is not installed, please install it by: ' 'pip install git+https://github.com/cocodataset/' 'panopticapi.git.') if file_client is None: file_client_args = dict(backend='disk') file_client = FileClient(**file_client_args) pq_stat = PQStat() idx = 0 for gt_ann, pred_ann in annotation_set: if print_log and idx % 100 == 0: print('Core: {}, {} from {} images processed'.format( proc_id, idx, len(annotation_set))) idx += 1 # The gt images can be on the local disk or `ceph`, so we use # file_client here. img_bytes = file_client.get( os.path.join(gt_folder, gt_ann['file_name'])) pan_gt = mmcv.imfrombytes(img_bytes, flag='color', channel_order='rgb') pan_gt = rgb2id(pan_gt) # The predictions can only be on the local dist now. pan_pred = mmcv.imread( os.path.join(pred_folder, pred_ann['file_name']), flag='color', channel_order='rgb') pan_pred = rgb2id(pan_pred) gt_segms = {el['id']: el for el in gt_ann['segments_info']} pred_segms = {el['id']: el for el in pred_ann['segments_info']} # predicted segments area calculation + prediction sanity checks pred_labels_set = set(el['id'] for el in pred_ann['segments_info']) labels, labels_cnt = np.unique(pan_pred, return_counts=True) for label, label_cnt in zip(labels, labels_cnt): if label not in pred_segms: if label == VOID: continue raise KeyError( 'In the image with ID {} segment with ID {} is ' 'presented in PNG and not presented in JSON.'.format( gt_ann['image_id'], label)) pred_segms[label]['area'] = label_cnt pred_labels_set.remove(label) if pred_segms[label]['category_id'] not in categories: raise KeyError( 'In the image with ID {} segment with ID {} has ' 'unknown category_id {}.'.format( gt_ann['image_id'], label, pred_segms[label]['category_id'])) if len(pred_labels_set) != 0: raise KeyError( 'In the image with ID {} the following segment IDs {} ' 'are presented in JSON and not presented in PNG.'.format( gt_ann['image_id'], list(pred_labels_set))) # confusion matrix calculation pan_gt_pred = pan_gt.astype(np.uint64) * OFFSET + pan_pred.astype( np.uint64) gt_pred_map = {} labels, labels_cnt = np.unique(pan_gt_pred, return_counts=True) for label, intersection in zip(labels, labels_cnt): gt_id = label // OFFSET pred_id = label % OFFSET gt_pred_map[(gt_id, pred_id)] = intersection # count all matched pairs gt_matched = set() pred_matched = set() for label_tuple, intersection in gt_pred_map.items(): gt_label, pred_label = label_tuple if gt_label not in gt_segms: continue if pred_label not in pred_segms: continue if gt_segms[gt_label]['iscrowd'] == 1: continue if gt_segms[gt_label]['category_id'] != pred_segms[pred_label][ 'category_id']: continue union = pred_segms[pred_label]['area'] + gt_segms[gt_label][ 'area'] - intersection - gt_pred_map.get((VOID, pred_label), 0) iou = intersection / union if iou > 0.5: pq_stat[gt_segms[gt_label]['category_id']].tp += 1 pq_stat[gt_segms[gt_label]['category_id']].iou += iou gt_matched.add(gt_label) pred_matched.add(pred_label) # count false positives crowd_labels_dict = {} for gt_label, gt_info in gt_segms.items(): if gt_label in gt_matched: continue # crowd segments are ignored if gt_info['iscrowd'] == 1: crowd_labels_dict[gt_info['category_id']] = gt_label continue pq_stat[gt_info['category_id']].fn += 1 # count false positives for pred_label, pred_info in pred_segms.items(): if pred_label in pred_matched: continue # intersection of the segment with VOID intersection = gt_pred_map.get((VOID, pred_label), 0) # plus intersection with corresponding CROWD region if it exists if pred_info['category_id'] in crowd_labels_dict: intersection += gt_pred_map.get( (crowd_labels_dict[pred_info['category_id']], pred_label), 0) # predicted segment is ignored if more than half of # the segment correspond to VOID and CROWD regions if intersection / pred_info['area'] > 0.5: continue pq_stat[pred_info['category_id']].fp += 1 if print_log: print('Core: {}, all {} images processed'.format( proc_id, len(annotation_set))) return pq_stat def pq_compute_multi_core(matched_annotations_list, gt_folder, pred_folder, categories, file_client=None, nproc=32): """Evaluate the metrics of Panoptic Segmentation with multithreading. Same as the function with the same name in `panopticapi`. Args: matched_annotations_list (list): The matched annotation list. Each element is a tuple of annotations of the same image with the format (gt_anns, pred_anns). gt_folder (str): The path of the ground truth images. pred_folder (str): The path of the prediction images. categories (str): The categories of the dataset. file_client (object): The file client of the dataset. If None, the backend will be set to `disk`. nproc (int): Number of processes for panoptic quality computing. Defaults to 32. When `nproc` exceeds the number of cpu cores, the number of cpu cores is used. """ if PQStat is None: raise RuntimeError( 'panopticapi is not installed, please install it by: ' 'pip install git+https://github.com/cocodataset/' 'panopticapi.git.') if file_client is None: file_client_args = dict(backend='disk') file_client = FileClient(**file_client_args) cpu_num = min(nproc, multiprocessing.cpu_count()) annotations_split = np.array_split(matched_annotations_list, cpu_num) print('Number of cores: {}, images per core: {}'.format( cpu_num, len(annotations_split[0]))) workers = multiprocessing.Pool(processes=cpu_num) processes = [] for proc_id, annotation_set in enumerate(annotations_split): p = workers.apply_async(pq_compute_single_core, (proc_id, annotation_set, gt_folder, pred_folder, categories, file_client)) processes.append(p) # Close the process pool, otherwise it will lead to memory # leaking problems. workers.close() workers.join() pq_stat = PQStat() for p in processes: pq_stat += p.get() return pq_stat ================================================ FILE: mmdet/evaluation/functional/recall.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from collections.abc import Sequence import numpy as np from mmengine.logging import print_log from terminaltables import AsciiTable from .bbox_overlaps import bbox_overlaps def _recalls(all_ious, proposal_nums, thrs): img_num = all_ious.shape[0] total_gt_num = sum([ious.shape[0] for ious in all_ious]) _ious = np.zeros((proposal_nums.size, total_gt_num), dtype=np.float32) for k, proposal_num in enumerate(proposal_nums): tmp_ious = np.zeros(0) for i in range(img_num): ious = all_ious[i][:, :proposal_num].copy() gt_ious = np.zeros((ious.shape[0])) if ious.size == 0: tmp_ious = np.hstack((tmp_ious, gt_ious)) continue for j in range(ious.shape[0]): gt_max_overlaps = ious.argmax(axis=1) max_ious = ious[np.arange(0, ious.shape[0]), gt_max_overlaps] gt_idx = max_ious.argmax() gt_ious[j] = max_ious[gt_idx] box_idx = gt_max_overlaps[gt_idx] ious[gt_idx, :] = -1 ious[:, box_idx] = -1 tmp_ious = np.hstack((tmp_ious, gt_ious)) _ious[k, :] = tmp_ious _ious = np.fliplr(np.sort(_ious, axis=1)) recalls = np.zeros((proposal_nums.size, thrs.size)) for i, thr in enumerate(thrs): recalls[:, i] = (_ious >= thr).sum(axis=1) / float(total_gt_num) return recalls def set_recall_param(proposal_nums, iou_thrs): """Check proposal_nums and iou_thrs and set correct format.""" if isinstance(proposal_nums, Sequence): _proposal_nums = np.array(proposal_nums) elif isinstance(proposal_nums, int): _proposal_nums = np.array([proposal_nums]) else: _proposal_nums = proposal_nums if iou_thrs is None: _iou_thrs = np.array([0.5]) elif isinstance(iou_thrs, Sequence): _iou_thrs = np.array(iou_thrs) elif isinstance(iou_thrs, float): _iou_thrs = np.array([iou_thrs]) else: _iou_thrs = iou_thrs return _proposal_nums, _iou_thrs def eval_recalls(gts, proposals, proposal_nums=None, iou_thrs=0.5, logger=None, use_legacy_coordinate=False): """Calculate recalls. Args: gts (list[ndarray]): a list of arrays of shape (n, 4) proposals (list[ndarray]): a list of arrays of shape (k, 4) or (k, 5) proposal_nums (int | Sequence[int]): Top N proposals to be evaluated. iou_thrs (float | Sequence[float]): IoU thresholds. Default: 0.5. logger (logging.Logger | str | None): The way to print the recall summary. See `mmengine.logging.print_log()` for details. Default: None. use_legacy_coordinate (bool): Whether use coordinate system in mmdet v1.x. "1" was added to both height and width which means w, h should be computed as 'x2 - x1 + 1` and 'y2 - y1 + 1'. Default: False. Returns: ndarray: recalls of different ious and proposal nums """ img_num = len(gts) assert img_num == len(proposals) proposal_nums, iou_thrs = set_recall_param(proposal_nums, iou_thrs) all_ious = [] for i in range(img_num): if proposals[i].ndim == 2 and proposals[i].shape[1] == 5: scores = proposals[i][:, 4] sort_idx = np.argsort(scores)[::-1] img_proposal = proposals[i][sort_idx, :] else: img_proposal = proposals[i] prop_num = min(img_proposal.shape[0], proposal_nums[-1]) if gts[i] is None or gts[i].shape[0] == 0: ious = np.zeros((0, img_proposal.shape[0]), dtype=np.float32) else: ious = bbox_overlaps( gts[i], img_proposal[:prop_num, :4], use_legacy_coordinate=use_legacy_coordinate) all_ious.append(ious) all_ious = np.array(all_ious) recalls = _recalls(all_ious, proposal_nums, iou_thrs) print_recall_summary(recalls, proposal_nums, iou_thrs, logger=logger) return recalls def print_recall_summary(recalls, proposal_nums, iou_thrs, row_idxs=None, col_idxs=None, logger=None): """Print recalls in a table. Args: recalls (ndarray): calculated from `bbox_recalls` proposal_nums (ndarray or list): top N proposals iou_thrs (ndarray or list): iou thresholds row_idxs (ndarray): which rows(proposal nums) to print col_idxs (ndarray): which cols(iou thresholds) to print logger (logging.Logger | str | None): The way to print the recall summary. See `mmengine.logging.print_log()` for details. Default: None. """ proposal_nums = np.array(proposal_nums, dtype=np.int32) iou_thrs = np.array(iou_thrs) if row_idxs is None: row_idxs = np.arange(proposal_nums.size) if col_idxs is None: col_idxs = np.arange(iou_thrs.size) row_header = [''] + iou_thrs[col_idxs].tolist() table_data = [row_header] for i, num in enumerate(proposal_nums[row_idxs]): row = [f'{val:.3f}' for val in recalls[row_idxs[i], col_idxs].tolist()] row.insert(0, num) table_data.append(row) table = AsciiTable(table_data) print_log('\n' + table.table, logger=logger) def plot_num_recall(recalls, proposal_nums): """Plot Proposal_num-Recalls curve. Args: recalls(ndarray or list): shape (k,) proposal_nums(ndarray or list): same shape as `recalls` """ if isinstance(proposal_nums, np.ndarray): _proposal_nums = proposal_nums.tolist() else: _proposal_nums = proposal_nums if isinstance(recalls, np.ndarray): _recalls = recalls.tolist() else: _recalls = recalls import matplotlib.pyplot as plt f = plt.figure() plt.plot([0] + _proposal_nums, [0] + _recalls) plt.xlabel('Proposal num') plt.ylabel('Recall') plt.axis([0, proposal_nums.max(), 0, 1]) f.show() def plot_iou_recall(recalls, iou_thrs): """Plot IoU-Recalls curve. Args: recalls(ndarray or list): shape (k,) iou_thrs(ndarray or list): same shape as `recalls` """ if isinstance(iou_thrs, np.ndarray): _iou_thrs = iou_thrs.tolist() else: _iou_thrs = iou_thrs if isinstance(recalls, np.ndarray): _recalls = recalls.tolist() else: _recalls = recalls import matplotlib.pyplot as plt f = plt.figure() plt.plot(_iou_thrs + [1.0], _recalls + [0.]) plt.xlabel('IoU') plt.ylabel('Recall') plt.axis([iou_thrs.min(), 1, 0, 1]) f.show() ================================================ FILE: mmdet/evaluation/metrics/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .cityscapes_metric import CityScapesMetric from .coco_metric import CocoMetric from .coco_occluded_metric import CocoOccludedSeparatedMetric from .coco_panoptic_metric import CocoPanopticMetric from .crowdhuman_metric import CrowdHumanMetric from .dump_det_results import DumpDetResults from .dump_proposals_metric import DumpProposals from .lvis_metric import LVISMetric from .openimages_metric import OpenImagesMetric from .voc_metric import VOCMetric __all__ = [ 'CityScapesMetric', 'CocoMetric', 'CocoPanopticMetric', 'OpenImagesMetric', 'VOCMetric', 'LVISMetric', 'CrowdHumanMetric', 'DumpProposals', 'CocoOccludedSeparatedMetric', 'DumpDetResults' ] ================================================ FILE: mmdet/evaluation/metrics/cityscapes_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os import os.path as osp import shutil from collections import OrderedDict from typing import Dict, Optional, Sequence import mmcv import numpy as np from mmengine.dist import is_main_process, master_only from mmengine.evaluator import BaseMetric from mmengine.logging import MMLogger from mmdet.registry import METRICS try: import cityscapesscripts from cityscapesscripts.evaluation import \ evalInstanceLevelSemanticLabeling as CSEval from cityscapesscripts.helpers import labels as CSLabels except ImportError: cityscapesscripts = None CSLabels = None CSEval = None @METRICS.register_module() class CityScapesMetric(BaseMetric): """CityScapes metric for instance segmentation. Args: outfile_prefix (str): The prefix of txt and png files. The txt and png file will be save in a directory whose path is "outfile_prefix.results/". seg_prefix (str, optional): Path to the directory which contains the cityscapes instance segmentation masks. It's necessary when training and validation. It could be None when infer on test dataset. Defaults to None. format_only (bool): Format the output results without perform evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to False. keep_results (bool): Whether to keep the results. When ``format_only`` is True, ``keep_results`` must be True. Defaults to False. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. prefix (str, optional): The prefix that will be added in the metric names to disambiguate homonymous metrics of different evaluators. If prefix is not provided in the argument, self.default_prefix will be used instead. Defaults to None. """ default_prefix: Optional[str] = 'cityscapes' def __init__(self, outfile_prefix: str, seg_prefix: Optional[str] = None, format_only: bool = False, keep_results: bool = False, collect_device: str = 'cpu', prefix: Optional[str] = None) -> None: if cityscapesscripts is None: raise RuntimeError('Please run "pip install cityscapesscripts" to ' 'install cityscapesscripts first.') assert outfile_prefix, 'outfile_prefix must be not None.' if format_only: assert keep_results, 'keep_results must be True when ' 'format_only is True' super().__init__(collect_device=collect_device, prefix=prefix) self.format_only = format_only self.keep_results = keep_results self.seg_out_dir = osp.abspath(f'{outfile_prefix}.results') self.seg_prefix = seg_prefix if is_main_process(): os.makedirs(self.seg_out_dir, exist_ok=True) @master_only def __del__(self) -> None: """Clean up.""" if not self.keep_results: shutil.rmtree(self.seg_out_dir) # TODO: data_batch is no longer needed, consider adjusting the # parameter position def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. The processed results should be stored in ``self.results``, which will be used to compute the metrics when all batches have been processed. Args: data_batch (dict): A batch of data from the dataloader. data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ for data_sample in data_samples: # parse pred result = dict() pred = data_sample['pred_instances'] filename = data_sample['img_path'] basename = osp.splitext(osp.basename(filename))[0] pred_txt = osp.join(self.seg_out_dir, basename + '_pred.txt') result['pred_txt'] = pred_txt labels = pred['labels'].cpu().numpy() masks = pred['masks'].cpu().numpy().astype(np.uint8) if 'mask_scores' in pred: # some detectors use different scores for bbox and mask mask_scores = pred['mask_scores'].cpu().numpy() else: mask_scores = pred['scores'].cpu().numpy() with open(pred_txt, 'w') as f: for i, (label, mask, mask_score) in enumerate( zip(labels, masks, mask_scores)): class_name = self.dataset_meta['classes'][label] class_id = CSLabels.name2label[class_name].id png_filename = osp.join( self.seg_out_dir, basename + f'_{i}_{class_name}.png') mmcv.imwrite(mask, png_filename) f.write(f'{osp.basename(png_filename)} ' f'{class_id} {mask_score}\n') # parse gt gt = dict() img_path = filename.replace('leftImg8bit.png', 'gtFine_instanceIds.png') img_path = img_path.replace('leftImg8bit', 'gtFine') gt['file_name'] = osp.join(self.seg_prefix, img_path) self.results.append((gt, result)) def compute_metrics(self, results: list) -> Dict[str, float]: """Compute the metrics from processed results. Args: results (list): The processed results of each batch. Returns: Dict[str, float]: The computed metrics. The keys are the names of the metrics, and the values are corresponding results. """ logger: MMLogger = MMLogger.get_current_instance() if self.format_only: logger.info( f'results are saved to {osp.dirname(self.seg_out_dir)}') return OrderedDict() logger.info('starts to compute metric') gts, preds = zip(*results) # set global states in cityscapes evaluation API CSEval.args.cityscapesPath = osp.join(self.seg_prefix, '../..') CSEval.args.predictionPath = self.seg_out_dir CSEval.args.predictionWalk = None CSEval.args.JSONOutput = False CSEval.args.colorized = False CSEval.args.gtInstancesFile = osp.join(self.seg_out_dir, 'gtInstances.json') groundTruthImgList = [gt['file_name'] for gt in gts] predictionImgList = [pred['pred_txt'] for pred in preds] CSEval_results = CSEval.evaluateImgLists(predictionImgList, groundTruthImgList, CSEval.args)['averages'] eval_results = OrderedDict() eval_results['mAP'] = CSEval_results['allAp'] eval_results['AP@50'] = CSEval_results['allAp50%'] return eval_results ================================================ FILE: mmdet/evaluation/metrics/coco_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import datetime import itertools import os.path as osp import tempfile from collections import OrderedDict from typing import Dict, List, Optional, Sequence, Union import numpy as np import torch from mmengine.evaluator import BaseMetric from mmengine.fileio import FileClient, dump, load from mmengine.logging import MMLogger from terminaltables import AsciiTable from mmdet.datasets.api_wrappers import COCO, COCOeval from mmdet.registry import METRICS from mmdet.structures.mask import encode_mask_results from ..functional import eval_recalls @METRICS.register_module() class CocoMetric(BaseMetric): """COCO evaluation metric. Evaluate AR, AP, and mAP for detection tasks including proposal/box detection and instance segmentation. Please refer to https://cocodataset.org/#detection-eval for more details. Args: ann_file (str, optional): Path to the coco format annotation file. If not specified, ground truth annotations from the dataset will be converted to coco format. Defaults to None. metric (str | List[str]): Metrics to be evaluated. Valid metrics include 'bbox', 'segm', 'proposal', and 'proposal_fast'. Defaults to 'bbox'. classwise (bool): Whether to evaluate the metric class-wise. Defaults to False. proposal_nums (Sequence[int]): Numbers of proposals to be evaluated. Defaults to (100, 300, 1000). iou_thrs (float | List[float], optional): IoU threshold to compute AP and AR. If not specified, IoUs from 0.5 to 0.95 will be used. Defaults to None. metric_items (List[str], optional): Metric result names to be recorded in the evaluation result. Defaults to None. format_only (bool): Format the output results without perform evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to False. outfile_prefix (str, optional): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Defaults to None. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. prefix (str, optional): The prefix that will be added in the metric names to disambiguate homonymous metrics of different evaluators. If prefix is not provided in the argument, self.default_prefix will be used instead. Defaults to None. sort_categories (bool): Whether sort categories in annotations. Only used for `Objects365V1Dataset`. Defaults to False. """ default_prefix: Optional[str] = 'coco' def __init__(self, ann_file: Optional[str] = None, metric: Union[str, List[str]] = 'bbox', classwise: bool = False, proposal_nums: Sequence[int] = (100, 300, 1000), iou_thrs: Optional[Union[float, Sequence[float]]] = None, metric_items: Optional[Sequence[str]] = None, format_only: bool = False, outfile_prefix: Optional[str] = None, file_client_args: dict = dict(backend='disk'), collect_device: str = 'cpu', prefix: Optional[str] = None, sort_categories: bool = False) -> None: super().__init__(collect_device=collect_device, prefix=prefix) # coco evaluation metrics self.metrics = metric if isinstance(metric, list) else [metric] allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] for metric in self.metrics: if metric not in allowed_metrics: raise KeyError( "metric should be one of 'bbox', 'segm', 'proposal', " f"'proposal_fast', but got {metric}.") # do class wise evaluation, default False self.classwise = classwise # proposal_nums used to compute recall or precision. self.proposal_nums = list(proposal_nums) # iou_thrs used to compute recall or precision. if iou_thrs is None: iou_thrs = np.linspace( .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) self.iou_thrs = iou_thrs self.metric_items = metric_items self.format_only = format_only if self.format_only: assert outfile_prefix is not None, 'outfile_prefix must be not' 'None when format_only is True, otherwise the result files will' 'be saved to a temp directory which will be cleaned up at the end.' self.outfile_prefix = outfile_prefix self.file_client_args = file_client_args self.file_client = FileClient(**file_client_args) # if ann_file is not specified, # initialize coco api with the converted dataset if ann_file is not None: with self.file_client.get_local_path(ann_file) as local_path: self._coco_api = COCO(local_path) if sort_categories: # 'categories' list in objects365_train.json and # objects365_val.json is inconsistent, need sort # list(or dict) before get cat_ids. cats = self._coco_api.cats sorted_cats = {i: cats[i] for i in sorted(cats)} self._coco_api.cats = sorted_cats categories = self._coco_api.dataset['categories'] sorted_categories = sorted( categories, key=lambda i: i['id']) self._coco_api.dataset['categories'] = sorted_categories else: self._coco_api = None # handle dataset lazy init self.cat_ids = None self.img_ids = None def fast_eval_recall(self, results: List[dict], proposal_nums: Sequence[int], iou_thrs: Sequence[float], logger: Optional[MMLogger] = None) -> np.ndarray: """Evaluate proposal recall with COCO's fast_eval_recall. Args: results (List[dict]): Results of the dataset. proposal_nums (Sequence[int]): Proposal numbers used for evaluation. iou_thrs (Sequence[float]): IoU thresholds used for evaluation. logger (MMLogger, optional): Logger used for logging the recall summary. Returns: np.ndarray: Averaged recall results. """ gt_bboxes = [] pred_bboxes = [result['bboxes'] for result in results] for i in range(len(self.img_ids)): ann_ids = self._coco_api.get_ann_ids(img_ids=self.img_ids[i]) ann_info = self._coco_api.load_anns(ann_ids) if len(ann_info) == 0: gt_bboxes.append(np.zeros((0, 4))) continue bboxes = [] for ann in ann_info: if ann.get('ignore', False) or ann['iscrowd']: continue x1, y1, w, h = ann['bbox'] bboxes.append([x1, y1, x1 + w, y1 + h]) bboxes = np.array(bboxes, dtype=np.float32) if bboxes.shape[0] == 0: bboxes = np.zeros((0, 4)) gt_bboxes.append(bboxes) recalls = eval_recalls( gt_bboxes, pred_bboxes, proposal_nums, iou_thrs, logger=logger) ar = recalls.mean(axis=1) return ar def xyxy2xywh(self, bbox: np.ndarray) -> list: """Convert ``xyxy`` style bounding boxes to ``xywh`` style for COCO evaluation. Args: bbox (numpy.ndarray): The bounding boxes, shape (4, ), in ``xyxy`` order. Returns: list[float]: The converted bounding boxes, in ``xywh`` order. """ _bbox: List = bbox.tolist() return [ _bbox[0], _bbox[1], _bbox[2] - _bbox[0], _bbox[3] - _bbox[1], ] def results2json(self, results: Sequence[dict], outfile_prefix: str) -> dict: """Dump the detection results to a COCO style json file. There are 3 types of results: proposals, bbox predictions, mask predictions, and they have different data types. This method will automatically recognize the type, and dump them to json files. Args: results (Sequence[dict]): Testing results of the dataset. outfile_prefix (str): The filename prefix of the json files. If the prefix is "somepath/xxx", the json files will be named "somepath/xxx.bbox.json", "somepath/xxx.segm.json", "somepath/xxx.proposal.json". Returns: dict: Possible keys are "bbox", "segm", "proposal", and values are corresponding filenames. """ bbox_json_results = [] segm_json_results = [] if 'masks' in results[0] else None for idx, result in enumerate(results): image_id = result.get('img_id', idx) labels = result['labels'] bboxes = result['bboxes'] scores = result['scores'] # bbox results for i, label in enumerate(labels): data = dict() data['image_id'] = image_id data['bbox'] = self.xyxy2xywh(bboxes[i]) data['score'] = float(scores[i]) data['category_id'] = self.cat_ids[label] bbox_json_results.append(data) if segm_json_results is None: continue # segm results masks = result['masks'] mask_scores = result.get('mask_scores', scores) for i, label in enumerate(labels): data = dict() data['image_id'] = image_id data['bbox'] = self.xyxy2xywh(bboxes[i]) data['score'] = float(mask_scores[i]) data['category_id'] = self.cat_ids[label] if isinstance(masks[i]['counts'], bytes): masks[i]['counts'] = masks[i]['counts'].decode() data['segmentation'] = masks[i] segm_json_results.append(data) result_files = dict() result_files['bbox'] = f'{outfile_prefix}.bbox.json' result_files['proposal'] = f'{outfile_prefix}.bbox.json' dump(bbox_json_results, result_files['bbox']) if segm_json_results is not None: result_files['segm'] = f'{outfile_prefix}.segm.json' dump(segm_json_results, result_files['segm']) return result_files def gt_to_coco_json(self, gt_dicts: Sequence[dict], outfile_prefix: str) -> str: """Convert ground truth to coco format json file. Args: gt_dicts (Sequence[dict]): Ground truth of the dataset. outfile_prefix (str): The filename prefix of the json files. If the prefix is "somepath/xxx", the json file will be named "somepath/xxx.gt.json". Returns: str: The filename of the json file. """ categories = [ dict(id=id, name=name) for id, name in enumerate(self.dataset_meta['classes']) ] image_infos = [] annotations = [] for idx, gt_dict in enumerate(gt_dicts): img_id = gt_dict.get('img_id', idx) image_info = dict( id=img_id, width=gt_dict['width'], height=gt_dict['height'], file_name='') image_infos.append(image_info) for ann in gt_dict['anns']: label = ann['bbox_label'] bbox = ann['bbox'] coco_bbox = [ bbox[0], bbox[1], bbox[2] - bbox[0], bbox[3] - bbox[1], ] annotation = dict( id=len(annotations) + 1, # coco api requires id starts with 1 image_id=img_id, bbox=coco_bbox, iscrowd=ann.get('ignore_flag', 0), category_id=int(label), area=coco_bbox[2] * coco_bbox[3]) if ann.get('mask', None): mask = ann['mask'] # area = mask_util.area(mask) if isinstance(mask, dict) and isinstance( mask['counts'], bytes): mask['counts'] = mask['counts'].decode() annotation['segmentation'] = mask # annotation['area'] = float(area) annotations.append(annotation) info = dict( date_created=str(datetime.datetime.now()), description='Coco json file converted by mmdet CocoMetric.') coco_json = dict( info=info, images=image_infos, categories=categories, licenses=None, ) if len(annotations) > 0: coco_json['annotations'] = annotations converted_json_path = f'{outfile_prefix}.gt.json' dump(coco_json, converted_json_path) return converted_json_path # TODO: data_batch is no longer needed, consider adjusting the # parameter position def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. The processed results should be stored in ``self.results``, which will be used to compute the metrics when all batches have been processed. Args: data_batch (dict): A batch of data from the dataloader. data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ for data_sample in data_samples: result = dict() pred = data_sample['pred_instances'] result['img_id'] = data_sample['img_id'] result['bboxes'] = pred['bboxes'].cpu().numpy() result['scores'] = pred['scores'].cpu().numpy() result['labels'] = pred['labels'].cpu().numpy() # encode mask to RLE if 'masks' in pred: result['masks'] = encode_mask_results( pred['masks'].detach().cpu().numpy()) if isinstance( pred['masks'], torch.Tensor) else pred['masks'] # some detectors use different scores for bbox and mask if 'mask_scores' in pred: result['mask_scores'] = pred['mask_scores'].cpu().numpy() # parse gt gt = dict() gt['width'] = data_sample['ori_shape'][1] gt['height'] = data_sample['ori_shape'][0] gt['img_id'] = data_sample['img_id'] if self._coco_api is None: # TODO: Need to refactor to support LoadAnnotations assert 'instances' in data_sample, \ 'ground truth is required for evaluation when ' \ '`ann_file` is not provided' gt['anns'] = data_sample['instances'] # add converted result to the results list self.results.append((gt, result)) def compute_metrics(self, results: list) -> Dict[str, float]: """Compute the metrics from processed results. Args: results (list): The processed results of each batch. Returns: Dict[str, float]: The computed metrics. The keys are the names of the metrics, and the values are corresponding results. """ logger: MMLogger = MMLogger.get_current_instance() # split gt and prediction list gts, preds = zip(*results) tmp_dir = None if self.outfile_prefix is None: tmp_dir = tempfile.TemporaryDirectory() outfile_prefix = osp.join(tmp_dir.name, 'results') else: outfile_prefix = self.outfile_prefix if self._coco_api is None: # use converted gt json file to initialize coco api logger.info('Converting ground truth to coco format...') coco_json_path = self.gt_to_coco_json( gt_dicts=gts, outfile_prefix=outfile_prefix) self._coco_api = COCO(coco_json_path) # handle lazy init if self.cat_ids is None: self.cat_ids = self._coco_api.get_cat_ids( cat_names=self.dataset_meta['classes']) if self.img_ids is None: self.img_ids = self._coco_api.get_img_ids() # convert predictions to coco format and dump to json file result_files = self.results2json(preds, outfile_prefix) eval_results = OrderedDict() if self.format_only: logger.info('results are saved in ' f'{osp.dirname(outfile_prefix)}') return eval_results for metric in self.metrics: logger.info(f'Evaluating {metric}...') # TODO: May refactor fast_eval_recall to an independent metric? # fast eval recall if metric == 'proposal_fast': ar = self.fast_eval_recall( preds, self.proposal_nums, self.iou_thrs, logger=logger) log_msg = [] for i, num in enumerate(self.proposal_nums): eval_results[f'AR@{num}'] = ar[i] log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}') log_msg = ''.join(log_msg) logger.info(log_msg) continue # evaluate proposal, bbox and segm iou_type = 'bbox' if metric == 'proposal' else metric if metric not in result_files: raise KeyError(f'{metric} is not in results') try: predictions = load(result_files[metric]) if iou_type == 'segm': # Refer to https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/coco.py#L331 # noqa # When evaluating mask AP, if the results contain bbox, # cocoapi will use the box area instead of the mask area # for calculating the instance area. Though the overall AP # is not affected, this leads to different # small/medium/large mask AP results. for x in predictions: x.pop('bbox') coco_dt = self._coco_api.loadRes(predictions) except IndexError: logger.error( 'The testing results of the whole dataset is empty.') break coco_eval = COCOeval(self._coco_api, coco_dt, iou_type) coco_eval.params.catIds = self.cat_ids coco_eval.params.imgIds = self.img_ids coco_eval.params.maxDets = list(self.proposal_nums) coco_eval.params.iouThrs = self.iou_thrs # mapping of cocoEval.stats coco_metric_names = { 'mAP': 0, 'mAP_50': 1, 'mAP_75': 2, 'mAP_s': 3, 'mAP_m': 4, 'mAP_l': 5, 'AR@100': 6, 'AR@300': 7, 'AR@1000': 8, 'AR_s@1000': 9, 'AR_m@1000': 10, 'AR_l@1000': 11 } metric_items = self.metric_items if metric_items is not None: for metric_item in metric_items: if metric_item not in coco_metric_names: raise KeyError( f'metric item "{metric_item}" is not supported') if metric == 'proposal': coco_eval.params.useCats = 0 coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() if metric_items is None: metric_items = [ 'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ] for item in metric_items: val = float( f'{coco_eval.stats[coco_metric_names[item]]:.3f}') eval_results[item] = val else: coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() if self.classwise: # Compute per-category AP # Compute per-category AP # from https://github.com/facebookresearch/detectron2/ precisions = coco_eval.eval['precision'] # precision: (iou, recall, cls, area range, max dets) assert len(self.cat_ids) == precisions.shape[2] results_per_category = [] for idx, cat_id in enumerate(self.cat_ids): # area range index 0: all area ranges # max dets index -1: typically 100 per image nm = self._coco_api.loadCats(cat_id)[0] precision = precisions[:, :, idx, 0, -1] precision = precision[precision > -1] if precision.size: ap = np.mean(precision) else: ap = float('nan') results_per_category.append( (f'{nm["name"]}', f'{round(ap, 3)}')) eval_results[f'{nm["name"]}_precision'] = round(ap, 3) num_columns = min(6, len(results_per_category) * 2) results_flatten = list( itertools.chain(*results_per_category)) headers = ['category', 'AP'] * (num_columns // 2) results_2d = itertools.zip_longest(*[ results_flatten[i::num_columns] for i in range(num_columns) ]) table_data = [headers] table_data += [result for result in results_2d] table = AsciiTable(table_data) logger.info('\n' + table.table) if metric_items is None: metric_items = [ 'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l' ] for metric_item in metric_items: key = f'{metric}_{metric_item}' val = coco_eval.stats[coco_metric_names[metric_item]] eval_results[key] = float(f'{round(val, 3)}') ap = coco_eval.stats[:6] logger.info(f'{metric}_mAP_copypaste: {ap[0]:.3f} ' f'{ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} ' f'{ap[4]:.3f} {ap[5]:.3f}') if tmp_dir is not None: tmp_dir.cleanup() return eval_results ================================================ FILE: mmdet/evaluation/metrics/coco_occluded_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp from typing import Dict, List, Optional, Union import mmengine import numpy as np from mmengine.fileio import load from mmengine.logging import print_log from pycocotools import mask as coco_mask from terminaltables import AsciiTable from mmdet.registry import METRICS from .coco_metric import CocoMetric @METRICS.register_module() class CocoOccludedSeparatedMetric(CocoMetric): """Metric of separated and occluded masks which presented in paper `A Tri- Layer Plugin to Improve Occluded Detection. `_. Separated COCO and Occluded COCO are automatically generated subsets of COCO val dataset, collecting separated objects and partially occluded objects for a large variety of categories. In this way, we define occlusion into two major categories: separated and partially occluded. - Separation: target object segmentation mask is separated into distinct regions by the occluder. - Partial Occlusion: target object is partially occluded but the segmentation mask is connected. These two new scalable real-image datasets are to benchmark a model's capability to detect occluded objects of 80 common categories. Please cite the paper if you use this dataset: @article{zhan2022triocc, title={A Tri-Layer Plugin to Improve Occluded Detection}, author={Zhan, Guanqi and Xie, Weidi and Zisserman, Andrew}, journal={British Machine Vision Conference}, year={2022} } Args: occluded_ann (str): Path to the occluded coco annotation file. separated_ann (str): Path to the separated coco annotation file. score_thr (float): Score threshold of the detection masks. Defaults to 0.3. iou_thr (float): IoU threshold for the recall calculation. Defaults to 0.75. metric (str | List[str]): Metrics to be evaluated. Valid metrics include 'bbox', 'segm', 'proposal', and 'proposal_fast'. Defaults to 'bbox'. """ default_prefix: Optional[str] = 'coco' def __init__( self, *args, occluded_ann: str = 'https://www.robots.ox.ac.uk/~vgg/research/tpod/datasets/occluded_coco.pkl', # noqa separated_ann: str = 'https://www.robots.ox.ac.uk/~vgg/research/tpod/datasets/separated_coco.pkl', # noqa score_thr: float = 0.3, iou_thr: float = 0.75, metric: Union[str, List[str]] = ['bbox', 'segm'], **kwargs) -> None: super().__init__(*args, metric=metric, **kwargs) # load from local file if osp.isfile(occluded_ann) and not osp.isabs(occluded_ann): occluded_ann = osp.join(self.data_root, occluded_ann) if osp.isfile(separated_ann) and not osp.isabs(separated_ann): separated_ann = osp.join(self.data_root, separated_ann) self.occluded_ann = load(occluded_ann) self.separated_ann = load(separated_ann) self.score_thr = score_thr self.iou_thr = iou_thr def compute_metrics(self, results: list) -> Dict[str, float]: """Compute the metrics from processed results. Args: results (list): The processed results of each batch. Returns: Dict[str, float]: The computed metrics. The keys are the names of the metrics, and the values are corresponding results. """ coco_metric_res = super().compute_metrics(results) eval_res = self.evaluate_occluded_separated(results) coco_metric_res.update(eval_res) return coco_metric_res def evaluate_occluded_separated(self, results: List[tuple]) -> dict: """Compute the recall of occluded and separated masks. Args: results (list[tuple]): Testing results of the dataset. Returns: dict[str, float]: The recall of occluded and separated masks. """ dict_det = {} print_log('processing detection results...') prog_bar = mmengine.ProgressBar(len(results)) for i in range(len(results)): gt, dt = results[i] img_id = dt['img_id'] cur_img_name = self._coco_api.imgs[img_id]['file_name'] if cur_img_name not in dict_det.keys(): dict_det[cur_img_name] = [] for bbox, score, label, mask in zip(dt['bboxes'], dt['scores'], dt['labels'], dt['masks']): cur_binary_mask = coco_mask.decode(mask) dict_det[cur_img_name].append([ score, self.dataset_meta['classes'][label], cur_binary_mask, bbox ]) dict_det[cur_img_name].sort( key=lambda x: (-x[0], x[3][0], x[3][1]) ) # rank by confidence from high to low, avoid same confidence prog_bar.update() print_log('\ncomputing occluded mask recall...', logger='current') occluded_correct_num, occluded_recall = self.compute_recall( dict_det, gt_ann=self.occluded_ann, is_occ=True) print_log( f'\nCOCO occluded mask recall: {occluded_recall:.2f}%', logger='current') print_log( f'COCO occluded mask success num: {occluded_correct_num}', logger='current') print_log('computing separated mask recall...', logger='current') separated_correct_num, separated_recall = self.compute_recall( dict_det, gt_ann=self.separated_ann, is_occ=False) print_log( f'\nCOCO separated mask recall: {separated_recall:.2f}%', logger='current') print_log( f'COCO separated mask success num: {separated_correct_num}', logger='current') table_data = [ ['mask type', 'recall', 'num correct'], ['occluded', f'{occluded_recall:.2f}%', occluded_correct_num], ['separated', f'{separated_recall:.2f}%', separated_correct_num] ] table = AsciiTable(table_data) print_log('\n' + table.table, logger='current') return dict( occluded_recall=occluded_recall, separated_recall=separated_recall) def compute_recall(self, result_dict: dict, gt_ann: list, is_occ: bool = True) -> tuple: """Compute the recall of occluded or separated masks. Args: result_dict (dict): Processed mask results. gt_ann (list): Occluded or separated coco annotations. is_occ (bool): Whether the annotation is occluded mask. Defaults to True. Returns: tuple: number of correct masks and the recall. """ correct = 0 prog_bar = mmengine.ProgressBar(len(gt_ann)) for iter_i in range(len(gt_ann)): cur_item = gt_ann[iter_i] cur_img_name = cur_item[0] cur_gt_bbox = cur_item[3] if is_occ: cur_gt_bbox = [ cur_gt_bbox[0], cur_gt_bbox[1], cur_gt_bbox[0] + cur_gt_bbox[2], cur_gt_bbox[1] + cur_gt_bbox[3] ] cur_gt_class = cur_item[1] cur_gt_mask = coco_mask.decode(cur_item[4]) assert cur_img_name in result_dict.keys() cur_detections = result_dict[cur_img_name] correct_flag = False for i in range(len(cur_detections)): cur_det_confidence = cur_detections[i][0] if cur_det_confidence < self.score_thr: break cur_det_class = cur_detections[i][1] if cur_det_class != cur_gt_class: continue cur_det_mask = cur_detections[i][2] cur_iou = self.mask_iou(cur_det_mask, cur_gt_mask) if cur_iou >= self.iou_thr: correct_flag = True break if correct_flag: correct += 1 prog_bar.update() recall = correct / len(gt_ann) * 100 return correct, recall def mask_iou(self, mask1: np.ndarray, mask2: np.ndarray) -> np.ndarray: """Compute IoU between two masks.""" mask1_area = np.count_nonzero(mask1 == 1) mask2_area = np.count_nonzero(mask2 == 1) intersection = np.count_nonzero(np.logical_and(mask1 == 1, mask2 == 1)) iou = intersection / (mask1_area + mask2_area - intersection) return iou ================================================ FILE: mmdet/evaluation/metrics/coco_panoptic_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import datetime import itertools import os.path as osp import tempfile from typing import Dict, Optional, Sequence, Tuple, Union import mmcv import numpy as np from mmengine.evaluator import BaseMetric from mmengine.fileio import FileClient, dump, load from mmengine.logging import MMLogger, print_log from terminaltables import AsciiTable from mmdet.datasets.api_wrappers import COCOPanoptic from mmdet.registry import METRICS from ..functional import (INSTANCE_OFFSET, pq_compute_multi_core, pq_compute_single_core) try: import panopticapi from panopticapi.evaluation import VOID, PQStat from panopticapi.utils import id2rgb, rgb2id except ImportError: panopticapi = None id2rgb = None rgb2id = None VOID = None PQStat = None @METRICS.register_module() class CocoPanopticMetric(BaseMetric): """COCO panoptic segmentation evaluation metric. Evaluate PQ, SQ RQ for panoptic segmentation tasks. Please refer to https://cocodataset.org/#panoptic-eval for more details. Args: ann_file (str, optional): Path to the coco format annotation file. If not specified, ground truth annotations from the dataset will be converted to coco format. Defaults to None. seg_prefix (str, optional): Path to the directory which contains the coco panoptic segmentation mask. It should be specified when evaluate. Defaults to None. classwise (bool): Whether to evaluate the metric class-wise. Defaults to False. outfile_prefix (str, optional): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. It should be specified when format_only is True. Defaults to None. format_only (bool): Format the output results without perform evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to False. nproc (int): Number of processes for panoptic quality computing. Defaults to 32. When ``nproc`` exceeds the number of cpu cores, the number of cpu cores is used. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. prefix (str, optional): The prefix that will be added in the metric names to disambiguate homonymous metrics of different evaluators. If prefix is not provided in the argument, self.default_prefix will be used instead. Defaults to None. """ default_prefix: Optional[str] = 'coco_panoptic' def __init__(self, ann_file: Optional[str] = None, seg_prefix: Optional[str] = None, classwise: bool = False, format_only: bool = False, outfile_prefix: Optional[str] = None, nproc: int = 32, file_client_args: dict = dict(backend='disk'), collect_device: str = 'cpu', prefix: Optional[str] = None) -> None: if panopticapi is None: raise RuntimeError( 'panopticapi is not installed, please install it by: ' 'pip install git+https://github.com/cocodataset/' 'panopticapi.git.') super().__init__(collect_device=collect_device, prefix=prefix) self.classwise = classwise self.format_only = format_only if self.format_only: assert outfile_prefix is not None, 'outfile_prefix must be not' 'None when format_only is True, otherwise the result files will' 'be saved to a temp directory which will be cleaned up at the end.' self.tmp_dir = None # outfile_prefix should be a prefix of a path which points to a shared # storage when train or test with multi nodes. self.outfile_prefix = outfile_prefix if outfile_prefix is None: self.tmp_dir = tempfile.TemporaryDirectory() self.outfile_prefix = osp.join(self.tmp_dir.name, 'results') # the directory to save predicted panoptic segmentation mask self.seg_out_dir = f'{self.outfile_prefix}.panoptic' self.nproc = nproc self.seg_prefix = seg_prefix self.cat_ids = None self.cat2label = None self.file_client_args = file_client_args self.file_client = FileClient(**file_client_args) if ann_file: with self.file_client.get_local_path(ann_file) as local_path: self._coco_api = COCOPanoptic(local_path) self.categories = self._coco_api.cats else: self._coco_api = None self.categories = None self.file_client = FileClient(**file_client_args) def __del__(self) -> None: """Clean up.""" if self.tmp_dir is not None: self.tmp_dir.cleanup() def gt_to_coco_json(self, gt_dicts: Sequence[dict], outfile_prefix: str) -> Tuple[str, str]: """Convert ground truth to coco panoptic segmentation format json file. Args: gt_dicts (Sequence[dict]): Ground truth of the dataset. outfile_prefix (str): The filename prefix of the json file. If the prefix is "somepath/xxx", the json file will be named "somepath/xxx.gt.json". Returns: Tuple[str, str]: The filename of the json file and the name of the\ directory which contains panoptic segmentation masks. """ assert len(gt_dicts) > 0, 'gt_dicts is empty.' gt_folder = osp.dirname(gt_dicts[0]['seg_map_path']) converted_json_path = f'{outfile_prefix}.gt.json' categories = [] for id, name in enumerate(self.dataset_meta['classes']): isthing = 1 if name in self.dataset_meta['thing_classes'] else 0 categories.append({'id': id, 'name': name, 'isthing': isthing}) image_infos = [] annotations = [] for gt_dict in gt_dicts: img_id = gt_dict['image_id'] image_info = { 'id': img_id, 'width': gt_dict['width'], 'height': gt_dict['height'], 'file_name': osp.split(gt_dict['seg_map_path'])[-1] } image_infos.append(image_info) pan_png = mmcv.imread(gt_dict['seg_map_path']).squeeze() pan_png = pan_png[:, :, ::-1] pan_png = rgb2id(pan_png) segments_info = [] for segment_info in gt_dict['segments_info']: id = segment_info['id'] label = segment_info['category'] mask = pan_png == id isthing = categories[label]['isthing'] if isthing: iscrowd = 1 if not segment_info['is_thing'] else 0 else: iscrowd = 0 new_segment_info = { 'id': id, 'category_id': label, 'isthing': isthing, 'iscrowd': iscrowd, 'area': mask.sum() } segments_info.append(new_segment_info) segm_file = image_info['file_name'].replace('jpg', 'png') annotation = dict( image_id=img_id, segments_info=segments_info, file_name=segm_file) annotations.append(annotation) pan_png = id2rgb(pan_png) info = dict( date_created=str(datetime.datetime.now()), description='Coco json file converted by mmdet CocoPanopticMetric.' ) coco_json = dict( info=info, images=image_infos, categories=categories, licenses=None, ) if len(annotations) > 0: coco_json['annotations'] = annotations dump(coco_json, converted_json_path) return converted_json_path, gt_folder def result2json(self, results: Sequence[dict], outfile_prefix: str) -> Tuple[str, str]: """Dump the panoptic results to a COCO style json file and a directory. Args: results (Sequence[dict]): Testing results of the dataset. outfile_prefix (str): The filename prefix of the json files and the directory. Returns: Tuple[str, str]: The json file and the directory which contains \ panoptic segmentation masks. The filename of the json is "somepath/xxx.panoptic.json" and name of the directory is "somepath/xxx.panoptic". """ label2cat = dict((v, k) for (k, v) in self.cat2label.items()) pred_annotations = [] for idx in range(len(results)): result = results[idx] for segment_info in result['segments_info']: sem_label = segment_info['category_id'] # convert sem_label to json label cat_id = label2cat[sem_label] segment_info['category_id'] = label2cat[sem_label] is_thing = self.categories[cat_id]['isthing'] segment_info['isthing'] = is_thing pred_annotations.append(result) pan_json_results = dict(annotations=pred_annotations) json_filename = f'{outfile_prefix}.panoptic.json' dump(pan_json_results, json_filename) return json_filename, ( self.seg_out_dir if self.tmp_dir is None else tempfile.gettempdir()) def _parse_predictions(self, pred: dict, img_id: int, segm_file: str, label2cat=None) -> dict: """Parse panoptic segmentation predictions. Args: pred (dict): Panoptic segmentation predictions. img_id (int): Image id. segm_file (str): Segmentation file name. label2cat (dict): Mapping from label to category id. Defaults to None. Returns: dict: Parsed predictions. """ result = dict() result['img_id'] = img_id # shape (1, H, W) -> (H, W) pan = pred['pred_panoptic_seg']['sem_seg'].cpu().numpy()[0] pan_labels = np.unique(pan) segments_info = [] for pan_label in pan_labels: sem_label = pan_label % INSTANCE_OFFSET # We reserve the length of dataset_meta['classes'] for VOID label if sem_label == len(self.dataset_meta['classes']): continue mask = pan == pan_label area = mask.sum() segments_info.append({ 'id': int(pan_label), # when ann_file provided, sem_label should be cat_id, otherwise # sem_label should be a continuous id, not the cat_id # defined in dataset 'category_id': label2cat[sem_label] if label2cat else sem_label, 'area': int(area) }) # evaluation script uses 0 for VOID label. pan[pan % INSTANCE_OFFSET == len(self.dataset_meta['classes'])] = VOID pan = id2rgb(pan).astype(np.uint8) mmcv.imwrite(pan[:, :, ::-1], osp.join(self.seg_out_dir, segm_file)) result = { 'image_id': img_id, 'segments_info': segments_info, 'file_name': segm_file } return result def _compute_batch_pq_stats(self, data_samples: Sequence[dict]): """Process gts and predictions when ``outfile_prefix`` is not set, gts are from dataset or a json file which is defined by ``ann_file``. Intermediate results, ``pq_stats``, are computed here and put into ``self.results``. """ if self._coco_api is None: categories = dict() for id, name in enumerate(self.dataset_meta['classes']): isthing = 1 if name in self.dataset_meta['thing_classes']\ else 0 categories[id] = {'id': id, 'name': name, 'isthing': isthing} label2cat = None else: categories = self.categories cat_ids = self._coco_api.get_cat_ids( cat_names=self.dataset_meta['classes']) label2cat = {i: cat_id for i, cat_id in enumerate(cat_ids)} for data_sample in data_samples: # parse pred img_id = data_sample['img_id'] segm_file = osp.basename(data_sample['img_path']).replace( 'jpg', 'png') result = self._parse_predictions( pred=data_sample, img_id=img_id, segm_file=segm_file, label2cat=label2cat) # parse gt gt = dict() gt['image_id'] = img_id gt['width'] = data_sample['ori_shape'][1] gt['height'] = data_sample['ori_shape'][0] gt['file_name'] = segm_file if self._coco_api is None: # get segments_info from data_sample seg_map_path = osp.join(self.seg_prefix, segm_file) pan_png = mmcv.imread(seg_map_path).squeeze() pan_png = pan_png[:, :, ::-1] pan_png = rgb2id(pan_png) segments_info = [] for segment_info in data_sample['segments_info']: id = segment_info['id'] label = segment_info['category'] mask = pan_png == id isthing = categories[label]['isthing'] if isthing: iscrowd = 1 if not segment_info['is_thing'] else 0 else: iscrowd = 0 new_segment_info = { 'id': id, 'category_id': label, 'isthing': isthing, 'iscrowd': iscrowd, 'area': mask.sum() } segments_info.append(new_segment_info) else: # get segments_info from annotation file segments_info = self._coco_api.imgToAnns[img_id] gt['segments_info'] = segments_info pq_stats = pq_compute_single_core( proc_id=0, annotation_set=[(gt, result)], gt_folder=self.seg_prefix, pred_folder=self.seg_out_dir, categories=categories, file_client=self.file_client) self.results.append(pq_stats) def _process_gt_and_predictions(self, data_samples: Sequence[dict]): """Process gts and predictions when ``outfile_prefix`` is set. The predictions will be saved to directory specified by ``outfile_predfix``. The matched pair (gt, result) will be put into ``self.results``. """ for data_sample in data_samples: # parse pred img_id = data_sample['img_id'] segm_file = osp.basename(data_sample['img_path']).replace( 'jpg', 'png') result = self._parse_predictions( pred=data_sample, img_id=img_id, segm_file=segm_file) # parse gt gt = dict() gt['image_id'] = img_id gt['width'] = data_sample['ori_shape'][1] gt['height'] = data_sample['ori_shape'][0] if self._coco_api is None: # get segments_info from dataset gt['segments_info'] = data_sample['segments_info'] gt['seg_map_path'] = data_sample['seg_map_path'] self.results.append((gt, result)) # TODO: data_batch is no longer needed, consider adjusting the # parameter position def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. The processed results should be stored in ``self.results``, which will be used to compute the metrics when all batches have been processed. Args: data_batch (dict): A batch of data from the dataloader. data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ # If ``self.tmp_dir`` is none, it will save gt and predictions to # self.results, otherwise, it will compute pq_stats here. if self.tmp_dir is None: self._process_gt_and_predictions(data_samples) else: self._compute_batch_pq_stats(data_samples) def compute_metrics(self, results: list) -> Dict[str, float]: """Compute the metrics from processed results. Args: results (list): The processed results of each batch. There are two cases: - When ``outfile_prefix`` is not provided, the elements in results are pq_stats which can be summed directly to get PQ. - When ``outfile_prefix`` is provided, the elements in results are tuples like (gt, pred). Returns: Dict[str, float]: The computed metrics. The keys are the names of the metrics, and the values are corresponding results. """ logger: MMLogger = MMLogger.get_current_instance() if self.tmp_dir is None: # do evaluation after collect all the results # split gt and prediction list gts, preds = zip(*results) if self._coco_api is None: # use converted gt json file to initialize coco api logger.info('Converting ground truth to coco format...') coco_json_path, gt_folder = self.gt_to_coco_json( gt_dicts=gts, outfile_prefix=self.outfile_prefix) self._coco_api = COCOPanoptic(coco_json_path) else: gt_folder = self.seg_prefix self.cat_ids = self._coco_api.get_cat_ids( cat_names=self.dataset_meta['classes']) self.cat2label = { cat_id: i for i, cat_id in enumerate(self.cat_ids) } self.img_ids = self._coco_api.get_img_ids() self.categories = self._coco_api.cats # convert predictions to coco format and dump to json file json_filename, pred_folder = self.result2json( results=preds, outfile_prefix=self.outfile_prefix) if self.format_only: logger.info('results are saved in ' f'{osp.dirname(self.outfile_prefix)}') return dict() imgs = self._coco_api.imgs gt_json = self._coco_api.img_ann_map gt_json = [{ 'image_id': k, 'segments_info': v, 'file_name': imgs[k]['segm_file'] } for k, v in gt_json.items()] pred_json = load(json_filename) pred_json = dict( (el['image_id'], el) for el in pred_json['annotations']) # match the gt_anns and pred_anns in the same image matched_annotations_list = [] for gt_ann in gt_json: img_id = gt_ann['image_id'] if img_id not in pred_json.keys(): raise Exception('no prediction for the image' ' with id: {}'.format(img_id)) matched_annotations_list.append((gt_ann, pred_json[img_id])) pq_stat = pq_compute_multi_core( matched_annotations_list, gt_folder, pred_folder, self.categories, file_client=self.file_client, nproc=self.nproc) else: # aggregate the results generated in process if self._coco_api is None: categories = dict() for id, name in enumerate(self.dataset_meta['classes']): isthing = 1 if name in self.dataset_meta[ 'thing_classes'] else 0 categories[id] = { 'id': id, 'name': name, 'isthing': isthing } self.categories = categories pq_stat = PQStat() for result in results: pq_stat += result metrics = [('All', None), ('Things', True), ('Stuff', False)] pq_results = {} for name, isthing in metrics: pq_results[name], classwise_results = pq_stat.pq_average( self.categories, isthing=isthing) if name == 'All': pq_results['classwise'] = classwise_results classwise_results = None if self.classwise: classwise_results = { k: v for k, v in zip(self.dataset_meta['classes'], pq_results['classwise'].values()) } print_panoptic_table(pq_results, classwise_results, logger=logger) results = parse_pq_results(pq_results) return results def parse_pq_results(pq_results: dict) -> dict: """Parse the Panoptic Quality results. Args: pq_results (dict): Panoptic Quality results. Returns: dict: Panoptic Quality results parsed. """ result = dict() result['PQ'] = 100 * pq_results['All']['pq'] result['SQ'] = 100 * pq_results['All']['sq'] result['RQ'] = 100 * pq_results['All']['rq'] result['PQ_th'] = 100 * pq_results['Things']['pq'] result['SQ_th'] = 100 * pq_results['Things']['sq'] result['RQ_th'] = 100 * pq_results['Things']['rq'] result['PQ_st'] = 100 * pq_results['Stuff']['pq'] result['SQ_st'] = 100 * pq_results['Stuff']['sq'] result['RQ_st'] = 100 * pq_results['Stuff']['rq'] return result def print_panoptic_table( pq_results: dict, classwise_results: Optional[dict] = None, logger: Optional[Union['MMLogger', str]] = None) -> None: """Print the panoptic evaluation results table. Args: pq_results(dict): The Panoptic Quality results. classwise_results(dict, optional): The classwise Panoptic Quality. results. The keys are class names and the values are metrics. Defaults to None. logger (:obj:`MMLogger` | str, optional): Logger used for printing related information during evaluation. Default: None. """ headers = ['', 'PQ', 'SQ', 'RQ', 'categories'] data = [headers] for name in ['All', 'Things', 'Stuff']: numbers = [ f'{(pq_results[name][k] * 100):0.3f}' for k in ['pq', 'sq', 'rq'] ] row = [name] + numbers + [pq_results[name]['n']] data.append(row) table = AsciiTable(data) print_log('Panoptic Evaluation Results:\n' + table.table, logger=logger) if classwise_results is not None: class_metrics = [(name, ) + tuple(f'{(metrics[k] * 100):0.3f}' for k in ['pq', 'sq', 'rq']) for name, metrics in classwise_results.items()] num_columns = min(8, len(class_metrics) * 4) results_flatten = list(itertools.chain(*class_metrics)) headers = ['category', 'PQ', 'SQ', 'RQ'] * (num_columns // 4) results_2d = itertools.zip_longest( *[results_flatten[i::num_columns] for i in range(num_columns)]) data = [headers] data += [result for result in results_2d] table = AsciiTable(data) print_log( 'Classwise Panoptic Evaluation Results:\n' + table.table, logger=logger) ================================================ FILE: mmdet/evaluation/metrics/crowdhuman_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import json import os.path as osp import tempfile from collections import OrderedDict from multiprocessing import Process, Queue from typing import Dict, List, Optional, Sequence, Union import numpy as np from mmengine.evaluator import BaseMetric from mmengine.fileio import FileClient, dump, load from mmengine.logging import MMLogger from scipy.sparse import csr_matrix from scipy.sparse.csgraph import maximum_bipartite_matching from mmdet.evaluation.functional.bbox_overlaps import bbox_overlaps from mmdet.registry import METRICS PERSON_CLASSES = ['background', 'person'] @METRICS.register_module() class CrowdHumanMetric(BaseMetric): """CrowdHuman evaluation metric. Evaluate Average Precision (AP), Miss Rate (MR) and Jaccard Index (JI) for detection tasks. Args: ann_file (str): Path to the annotation file. metric (str | List[str]): Metrics to be evaluated. Valid metrics include 'AP', 'MR' and 'JI'. Defaults to 'AP'. format_only (bool): Format the output results without perform evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to False. outfile_prefix (str, optional): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Defaults to None. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. prefix (str, optional): The prefix that will be added in the metric names to disambiguate homonymous metrics of different evaluators. If prefix is not provided in the argument, self.default_prefix will be used instead. Defaults to None. eval_mode (int): Select the mode of evaluate. Valid mode include 0(just body box), 1(just head box) and 2(both of them). Defaults to 0. iou_thres (float): IoU threshold. Defaults to 0.5. compare_matching_method (str, optional): Matching method to compare the detection results with the ground_truth when compute 'AP' and 'MR'.Valid method include VOC and None(CALTECH). Default to None. mr_ref (str): Different parameter selection to calculate MR. Valid ref include CALTECH_-2 and CALTECH_-4. Defaults to CALTECH_-2. num_ji_process (int): The number of processes to evaluation JI. Defaults to 10. """ default_prefix: Optional[str] = 'crowd_human' def __init__(self, ann_file: str, metric: Union[str, List[str]] = ['AP', 'MR', 'JI'], format_only: bool = False, outfile_prefix: Optional[str] = None, file_client_args: dict = dict(backend='disk'), collect_device: str = 'cpu', prefix: Optional[str] = None, eval_mode: int = 0, iou_thres: float = 0.5, compare_matching_method: Optional[str] = None, mr_ref: str = 'CALTECH_-2', num_ji_process: int = 10) -> None: super().__init__(collect_device=collect_device, prefix=prefix) self.ann_file = ann_file # crowdhuman evaluation metrics self.metrics = metric if isinstance(metric, list) else [metric] allowed_metrics = ['MR', 'AP', 'JI'] for metric in self.metrics: if metric not in allowed_metrics: raise KeyError(f"metric should be one of 'MR', 'AP', 'JI'," f'but got {metric}.') self.format_only = format_only if self.format_only: assert outfile_prefix is not None, 'outfile_prefix must be not' 'None when format_only is True, otherwise the result files will' 'be saved to a temp directory which will be cleaned up at the end.' self.outfile_prefix = outfile_prefix self.file_client_args = file_client_args self.file_client = FileClient(**file_client_args) assert eval_mode in [0, 1, 2], \ "Unknown eval mode. mr_ref should be one of '0', '1', '2'." assert compare_matching_method is None or \ compare_matching_method == 'VOC', \ 'The alternative compare_matching_method is VOC.' \ 'This parameter defaults to CALTECH(None)' assert mr_ref == 'CALTECH_-2' or mr_ref == 'CALTECH_-4', \ "mr_ref should be one of 'CALTECH_-2', 'CALTECH_-4'." self.eval_mode = eval_mode self.iou_thres = iou_thres self.compare_matching_method = compare_matching_method self.mr_ref = mr_ref self.num_ji_process = num_ji_process @staticmethod def results2json(results: Sequence[dict], outfile_prefix: str) -> str: """Dump the detection results to a json file.""" result_file_path = f'{outfile_prefix}.json' bbox_json_results = [] for i, result in enumerate(results): ann, pred = result dump_dict = dict() dump_dict['ID'] = ann['ID'] dump_dict['width'] = ann['width'] dump_dict['height'] = ann['height'] dtboxes = [] bboxes = pred.tolist() for _, single_bbox in enumerate(bboxes): temp_dict = dict() x1, y1, x2, y2, score = single_bbox temp_dict['box'] = [x1, y1, x2 - x1, y2 - y1] temp_dict['score'] = score temp_dict['tag'] = 1 dtboxes.append(temp_dict) dump_dict['dtboxes'] = dtboxes bbox_json_results.append(dump_dict) dump(bbox_json_results, result_file_path) return result_file_path def process(self, data_batch: Sequence[dict], data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. The processed results should be stored in ``self.results``, which will be used to compute the metrics when all batches have been processed. Args: data_batch (dict): A batch of data from the dataloader. data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ for data_sample in data_samples: ann = dict() ann['ID'] = data_sample['img_id'] ann['width'] = data_sample['ori_shape'][1] ann['height'] = data_sample['ori_shape'][0] pred_bboxes = data_sample['pred_instances']['bboxes'].cpu().numpy() pred_scores = data_sample['pred_instances']['scores'].cpu().numpy() pred_bbox_scores = np.hstack( [pred_bboxes, pred_scores.reshape((-1, 1))]) self.results.append((ann, pred_bbox_scores)) def compute_metrics(self, results: list) -> Dict[str, float]: """Compute the metrics from processed results. Args: results (list): The processed results of each batch. Returns: eval_results(Dict[str, float]): The computed metrics. The keys are the names of the metrics, and the values are corresponding results. """ logger: MMLogger = MMLogger.get_current_instance() tmp_dir = None if self.outfile_prefix is None: tmp_dir = tempfile.TemporaryDirectory() outfile_prefix = osp.join(tmp_dir.name, 'result') else: outfile_prefix = self.outfile_prefix # convert predictions to coco format and dump to json file result_file = self.results2json(results, outfile_prefix) eval_results = OrderedDict() if self.format_only: logger.info(f'results are saved in {osp.dirname(outfile_prefix)}') return eval_results # load evaluation samples eval_samples = self.load_eval_samples(result_file) if 'AP' in self.metrics or 'MR' in self.metrics: score_list = self.compare(eval_samples) gt_num = sum([eval_samples[i].gt_num for i in eval_samples]) ign_num = sum([eval_samples[i].ign_num for i in eval_samples]) gt_num = gt_num - ign_num img_num = len(eval_samples) for metric in self.metrics: logger.info(f'Evaluating {metric}...') if metric == 'AP': AP = self.eval_ap(score_list, gt_num, img_num) eval_results['mAP'] = float(f'{round(AP, 4)}') if metric == 'MR': MR = self.eval_mr(score_list, gt_num, img_num) eval_results['mMR'] = float(f'{round(MR, 4)}') if metric == 'JI': JI = self.eval_ji(eval_samples) eval_results['JI'] = float(f'{round(JI, 4)}') if tmp_dir is not None: tmp_dir.cleanup() return eval_results def load_eval_samples(self, result_file): """Load data from annotations file and detection results. Args: result_file (str): The file path of the saved detection results. Returns: Dict[Image]: The detection result packaged by Image """ gt_str = self.file_client.get_text(self.ann_file).strip().split('\n') gt_records = [json.loads(line) for line in gt_str] pred_records = load(result_file) eval_samples = dict() for gt_record, pred_record in zip(gt_records, pred_records): assert gt_record['ID'] == pred_record['ID'], \ 'please set val_dataloader.sampler.shuffle=False and try again' eval_samples[pred_record['ID']] = Image(self.eval_mode) eval_samples[pred_record['ID']].load(gt_record, 'box', None, PERSON_CLASSES, True) eval_samples[pred_record['ID']].load(pred_record, 'box', None, PERSON_CLASSES, False) eval_samples[pred_record['ID']].clip_all_boader() return eval_samples def compare(self, samples): """Match the detection results with the ground_truth. Args: samples (dict[Image]): The detection result packaged by Image. Returns: score_list(list[tuple[ndarray, int, str]]): Matching result. a list of tuples (dtbox, label, imgID) in the descending sort of dtbox.score. """ score_list = list() for id in samples: if self.compare_matching_method == 'VOC': result = samples[id].compare_voc(self.iou_thres) else: result = samples[id].compare_caltech(self.iou_thres) score_list.extend(result) # In the descending sort of dtbox score. score_list.sort(key=lambda x: x[0][-1], reverse=True) return score_list @staticmethod def eval_ap(score_list, gt_num, img_num): """Evaluate by average precision. Args: score_list(list[tuple[ndarray, int, str]]): Matching result. a list of tuples (dtbox, label, imgID) in the descending sort of dtbox.score. gt_num(int): The number of gt boxes in the entire dataset. img_num(int)： The number of images in the entire dataset. Returns: ap(float): result of average precision. """ # calculate general ap score def _calculate_map(_recall, _precision): assert len(_recall) == len(_precision) area = 0 for k in range(1, len(_recall)): delta_h = (_precision[k - 1] + _precision[k]) / 2 delta_w = _recall[k] - _recall[k - 1] area += delta_w * delta_h return area tp, fp = 0.0, 0.0 rpX, rpY = list(), list() fpn = [] recalln = [] thr = [] fppi = [] for i, item in enumerate(score_list): if item[1] == 1: tp += 1.0 elif item[1] == 0: fp += 1.0 fn = gt_num - tp recall = tp / (tp + fn) precision = tp / (tp + fp) rpX.append(recall) rpY.append(precision) fpn.append(fp) recalln.append(tp) thr.append(item[0][-1]) fppi.append(fp / img_num) ap = _calculate_map(rpX, rpY) return ap def eval_mr(self, score_list, gt_num, img_num): """Evaluate by Caltech-style log-average miss rate. Args: score_list(list[tuple[ndarray, int, str]]): Matching result. a list of tuples (dtbox, label, imgID) in the descending sort of dtbox.score. gt_num(int): The number of gt boxes in the entire dataset. img_num(int): The number of image in the entire dataset. Returns: mr(float): result of miss rate. """ # find greater_than def _find_gt(lst, target): for idx, _item in enumerate(lst): if _item >= target: return idx return len(lst) - 1 if self.mr_ref == 'CALTECH_-2': # CALTECH_MRREF_2: anchor points (from 10^-2 to 1) as in # P.Dollar's paper ref = [ 0.0100, 0.0178, 0.03160, 0.0562, 0.1000, 0.1778, 0.3162, 0.5623, 1.000 ] else: # CALTECH_MRREF_4: anchor points (from 10^-4 to 1) as in # S.Zhang's paper ref = [ 0.0001, 0.0003, 0.00100, 0.0032, 0.0100, 0.0316, 0.1000, 0.3162, 1.000 ] tp, fp = 0.0, 0.0 fppiX, fppiY = list(), list() for i, item in enumerate(score_list): if item[1] == 1: tp += 1.0 elif item[1] == 0: fp += 1.0 fn = gt_num - tp recall = tp / (tp + fn) missrate = 1.0 - recall fppi = fp / img_num fppiX.append(fppi) fppiY.append(missrate) score = list() for pos in ref: argmin = _find_gt(fppiX, pos) if argmin >= 0: score.append(fppiY[argmin]) score = np.array(score) mr = np.exp(np.log(score).mean()) return mr def eval_ji(self, samples): """Evaluate by JI using multi_process. Args: samples(Dict[str, Image]): The detection result packaged by Image. Returns: ji(float): result of jaccard index. """ import math res_line = [] res_ji = [] for i in range(10): score_thr = 1e-1 * i total = len(samples) stride = math.ceil(total / self.num_ji_process) result_queue = Queue(10000) results, procs = [], [] records = list(samples.items()) for i in range(self.num_ji_process): start = i * stride end = np.min([start + stride, total]) sample_data = dict(records[start:end]) p = Process( target=self.compute_ji_with_ignore, args=(result_queue, sample_data, score_thr)) p.start() procs.append(p) for i in range(total): t = result_queue.get() results.append(t) for p in procs: p.join() line, mean_ratio = self.gather(results) line = 'score_thr:{:.1f}, {}'.format(score_thr, line) res_line.append(line) res_ji.append(mean_ratio) return max(res_ji) def compute_ji_with_ignore(self, result_queue, dt_result, score_thr): """Compute JI with ignore. Args: result_queue(Queue): The Queue for save compute result when multi_process. dt_result(dict[Image]): Detection result packaged by Image. score_thr(float): The threshold of detection score. Returns: dict: compute result. """ for ID, record in dt_result.items(): gt_boxes = record.gt_boxes dt_boxes = record.dt_boxes keep = dt_boxes[:, -1] > score_thr dt_boxes = dt_boxes[keep][:, :-1] gt_tag = np.array(gt_boxes[:, -1] != -1) matches = self.compute_ji_matching(dt_boxes, gt_boxes[gt_tag, :4]) # get the unmatched_indices matched_indices = np.array([j for (j, _) in matches]) unmatched_indices = list( set(np.arange(dt_boxes.shape[0])) - set(matched_indices)) num_ignore_dt = self.get_ignores(dt_boxes[unmatched_indices], gt_boxes[~gt_tag, :4]) matched_indices = np.array([j for (_, j) in matches]) unmatched_indices = list( set(np.arange(gt_boxes[gt_tag].shape[0])) - set(matched_indices)) num_ignore_gt = self.get_ignores( gt_boxes[gt_tag][unmatched_indices], gt_boxes[~gt_tag, :4]) # compute results eps = 1e-6 k = len(matches) m = gt_tag.sum() - num_ignore_gt n = dt_boxes.shape[0] - num_ignore_dt ratio = k / (m + n - k + eps) recall = k / (m + eps) cover = k / (n + eps) noise = 1 - cover result_dict = dict( ratio=ratio, recall=recall, cover=cover, noise=noise, k=k, m=m, n=n) result_queue.put_nowait(result_dict) @staticmethod def gather(results): """Integrate test results.""" assert len(results) img_num = 0 for result in results: if result['n'] != 0 or result['m'] != 0: img_num += 1 mean_ratio = np.sum([rb['ratio'] for rb in results]) / img_num valids = np.sum([rb['k'] for rb in results]) total = np.sum([rb['n'] for rb in results]) gtn = np.sum([rb['m'] for rb in results]) line = 'mean_ratio:{:.4f}, valids:{}, total:{}, gtn:{}'\ .format(mean_ratio, valids, total, gtn) return line, mean_ratio def compute_ji_matching(self, dt_boxes, gt_boxes): """Match the annotation box for each detection box. Args: dt_boxes(ndarray): Detection boxes. gt_boxes(ndarray): Ground_truth boxes. Returns: matches_(list[tuple[int, int]]): Match result. """ assert dt_boxes.shape[-1] > 3 and gt_boxes.shape[-1] > 3 if dt_boxes.shape[0] < 1 or gt_boxes.shape[0] < 1: return list() ious = bbox_overlaps(dt_boxes, gt_boxes, mode='iou') input_ = copy.deepcopy(ious) input_[input_ < self.iou_thres] = 0 match_scipy = maximum_bipartite_matching( csr_matrix(input_), perm_type='column') matches_ = [] for i in range(len(match_scipy)): if match_scipy[i] != -1: matches_.append((i, int(match_scipy[i]))) return matches_ def get_ignores(self, dt_boxes, gt_boxes): """Get the number of ignore bboxes.""" if gt_boxes.size: ioas = bbox_overlaps(dt_boxes, gt_boxes, mode='iof') ioas = np.max(ioas, axis=1) rows = np.where(ioas > self.iou_thres)[0] return len(rows) else: return 0 class Image(object): """Data structure for evaluation of CrowdHuman. Note: This implementation is modified from https://github.com/Purkialo/ CrowdDet/blob/master/lib/evaluate/APMRToolkits/image.py Args: mode (int): Select the mode of evaluate. Valid mode include 0(just body box), 1(just head box) and 2(both of them). Defaults to 0. """ def __init__(self, mode): self.ID = None self.width = None self.height = None self.dt_boxes = None self.gt_boxes = None self.eval_mode = mode self.ign_num = None self.gt_num = None self.dt_num = None def load(self, record, body_key, head_key, class_names, gt_flag): """Loading information for evaluation. Args: record (dict): Label information or test results. The format might look something like this: { 'ID': '273271,c9db000d5146c15', 'gtboxes': [ {'fbox': [72, 202, 163, 503], 'tag': 'person', ...}, {'fbox': [199, 180, 144, 499], 'tag': 'person', ...}, ... ] } or: { 'ID': '273271,c9db000d5146c15', 'width': 800, 'height': 1067, 'dtboxes': [ { 'box': [306.22, 205.95, 164.05, 394.04], 'score': 0.99, 'tag': 1 }, { 'box': [403.60, 178.66, 157.15, 421.33], 'score': 0.99, 'tag': 1 }, ... ] } body_key (str, None): key of detection body box. Valid when loading detection results and self.eval_mode!=1. head_key (str, None): key of detection head box. Valid when loading detection results and self.eval_mode!=0. class_names (list[str]):class names of data set. Defaults to ['background', 'person']. gt_flag (bool): Indicate whether record is ground truth or predicting the outcome. """ if 'ID' in record and self.ID is None: self.ID = record['ID'] if 'width' in record and self.width is None: self.width = record['width'] if 'height' in record and self.height is None: self.height = record['height'] if gt_flag: self.gt_num = len(record['gtboxes']) body_bbox, head_bbox = self.load_gt_boxes(record, 'gtboxes', class_names) if self.eval_mode == 0: self.gt_boxes = body_bbox self.ign_num = (body_bbox[:, -1] == -1).sum() elif self.eval_mode == 1: self.gt_boxes = head_bbox self.ign_num = (head_bbox[:, -1] == -1).sum() else: gt_tag = np.array([ body_bbox[i, -1] != -1 and head_bbox[i, -1] != -1 for i in range(len(body_bbox)) ]) self.ign_num = (gt_tag == 0).sum() self.gt_boxes = np.hstack( (body_bbox[:, :-1], head_bbox[:, :-1], gt_tag.reshape(-1, 1))) if not gt_flag: self.dt_num = len(record['dtboxes']) if self.eval_mode == 0: self.dt_boxes = self.load_det_boxes(record, 'dtboxes', body_key, 'score') elif self.eval_mode == 1: self.dt_boxes = self.load_det_boxes(record, 'dtboxes', head_key, 'score') else: body_dtboxes = self.load_det_boxes(record, 'dtboxes', body_key, 'score') head_dtboxes = self.load_det_boxes(record, 'dtboxes', head_key, 'score') self.dt_boxes = np.hstack((body_dtboxes, head_dtboxes)) @staticmethod def load_gt_boxes(dict_input, key_name, class_names): """load ground_truth and transform [x, y, w, h] to [x1, y1, x2, y2]""" assert key_name in dict_input if len(dict_input[key_name]) < 1: return np.empty([0, 5]) head_bbox = [] body_bbox = [] for rb in dict_input[key_name]: if rb['tag'] in class_names: body_tag = class_names.index(rb['tag']) head_tag = copy.deepcopy(body_tag) else: body_tag = -1 head_tag = -1 if 'extra' in rb: if 'ignore' in rb['extra']: if rb['extra']['ignore'] != 0: body_tag = -1 head_tag = -1 if 'head_attr' in rb: if 'ignore' in rb['head_attr']: if rb['head_attr']['ignore'] != 0: head_tag = -1 head_bbox.append(np.hstack((rb['hbox'], head_tag))) body_bbox.append(np.hstack((rb['fbox'], body_tag))) head_bbox = np.array(head_bbox) head_bbox[:, 2:4] += head_bbox[:, :2] body_bbox = np.array(body_bbox) body_bbox[:, 2:4] += body_bbox[:, :2] return body_bbox, head_bbox @staticmethod def load_det_boxes(dict_input, key_name, key_box, key_score, key_tag=None): """load detection boxes.""" assert key_name in dict_input if len(dict_input[key_name]) < 1: return np.empty([0, 5]) else: assert key_box in dict_input[key_name][0] if key_score: assert key_score in dict_input[key_name][0] if key_tag: assert key_tag in dict_input[key_name][0] if key_score: if key_tag: bboxes = np.vstack([ np.hstack((rb[key_box], rb[key_score], rb[key_tag])) for rb in dict_input[key_name] ]) else: bboxes = np.vstack([ np.hstack((rb[key_box], rb[key_score])) for rb in dict_input[key_name] ]) else: if key_tag: bboxes = np.vstack([ np.hstack((rb[key_box], rb[key_tag])) for rb in dict_input[key_name] ]) else: bboxes = np.vstack( [rb[key_box] for rb in dict_input[key_name]]) bboxes[:, 2:4] += bboxes[:, :2] return bboxes def clip_all_boader(self): """Make sure boxes are within the image range.""" def _clip_boundary(boxes, height, width): assert boxes.shape[-1] >= 4 boxes[:, 0] = np.minimum(np.maximum(boxes[:, 0], 0), width - 1) boxes[:, 1] = np.minimum(np.maximum(boxes[:, 1], 0), height - 1) boxes[:, 2] = np.maximum(np.minimum(boxes[:, 2], width), 0) boxes[:, 3] = np.maximum(np.minimum(boxes[:, 3], height), 0) return boxes assert self.dt_boxes.shape[-1] >= 4 assert self.gt_boxes.shape[-1] >= 4 assert self.width is not None and self.height is not None if self.eval_mode == 2: self.dt_boxes[:, :4] = _clip_boundary(self.dt_boxes[:, :4], self.height, self.width) self.gt_boxes[:, :4] = _clip_boundary(self.gt_boxes[:, :4], self.height, self.width) self.dt_boxes[:, 4:8] = _clip_boundary(self.dt_boxes[:, 4:8], self.height, self.width) self.gt_boxes[:, 4:8] = _clip_boundary(self.gt_boxes[:, 4:8], self.height, self.width) else: self.dt_boxes = _clip_boundary(self.dt_boxes, self.height, self.width) self.gt_boxes = _clip_boundary(self.gt_boxes, self.height, self.width) def compare_voc(self, thres): """Match the detection results with the ground_truth by VOC. Args: thres (float): IOU threshold. Returns: score_list(list[tuple[ndarray, int, str]]): Matching result. a list of tuples (dtbox, label, imgID) in the descending sort of dtbox.score. """ if self.dt_boxes is None: return list() dtboxes = self.dt_boxes gtboxes = self.gt_boxes if self.gt_boxes is not None else list() dtboxes.sort(key=lambda x: x.score, reverse=True) gtboxes.sort(key=lambda x: x.ign) score_list = list() for i, dt in enumerate(dtboxes): maxpos = -1 maxiou = thres for j, gt in enumerate(gtboxes): overlap = dt.iou(gt) if overlap > maxiou: maxiou = overlap maxpos = j if maxpos >= 0: if gtboxes[maxpos].ign == 0: gtboxes[maxpos].matched = 1 dtboxes[i].matched = 1 score_list.append((dt, self.ID)) else: dtboxes[i].matched = -1 else: dtboxes[i].matched = 0 score_list.append((dt, self.ID)) return score_list def compare_caltech(self, thres): """Match the detection results with the ground_truth by Caltech matching strategy. Args: thres (float): IOU threshold. Returns: score_list(list[tuple[ndarray, int, str]]): Matching result. a list of tuples (dtbox, label, imgID) in the descending sort of dtbox.score. """ if self.dt_boxes is None or self.gt_boxes is None: return list() dtboxes = self.dt_boxes if self.dt_boxes is not None else list() gtboxes = self.gt_boxes if self.gt_boxes is not None else list() dt_matched = np.zeros(dtboxes.shape[0]) gt_matched = np.zeros(gtboxes.shape[0]) dtboxes = np.array(sorted(dtboxes, key=lambda x: x[-1], reverse=True)) gtboxes = np.array(sorted(gtboxes, key=lambda x: x[-1], reverse=True)) if len(dtboxes): overlap_iou = bbox_overlaps(dtboxes, gtboxes, mode='iou') overlap_ioa = bbox_overlaps(dtboxes, gtboxes, mode='iof') else: return list() score_list = list() for i, dt in enumerate(dtboxes): maxpos = -1 maxiou = thres for j, gt in enumerate(gtboxes): if gt_matched[j] == 1: continue if gt[-1] > 0: overlap = overlap_iou[i][j] if overlap > maxiou: maxiou = overlap maxpos = j else: if maxpos >= 0: break else: overlap = overlap_ioa[i][j] if overlap > thres: maxiou = overlap maxpos = j if maxpos >= 0: if gtboxes[maxpos, -1] > 0: gt_matched[maxpos] = 1 dt_matched[i] = 1 score_list.append((dt, 1, self.ID)) else: dt_matched[i] = -1 else: dt_matched[i] = 0 score_list.append((dt, 0, self.ID)) return score_list ================================================ FILE: mmdet/evaluation/metrics/dump_det_results.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings from typing import Sequence from mmengine.evaluator import DumpResults from mmengine.evaluator.metric import _to_cpu from mmdet.registry import METRICS from mmdet.structures.mask import encode_mask_results @METRICS.register_module() class DumpDetResults(DumpResults): """Dump model predictions to a pickle file for offline evaluation. Different from `DumpResults` in MMEngine, it compresses instance segmentation masks into RLE format. Args: out_file_path (str): Path of the dumped file. Must end with '.pkl' or '.pickle'. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. """ def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: """transfer tensors in predictions to CPU.""" data_samples = _to_cpu(data_samples) for data_sample in data_samples: # remove gt data_sample.pop('gt_instances', None) data_sample.pop('ignored_instances', None) data_sample.pop('gt_panoptic_seg', None) if 'pred_instances' in data_sample: pred = data_sample['pred_instances'] # encode mask to RLE if 'masks' in pred: pred['masks'] = encode_mask_results(pred['masks'].numpy()) if 'pred_panoptic_seg' in data_sample: warnings.warn( 'Panoptic segmentation map will not be compressed. ' 'The dumped file will be extremely large! ' 'Suggest using `CocoPanopticMetric` to save the coco ' 'format json and segmentation png files directly.') self.results.extend(data_samples) ================================================ FILE: mmdet/evaluation/metrics/dump_proposals_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os import os.path as osp from typing import Optional, Sequence from mmengine.dist import is_main_process from mmengine.evaluator import BaseMetric from mmengine.fileio import dump from mmengine.logging import MMLogger from mmengine.structures import InstanceData from mmdet.registry import METRICS @METRICS.register_module() class DumpProposals(BaseMetric): """Dump proposals pseudo metric. Args: output_dir (str): The root directory for ``proposals_file``. Defaults to ''. proposals_file (str): Proposals file path. Defaults to 'proposals.pkl'. num_max_proposals (int, optional): Maximum number of proposals to dump. If not specified, all proposals will be dumped. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. prefix (str, optional): The prefix that will be added in the metric names to disambiguate homonymous metrics of different evaluators. If prefix is not provided in the argument, self.default_prefix will be used instead. Defaults to None. """ default_prefix: Optional[str] = 'dump_proposals' def __init__(self, output_dir: str = '', proposals_file: str = 'proposals.pkl', num_max_proposals: Optional[int] = None, file_client_args: dict = dict(backend='disk'), collect_device: str = 'cpu', prefix: Optional[str] = None) -> None: super().__init__(collect_device=collect_device, prefix=prefix) self.num_max_proposals = num_max_proposals # TODO: update after mmengine finish refactor fileio. self.file_client_args = file_client_args self.output_dir = output_dir assert proposals_file.endswith(('.pkl', '.pickle')), \ 'The output file must be a pkl file.' self.proposals_file = os.path.join(self.output_dir, proposals_file) if is_main_process(): os.makedirs(self.output_dir, exist_ok=True) def process(self, data_batch: Sequence[dict], data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. The processed results should be stored in ``self.results``, which will be used to compute the metrics when all batches have been processed. Args: data_batch (dict): A batch of data from the dataloader. data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ for data_sample in data_samples: pred = data_sample['pred_instances'] # `bboxes` is sorted by `scores` ranked_scores, rank_inds = pred['scores'].sort(descending=True) ranked_bboxes = pred['bboxes'][rank_inds, :] ranked_bboxes = ranked_bboxes.cpu().numpy() ranked_scores = ranked_scores.cpu().numpy() pred_instance = InstanceData() pred_instance.bboxes = ranked_bboxes pred_instance.scores = ranked_scores if self.num_max_proposals is not None: pred_instance = pred_instance[:self.num_max_proposals] img_path = data_sample['img_path'] # `file_name` is the key to obtain the proposals from the # `proposals_list`. file_name = osp.join( osp.split(osp.split(img_path)[0])[-1], osp.split(img_path)[-1]) result = {file_name: pred_instance} self.results.append(result) def compute_metrics(self, results: list) -> dict: """Dump the processed results. Args: results (list): The processed results of each batch. Returns: dict: An empty dict. """ logger: MMLogger = MMLogger.get_current_instance() dump_results = {} for result in results: dump_results.update(result) dump( dump_results, file=self.proposals_file, file_client_args=self.file_client_args) logger.info(f'Results are saved at {self.proposals_file}') return {} ================================================ FILE: mmdet/evaluation/metrics/lvis_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import itertools import os.path as osp import tempfile import warnings from collections import OrderedDict from typing import Dict, List, Optional, Sequence, Union import numpy as np from mmengine.logging import MMLogger from terminaltables import AsciiTable from mmdet.registry import METRICS from mmdet.structures.mask import encode_mask_results from ..functional import eval_recalls from .coco_metric import CocoMetric try: import lvis if getattr(lvis, '__version__', '0') >= '10.5.3': warnings.warn( 'mmlvis is deprecated, please install official lvis-api by "pip install git+https://github.com/lvis-dataset/lvis-api.git"', # noqa: E501 UserWarning) from lvis import LVIS, LVISEval, LVISResults except ImportError: lvis = None LVISEval = None LVISResults = None @METRICS.register_module() class LVISMetric(CocoMetric): """LVIS evaluation metric. Args: ann_file (str, optional): Path to the coco format annotation file. If not specified, ground truth annotations from the dataset will be converted to coco format. Defaults to None. metric (str | List[str]): Metrics to be evaluated. Valid metrics include 'bbox', 'segm', 'proposal', and 'proposal_fast'. Defaults to 'bbox'. classwise (bool): Whether to evaluate the metric class-wise. Defaults to False. proposal_nums (Sequence[int]): Numbers of proposals to be evaluated. Defaults to (100, 300, 1000). iou_thrs (float | List[float], optional): IoU threshold to compute AP and AR. If not specified, IoUs from 0.5 to 0.95 will be used. Defaults to None. metric_items (List[str], optional): Metric result names to be recorded in the evaluation result. Defaults to None. format_only (bool): Format the output results without perform evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to False. outfile_prefix (str, optional): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Defaults to None. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. prefix (str, optional): The prefix that will be added in the metric names to disambiguate homonymous metrics of different evaluators. If prefix is not provided in the argument, self.default_prefix will be used instead. Defaults to None. """ default_prefix: Optional[str] = 'lvis' def __init__(self, ann_file: Optional[str] = None, metric: Union[str, List[str]] = 'bbox', classwise: bool = False, proposal_nums: Sequence[int] = (100, 300, 1000), iou_thrs: Optional[Union[float, Sequence[float]]] = None, metric_items: Optional[Sequence[str]] = None, format_only: bool = False, outfile_prefix: Optional[str] = None, collect_device: str = 'cpu', prefix: Optional[str] = None) -> None: if lvis is None: raise RuntimeError( 'Package lvis is not installed. Please run "pip install ' 'git+https://github.com/lvis-dataset/lvis-api.git".') super().__init__(collect_device=collect_device, prefix=prefix) # coco evaluation metrics self.metrics = metric if isinstance(metric, list) else [metric] allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] for metric in self.metrics: if metric not in allowed_metrics: raise KeyError( "metric should be one of 'bbox', 'segm', 'proposal', " f"'proposal_fast', but got {metric}.") # do class wise evaluation, default False self.classwise = classwise # proposal_nums used to compute recall or precision. self.proposal_nums = list(proposal_nums) # iou_thrs used to compute recall or precision. if iou_thrs is None: iou_thrs = np.linspace( .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) self.iou_thrs = iou_thrs self.metric_items = metric_items self.format_only = format_only if self.format_only: assert outfile_prefix is not None, 'outfile_prefix must be not' 'None when format_only is True, otherwise the result files will' 'be saved to a temp directory which will be cleaned up at the end.' self.outfile_prefix = outfile_prefix # if ann_file is not specified, # initialize lvis api with the converted dataset self._lvis_api = LVIS(ann_file) if ann_file else None # handle dataset lazy init self.cat_ids = None self.img_ids = None def fast_eval_recall(self, results: List[dict], proposal_nums: Sequence[int], iou_thrs: Sequence[float], logger: Optional[MMLogger] = None) -> np.ndarray: """Evaluate proposal recall with LVIS's fast_eval_recall. Args: results (List[dict]): Results of the dataset. proposal_nums (Sequence[int]): Proposal numbers used for evaluation. iou_thrs (Sequence[float]): IoU thresholds used for evaluation. logger (MMLogger, optional): Logger used for logging the recall summary. Returns: np.ndarray: Averaged recall results. """ gt_bboxes = [] pred_bboxes = [result['bboxes'] for result in results] for i in range(len(self.img_ids)): ann_ids = self._lvis_api.get_ann_ids(img_ids=[self.img_ids[i]]) ann_info = self._lvis_api.load_anns(ann_ids) if len(ann_info) == 0: gt_bboxes.append(np.zeros((0, 4))) continue bboxes = [] for ann in ann_info: x1, y1, w, h = ann['bbox'] bboxes.append([x1, y1, x1 + w, y1 + h]) bboxes = np.array(bboxes, dtype=np.float32) if bboxes.shape[0] == 0: bboxes = np.zeros((0, 4)) gt_bboxes.append(bboxes) recalls = eval_recalls( gt_bboxes, pred_bboxes, proposal_nums, iou_thrs, logger=logger) ar = recalls.mean(axis=1) return ar # TODO: data_batch is no longer needed, consider adjusting the # parameter position def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. The processed results should be stored in ``self.results``, which will be used to compute the metrics when all batches have been processed. Args: data_batch (dict): A batch of data from the dataloader. data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ for data_sample in data_samples: result = dict() pred = data_sample['pred_instances'] result['img_id'] = data_sample['img_id'] result['bboxes'] = pred['bboxes'].cpu().numpy() result['scores'] = pred['scores'].cpu().numpy() result['labels'] = pred['labels'].cpu().numpy() # encode mask to RLE if 'masks' in pred: result['masks'] = encode_mask_results( pred['masks'].detach().cpu().numpy()) # some detectors use different scores for bbox and mask if 'mask_scores' in pred: result['mask_scores'] = pred['mask_scores'].cpu().numpy() # parse gt gt = dict() gt['width'] = data_sample['ori_shape'][1] gt['height'] = data_sample['ori_shape'][0] gt['img_id'] = data_sample['img_id'] if self._lvis_api is None: # TODO: Need to refactor to support LoadAnnotations assert 'instances' in data_sample, \ 'ground truth is required for evaluation when ' \ '`ann_file` is not provided' gt['anns'] = data_sample['instances'] # add converted result to the results list self.results.append((gt, result)) def compute_metrics(self, results: list) -> Dict[str, float]: """Compute the metrics from processed results. Args: results (list): The processed results of each batch. Returns: Dict[str, float]: The computed metrics. The keys are the names of the metrics, and the values are corresponding results. """ logger: MMLogger = MMLogger.get_current_instance() # split gt and prediction list gts, preds = zip(*results) tmp_dir = None if self.outfile_prefix is None: tmp_dir = tempfile.TemporaryDirectory() outfile_prefix = osp.join(tmp_dir.name, 'results') else: outfile_prefix = self.outfile_prefix if self._lvis_api is None: # use converted gt json file to initialize coco api logger.info('Converting ground truth to coco format...') coco_json_path = self.gt_to_coco_json( gt_dicts=gts, outfile_prefix=outfile_prefix) self._lvis_api = LVIS(coco_json_path) # handle lazy init if self.cat_ids is None: self.cat_ids = self._lvis_api.get_cat_ids() if self.img_ids is None: self.img_ids = self._lvis_api.get_img_ids() # convert predictions to coco format and dump to json file result_files = self.results2json(preds, outfile_prefix) eval_results = OrderedDict() if self.format_only: logger.info('results are saved in ' f'{osp.dirname(outfile_prefix)}') return eval_results lvis_gt = self._lvis_api for metric in self.metrics: logger.info(f'Evaluating {metric}...') # TODO: May refactor fast_eval_recall to an independent metric? # fast eval recall if metric == 'proposal_fast': ar = self.fast_eval_recall( preds, self.proposal_nums, self.iou_thrs, logger=logger) log_msg = [] for i, num in enumerate(self.proposal_nums): eval_results[f'AR@{num}'] = ar[i] log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}') log_msg = ''.join(log_msg) logger.info(log_msg) continue try: lvis_dt = LVISResults(lvis_gt, result_files[metric]) except IndexError: logger.info( 'The testing results of the whole dataset is empty.') break iou_type = 'bbox' if metric == 'proposal' else metric lvis_eval = LVISEval(lvis_gt, lvis_dt, iou_type) lvis_eval.params.imgIds = self.img_ids metric_items = self.metric_items if metric == 'proposal': lvis_eval.params.useCats = 0 lvis_eval.params.maxDets = list(self.proposal_nums) lvis_eval.evaluate() lvis_eval.accumulate() lvis_eval.summarize() if metric_items is None: metric_items = ['AR@300', 'ARs@300', 'ARm@300', 'ARl@300'] for k, v in lvis_eval.get_results().items(): if k in metric_items: val = float('{:.3f}'.format(float(v))) eval_results[k] = val else: lvis_eval.evaluate() lvis_eval.accumulate() lvis_eval.summarize() lvis_results = lvis_eval.get_results() if self.classwise: # Compute per-category AP # Compute per-category AP # from https://github.com/facebookresearch/detectron2/ precisions = lvis_eval.eval['precision'] # precision: (iou, recall, cls, area range, max dets) assert len(self.cat_ids) == precisions.shape[2] results_per_category = [] for idx, catId in enumerate(self.cat_ids): # area range index 0: all area ranges # max dets index -1: typically 100 per image # the dimensions of precisions are # [num_thrs, num_recalls, num_cats, num_area_rngs] nm = self._lvis_api.load_cats([catId])[0] precision = precisions[:, :, idx, 0] precision = precision[precision > -1] if precision.size: ap = np.mean(precision) else: ap = float('nan') results_per_category.append( (f'{nm["name"]}', f'{float(ap):0.3f}')) eval_results[f'{nm["name"]}_precision'] = round(ap, 3) num_columns = min(6, len(results_per_category) * 2) results_flatten = list( itertools.chain(*results_per_category)) headers = ['category', 'AP'] * (num_columns // 2) results_2d = itertools.zip_longest(*[ results_flatten[i::num_columns] for i in range(num_columns) ]) table_data = [headers] table_data += [result for result in results_2d] table = AsciiTable(table_data) logger.info('\n' + table.table) if metric_items is None: metric_items = [ 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'APr', 'APc', 'APf' ] for k, v in lvis_results.items(): if k in metric_items: key = '{}_{}'.format(metric, k) val = float('{:.3f}'.format(float(v))) eval_results[key] = val lvis_eval.print_results() if tmp_dir is not None: tmp_dir.cleanup() return eval_results ================================================ FILE: mmdet/evaluation/metrics/openimages_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from collections import OrderedDict from typing import List, Optional, Sequence, Union import numpy as np from mmengine.evaluator import BaseMetric from mmengine.logging import MMLogger, print_log from mmdet.registry import METRICS from ..functional import eval_map @METRICS.register_module() class OpenImagesMetric(BaseMetric): """OpenImages evaluation metric. Evaluate detection mAP for OpenImages. Please refer to https://storage.googleapis.com/openimages/web/evaluation.html for more details. Args: iou_thrs (float or List[float]): IoU threshold. Defaults to 0.5. ioa_thrs (float or List[float]): IoA threshold. Defaults to 0.5. scale_ranges (List[tuple], optional): Scale ranges for evaluating mAP. If not specified, all bounding boxes would be included in evaluation. Defaults to None use_group_of (bool): Whether consider group of groud truth bboxes during evaluating. Defaults to True. get_supercategory (bool): Whether to get parent class of the current class. Default: True. filter_labels (bool): Whether filter unannotated classes. Default: True. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. prefix (str, optional): The prefix that will be added in the metric names to disambiguate homonymous metrics of different evaluators. If prefix is not provided in the argument, self.default_prefix will be used instead. Defaults to None. """ default_prefix: Optional[str] = 'openimages' def __init__(self, iou_thrs: Union[float, List[float]] = 0.5, ioa_thrs: Union[float, List[float]] = 0.5, scale_ranges: Optional[List[tuple]] = None, use_group_of: bool = True, get_supercategory: bool = True, filter_labels: bool = True, collect_device: str = 'cpu', prefix: Optional[str] = None) -> None: super().__init__(collect_device=collect_device, prefix=prefix) self.iou_thrs = [iou_thrs] if isinstance(iou_thrs, float) else iou_thrs self.ioa_thrs = [ioa_thrs] if (isinstance(ioa_thrs, float) or ioa_thrs is None) else ioa_thrs assert isinstance(self.iou_thrs, list) and isinstance( self.ioa_thrs, list) assert len(self.iou_thrs) == len(self.ioa_thrs) self.scale_ranges = scale_ranges self.use_group_of = use_group_of self.get_supercategory = get_supercategory self.filter_labels = filter_labels def _get_supercategory_ann(self, instances: List[dict]) -> List[dict]: """Get parent classes's annotation of the corresponding class. Args: instances (List[dict]): A list of annotations of the instances. Returns: List[dict]: Annotations extended with super-category. """ supercat_instances = [] relation_matrix = self.dataset_meta['RELATION_MATRIX'] for instance in instances: labels = np.where(relation_matrix[instance['bbox_label']])[0] for label in labels: if label == instance['bbox_label']: continue new_instance = copy.deepcopy(instance) new_instance['bbox_label'] = label supercat_instances.append(new_instance) return supercat_instances def _process_predictions(self, pred_bboxes: np.ndarray, pred_scores: np.ndarray, pred_labels: np.ndarray, gt_instances: list, image_level_labels: np.ndarray) -> tuple: """Process results of the corresponding class of the detection bboxes. Note: It will choose to do the following two processing according to the parameters: 1. Whether to add parent classes of the corresponding class of the detection bboxes. 2. Whether to ignore the classes that unannotated on that image. Args: pred_bboxes (np.ndarray): bboxes predicted by the model pred_scores (np.ndarray): scores predicted by the model pred_labels (np.ndarray): labels predicted by the model gt_instances (list): ground truth annotations image_level_labels (np.ndarray): human-verified image level labels Returns: tuple: Processed bboxes, scores, and labels. """ processed_bboxes = copy.deepcopy(pred_bboxes) processed_scores = copy.deepcopy(pred_scores) processed_labels = copy.deepcopy(pred_labels) gt_labels = np.array([ins['bbox_label'] for ins in gt_instances], dtype=np.int64) if image_level_labels is not None: allowed_classes = np.unique( np.append(gt_labels, image_level_labels)) else: allowed_classes = np.unique(gt_labels) relation_matrix = self.dataset_meta['RELATION_MATRIX'] pred_classes = np.unique(pred_labels) for pred_class in pred_classes: classes = np.where(relation_matrix[pred_class])[0] for cls in classes: if (cls in allowed_classes and cls != pred_class and self.get_supercategory): # add super-supercategory preds index = np.where(pred_labels == pred_class)[0] processed_scores = np.concatenate( [processed_scores, pred_scores[index]]) processed_bboxes = np.concatenate( [processed_bboxes, pred_bboxes[index]]) extend_labels = np.full(index.shape, cls, dtype=np.int64) processed_labels = np.concatenate( [processed_labels, extend_labels]) elif cls not in allowed_classes and self.filter_labels: # remove unannotated preds index = np.where(processed_labels != cls)[0] processed_scores = processed_scores[index] processed_bboxes = processed_bboxes[index] processed_labels = processed_labels[index] return processed_bboxes, processed_scores, processed_labels # TODO: data_batch is no longer needed, consider adjusting the # parameter position def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. The processed results should be stored in ``self.results``, which will be used to compute the metrics when all batches have been processed. Args: data_batch (dict): A batch of data from the dataloader. data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ for data_sample in data_samples: gt = copy.deepcopy(data_sample) # add super-category instances # TODO: Need to refactor to support LoadAnnotations instances = gt['instances'] if self.get_supercategory: supercat_instances = self._get_supercategory_ann(instances) instances.extend(supercat_instances) gt_labels = [] gt_bboxes = [] is_group_ofs = [] for ins in instances: gt_labels.append(ins['bbox_label']) gt_bboxes.append(ins['bbox']) is_group_ofs.append(ins['is_group_of']) ann = dict( labels=np.array(gt_labels, dtype=np.int64), bboxes=np.array(gt_bboxes, dtype=np.float32).reshape((-1, 4)), gt_is_group_ofs=np.array(is_group_ofs, dtype=bool)) image_level_labels = gt.get('image_level_labels', None) pred = data_sample['pred_instances'] pred_bboxes = pred['bboxes'].cpu().numpy() pred_scores = pred['scores'].cpu().numpy() pred_labels = pred['labels'].cpu().numpy() pred_bboxes, pred_scores, pred_labels = self._process_predictions( pred_bboxes, pred_scores, pred_labels, instances, image_level_labels) dets = [] for label in range(len(self.dataset_meta['classes'])): index = np.where(pred_labels == label)[0] pred_bbox_scores = np.hstack( [pred_bboxes[index], pred_scores[index].reshape((-1, 1))]) dets.append(pred_bbox_scores) self.results.append((ann, dets)) def compute_metrics(self, results: list) -> dict: """Compute the metrics from processed results. Args: results (list): The processed results of each batch. Returns: dict: The computed metrics. The keys are the names of the metrics, and the values are corresponding results. """ logger = MMLogger.get_current_instance() gts, preds = zip(*results) eval_results = OrderedDict() # get dataset type dataset_type = self.dataset_meta.get('dataset_type') if dataset_type not in ['oid_challenge', 'oid_v6']: dataset_type = 'oid_v6' print_log( 'Cannot infer dataset type from the length of the' ' classes. Set `oid_v6` as dataset type.', logger='current') mean_aps = [] for i, (iou_thr, ioa_thr) in enumerate(zip(self.iou_thrs, self.ioa_thrs)): if self.use_group_of: assert ioa_thr is not None, 'ioa_thr must have value when' \ ' using group_of in evaluation.' print_log(f'\n{"-" * 15}iou_thr, ioa_thr: {iou_thr}, {ioa_thr}' f'{"-" * 15}') mean_ap, _ = eval_map( preds, gts, scale_ranges=self.scale_ranges, iou_thr=iou_thr, ioa_thr=ioa_thr, dataset=dataset_type, logger=logger, use_group_of=self.use_group_of) mean_aps.append(mean_ap) eval_results[f'AP{int(iou_thr * 100):02d}'] = round(mean_ap, 3) eval_results['mAP'] = sum(mean_aps) / len(mean_aps) return eval_results ================================================ FILE: mmdet/evaluation/metrics/voc_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import warnings from collections import OrderedDict from typing import List, Optional, Sequence, Union import numpy as np from mmengine.evaluator import BaseMetric from mmengine.logging import MMLogger from mmdet.registry import METRICS from ..functional import eval_map, eval_recalls @METRICS.register_module() class VOCMetric(BaseMetric): """Pascal VOC evaluation metric. Args: iou_thrs (float or List[float]): IoU threshold. Defaults to 0.5. scale_ranges (List[tuple], optional): Scale ranges for evaluating mAP. If not specified, all bounding boxes would be included in evaluation. Defaults to None. metric (str | list[str]): Metrics to be evaluated. Options are 'mAP', 'recall'. If is list, the first setting in the list will be used to evaluate metric. proposal_nums (Sequence[int]): Proposal number used for evaluating recalls, such as recall@100, recall@1000. Default: (100, 300, 1000). eval_mode (str): 'area' or '11points', 'area' means calculating the area under precision-recall curve, '11points' means calculating the average precision of recalls at [0, 0.1, ..., 1]. The PASCAL VOC2007 defaults to use '11points', while PASCAL VOC2012 defaults to use 'area'. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. prefix (str, optional): The prefix that will be added in the metric names to disambiguate homonymous metrics of different evaluators. If prefix is not provided in the argument, self.default_prefix will be used instead. Defaults to None. """ default_prefix: Optional[str] = 'pascal_voc' def __init__(self, iou_thrs: Union[float, List[float]] = 0.5, scale_ranges: Optional[List[tuple]] = None, metric: Union[str, List[str]] = 'mAP', proposal_nums: Sequence[int] = (100, 300, 1000), eval_mode: str = '11points', collect_device: str = 'cpu', prefix: Optional[str] = None) -> None: super().__init__(collect_device=collect_device, prefix=prefix) self.iou_thrs = [iou_thrs] if isinstance(iou_thrs, float) \ else iou_thrs self.scale_ranges = scale_ranges # voc evaluation metrics if not isinstance(metric, str): assert len(metric) == 1 metric = metric[0] allowed_metrics = ['recall', 'mAP'] if metric not in allowed_metrics: raise KeyError( f"metric should be one of 'recall', 'mAP', but got {metric}.") self.metric = metric self.proposal_nums = proposal_nums assert eval_mode in ['area', '11points'], \ 'Unrecognized mode, only "area" and "11points" are supported' self.eval_mode = eval_mode # TODO: data_batch is no longer needed, consider adjusting the # parameter position def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. The processed results should be stored in ``self.results``, which will be used to compute the metrics when all batches have been processed. Args: data_batch (dict): A batch of data from the dataloader. data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ for data_sample in data_samples: gt = copy.deepcopy(data_sample) # TODO: Need to refactor to support LoadAnnotations gt_instances = gt['gt_instances'] gt_ignore_instances = gt['ignored_instances'] ann = dict( labels=gt_instances['labels'].cpu().numpy(), bboxes=gt_instances['bboxes'].cpu().numpy(), bboxes_ignore=gt_ignore_instances['bboxes'].cpu().numpy(), labels_ignore=gt_ignore_instances['labels'].cpu().numpy()) pred = data_sample['pred_instances'] pred_bboxes = pred['bboxes'].cpu().numpy() pred_scores = pred['scores'].cpu().numpy() pred_labels = pred['labels'].cpu().numpy() dets = [] for label in range(len(self.dataset_meta['classes'])): index = np.where(pred_labels == label)[0] pred_bbox_scores = np.hstack( [pred_bboxes[index], pred_scores[index].reshape((-1, 1))]) dets.append(pred_bbox_scores) self.results.append((ann, dets)) def compute_metrics(self, results: list) -> dict: """Compute the metrics from processed results. Args: results (list): The processed results of each batch. Returns: dict: The computed metrics. The keys are the names of the metrics, and the values are corresponding results. """ logger: MMLogger = MMLogger.get_current_instance() gts, preds = zip(*results) eval_results = OrderedDict() if self.metric == 'mAP': assert isinstance(self.iou_thrs, list) dataset_type = self.dataset_meta.get('dataset_type') if dataset_type in ['VOC2007', 'VOC2012']: dataset_name = 'voc' if dataset_type == 'VOC2007' and self.eval_mode != '11points': warnings.warn('Pascal VOC2007 uses `11points` as default ' 'evaluate mode, but you are using ' f'{self.eval_mode}.') elif dataset_type == 'VOC2012' and self.eval_mode != 'area': warnings.warn('Pascal VOC2012 uses `area` as default ' 'evaluate mode, but you are using ' f'{self.eval_mode}.') else: dataset_name = self.dataset_meta['classes'] mean_aps = [] for iou_thr in self.iou_thrs: logger.info(f'\n{"-" * 15}iou_thr: {iou_thr}{"-" * 15}') # Follow the official implementation, # http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCdevkit_18-May-2011.tar # we should use the legacy coordinate system in mmdet 1.x, # which means w, h should be computed as 'x2 - x1 + 1` and # `y2 - y1 + 1` mean_ap, _ = eval_map( preds, gts, scale_ranges=self.scale_ranges, iou_thr=iou_thr, dataset=dataset_name, logger=logger, eval_mode=self.eval_mode, use_legacy_coordinate=True) mean_aps.append(mean_ap) eval_results[f'AP{int(iou_thr * 100):02d}'] = round(mean_ap, 3) eval_results['mAP'] = sum(mean_aps) / len(mean_aps) eval_results.move_to_end('mAP', last=False) elif self.metric == 'recall': # TODO: Currently not checked. gt_bboxes = [ann['bboxes'] for ann in self.annotations] recalls = eval_recalls( gt_bboxes, results, self.proposal_nums, self.iou_thrs, logger=logger, use_legacy_coordinate=True) for i, num in enumerate(self.proposal_nums): for j, iou_thr in enumerate(self.iou_thrs): eval_results[f'recall@{num}@{iou_thr}'] = recalls[i, j] if recalls.shape[1] > 1: ar = recalls.mean(axis=1) for i, num in enumerate(self.proposal_nums): eval_results[f'AR@{num}'] = ar[i] return eval_results ================================================ FILE: mmdet/models/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .backbones import * # noqa: F401,F403 from .data_preprocessors import * # noqa: F401,F403 from .dense_heads import * # noqa: F401,F403 from .detectors import * # noqa: F401,F403 from .layers import * # noqa: F401,F403 from .losses import * # noqa: F401,F403 from .necks import * # noqa: F401,F403 from .roi_heads import * # noqa: F401,F403 from .seg_heads import * # noqa: F401,F403 from .task_modules import * # noqa: F401,F403 from .test_time_augs import * # noqa: F401,F403 ================================================ FILE: mmdet/models/backbones/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .csp_darknet import CSPDarknet from .cspnext import CSPNeXt from .darknet import Darknet from .detectors_resnet import DetectoRS_ResNet from .detectors_resnext import DetectoRS_ResNeXt from .efficientnet import EfficientNet from .hourglass import HourglassNet from .hrnet import HRNet from .mobilenet_v2 import MobileNetV2 from .pvt import PyramidVisionTransformer, PyramidVisionTransformerV2 from .regnet import RegNet from .res2net import Res2Net from .resnest import ResNeSt from .resnet import ResNet, ResNetV1d from .resnext import ResNeXt from .ssd_vgg import SSDVGG from .swin import SwinTransformer from .trident_resnet import TridentResNet __all__ = [ 'RegNet', 'ResNet', 'ResNetV1d', 'ResNeXt', 'SSDVGG', 'HRNet', 'MobileNetV2', 'Res2Net', 'HourglassNet', 'DetectoRS_ResNet', 'DetectoRS_ResNeXt', 'Darknet', 'ResNeSt', 'TridentResNet', 'CSPDarknet', 'SwinTransformer', 'PyramidVisionTransformer', 'PyramidVisionTransformerV2', 'EfficientNet', 'CSPNeXt' ] ================================================ FILE: mmdet/models/backbones/csp_darknet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math import torch import torch.nn as nn from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule from mmengine.model import BaseModule from torch.nn.modules.batchnorm import _BatchNorm from mmdet.registry import MODELS from ..layers import CSPLayer class Focus(nn.Module): """Focus width and height information into channel space. Args: in_channels (int): The input channels of this Module. out_channels (int): The output channels of this Module. kernel_size (int): The kernel size of the convolution. Default: 1 stride (int): The stride of the convolution. Default: 1 conv_cfg (dict): Config dict for convolution layer. Default: None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Default: dict(type='BN', momentum=0.03, eps=0.001). act_cfg (dict): Config dict for activation layer. Default: dict(type='Swish'). """ def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, conv_cfg=None, norm_cfg=dict(type='BN', momentum=0.03, eps=0.001), act_cfg=dict(type='Swish')): super().__init__() self.conv = ConvModule( in_channels * 4, out_channels, kernel_size, stride, padding=(kernel_size - 1) // 2, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) def forward(self, x): # shape of x (b,c,w,h) -> y(b,4c,w/2,h/2) patch_top_left = x[..., ::2, ::2] patch_top_right = x[..., ::2, 1::2] patch_bot_left = x[..., 1::2, ::2] patch_bot_right = x[..., 1::2, 1::2] x = torch.cat( ( patch_top_left, patch_bot_left, patch_top_right, patch_bot_right, ), dim=1, ) return self.conv(x) class SPPBottleneck(BaseModule): """Spatial pyramid pooling layer used in YOLOv3-SPP. Args: in_channels (int): The input channels of this Module. out_channels (int): The output channels of this Module. kernel_sizes (tuple[int]): Sequential of kernel sizes of pooling layers. Default: (5, 9, 13). conv_cfg (dict): Config dict for convolution layer. Default: None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Default: dict(type='BN'). act_cfg (dict): Config dict for activation layer. Default: dict(type='Swish'). init_cfg (dict or list[dict], optional): Initialization config dict. Default: None. """ def __init__(self, in_channels, out_channels, kernel_sizes=(5, 9, 13), conv_cfg=None, norm_cfg=dict(type='BN', momentum=0.03, eps=0.001), act_cfg=dict(type='Swish'), init_cfg=None): super().__init__(init_cfg) mid_channels = in_channels // 2 self.conv1 = ConvModule( in_channels, mid_channels, 1, stride=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.poolings = nn.ModuleList([ nn.MaxPool2d(kernel_size=ks, stride=1, padding=ks // 2) for ks in kernel_sizes ]) conv2_channels = mid_channels * (len(kernel_sizes) + 1) self.conv2 = ConvModule( conv2_channels, out_channels, 1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) def forward(self, x): x = self.conv1(x) with torch.cuda.amp.autocast(enabled=False): x = torch.cat( [x] + [pooling(x) for pooling in self.poolings], dim=1) x = self.conv2(x) return x @MODELS.register_module() class CSPDarknet(BaseModule): """CSP-Darknet backbone used in YOLOv5 and YOLOX. Args: arch (str): Architecture of CSP-Darknet, from {P5, P6}. Default: P5. deepen_factor (float): Depth multiplier, multiply number of blocks in CSP layer by this amount. Default: 1.0. widen_factor (float): Width multiplier, multiply number of channels in each layer by this amount. Default: 1.0. out_indices (Sequence[int]): Output from which stages. Default: (2, 3, 4). frozen_stages (int): Stages to be frozen (stop grad and set eval mode). -1 means not freezing any parameters. Default: -1. use_depthwise (bool): Whether to use depthwise separable convolution. Default: False. arch_ovewrite(list): Overwrite default arch settings. Default: None. spp_kernal_sizes: (tuple[int]): Sequential of kernel sizes of SPP layers. Default: (5, 9, 13). conv_cfg (dict): Config dict for convolution layer. Default: None. norm_cfg (dict): Dictionary to construct and config norm layer. Default: dict(type='BN', requires_grad=True). act_cfg (dict): Config dict for activation layer. Default: dict(type='LeakyReLU', negative_slope=0.1). norm_eval (bool): Whether to set norm layers to eval mode, namely, freeze running stats (mean and var). Note: Effect on Batch Norm and its variants only. init_cfg (dict or list[dict], optional): Initialization config dict. Default: None. Example: >>> from mmdet.models import CSPDarknet >>> import torch >>> self = CSPDarknet(depth=53) >>> self.eval() >>> inputs = torch.rand(1, 3, 416, 416) >>> level_outputs = self.forward(inputs) >>> for level_out in level_outputs: ... print(tuple(level_out.shape)) ... (1, 256, 52, 52) (1, 512, 26, 26) (1, 1024, 13, 13) """ # From left to right: # in_channels, out_channels, num_blocks, add_identity, use_spp arch_settings = { 'P5': [[64, 128, 3, True, False], [128, 256, 9, True, False], [256, 512, 9, True, False], [512, 1024, 3, False, True]], 'P6': [[64, 128, 3, True, False], [128, 256, 9, True, False], [256, 512, 9, True, False], [512, 768, 3, True, False], [768, 1024, 3, False, True]] } def __init__(self, arch='P5', deepen_factor=1.0, widen_factor=1.0, out_indices=(2, 3, 4), frozen_stages=-1, use_depthwise=False, arch_ovewrite=None, spp_kernal_sizes=(5, 9, 13), conv_cfg=None, norm_cfg=dict(type='BN', momentum=0.03, eps=0.001), act_cfg=dict(type='Swish'), norm_eval=False, init_cfg=dict( type='Kaiming', layer='Conv2d', a=math.sqrt(5), distribution='uniform', mode='fan_in', nonlinearity='leaky_relu')): super().__init__(init_cfg) arch_setting = self.arch_settings[arch] if arch_ovewrite: arch_setting = arch_ovewrite assert set(out_indices).issubset( i for i in range(len(arch_setting) + 1)) if frozen_stages not in range(-1, len(arch_setting) + 1): raise ValueError('frozen_stages must be in range(-1, ' 'len(arch_setting) + 1). But received ' f'{frozen_stages}') self.out_indices = out_indices self.frozen_stages = frozen_stages self.use_depthwise = use_depthwise self.norm_eval = norm_eval conv = DepthwiseSeparableConvModule if use_depthwise else ConvModule self.stem = Focus( 3, int(arch_setting[0][0] * widen_factor), kernel_size=3, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.layers = ['stem'] for i, (in_channels, out_channels, num_blocks, add_identity, use_spp) in enumerate(arch_setting): in_channels = int(in_channels * widen_factor) out_channels = int(out_channels * widen_factor) num_blocks = max(round(num_blocks * deepen_factor), 1) stage = [] conv_layer = conv( in_channels, out_channels, 3, stride=2, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) stage.append(conv_layer) if use_spp: spp = SPPBottleneck( out_channels, out_channels, kernel_sizes=spp_kernal_sizes, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) stage.append(spp) csp_layer = CSPLayer( out_channels, out_channels, num_blocks=num_blocks, add_identity=add_identity, use_depthwise=use_depthwise, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) stage.append(csp_layer) self.add_module(f'stage{i + 1}', nn.Sequential(*stage)) self.layers.append(f'stage{i + 1}') def _freeze_stages(self): if self.frozen_stages >= 0: for i in range(self.frozen_stages + 1): m = getattr(self, self.layers[i]) m.eval() for param in m.parameters(): param.requires_grad = False def train(self, mode=True): super(CSPDarknet, self).train(mode) self._freeze_stages() if mode and self.norm_eval: for m in self.modules(): if isinstance(m, _BatchNorm): m.eval() def forward(self, x): outs = [] for i, layer_name in enumerate(self.layers): layer = getattr(self, layer_name) x = layer(x) if i in self.out_indices: outs.append(x) return tuple(outs) ================================================ FILE: mmdet/models/backbones/cspnext.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from typing import Sequence, Tuple import torch.nn as nn from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule from mmengine.model import BaseModule from torch import Tensor from torch.nn.modules.batchnorm import _BatchNorm from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from ..layers import CSPLayer from .csp_darknet import SPPBottleneck @MODELS.register_module() class CSPNeXt(BaseModule): """CSPNeXt backbone used in RTMDet. Args: arch (str): Architecture of CSPNeXt, from {P5, P6}. Defaults to P5. expand_ratio (float): Ratio to adjust the number of channels of the hidden layer. Defaults to 0.5. deepen_factor (float): Depth multiplier, multiply number of blocks in CSP layer by this amount. Defaults to 1.0. widen_factor (float): Width multiplier, multiply number of channels in each layer by this amount. Defaults to 1.0. out_indices (Sequence[int]): Output from which stages. Defaults to (2, 3, 4). frozen_stages (int): Stages to be frozen (stop grad and set eval mode). -1 means not freezing any parameters. Defaults to -1. use_depthwise (bool): Whether to use depthwise separable convolution. Defaults to False. arch_ovewrite (list): Overwrite default arch settings. Defaults to None. spp_kernel_sizes: (tuple[int]): Sequential of kernel sizes of SPP layers. Defaults to (5, 9, 13). channel_attention (bool): Whether to add channel attention in each stage. Defaults to True. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict): Dictionary to construct and config norm layer. Defaults to dict(type='BN', requires_grad=True). act_cfg (:obj:`ConfigDict` or dict): Config dict for activation layer. Defaults to dict(type='SiLU'). norm_eval (bool): Whether to set norm layers to eval mode, namely, freeze running stats (mean and var). Note: Effect on Batch Norm and its variants only. init_cfg (:obj:`ConfigDict` or dict or list[dict] or list[:obj:`ConfigDict`]): Initialization config dict. """ # From left to right: # in_channels, out_channels, num_blocks, add_identity, use_spp arch_settings = { 'P5': [[64, 128, 3, True, False], [128, 256, 6, True, False], [256, 512, 6, True, False], [512, 1024, 3, False, True]], 'P6': [[64, 128, 3, True, False], [128, 256, 6, True, False], [256, 512, 6, True, False], [512, 768, 3, True, False], [768, 1024, 3, False, True]] } def __init__( self, arch: str = 'P5', deepen_factor: float = 1.0, widen_factor: float = 1.0, out_indices: Sequence[int] = (2, 3, 4), frozen_stages: int = -1, use_depthwise: bool = False, expand_ratio: float = 0.5, arch_ovewrite: dict = None, spp_kernel_sizes: Sequence[int] = (5, 9, 13), channel_attention: bool = True, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN', momentum=0.03, eps=0.001), act_cfg: ConfigType = dict(type='SiLU'), norm_eval: bool = False, init_cfg: OptMultiConfig = dict( type='Kaiming', layer='Conv2d', a=math.sqrt(5), distribution='uniform', mode='fan_in', nonlinearity='leaky_relu') ) -> None: super().__init__(init_cfg=init_cfg) arch_setting = self.arch_settings[arch] if arch_ovewrite: arch_setting = arch_ovewrite assert set(out_indices).issubset( i for i in range(len(arch_setting) + 1)) if frozen_stages not in range(-1, len(arch_setting) + 1): raise ValueError('frozen_stages must be in range(-1, ' 'len(arch_setting) + 1). But received ' f'{frozen_stages}') self.out_indices = out_indices self.frozen_stages = frozen_stages self.use_depthwise = use_depthwise self.norm_eval = norm_eval conv = DepthwiseSeparableConvModule if use_depthwise else ConvModule self.stem = nn.Sequential( ConvModule( 3, int(arch_setting[0][0] * widen_factor // 2), 3, padding=1, stride=2, norm_cfg=norm_cfg, act_cfg=act_cfg), ConvModule( int(arch_setting[0][0] * widen_factor // 2), int(arch_setting[0][0] * widen_factor // 2), 3, padding=1, stride=1, norm_cfg=norm_cfg, act_cfg=act_cfg), ConvModule( int(arch_setting[0][0] * widen_factor // 2), int(arch_setting[0][0] * widen_factor), 3, padding=1, stride=1, norm_cfg=norm_cfg, act_cfg=act_cfg)) self.layers = ['stem'] for i, (in_channels, out_channels, num_blocks, add_identity, use_spp) in enumerate(arch_setting): in_channels = int(in_channels * widen_factor) out_channels = int(out_channels * widen_factor) num_blocks = max(round(num_blocks * deepen_factor), 1) stage = [] conv_layer = conv( in_channels, out_channels, 3, stride=2, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) stage.append(conv_layer) if use_spp: spp = SPPBottleneck( out_channels, out_channels, kernel_sizes=spp_kernel_sizes, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) stage.append(spp) csp_layer = CSPLayer( out_channels, out_channels, num_blocks=num_blocks, add_identity=add_identity, use_depthwise=use_depthwise, use_cspnext_block=True, expand_ratio=expand_ratio, channel_attention=channel_attention, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) stage.append(csp_layer) self.add_module(f'stage{i + 1}', nn.Sequential(*stage)) self.layers.append(f'stage{i + 1}') def _freeze_stages(self) -> None: if self.frozen_stages >= 0: for i in range(self.frozen_stages + 1): m = getattr(self, self.layers[i]) m.eval() for param in m.parameters(): param.requires_grad = False def train(self, mode=True) -> None: super().train(mode) self._freeze_stages() if mode and self.norm_eval: for m in self.modules(): if isinstance(m, _BatchNorm): m.eval() def forward(self, x: Tuple[Tensor, ...]) -> Tuple[Tensor, ...]: outs = [] for i, layer_name in enumerate(self.layers): layer = getattr(self, layer_name) x = layer(x) if i in self.out_indices: outs.append(x) return tuple(outs) ================================================ FILE: mmdet/models/backbones/darknet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Copyright (c) 2019 Western Digital Corporation or its affiliates. import warnings import torch.nn as nn from mmcv.cnn import ConvModule from mmengine.model import BaseModule from torch.nn.modules.batchnorm import _BatchNorm from mmdet.registry import MODELS class ResBlock(BaseModule): """The basic residual block used in Darknet. Each ResBlock consists of two ConvModules and the input is added to the final output. Each ConvModule is composed of Conv, BN, and LeakyReLU. In YoloV3 paper, the first convLayer has half of the number of the filters as much as the second convLayer. The first convLayer has filter size of 1x1 and the second one has the filter size of 3x3. Args: in_channels (int): The input channels. Must be even. conv_cfg (dict): Config dict for convolution layer. Default: None. norm_cfg (dict): Dictionary to construct and config norm layer. Default: dict(type='BN', requires_grad=True) act_cfg (dict): Config dict for activation layer. Default: dict(type='LeakyReLU', negative_slope=0.1). init_cfg (dict or list[dict], optional): Initialization config dict. Default: None """ def __init__(self, in_channels, conv_cfg=None, norm_cfg=dict(type='BN', requires_grad=True), act_cfg=dict(type='LeakyReLU', negative_slope=0.1), init_cfg=None): super(ResBlock, self).__init__(init_cfg) assert in_channels % 2 == 0 # ensure the in_channels is even half_in_channels = in_channels // 2 # shortcut cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.conv1 = ConvModule(in_channels, half_in_channels, 1, **cfg) self.conv2 = ConvModule( half_in_channels, in_channels, 3, padding=1, **cfg) def forward(self, x): residual = x out = self.conv1(x) out = self.conv2(out) out = out + residual return out @MODELS.register_module() class Darknet(BaseModule): """Darknet backbone. Args: depth (int): Depth of Darknet. Currently only support 53. out_indices (Sequence[int]): Output from which stages. frozen_stages (int): Stages to be frozen (stop grad and set eval mode). -1 means not freezing any parameters. Default: -1. conv_cfg (dict): Config dict for convolution layer. Default: None. norm_cfg (dict): Dictionary to construct and config norm layer. Default: dict(type='BN', requires_grad=True) act_cfg (dict): Config dict for activation layer. Default: dict(type='LeakyReLU', negative_slope=0.1). norm_eval (bool): Whether to set norm layers to eval mode, namely, freeze running stats (mean and var). Note: Effect on Batch Norm and its variants only. pretrained (str, optional): model pretrained path. Default: None init_cfg (dict or list[dict], optional): Initialization config dict. Default: None Example: >>> from mmdet.models import Darknet >>> import torch >>> self = Darknet(depth=53) >>> self.eval() >>> inputs = torch.rand(1, 3, 416, 416) >>> level_outputs = self.forward(inputs) >>> for level_out in level_outputs: ... print(tuple(level_out.shape)) ... (1, 256, 52, 52) (1, 512, 26, 26) (1, 1024, 13, 13) """ # Dict(depth: (layers, channels)) arch_settings = { 53: ((1, 2, 8, 8, 4), ((32, 64), (64, 128), (128, 256), (256, 512), (512, 1024))) } def __init__(self, depth=53, out_indices=(3, 4, 5), frozen_stages=-1, conv_cfg=None, norm_cfg=dict(type='BN', requires_grad=True), act_cfg=dict(type='LeakyReLU', negative_slope=0.1), norm_eval=True, pretrained=None, init_cfg=None): super(Darknet, self).__init__(init_cfg) if depth not in self.arch_settings: raise KeyError(f'invalid depth {depth} for darknet') self.depth = depth self.out_indices = out_indices self.frozen_stages = frozen_stages self.layers, self.channels = self.arch_settings[depth] cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.conv1 = ConvModule(3, 32, 3, padding=1, **cfg) self.cr_blocks = ['conv1'] for i, n_layers in enumerate(self.layers): layer_name = f'conv_res_block{i + 1}' in_c, out_c = self.channels[i] self.add_module( layer_name, self.make_conv_res_block(in_c, out_c, n_layers, **cfg)) self.cr_blocks.append(layer_name) self.norm_eval = norm_eval assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be specified at the same time' if isinstance(pretrained, str): warnings.warn('DeprecationWarning: pretrained is deprecated, ' 'please use "init_cfg" instead') self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) elif pretrained is None: if init_cfg is None: self.init_cfg = [ dict(type='Kaiming', layer='Conv2d'), dict( type='Constant', val=1, layer=['_BatchNorm', 'GroupNorm']) ] else: raise TypeError('pretrained must be a str or None') def forward(self, x): outs = [] for i, layer_name in enumerate(self.cr_blocks): cr_block = getattr(self, layer_name) x = cr_block(x) if i in self.out_indices: outs.append(x) return tuple(outs) def _freeze_stages(self): if self.frozen_stages >= 0: for i in range(self.frozen_stages): m = getattr(self, self.cr_blocks[i]) m.eval() for param in m.parameters(): param.requires_grad = False def train(self, mode=True): super(Darknet, self).train(mode) self._freeze_stages() if mode and self.norm_eval: for m in self.modules(): if isinstance(m, _BatchNorm): m.eval() @staticmethod def make_conv_res_block(in_channels, out_channels, res_repeat, conv_cfg=None, norm_cfg=dict(type='BN', requires_grad=True), act_cfg=dict(type='LeakyReLU', negative_slope=0.1)): """In Darknet backbone, ConvLayer is usually followed by ResBlock. This function will make that. The Conv layers always have 3x3 filters with stride=2. The number of the filters in Conv layer is the same as the out channels of the ResBlock. Args: in_channels (int): The number of input channels. out_channels (int): The number of output channels. res_repeat (int): The number of ResBlocks. conv_cfg (dict): Config dict for convolution layer. Default: None. norm_cfg (dict): Dictionary to construct and config norm layer. Default: dict(type='BN', requires_grad=True) act_cfg (dict): Config dict for activation layer. Default: dict(type='LeakyReLU', negative_slope=0.1). """ cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) model = nn.Sequential() model.add_module( 'conv', ConvModule( in_channels, out_channels, 3, stride=2, padding=1, **cfg)) for idx in range(res_repeat): model.add_module('res{}'.format(idx), ResBlock(out_channels, **cfg)) return model ================================================ FILE: mmdet/models/backbones/detectors_resnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn as nn import torch.utils.checkpoint as cp from mmcv.cnn import build_conv_layer, build_norm_layer from mmengine.logging import MMLogger from mmengine.model import Sequential, constant_init, kaiming_init from mmengine.runner.checkpoint import load_checkpoint from torch.nn.modules.batchnorm import _BatchNorm from mmdet.registry import MODELS from .resnet import BasicBlock from .resnet import Bottleneck as _Bottleneck from .resnet import ResNet class Bottleneck(_Bottleneck): r"""Bottleneck for the ResNet backbone in `DetectoRS `_. This bottleneck allows the users to specify whether to use SAC (Switchable Atrous Convolution) and RFP (Recursive Feature Pyramid). Args: inplanes (int): The number of input channels. planes (int): The number of output channels before expansion. rfp_inplanes (int, optional): The number of channels from RFP. Default: None. If specified, an additional conv layer will be added for ``rfp_feat``. Otherwise, the structure is the same as base class. sac (dict, optional): Dictionary to construct SAC. Default: None. init_cfg (dict or list[dict], optional): Initialization config dict. Default: None """ expansion = 4 def __init__(self, inplanes, planes, rfp_inplanes=None, sac=None, init_cfg=None, **kwargs): super(Bottleneck, self).__init__( inplanes, planes, init_cfg=init_cfg, **kwargs) assert sac is None or isinstance(sac, dict) self.sac = sac self.with_sac = sac is not None if self.with_sac: self.conv2 = build_conv_layer( self.sac, planes, planes, kernel_size=3, stride=self.conv2_stride, padding=self.dilation, dilation=self.dilation, bias=False) self.rfp_inplanes = rfp_inplanes if self.rfp_inplanes: self.rfp_conv = build_conv_layer( None, self.rfp_inplanes, planes * self.expansion, 1, stride=1, bias=True) if init_cfg is None: self.init_cfg = dict( type='Constant', val=0, override=dict(name='rfp_conv')) def rfp_forward(self, x, rfp_feat): """The forward function that also takes the RFP features as input.""" def _inner_forward(x): identity = x out = self.conv1(x) out = self.norm1(out) out = self.relu(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv1_plugin_names) out = self.conv2(out) out = self.norm2(out) out = self.relu(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv2_plugin_names) out = self.conv3(out) out = self.norm3(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv3_plugin_names) if self.downsample is not None: identity = self.downsample(x) out += identity return out if self.with_cp and x.requires_grad: out = cp.checkpoint(_inner_forward, x) else: out = _inner_forward(x) if self.rfp_inplanes: rfp_feat = self.rfp_conv(rfp_feat) out = out + rfp_feat out = self.relu(out) return out class ResLayer(Sequential): """ResLayer to build ResNet style backbone for RPF in detectoRS. The difference between this module and base class is that we pass ``rfp_inplanes`` to the first block. Args: block (nn.Module): block used to build ResLayer. inplanes (int): inplanes of block. planes (int): planes of block. num_blocks (int): number of blocks. stride (int): stride of the first block. Default: 1 avg_down (bool): Use AvgPool instead of stride conv when downsampling in the bottleneck. Default: False conv_cfg (dict): dictionary to construct and config conv layer. Default: None norm_cfg (dict): dictionary to construct and config norm layer. Default: dict(type='BN') downsample_first (bool): Downsample at the first block or last block. False for Hourglass, True for ResNet. Default: True rfp_inplanes (int, optional): The number of channels from RFP. Default: None. If specified, an additional conv layer will be added for ``rfp_feat``. Otherwise, the structure is the same as base class. """ def __init__(self, block, inplanes, planes, num_blocks, stride=1, avg_down=False, conv_cfg=None, norm_cfg=dict(type='BN'), downsample_first=True, rfp_inplanes=None, **kwargs): self.block = block assert downsample_first, f'downsample_first={downsample_first} is ' \ 'not supported in DetectoRS' downsample = None if stride != 1 or inplanes != planes * block.expansion: downsample = [] conv_stride = stride if avg_down and stride != 1: conv_stride = 1 downsample.append( nn.AvgPool2d( kernel_size=stride, stride=stride, ceil_mode=True, count_include_pad=False)) downsample.extend([ build_conv_layer( conv_cfg, inplanes, planes * block.expansion, kernel_size=1, stride=conv_stride, bias=False), build_norm_layer(norm_cfg, planes * block.expansion)[1] ]) downsample = nn.Sequential(*downsample) layers = [] layers.append( block( inplanes=inplanes, planes=planes, stride=stride, downsample=downsample, conv_cfg=conv_cfg, norm_cfg=norm_cfg, rfp_inplanes=rfp_inplanes, **kwargs)) inplanes = planes * block.expansion for _ in range(1, num_blocks): layers.append( block( inplanes=inplanes, planes=planes, stride=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, **kwargs)) super(ResLayer, self).__init__(*layers) @MODELS.register_module() class DetectoRS_ResNet(ResNet): """ResNet backbone for DetectoRS. Args: sac (dict, optional): Dictionary to construct SAC (Switchable Atrous Convolution). Default: None. stage_with_sac (list): Which stage to use sac. Default: (False, False, False, False). rfp_inplanes (int, optional): The number of channels from RFP. Default: None. If specified, an additional conv layer will be added for ``rfp_feat``. Otherwise, the structure is the same as base class. output_img (bool): If ``True``, the input image will be inserted into the starting position of output. Default: False. """ arch_settings = { 50: (Bottleneck, (3, 4, 6, 3)), 101: (Bottleneck, (3, 4, 23, 3)), 152: (Bottleneck, (3, 8, 36, 3)) } def __init__(self, sac=None, stage_with_sac=(False, False, False, False), rfp_inplanes=None, output_img=False, pretrained=None, init_cfg=None, **kwargs): assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be specified at the same time' self.pretrained = pretrained if init_cfg is not None: assert isinstance(init_cfg, dict), \ f'init_cfg must be a dict, but got {type(init_cfg)}' if 'type' in init_cfg: assert init_cfg.get('type') == 'Pretrained', \ 'Only can initialize module by loading a pretrained model' else: raise KeyError('`init_cfg` must contain the key "type"') self.pretrained = init_cfg.get('checkpoint') self.sac = sac self.stage_with_sac = stage_with_sac self.rfp_inplanes = rfp_inplanes self.output_img = output_img super(DetectoRS_ResNet, self).__init__(**kwargs) self.inplanes = self.stem_channels self.res_layers = [] for i, num_blocks in enumerate(self.stage_blocks): stride = self.strides[i] dilation = self.dilations[i] dcn = self.dcn if self.stage_with_dcn[i] else None sac = self.sac if self.stage_with_sac[i] else None if self.plugins is not None: stage_plugins = self.make_stage_plugins(self.plugins, i) else: stage_plugins = None planes = self.base_channels * 2**i res_layer = self.make_res_layer( block=self.block, inplanes=self.inplanes, planes=planes, num_blocks=num_blocks, stride=stride, dilation=dilation, style=self.style, avg_down=self.avg_down, with_cp=self.with_cp, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, dcn=dcn, sac=sac, rfp_inplanes=rfp_inplanes if i > 0 else None, plugins=stage_plugins) self.inplanes = planes * self.block.expansion layer_name = f'layer{i + 1}' self.add_module(layer_name, res_layer) self.res_layers.append(layer_name) self._freeze_stages() # In order to be properly initialized by RFP def init_weights(self): # Calling this method will cause parameter initialization exception # super(DetectoRS_ResNet, self).init_weights() if isinstance(self.pretrained, str): logger = MMLogger.get_current_instance() load_checkpoint(self, self.pretrained, strict=False, logger=logger) elif self.pretrained is None: for m in self.modules(): if isinstance(m, nn.Conv2d): kaiming_init(m) elif isinstance(m, (_BatchNorm, nn.GroupNorm)): constant_init(m, 1) if self.dcn is not None: for m in self.modules(): if isinstance(m, Bottleneck) and hasattr( m.conv2, 'conv_offset'): constant_init(m.conv2.conv_offset, 0) if self.zero_init_residual: for m in self.modules(): if isinstance(m, Bottleneck): constant_init(m.norm3, 0) elif isinstance(m, BasicBlock): constant_init(m.norm2, 0) else: raise TypeError('pretrained must be a str or None') def make_res_layer(self, **kwargs): """Pack all blocks in a stage into a ``ResLayer`` for DetectoRS.""" return ResLayer(**kwargs) def forward(self, x): """Forward function.""" outs = list(super(DetectoRS_ResNet, self).forward(x)) if self.output_img: outs.insert(0, x) return tuple(outs) def rfp_forward(self, x, rfp_feats): """Forward function for RFP.""" if self.deep_stem: x = self.stem(x) else: x = self.conv1(x) x = self.norm1(x) x = self.relu(x) x = self.maxpool(x) outs = [] for i, layer_name in enumerate(self.res_layers): res_layer = getattr(self, layer_name) rfp_feat = rfp_feats[i] if i > 0 else None for layer in res_layer: x = layer.rfp_forward(x, rfp_feat) if i in self.out_indices: outs.append(x) return tuple(outs) ================================================ FILE: mmdet/models/backbones/detectors_resnext.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from mmcv.cnn import build_conv_layer, build_norm_layer from mmdet.registry import MODELS from .detectors_resnet import Bottleneck as _Bottleneck from .detectors_resnet import DetectoRS_ResNet class Bottleneck(_Bottleneck): expansion = 4 def __init__(self, inplanes, planes, groups=1, base_width=4, base_channels=64, **kwargs): """Bottleneck block for ResNeXt. If style is "pytorch", the stride-two layer is the 3x3 conv layer, if it is "caffe", the stride-two layer is the first 1x1 conv layer. """ super(Bottleneck, self).__init__(inplanes, planes, **kwargs) if groups == 1: width = self.planes else: width = math.floor(self.planes * (base_width / base_channels)) * groups self.norm1_name, norm1 = build_norm_layer( self.norm_cfg, width, postfix=1) self.norm2_name, norm2 = build_norm_layer( self.norm_cfg, width, postfix=2) self.norm3_name, norm3 = build_norm_layer( self.norm_cfg, self.planes * self.expansion, postfix=3) self.conv1 = build_conv_layer( self.conv_cfg, self.inplanes, width, kernel_size=1, stride=self.conv1_stride, bias=False) self.add_module(self.norm1_name, norm1) fallback_on_stride = False self.with_modulated_dcn = False if self.with_dcn: fallback_on_stride = self.dcn.pop('fallback_on_stride', False) if self.with_sac: self.conv2 = build_conv_layer( self.sac, width, width, kernel_size=3, stride=self.conv2_stride, padding=self.dilation, dilation=self.dilation, groups=groups, bias=False) elif not self.with_dcn or fallback_on_stride: self.conv2 = build_conv_layer( self.conv_cfg, width, width, kernel_size=3, stride=self.conv2_stride, padding=self.dilation, dilation=self.dilation, groups=groups, bias=False) else: assert self.conv_cfg is None, 'conv_cfg must be None for DCN' self.conv2 = build_conv_layer( self.dcn, width, width, kernel_size=3, stride=self.conv2_stride, padding=self.dilation, dilation=self.dilation, groups=groups, bias=False) self.add_module(self.norm2_name, norm2) self.conv3 = build_conv_layer( self.conv_cfg, width, self.planes * self.expansion, kernel_size=1, bias=False) self.add_module(self.norm3_name, norm3) @MODELS.register_module() class DetectoRS_ResNeXt(DetectoRS_ResNet): """ResNeXt backbone for DetectoRS. Args: groups (int): The number of groups in ResNeXt. base_width (int): The base width of ResNeXt. """ arch_settings = { 50: (Bottleneck, (3, 4, 6, 3)), 101: (Bottleneck, (3, 4, 23, 3)), 152: (Bottleneck, (3, 8, 36, 3)) } def __init__(self, groups=1, base_width=4, **kwargs): self.groups = groups self.base_width = base_width super(DetectoRS_ResNeXt, self).__init__(**kwargs) def make_res_layer(self, **kwargs): return super().make_res_layer( groups=self.groups, base_width=self.base_width, base_channels=self.base_channels, **kwargs) ================================================ FILE: mmdet/models/backbones/efficientnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import math from functools import partial import torch import torch.nn as nn import torch.utils.checkpoint as cp from mmcv.cnn.bricks import ConvModule, DropPath from mmengine.model import BaseModule, Sequential from mmdet.registry import MODELS from ..layers import InvertedResidual, SELayer from ..utils import make_divisible class EdgeResidual(BaseModule): """Edge Residual Block. Args: in_channels (int): The input channels of this module. out_channels (int): The output channels of this module. mid_channels (int): The input channels of the second convolution. kernel_size (int): The kernel size of the first convolution. Defaults to 3. stride (int): The stride of the first convolution. Defaults to 1. se_cfg (dict, optional): Config dict for se layer. Defaults to None, which means no se layer. with_residual (bool): Use residual connection. Defaults to True. conv_cfg (dict, optional): Config dict for convolution layer. Defaults to None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Defaults to ``dict(type='BN')``. act_cfg (dict): Config dict for activation layer. Defaults to ``dict(type='ReLU')``. drop_path_rate (float): stochastic depth rate. Defaults to 0. with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. Defaults to False. init_cfg (dict | list[dict], optional): Initialization config dict. """ def __init__(self, in_channels, out_channels, mid_channels, kernel_size=3, stride=1, se_cfg=None, with_residual=True, conv_cfg=None, norm_cfg=dict(type='BN'), act_cfg=dict(type='ReLU'), drop_path_rate=0., with_cp=False, init_cfg=None, **kwargs): super(EdgeResidual, self).__init__(init_cfg=init_cfg) assert stride in [1, 2] self.with_cp = with_cp self.drop_path = DropPath( drop_path_rate) if drop_path_rate > 0 else nn.Identity() self.with_se = se_cfg is not None self.with_residual = ( stride == 1 and in_channels == out_channels and with_residual) if self.with_se: assert isinstance(se_cfg, dict) self.conv1 = ConvModule( in_channels=in_channels, out_channels=mid_channels, kernel_size=kernel_size, stride=1, padding=kernel_size // 2, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) if self.with_se: self.se = SELayer(**se_cfg) self.conv2 = ConvModule( in_channels=mid_channels, out_channels=out_channels, kernel_size=1, stride=stride, padding=0, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=None) def forward(self, x): def _inner_forward(x): out = x out = self.conv1(out) if self.with_se: out = self.se(out) out = self.conv2(out) if self.with_residual: return x + self.drop_path(out) else: return out if self.with_cp and x.requires_grad: out = cp.checkpoint(_inner_forward, x) else: out = _inner_forward(x) return out def model_scaling(layer_setting, arch_setting): """Scaling operation to the layer's parameters according to the arch_setting.""" # scale width new_layer_setting = copy.deepcopy(layer_setting) for layer_cfg in new_layer_setting: for block_cfg in layer_cfg: block_cfg[1] = make_divisible(block_cfg[1] * arch_setting[0], 8) # scale depth split_layer_setting = [new_layer_setting[0]] for layer_cfg in new_layer_setting[1:-1]: tmp_index = [0] for i in range(len(layer_cfg) - 1): if layer_cfg[i + 1][1] != layer_cfg[i][1]: tmp_index.append(i + 1) tmp_index.append(len(layer_cfg)) for i in range(len(tmp_index) - 1): split_layer_setting.append(layer_cfg[tmp_index[i]:tmp_index[i + 1]]) split_layer_setting.append(new_layer_setting[-1]) num_of_layers = [len(layer_cfg) for layer_cfg in split_layer_setting[1:-1]] new_layers = [ int(math.ceil(arch_setting[1] * num)) for num in num_of_layers ] merge_layer_setting = [split_layer_setting[0]] for i, layer_cfg in enumerate(split_layer_setting[1:-1]): if new_layers[i] <= num_of_layers[i]: tmp_layer_cfg = layer_cfg[:new_layers[i]] else: tmp_layer_cfg = copy.deepcopy(layer_cfg) + [layer_cfg[-1]] * ( new_layers[i] - num_of_layers[i]) if tmp_layer_cfg[0][3] == 1 and i != 0: merge_layer_setting[-1] += tmp_layer_cfg.copy() else: merge_layer_setting.append(tmp_layer_cfg.copy()) merge_layer_setting.append(split_layer_setting[-1]) return merge_layer_setting @MODELS.register_module() class EfficientNet(BaseModule): """EfficientNet backbone. Args: arch (str): Architecture of efficientnet. Defaults to b0. out_indices (Sequence[int]): Output from which stages. Defaults to (6, ). frozen_stages (int): Stages to be frozen (all param fixed). Defaults to 0, which means not freezing any parameters. conv_cfg (dict): Config dict for convolution layer. Defaults to None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Defaults to dict(type='BN'). act_cfg (dict): Config dict for activation layer. Defaults to dict(type='Swish'). norm_eval (bool): Whether to set norm layers to eval mode, namely, freeze running stats (mean and var). Note: Effect on Batch Norm and its variants only. Defaults to False. with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. Defaults to False. """ # Parameters to build layers. # 'b' represents the architecture of normal EfficientNet family includes # 'b0', 'b1', 'b2', 'b3', 'b4', 'b5', 'b6', 'b7', 'b8'. # 'e' represents the architecture of EfficientNet-EdgeTPU including 'es', # 'em', 'el'. # 6 parameters are needed to construct a layer, From left to right: # - kernel_size: The kernel size of the block # - out_channel: The number of out_channels of the block # - se_ratio: The sequeeze ratio of SELayer. # - stride: The stride of the block # - expand_ratio: The expand_ratio of the mid_channels # - block_type: -1: Not a block, 0: InvertedResidual, 1: EdgeResidual layer_settings = { 'b': [[[3, 32, 0, 2, 0, -1]], [[3, 16, 4, 1, 1, 0]], [[3, 24, 4, 2, 6, 0], [3, 24, 4, 1, 6, 0]], [[5, 40, 4, 2, 6, 0], [5, 40, 4, 1, 6, 0]], [[3, 80, 4, 2, 6, 0], [3, 80, 4, 1, 6, 0], [3, 80, 4, 1, 6, 0], [5, 112, 4, 1, 6, 0], [5, 112, 4, 1, 6, 0], [5, 112, 4, 1, 6, 0]], [[5, 192, 4, 2, 6, 0], [5, 192, 4, 1, 6, 0], [5, 192, 4, 1, 6, 0], [5, 192, 4, 1, 6, 0], [3, 320, 4, 1, 6, 0]], [[1, 1280, 0, 1, 0, -1]] ], 'e': [[[3, 32, 0, 2, 0, -1]], [[3, 24, 0, 1, 3, 1]], [[3, 32, 0, 2, 8, 1], [3, 32, 0, 1, 8, 1]], [[3, 48, 0, 2, 8, 1], [3, 48, 0, 1, 8, 1], [3, 48, 0, 1, 8, 1], [3, 48, 0, 1, 8, 1]], [[5, 96, 0, 2, 8, 0], [5, 96, 0, 1, 8, 0], [5, 96, 0, 1, 8, 0], [5, 96, 0, 1, 8, 0], [5, 96, 0, 1, 8, 0], [5, 144, 0, 1, 8, 0], [5, 144, 0, 1, 8, 0], [5, 144, 0, 1, 8, 0], [5, 144, 0, 1, 8, 0]], [[5, 192, 0, 2, 8, 0], [5, 192, 0, 1, 8, 0]], [[1, 1280, 0, 1, 0, -1]] ] } # yapf: disable # Parameters to build different kinds of architecture. # From left to right: scaling factor for width, scaling factor for depth, # resolution. arch_settings = { 'b0': (1.0, 1.0, 224), 'b1': (1.0, 1.1, 240), 'b2': (1.1, 1.2, 260), 'b3': (1.2, 1.4, 300), 'b4': (1.4, 1.8, 380), 'b5': (1.6, 2.2, 456), 'b6': (1.8, 2.6, 528), 'b7': (2.0, 3.1, 600), 'b8': (2.2, 3.6, 672), 'es': (1.0, 1.0, 224), 'em': (1.0, 1.1, 240), 'el': (1.2, 1.4, 300) } def __init__(self, arch='b0', drop_path_rate=0., out_indices=(6, ), frozen_stages=0, conv_cfg=dict(type='Conv2dAdaptivePadding'), norm_cfg=dict(type='BN', eps=1e-3), act_cfg=dict(type='Swish'), norm_eval=False, with_cp=False, init_cfg=[ dict(type='Kaiming', layer='Conv2d'), dict( type='Constant', layer=['_BatchNorm', 'GroupNorm'], val=1) ]): super(EfficientNet, self).__init__(init_cfg) assert arch in self.arch_settings, \ f'"{arch}" is not one of the arch_settings ' \ f'({", ".join(self.arch_settings.keys())})' self.arch_setting = self.arch_settings[arch] self.layer_setting = self.layer_settings[arch[:1]] for index in out_indices: if index not in range(0, len(self.layer_setting)): raise ValueError('the item in out_indices must in ' f'range(0, {len(self.layer_setting)}). ' f'But received {index}') if frozen_stages not in range(len(self.layer_setting) + 1): raise ValueError('frozen_stages must be in range(0, ' f'{len(self.layer_setting) + 1}). ' f'But received {frozen_stages}') self.drop_path_rate = drop_path_rate self.out_indices = out_indices self.frozen_stages = frozen_stages self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.act_cfg = act_cfg self.norm_eval = norm_eval self.with_cp = with_cp self.layer_setting = model_scaling(self.layer_setting, self.arch_setting) block_cfg_0 = self.layer_setting[0][0] block_cfg_last = self.layer_setting[-1][0] self.in_channels = make_divisible(block_cfg_0[1], 8) self.out_channels = block_cfg_last[1] self.layers = nn.ModuleList() self.layers.append( ConvModule( in_channels=3, out_channels=self.in_channels, kernel_size=block_cfg_0[0], stride=block_cfg_0[3], padding=block_cfg_0[0] // 2, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) self.make_layer() # Avoid building unused layers in mmdetection. if len(self.layers) < max(self.out_indices) + 1: self.layers.append( ConvModule( in_channels=self.in_channels, out_channels=self.out_channels, kernel_size=block_cfg_last[0], stride=block_cfg_last[3], padding=block_cfg_last[0] // 2, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) def make_layer(self): # Without the first and the final conv block. layer_setting = self.layer_setting[1:-1] total_num_blocks = sum([len(x) for x in layer_setting]) block_idx = 0 dpr = [ x.item() for x in torch.linspace(0, self.drop_path_rate, total_num_blocks) ] # stochastic depth decay rule for i, layer_cfg in enumerate(layer_setting): # Avoid building unused layers in mmdetection. if i > max(self.out_indices) - 1: break layer = [] for i, block_cfg in enumerate(layer_cfg): (kernel_size, out_channels, se_ratio, stride, expand_ratio, block_type) = block_cfg mid_channels = int(self.in_channels * expand_ratio) out_channels = make_divisible(out_channels, 8) if se_ratio <= 0: se_cfg = None else: # In mmdetection, the `divisor` is deleted to align # the logic of SELayer with mmcls. se_cfg = dict( channels=mid_channels, ratio=expand_ratio * se_ratio, act_cfg=(self.act_cfg, dict(type='Sigmoid'))) if block_type == 1: # edge tpu if i > 0 and expand_ratio == 3: with_residual = False expand_ratio = 4 else: with_residual = True mid_channels = int(self.in_channels * expand_ratio) if se_cfg is not None: # In mmdetection, the `divisor` is deleted to align # the logic of SELayer with mmcls. se_cfg = dict( channels=mid_channels, ratio=se_ratio * expand_ratio, act_cfg=(self.act_cfg, dict(type='Sigmoid'))) block = partial(EdgeResidual, with_residual=with_residual) else: block = InvertedResidual layer.append( block( in_channels=self.in_channels, out_channels=out_channels, mid_channels=mid_channels, kernel_size=kernel_size, stride=stride, se_cfg=se_cfg, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg, drop_path_rate=dpr[block_idx], with_cp=self.with_cp, # In mmdetection, `with_expand_conv` is set to align # the logic of InvertedResidual with mmcls. with_expand_conv=(mid_channels != self.in_channels))) self.in_channels = out_channels block_idx += 1 self.layers.append(Sequential(*layer)) def forward(self, x): outs = [] for i, layer in enumerate(self.layers): x = layer(x) if i in self.out_indices: outs.append(x) return tuple(outs) def _freeze_stages(self): for i in range(self.frozen_stages): m = self.layers[i] m.eval() for param in m.parameters(): param.requires_grad = False def train(self, mode=True): super(EfficientNet, self).train(mode) self._freeze_stages() if mode and self.norm_eval: for m in self.modules(): if isinstance(m, nn.BatchNorm2d): m.eval() ================================================ FILE: mmdet/models/backbones/hourglass.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Sequence import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.model import BaseModule from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptMultiConfig from ..layers import ResLayer from .resnet import BasicBlock class HourglassModule(BaseModule): """Hourglass Module for HourglassNet backbone. Generate module recursively and use BasicBlock as the base unit. Args: depth (int): Depth of current HourglassModule. stage_channels (list[int]): Feature channels of sub-modules in current and follow-up HourglassModule. stage_blocks (list[int]): Number of sub-modules stacked in current and follow-up HourglassModule. norm_cfg (ConfigType): Dictionary to construct and config norm layer. Defaults to `dict(type='BN', requires_grad=True)` upsample_cfg (ConfigType): Config dict for interpolate layer. Defaults to `dict(mode='nearest')` init_cfg (dict or ConfigDict, optional): the config to control the initialization. """ def __init__(self, depth: int, stage_channels: List[int], stage_blocks: List[int], norm_cfg: ConfigType = dict(type='BN', requires_grad=True), upsample_cfg: ConfigType = dict(mode='nearest'), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg) self.depth = depth cur_block = stage_blocks[0] next_block = stage_blocks[1] cur_channel = stage_channels[0] next_channel = stage_channels[1] self.up1 = ResLayer( BasicBlock, cur_channel, cur_channel, cur_block, norm_cfg=norm_cfg) self.low1 = ResLayer( BasicBlock, cur_channel, next_channel, cur_block, stride=2, norm_cfg=norm_cfg) if self.depth > 1: self.low2 = HourglassModule(depth - 1, stage_channels[1:], stage_blocks[1:]) else: self.low2 = ResLayer( BasicBlock, next_channel, next_channel, next_block, norm_cfg=norm_cfg) self.low3 = ResLayer( BasicBlock, next_channel, cur_channel, cur_block, norm_cfg=norm_cfg, downsample_first=False) self.up2 = F.interpolate self.upsample_cfg = upsample_cfg def forward(self, x: torch.Tensor) -> nn.Module: """Forward function.""" up1 = self.up1(x) low1 = self.low1(x) low2 = self.low2(low1) low3 = self.low3(low2) # Fixing `scale factor` (e.g. 2) is common for upsampling, but # in some cases the spatial size is mismatched and error will arise. if 'scale_factor' in self.upsample_cfg: up2 = self.up2(low3, **self.upsample_cfg) else: shape = up1.shape[2:] up2 = self.up2(low3, size=shape, **self.upsample_cfg) return up1 + up2 @MODELS.register_module() class HourglassNet(BaseModule): """HourglassNet backbone. Stacked Hourglass Networks for Human Pose Estimation. More details can be found in the `paper `_ . Args: downsample_times (int): Downsample times in a HourglassModule. num_stacks (int): Number of HourglassModule modules stacked, 1 for Hourglass-52, 2 for Hourglass-104. stage_channels (Sequence[int]): Feature channel of each sub-module in a HourglassModule. stage_blocks (Sequence[int]): Number of sub-modules stacked in a HourglassModule. feat_channel (int): Feature channel of conv after a HourglassModule. norm_cfg (norm_cfg): Dictionary to construct and config norm layer. init_cfg (dict or ConfigDict, optional): the config to control the initialization. Example: >>> from mmdet.models import HourglassNet >>> import torch >>> self = HourglassNet() >>> self.eval() >>> inputs = torch.rand(1, 3, 511, 511) >>> level_outputs = self.forward(inputs) >>> for level_output in level_outputs: ... print(tuple(level_output.shape)) (1, 256, 128, 128) (1, 256, 128, 128) """ def __init__(self, downsample_times: int = 5, num_stacks: int = 2, stage_channels: Sequence = (256, 256, 384, 384, 384, 512), stage_blocks: Sequence = (2, 2, 2, 2, 2, 4), feat_channel: int = 256, norm_cfg: ConfigType = dict(type='BN', requires_grad=True), init_cfg: OptMultiConfig = None) -> None: assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' super().__init__(init_cfg) self.num_stacks = num_stacks assert self.num_stacks >= 1 assert len(stage_channels) == len(stage_blocks) assert len(stage_channels) > downsample_times cur_channel = stage_channels[0] self.stem = nn.Sequential( ConvModule( 3, cur_channel // 2, 7, padding=3, stride=2, norm_cfg=norm_cfg), ResLayer( BasicBlock, cur_channel // 2, cur_channel, 1, stride=2, norm_cfg=norm_cfg)) self.hourglass_modules = nn.ModuleList([ HourglassModule(downsample_times, stage_channels, stage_blocks) for _ in range(num_stacks) ]) self.inters = ResLayer( BasicBlock, cur_channel, cur_channel, num_stacks - 1, norm_cfg=norm_cfg) self.conv1x1s = nn.ModuleList([ ConvModule( cur_channel, cur_channel, 1, norm_cfg=norm_cfg, act_cfg=None) for _ in range(num_stacks - 1) ]) self.out_convs = nn.ModuleList([ ConvModule( cur_channel, feat_channel, 3, padding=1, norm_cfg=norm_cfg) for _ in range(num_stacks) ]) self.remap_convs = nn.ModuleList([ ConvModule( feat_channel, cur_channel, 1, norm_cfg=norm_cfg, act_cfg=None) for _ in range(num_stacks - 1) ]) self.relu = nn.ReLU(inplace=True) def init_weights(self) -> None: """Init module weights.""" # Training Centripetal Model needs to reset parameters for Conv2d super().init_weights() for m in self.modules(): if isinstance(m, nn.Conv2d): m.reset_parameters() def forward(self, x: torch.Tensor) -> List[torch.Tensor]: """Forward function.""" inter_feat = self.stem(x) out_feats = [] for ind in range(self.num_stacks): single_hourglass = self.hourglass_modules[ind] out_conv = self.out_convs[ind] hourglass_feat = single_hourglass(inter_feat) out_feat = out_conv(hourglass_feat) out_feats.append(out_feat) if ind < self.num_stacks - 1: inter_feat = self.conv1x1s[ind]( inter_feat) + self.remap_convs[ind]( out_feat) inter_feat = self.inters[ind](self.relu(inter_feat)) return out_feats ================================================ FILE: mmdet/models/backbones/hrnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings import torch.nn as nn from mmcv.cnn import build_conv_layer, build_norm_layer from mmengine.model import BaseModule, ModuleList, Sequential from torch.nn.modules.batchnorm import _BatchNorm from mmdet.registry import MODELS from .resnet import BasicBlock, Bottleneck class HRModule(BaseModule): """High-Resolution Module for HRNet. In this module, every branch has 4 BasicBlocks/Bottlenecks. Fusion/Exchange is in this module. """ def __init__(self, num_branches, blocks, num_blocks, in_channels, num_channels, multiscale_output=True, with_cp=False, conv_cfg=None, norm_cfg=dict(type='BN'), block_init_cfg=None, init_cfg=None): super(HRModule, self).__init__(init_cfg) self.block_init_cfg = block_init_cfg self._check_branches(num_branches, num_blocks, in_channels, num_channels) self.in_channels = in_channels self.num_branches = num_branches self.multiscale_output = multiscale_output self.norm_cfg = norm_cfg self.conv_cfg = conv_cfg self.with_cp = with_cp self.branches = self._make_branches(num_branches, blocks, num_blocks, num_channels) self.fuse_layers = self._make_fuse_layers() self.relu = nn.ReLU(inplace=False) def _check_branches(self, num_branches, num_blocks, in_channels, num_channels): if num_branches != len(num_blocks): error_msg = f'NUM_BRANCHES({num_branches}) ' \ f'!= NUM_BLOCKS({len(num_blocks)})' raise ValueError(error_msg) if num_branches != len(num_channels): error_msg = f'NUM_BRANCHES({num_branches}) ' \ f'!= NUM_CHANNELS({len(num_channels)})' raise ValueError(error_msg) if num_branches != len(in_channels): error_msg = f'NUM_BRANCHES({num_branches}) ' \ f'!= NUM_INCHANNELS({len(in_channels)})' raise ValueError(error_msg) def _make_one_branch(self, branch_index, block, num_blocks, num_channels, stride=1): downsample = None if stride != 1 or \ self.in_channels[branch_index] != \ num_channels[branch_index] * block.expansion: downsample = nn.Sequential( build_conv_layer( self.conv_cfg, self.in_channels[branch_index], num_channels[branch_index] * block.expansion, kernel_size=1, stride=stride, bias=False), build_norm_layer(self.norm_cfg, num_channels[branch_index] * block.expansion)[1]) layers = [] layers.append( block( self.in_channels[branch_index], num_channels[branch_index], stride, downsample=downsample, with_cp=self.with_cp, norm_cfg=self.norm_cfg, conv_cfg=self.conv_cfg, init_cfg=self.block_init_cfg)) self.in_channels[branch_index] = \ num_channels[branch_index] * block.expansion for i in range(1, num_blocks[branch_index]): layers.append( block( self.in_channels[branch_index], num_channels[branch_index], with_cp=self.with_cp, norm_cfg=self.norm_cfg, conv_cfg=self.conv_cfg, init_cfg=self.block_init_cfg)) return Sequential(*layers) def _make_branches(self, num_branches, block, num_blocks, num_channels): branches = [] for i in range(num_branches): branches.append( self._make_one_branch(i, block, num_blocks, num_channels)) return ModuleList(branches) def _make_fuse_layers(self): if self.num_branches == 1: return None num_branches = self.num_branches in_channels = self.in_channels fuse_layers = [] num_out_branches = num_branches if self.multiscale_output else 1 for i in range(num_out_branches): fuse_layer = [] for j in range(num_branches): if j > i: fuse_layer.append( nn.Sequential( build_conv_layer( self.conv_cfg, in_channels[j], in_channels[i], kernel_size=1, stride=1, padding=0, bias=False), build_norm_layer(self.norm_cfg, in_channels[i])[1], nn.Upsample( scale_factor=2**(j - i), mode='nearest'))) elif j == i: fuse_layer.append(None) else: conv_downsamples = [] for k in range(i - j): if k == i - j - 1: conv_downsamples.append( nn.Sequential( build_conv_layer( self.conv_cfg, in_channels[j], in_channels[i], kernel_size=3, stride=2, padding=1, bias=False), build_norm_layer(self.norm_cfg, in_channels[i])[1])) else: conv_downsamples.append( nn.Sequential( build_conv_layer( self.conv_cfg, in_channels[j], in_channels[j], kernel_size=3, stride=2, padding=1, bias=False), build_norm_layer(self.norm_cfg, in_channels[j])[1], nn.ReLU(inplace=False))) fuse_layer.append(nn.Sequential(*conv_downsamples)) fuse_layers.append(nn.ModuleList(fuse_layer)) return nn.ModuleList(fuse_layers) def forward(self, x): """Forward function.""" if self.num_branches == 1: return [self.branches[0](x[0])] for i in range(self.num_branches): x[i] = self.branches[i](x[i]) x_fuse = [] for i in range(len(self.fuse_layers)): y = 0 for j in range(self.num_branches): if i == j: y += x[j] else: y += self.fuse_layers[i][j](x[j]) x_fuse.append(self.relu(y)) return x_fuse @MODELS.register_module() class HRNet(BaseModule): """HRNet backbone. `High-Resolution Representations for Labeling Pixels and Regions arXiv: `_. Args: extra (dict): Detailed configuration for each stage of HRNet. There must be 4 stages, the configuration for each stage must have 5 keys: - num_modules(int): The number of HRModule in this stage. - num_branches(int): The number of branches in the HRModule. - block(str): The type of convolution block. - num_blocks(tuple): The number of blocks in each branch. The length must be equal to num_branches. - num_channels(tuple): The number of channels in each branch. The length must be equal to num_branches. in_channels (int): Number of input image channels. Default: 3. conv_cfg (dict): Dictionary to construct and config conv layer. norm_cfg (dict): Dictionary to construct and config norm layer. norm_eval (bool): Whether to set norm layers to eval mode, namely, freeze running stats (mean and var). Note: Effect on Batch Norm and its variants only. Default: True. with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. Default: False. zero_init_residual (bool): Whether to use zero init for last norm layer in resblocks to let them behave as identity. Default: False. multiscale_output (bool): Whether to output multi-level features produced by multiple branches. If False, only the first level feature will be output. Default: True. pretrained (str, optional): Model pretrained path. Default: None. init_cfg (dict or list[dict], optional): Initialization config dict. Default: None. Example: >>> from mmdet.models import HRNet >>> import torch >>> extra = dict( >>> stage1=dict( >>> num_modules=1, >>> num_branches=1, >>> block='BOTTLENECK', >>> num_blocks=(4, ), >>> num_channels=(64, )), >>> stage2=dict( >>> num_modules=1, >>> num_branches=2, >>> block='BASIC', >>> num_blocks=(4, 4), >>> num_channels=(32, 64)), >>> stage3=dict( >>> num_modules=4, >>> num_branches=3, >>> block='BASIC', >>> num_blocks=(4, 4, 4), >>> num_channels=(32, 64, 128)), >>> stage4=dict( >>> num_modules=3, >>> num_branches=4, >>> block='BASIC', >>> num_blocks=(4, 4, 4, 4), >>> num_channels=(32, 64, 128, 256))) >>> self = HRNet(extra, in_channels=1) >>> self.eval() >>> inputs = torch.rand(1, 1, 32, 32) >>> level_outputs = self.forward(inputs) >>> for level_out in level_outputs: ... print(tuple(level_out.shape)) (1, 32, 8, 8) (1, 64, 4, 4) (1, 128, 2, 2) (1, 256, 1, 1) """ blocks_dict = {'BASIC': BasicBlock, 'BOTTLENECK': Bottleneck} def __init__(self, extra, in_channels=3, conv_cfg=None, norm_cfg=dict(type='BN'), norm_eval=True, with_cp=False, zero_init_residual=False, multiscale_output=True, pretrained=None, init_cfg=None): super(HRNet, self).__init__(init_cfg) self.pretrained = pretrained assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be specified at the same time' if isinstance(pretrained, str): warnings.warn('DeprecationWarning: pretrained is deprecated, ' 'please use "init_cfg" instead') self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) elif pretrained is None: if init_cfg is None: self.init_cfg = [ dict(type='Kaiming', layer='Conv2d'), dict( type='Constant', val=1, layer=['_BatchNorm', 'GroupNorm']) ] else: raise TypeError('pretrained must be a str or None') # Assert configurations of 4 stages are in extra assert 'stage1' in extra and 'stage2' in extra \ and 'stage3' in extra and 'stage4' in extra # Assert whether the length of `num_blocks` and `num_channels` are # equal to `num_branches` for i in range(4): cfg = extra[f'stage{i + 1}'] assert len(cfg['num_blocks']) == cfg['num_branches'] and \ len(cfg['num_channels']) == cfg['num_branches'] self.extra = extra self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.norm_eval = norm_eval self.with_cp = with_cp self.zero_init_residual = zero_init_residual # stem net self.norm1_name, norm1 = build_norm_layer(self.norm_cfg, 64, postfix=1) self.norm2_name, norm2 = build_norm_layer(self.norm_cfg, 64, postfix=2) self.conv1 = build_conv_layer( self.conv_cfg, in_channels, 64, kernel_size=3, stride=2, padding=1, bias=False) self.add_module(self.norm1_name, norm1) self.conv2 = build_conv_layer( self.conv_cfg, 64, 64, kernel_size=3, stride=2, padding=1, bias=False) self.add_module(self.norm2_name, norm2) self.relu = nn.ReLU(inplace=True) # stage 1 self.stage1_cfg = self.extra['stage1'] num_channels = self.stage1_cfg['num_channels'][0] block_type = self.stage1_cfg['block'] num_blocks = self.stage1_cfg['num_blocks'][0] block = self.blocks_dict[block_type] stage1_out_channels = num_channels * block.expansion self.layer1 = self._make_layer(block, 64, num_channels, num_blocks) # stage 2 self.stage2_cfg = self.extra['stage2'] num_channels = self.stage2_cfg['num_channels'] block_type = self.stage2_cfg['block'] block = self.blocks_dict[block_type] num_channels = [channel * block.expansion for channel in num_channels] self.transition1 = self._make_transition_layer([stage1_out_channels], num_channels) self.stage2, pre_stage_channels = self._make_stage( self.stage2_cfg, num_channels) # stage 3 self.stage3_cfg = self.extra['stage3'] num_channels = self.stage3_cfg['num_channels'] block_type = self.stage3_cfg['block'] block = self.blocks_dict[block_type] num_channels = [channel * block.expansion for channel in num_channels] self.transition2 = self._make_transition_layer(pre_stage_channels, num_channels) self.stage3, pre_stage_channels = self._make_stage( self.stage3_cfg, num_channels) # stage 4 self.stage4_cfg = self.extra['stage4'] num_channels = self.stage4_cfg['num_channels'] block_type = self.stage4_cfg['block'] block = self.blocks_dict[block_type] num_channels = [channel * block.expansion for channel in num_channels] self.transition3 = self._make_transition_layer(pre_stage_channels, num_channels) self.stage4, pre_stage_channels = self._make_stage( self.stage4_cfg, num_channels, multiscale_output=multiscale_output) @property def norm1(self): """nn.Module: the normalization layer named "norm1" """ return getattr(self, self.norm1_name) @property def norm2(self): """nn.Module: the normalization layer named "norm2" """ return getattr(self, self.norm2_name) def _make_transition_layer(self, num_channels_pre_layer, num_channels_cur_layer): num_branches_cur = len(num_channels_cur_layer) num_branches_pre = len(num_channels_pre_layer) transition_layers = [] for i in range(num_branches_cur): if i < num_branches_pre: if num_channels_cur_layer[i] != num_channels_pre_layer[i]: transition_layers.append( nn.Sequential( build_conv_layer( self.conv_cfg, num_channels_pre_layer[i], num_channels_cur_layer[i], kernel_size=3, stride=1, padding=1, bias=False), build_norm_layer(self.norm_cfg, num_channels_cur_layer[i])[1], nn.ReLU(inplace=True))) else: transition_layers.append(None) else: conv_downsamples = [] for j in range(i + 1 - num_branches_pre): in_channels = num_channels_pre_layer[-1] out_channels = num_channels_cur_layer[i] \ if j == i - num_branches_pre else in_channels conv_downsamples.append( nn.Sequential( build_conv_layer( self.conv_cfg, in_channels, out_channels, kernel_size=3, stride=2, padding=1, bias=False), build_norm_layer(self.norm_cfg, out_channels)[1], nn.ReLU(inplace=True))) transition_layers.append(nn.Sequential(*conv_downsamples)) return nn.ModuleList(transition_layers) def _make_layer(self, block, inplanes, planes, blocks, stride=1): downsample = None if stride != 1 or inplanes != planes * block.expansion: downsample = nn.Sequential( build_conv_layer( self.conv_cfg, inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False), build_norm_layer(self.norm_cfg, planes * block.expansion)[1]) layers = [] block_init_cfg = None if self.pretrained is None and not hasattr( self, 'init_cfg') and self.zero_init_residual: if block is BasicBlock: block_init_cfg = dict( type='Constant', val=0, override=dict(name='norm2')) elif block is Bottleneck: block_init_cfg = dict( type='Constant', val=0, override=dict(name='norm3')) layers.append( block( inplanes, planes, stride, downsample=downsample, with_cp=self.with_cp, norm_cfg=self.norm_cfg, conv_cfg=self.conv_cfg, init_cfg=block_init_cfg, )) inplanes = planes * block.expansion for i in range(1, blocks): layers.append( block( inplanes, planes, with_cp=self.with_cp, norm_cfg=self.norm_cfg, conv_cfg=self.conv_cfg, init_cfg=block_init_cfg)) return Sequential(*layers) def _make_stage(self, layer_config, in_channels, multiscale_output=True): num_modules = layer_config['num_modules'] num_branches = layer_config['num_branches'] num_blocks = layer_config['num_blocks'] num_channels = layer_config['num_channels'] block = self.blocks_dict[layer_config['block']] hr_modules = [] block_init_cfg = None if self.pretrained is None and not hasattr( self, 'init_cfg') and self.zero_init_residual: if block is BasicBlock: block_init_cfg = dict( type='Constant', val=0, override=dict(name='norm2')) elif block is Bottleneck: block_init_cfg = dict( type='Constant', val=0, override=dict(name='norm3')) for i in range(num_modules): # multi_scale_output is only used for the last module if not multiscale_output and i == num_modules - 1: reset_multiscale_output = False else: reset_multiscale_output = True hr_modules.append( HRModule( num_branches, block, num_blocks, in_channels, num_channels, reset_multiscale_output, with_cp=self.with_cp, norm_cfg=self.norm_cfg, conv_cfg=self.conv_cfg, block_init_cfg=block_init_cfg)) return Sequential(*hr_modules), in_channels def forward(self, x): """Forward function.""" x = self.conv1(x) x = self.norm1(x) x = self.relu(x) x = self.conv2(x) x = self.norm2(x) x = self.relu(x) x = self.layer1(x) x_list = [] for i in range(self.stage2_cfg['num_branches']): if self.transition1[i] is not None: x_list.append(self.transition1[i](x)) else: x_list.append(x) y_list = self.stage2(x_list) x_list = [] for i in range(self.stage3_cfg['num_branches']): if self.transition2[i] is not None: x_list.append(self.transition2[i](y_list[-1])) else: x_list.append(y_list[i]) y_list = self.stage3(x_list) x_list = [] for i in range(self.stage4_cfg['num_branches']): if self.transition3[i] is not None: x_list.append(self.transition3[i](y_list[-1])) else: x_list.append(y_list[i]) y_list = self.stage4(x_list) return y_list def train(self, mode=True): """Convert the model into training mode will keeping the normalization layer freezed.""" super(HRNet, self).train(mode) if mode and self.norm_eval: for m in self.modules(): # trick: eval have effect on BatchNorm only if isinstance(m, _BatchNorm): m.eval() ================================================ FILE: mmdet/models/backbones/mobilenet_v2.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings import torch.nn as nn from mmcv.cnn import ConvModule from mmengine.model import BaseModule from torch.nn.modules.batchnorm import _BatchNorm from mmdet.registry import MODELS from ..layers import InvertedResidual from ..utils import make_divisible @MODELS.register_module() class MobileNetV2(BaseModule): """MobileNetV2 backbone. Args: widen_factor (float): Width multiplier, multiply number of channels in each layer by this amount. Default: 1.0. out_indices (Sequence[int], optional): Output from which stages. Default: (1, 2, 4, 7). frozen_stages (int): Stages to be frozen (all param fixed). Default: -1, which means not freezing any parameters. conv_cfg (dict, optional): Config dict for convolution layer. Default: None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Default: dict(type='BN'). act_cfg (dict): Config dict for activation layer. Default: dict(type='ReLU6'). norm_eval (bool): Whether to set norm layers to eval mode, namely, freeze running stats (mean and var). Note: Effect on Batch Norm and its variants only. Default: False. with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. Default: False. pretrained (str, optional): model pretrained path. Default: None init_cfg (dict or list[dict], optional): Initialization config dict. Default: None """ # Parameters to build layers. 4 parameters are needed to construct a # layer, from left to right: expand_ratio, channel, num_blocks, stride. arch_settings = [[1, 16, 1, 1], [6, 24, 2, 2], [6, 32, 3, 2], [6, 64, 4, 2], [6, 96, 3, 1], [6, 160, 3, 2], [6, 320, 1, 1]] def __init__(self, widen_factor=1., out_indices=(1, 2, 4, 7), frozen_stages=-1, conv_cfg=None, norm_cfg=dict(type='BN'), act_cfg=dict(type='ReLU6'), norm_eval=False, with_cp=False, pretrained=None, init_cfg=None): super(MobileNetV2, self).__init__(init_cfg) self.pretrained = pretrained assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be specified at the same time' if isinstance(pretrained, str): warnings.warn('DeprecationWarning: pretrained is deprecated, ' 'please use "init_cfg" instead') self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) elif pretrained is None: if init_cfg is None: self.init_cfg = [ dict(type='Kaiming', layer='Conv2d'), dict( type='Constant', val=1, layer=['_BatchNorm', 'GroupNorm']) ] else: raise TypeError('pretrained must be a str or None') self.widen_factor = widen_factor self.out_indices = out_indices if not set(out_indices).issubset(set(range(0, 8))): raise ValueError('out_indices must be a subset of range' f'(0, 8). But received {out_indices}') if frozen_stages not in range(-1, 8): raise ValueError('frozen_stages must be in range(-1, 8). ' f'But received {frozen_stages}') self.out_indices = out_indices self.frozen_stages = frozen_stages self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.act_cfg = act_cfg self.norm_eval = norm_eval self.with_cp = with_cp self.in_channels = make_divisible(32 * widen_factor, 8) self.conv1 = ConvModule( in_channels=3, out_channels=self.in_channels, kernel_size=3, stride=2, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg) self.layers = [] for i, layer_cfg in enumerate(self.arch_settings): expand_ratio, channel, num_blocks, stride = layer_cfg out_channels = make_divisible(channel * widen_factor, 8) inverted_res_layer = self.make_layer( out_channels=out_channels, num_blocks=num_blocks, stride=stride, expand_ratio=expand_ratio) layer_name = f'layer{i + 1}' self.add_module(layer_name, inverted_res_layer) self.layers.append(layer_name) if widen_factor > 1.0: self.out_channel = int(1280 * widen_factor) else: self.out_channel = 1280 layer = ConvModule( in_channels=self.in_channels, out_channels=self.out_channel, kernel_size=1, stride=1, padding=0, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg) self.add_module('conv2', layer) self.layers.append('conv2') def make_layer(self, out_channels, num_blocks, stride, expand_ratio): """Stack InvertedResidual blocks to build a layer for MobileNetV2. Args: out_channels (int): out_channels of block. num_blocks (int): number of blocks. stride (int): stride of the first block. Default: 1 expand_ratio (int): Expand the number of channels of the hidden layer in InvertedResidual by this ratio. Default: 6. """ layers = [] for i in range(num_blocks): if i >= 1: stride = 1 layers.append( InvertedResidual( self.in_channels, out_channels, mid_channels=int(round(self.in_channels * expand_ratio)), stride=stride, with_expand_conv=expand_ratio != 1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg, with_cp=self.with_cp)) self.in_channels = out_channels return nn.Sequential(*layers) def _freeze_stages(self): if self.frozen_stages >= 0: for param in self.conv1.parameters(): param.requires_grad = False for i in range(1, self.frozen_stages + 1): layer = getattr(self, f'layer{i}') layer.eval() for param in layer.parameters(): param.requires_grad = False def forward(self, x): """Forward function.""" x = self.conv1(x) outs = [] for i, layer_name in enumerate(self.layers): layer = getattr(self, layer_name) x = layer(x) if i in self.out_indices: outs.append(x) return tuple(outs) def train(self, mode=True): """Convert the model into training mode while keep normalization layer frozen.""" super(MobileNetV2, self).train(mode) self._freeze_stages() if mode and self.norm_eval: for m in self.modules(): # trick: eval have effect on BatchNorm only if isinstance(m, _BatchNorm): m.eval() ================================================ FILE: mmdet/models/backbones/pvt.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math import warnings from collections import OrderedDict import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import Conv2d, build_activation_layer, build_norm_layer from mmcv.cnn.bricks.drop import build_dropout from mmcv.cnn.bricks.transformer import MultiheadAttention from mmengine.logging import MMLogger from mmengine.model import (BaseModule, ModuleList, Sequential, constant_init, normal_init, trunc_normal_init) from mmengine.model.weight_init import trunc_normal_ from mmengine.runner.checkpoint import CheckpointLoader, load_state_dict from torch.nn.modules.utils import _pair as to_2tuple from mmdet.registry import MODELS from ..layers import PatchEmbed, nchw_to_nlc, nlc_to_nchw class MixFFN(BaseModule): """An implementation of MixFFN of PVT. The differences between MixFFN & FFN: 1. Use 1X1 Conv to replace Linear layer. 2. Introduce 3X3 Depth-wise Conv to encode positional information. Args: embed_dims (int): The feature dimension. Same as `MultiheadAttention`. feedforward_channels (int): The hidden dimension of FFNs. act_cfg (dict, optional): The activation config for FFNs. Default: dict(type='GELU'). ffn_drop (float, optional): Probability of an element to be zeroed in FFN. Default 0.0. dropout_layer (obj:`ConfigDict`): The dropout_layer used when adding the shortcut. Default: None. use_conv (bool): If True, add 3x3 DWConv between two Linear layers. Defaults: False. init_cfg (obj:`mmengine.ConfigDict`): The Config for initialization. Default: None. """ def __init__(self, embed_dims, feedforward_channels, act_cfg=dict(type='GELU'), ffn_drop=0., dropout_layer=None, use_conv=False, init_cfg=None): super(MixFFN, self).__init__(init_cfg=init_cfg) self.embed_dims = embed_dims self.feedforward_channels = feedforward_channels self.act_cfg = act_cfg activate = build_activation_layer(act_cfg) in_channels = embed_dims fc1 = Conv2d( in_channels=in_channels, out_channels=feedforward_channels, kernel_size=1, stride=1, bias=True) if use_conv: # 3x3 depth wise conv to provide positional encode information dw_conv = Conv2d( in_channels=feedforward_channels, out_channels=feedforward_channels, kernel_size=3, stride=1, padding=(3 - 1) // 2, bias=True, groups=feedforward_channels) fc2 = Conv2d( in_channels=feedforward_channels, out_channels=in_channels, kernel_size=1, stride=1, bias=True) drop = nn.Dropout(ffn_drop) layers = [fc1, activate, drop, fc2, drop] if use_conv: layers.insert(1, dw_conv) self.layers = Sequential(*layers) self.dropout_layer = build_dropout( dropout_layer) if dropout_layer else torch.nn.Identity() def forward(self, x, hw_shape, identity=None): out = nlc_to_nchw(x, hw_shape) out = self.layers(out) out = nchw_to_nlc(out) if identity is None: identity = x return identity + self.dropout_layer(out) class SpatialReductionAttention(MultiheadAttention): """An implementation of Spatial Reduction Attention of PVT. This module is modified from MultiheadAttention which is a module from mmcv.cnn.bricks.transformer. Args: embed_dims (int): The embedding dimension. num_heads (int): Parallel attention heads. attn_drop (float): A Dropout layer on attn_output_weights. Default: 0.0. proj_drop (float): A Dropout layer after `nn.MultiheadAttention`. Default: 0.0. dropout_layer (obj:`ConfigDict`): The dropout_layer used when adding the shortcut. Default: None. batch_first (bool): Key, Query and Value are shape of (batch, n, embed_dim) or (n, batch, embed_dim). Default: False. qkv_bias (bool): enable bias for qkv if True. Default: True. norm_cfg (dict): Config dict for normalization layer. Default: dict(type='LN'). sr_ratio (int): The ratio of spatial reduction of Spatial Reduction Attention of PVT. Default: 1. init_cfg (obj:`mmengine.ConfigDict`): The Config for initialization. Default: None. """ def __init__(self, embed_dims, num_heads, attn_drop=0., proj_drop=0., dropout_layer=None, batch_first=True, qkv_bias=True, norm_cfg=dict(type='LN'), sr_ratio=1, init_cfg=None): super().__init__( embed_dims, num_heads, attn_drop, proj_drop, batch_first=batch_first, dropout_layer=dropout_layer, bias=qkv_bias, init_cfg=init_cfg) self.sr_ratio = sr_ratio if sr_ratio > 1: self.sr = Conv2d( in_channels=embed_dims, out_channels=embed_dims, kernel_size=sr_ratio, stride=sr_ratio) # The ret[0] of build_norm_layer is norm name. self.norm = build_norm_layer(norm_cfg, embed_dims)[1] # handle the BC-breaking from https://github.com/open-mmlab/mmcv/pull/1418 # noqa from mmdet import digit_version, mmcv_version if mmcv_version < digit_version('1.3.17'): warnings.warn('The legacy version of forward function in' 'SpatialReductionAttention is deprecated in' 'mmcv>=1.3.17 and will no longer support in the' 'future. Please upgrade your mmcv.') self.forward = self.legacy_forward def forward(self, x, hw_shape, identity=None): x_q = x if self.sr_ratio > 1: x_kv = nlc_to_nchw(x, hw_shape) x_kv = self.sr(x_kv) x_kv = nchw_to_nlc(x_kv) x_kv = self.norm(x_kv) else: x_kv = x if identity is None: identity = x_q # Because the dataflow('key', 'query', 'value') of # ``torch.nn.MultiheadAttention`` is (num_queries, batch, # embed_dims), We should adjust the shape of dataflow from # batch_first (batch, num_queries, embed_dims) to num_queries_first # (num_queries ,batch, embed_dims), and recover ``attn_output`` # from num_queries_first to batch_first. if self.batch_first: x_q = x_q.transpose(0, 1) x_kv = x_kv.transpose(0, 1) out = self.attn(query=x_q, key=x_kv, value=x_kv)[0] if self.batch_first: out = out.transpose(0, 1) return identity + self.dropout_layer(self.proj_drop(out)) def legacy_forward(self, x, hw_shape, identity=None): """multi head attention forward in mmcv version < 1.3.17.""" x_q = x if self.sr_ratio > 1: x_kv = nlc_to_nchw(x, hw_shape) x_kv = self.sr(x_kv) x_kv = nchw_to_nlc(x_kv) x_kv = self.norm(x_kv) else: x_kv = x if identity is None: identity = x_q out = self.attn(query=x_q, key=x_kv, value=x_kv)[0] return identity + self.dropout_layer(self.proj_drop(out)) class PVTEncoderLayer(BaseModule): """Implements one encoder layer in PVT. Args: embed_dims (int): The feature dimension. num_heads (int): Parallel attention heads. feedforward_channels (int): The hidden dimension for FFNs. drop_rate (float): Probability of an element to be zeroed. after the feed forward layer. Default: 0.0. attn_drop_rate (float): The drop out rate for attention layer. Default: 0.0. drop_path_rate (float): stochastic depth rate. Default: 0.0. qkv_bias (bool): enable bias for qkv if True. Default: True. act_cfg (dict): The activation config for FFNs. Default: dict(type='GELU'). norm_cfg (dict): Config dict for normalization layer. Default: dict(type='LN'). sr_ratio (int): The ratio of spatial reduction of Spatial Reduction Attention of PVT. Default: 1. use_conv_ffn (bool): If True, use Convolutional FFN to replace FFN. Default: False. init_cfg (dict, optional): Initialization config dict. Default: None. """ def __init__(self, embed_dims, num_heads, feedforward_channels, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., qkv_bias=True, act_cfg=dict(type='GELU'), norm_cfg=dict(type='LN'), sr_ratio=1, use_conv_ffn=False, init_cfg=None): super(PVTEncoderLayer, self).__init__(init_cfg=init_cfg) # The ret[0] of build_norm_layer is norm name. self.norm1 = build_norm_layer(norm_cfg, embed_dims)[1] self.attn = SpatialReductionAttention( embed_dims=embed_dims, num_heads=num_heads, attn_drop=attn_drop_rate, proj_drop=drop_rate, dropout_layer=dict(type='DropPath', drop_prob=drop_path_rate), qkv_bias=qkv_bias, norm_cfg=norm_cfg, sr_ratio=sr_ratio) # The ret[0] of build_norm_layer is norm name. self.norm2 = build_norm_layer(norm_cfg, embed_dims)[1] self.ffn = MixFFN( embed_dims=embed_dims, feedforward_channels=feedforward_channels, ffn_drop=drop_rate, dropout_layer=dict(type='DropPath', drop_prob=drop_path_rate), use_conv=use_conv_ffn, act_cfg=act_cfg) def forward(self, x, hw_shape): x = self.attn(self.norm1(x), hw_shape, identity=x) x = self.ffn(self.norm2(x), hw_shape, identity=x) return x class AbsolutePositionEmbedding(BaseModule): """An implementation of the absolute position embedding in PVT. Args: pos_shape (int): The shape of the absolute position embedding. pos_dim (int): The dimension of the absolute position embedding. drop_rate (float): Probability of an element to be zeroed. Default: 0.0. """ def __init__(self, pos_shape, pos_dim, drop_rate=0., init_cfg=None): super().__init__(init_cfg=init_cfg) if isinstance(pos_shape, int): pos_shape = to_2tuple(pos_shape) elif isinstance(pos_shape, tuple): if len(pos_shape) == 1: pos_shape = to_2tuple(pos_shape[0]) assert len(pos_shape) == 2, \ f'The size of image should have length 1 or 2, ' \ f'but got {len(pos_shape)}' self.pos_shape = pos_shape self.pos_dim = pos_dim self.pos_embed = nn.Parameter( torch.zeros(1, pos_shape[0] * pos_shape[1], pos_dim)) self.drop = nn.Dropout(p=drop_rate) def init_weights(self): trunc_normal_(self.pos_embed, std=0.02) def resize_pos_embed(self, pos_embed, input_shape, mode='bilinear'): """Resize pos_embed weights. Resize pos_embed using bilinear interpolate method. Args: pos_embed (torch.Tensor): Position embedding weights. input_shape (tuple): Tuple for (downsampled input image height, downsampled input image width). mode (str): Algorithm used for upsampling: ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | ``'bicubic'`` | ``'trilinear'``. Default: ``'bilinear'``. Return: torch.Tensor: The resized pos_embed of shape [B, L_new, C]. """ assert pos_embed.ndim == 3, 'shape of pos_embed must be [B, L, C]' pos_h, pos_w = self.pos_shape pos_embed_weight = pos_embed[:, (-1 * pos_h * pos_w):] pos_embed_weight = pos_embed_weight.reshape( 1, pos_h, pos_w, self.pos_dim).permute(0, 3, 1, 2).contiguous() pos_embed_weight = F.interpolate( pos_embed_weight, size=input_shape, mode=mode) pos_embed_weight = torch.flatten(pos_embed_weight, 2).transpose(1, 2).contiguous() pos_embed = pos_embed_weight return pos_embed def forward(self, x, hw_shape, mode='bilinear'): pos_embed = self.resize_pos_embed(self.pos_embed, hw_shape, mode) return self.drop(x + pos_embed) @MODELS.register_module() class PyramidVisionTransformer(BaseModule): """Pyramid Vision Transformer (PVT) Implementation of `Pyramid Vision Transformer: A Versatile Backbone for Dense Prediction without Convolutions `_. Args: pretrain_img_size (int | tuple[int]): The size of input image when pretrain. Defaults: 224. in_channels (int): Number of input channels. Default: 3. embed_dims (int): Embedding dimension. Default: 64. num_stags (int): The num of stages. Default: 4. num_layers (Sequence[int]): The layer number of each transformer encode layer. Default: [3, 4, 6, 3]. num_heads (Sequence[int]): The attention heads of each transformer encode layer. Default: [1, 2, 5, 8]. patch_sizes (Sequence[int]): The patch_size of each patch embedding. Default: [4, 2, 2, 2]. strides (Sequence[int]): The stride of each patch embedding. Default: [4, 2, 2, 2]. paddings (Sequence[int]): The padding of each patch embedding. Default: [0, 0, 0, 0]. sr_ratios (Sequence[int]): The spatial reduction rate of each transformer encode layer. Default: [8, 4, 2, 1]. out_indices (Sequence[int] | int): Output from which stages. Default: (0, 1, 2, 3). mlp_ratios (Sequence[int]): The ratio of the mlp hidden dim to the embedding dim of each transformer encode layer. Default: [8, 8, 4, 4]. qkv_bias (bool): Enable bias for qkv if True. Default: True. drop_rate (float): Probability of an element to be zeroed. Default 0.0. attn_drop_rate (float): The drop out rate for attention layer. Default 0.0. drop_path_rate (float): stochastic depth rate. Default 0.1. use_abs_pos_embed (bool): If True, add absolute position embedding to the patch embedding. Defaults: True. use_conv_ffn (bool): If True, use Convolutional FFN to replace FFN. Default: False. act_cfg (dict): The activation config for FFNs. Default: dict(type='GELU'). norm_cfg (dict): Config dict for normalization layer. Default: dict(type='LN'). pretrained (str, optional): model pretrained path. Default: None. convert_weights (bool): The flag indicates whether the pre-trained model is from the original repo. We may need to convert some keys to make it compatible. Default: True. init_cfg (dict or list[dict], optional): Initialization config dict. Default: None. """ def __init__(self, pretrain_img_size=224, in_channels=3, embed_dims=64, num_stages=4, num_layers=[3, 4, 6, 3], num_heads=[1, 2, 5, 8], patch_sizes=[4, 2, 2, 2], strides=[4, 2, 2, 2], paddings=[0, 0, 0, 0], sr_ratios=[8, 4, 2, 1], out_indices=(0, 1, 2, 3), mlp_ratios=[8, 8, 4, 4], qkv_bias=True, drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1, use_abs_pos_embed=True, norm_after_stage=False, use_conv_ffn=False, act_cfg=dict(type='GELU'), norm_cfg=dict(type='LN', eps=1e-6), pretrained=None, convert_weights=True, init_cfg=None): super().__init__(init_cfg=init_cfg) self.convert_weights = convert_weights if isinstance(pretrain_img_size, int): pretrain_img_size = to_2tuple(pretrain_img_size) elif isinstance(pretrain_img_size, tuple): if len(pretrain_img_size) == 1: pretrain_img_size = to_2tuple(pretrain_img_size[0]) assert len(pretrain_img_size) == 2, \ f'The size of image should have length 1 or 2, ' \ f'but got {len(pretrain_img_size)}' assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be setting at the same time' if isinstance(pretrained, str): warnings.warn('DeprecationWarning: pretrained is deprecated, ' 'please use "init_cfg" instead') self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) elif pretrained is None: self.init_cfg = init_cfg else: raise TypeError('pretrained must be a str or None') self.embed_dims = embed_dims self.num_stages = num_stages self.num_layers = num_layers self.num_heads = num_heads self.patch_sizes = patch_sizes self.strides = strides self.sr_ratios = sr_ratios assert num_stages == len(num_layers) == len(num_heads) \ == len(patch_sizes) == len(strides) == len(sr_ratios) self.out_indices = out_indices assert max(out_indices) < self.num_stages self.pretrained = pretrained # transformer encoder dpr = [ x.item() for x in torch.linspace(0, drop_path_rate, sum(num_layers)) ] # stochastic num_layer decay rule cur = 0 self.layers = ModuleList() for i, num_layer in enumerate(num_layers): embed_dims_i = embed_dims * num_heads[i] patch_embed = PatchEmbed( in_channels=in_channels, embed_dims=embed_dims_i, kernel_size=patch_sizes[i], stride=strides[i], padding=paddings[i], bias=True, norm_cfg=norm_cfg) layers = ModuleList() if use_abs_pos_embed: pos_shape = pretrain_img_size // np.prod(patch_sizes[:i + 1]) pos_embed = AbsolutePositionEmbedding( pos_shape=pos_shape, pos_dim=embed_dims_i, drop_rate=drop_rate) layers.append(pos_embed) layers.extend([ PVTEncoderLayer( embed_dims=embed_dims_i, num_heads=num_heads[i], feedforward_channels=mlp_ratios[i] * embed_dims_i, drop_rate=drop_rate, attn_drop_rate=attn_drop_rate, drop_path_rate=dpr[cur + idx], qkv_bias=qkv_bias, act_cfg=act_cfg, norm_cfg=norm_cfg, sr_ratio=sr_ratios[i], use_conv_ffn=use_conv_ffn) for idx in range(num_layer) ]) in_channels = embed_dims_i # The ret[0] of build_norm_layer is norm name. if norm_after_stage: norm = build_norm_layer(norm_cfg, embed_dims_i)[1] else: norm = nn.Identity() self.layers.append(ModuleList([patch_embed, layers, norm])) cur += num_layer def init_weights(self): logger = MMLogger.get_current_instance() if self.init_cfg is None: logger.warn(f'No pre-trained weights for ' f'{self.__class__.__name__}, ' f'training start from scratch') for m in self.modules(): if isinstance(m, nn.Linear): trunc_normal_init(m, std=.02, bias=0.) elif isinstance(m, nn.LayerNorm): constant_init(m, 1.0) elif isinstance(m, nn.Conv2d): fan_out = m.kernel_size[0] * m.kernel_size[ 1] * m.out_channels fan_out //= m.groups normal_init(m, 0, math.sqrt(2.0 / fan_out)) elif isinstance(m, AbsolutePositionEmbedding): m.init_weights() else: assert 'checkpoint' in self.init_cfg, f'Only support ' \ f'specify `Pretrained` in ' \ f'`init_cfg` in ' \ f'{self.__class__.__name__} ' checkpoint = CheckpointLoader.load_checkpoint( self.init_cfg.checkpoint, logger=logger, map_location='cpu') logger.warn(f'Load pre-trained model for ' f'{self.__class__.__name__} from original repo') if 'state_dict' in checkpoint: state_dict = checkpoint['state_dict'] elif 'model' in checkpoint: state_dict = checkpoint['model'] else: state_dict = checkpoint if self.convert_weights: # Because pvt backbones are not supported by mmcls, # so we need to convert pre-trained weights to match this # implementation. state_dict = pvt_convert(state_dict) load_state_dict(self, state_dict, strict=False, logger=logger) def forward(self, x): outs = [] for i, layer in enumerate(self.layers): x, hw_shape = layer[0](x) for block in layer[1]: x = block(x, hw_shape) x = layer[2](x) x = nlc_to_nchw(x, hw_shape) if i in self.out_indices: outs.append(x) return outs @MODELS.register_module() class PyramidVisionTransformerV2(PyramidVisionTransformer): """Implementation of `PVTv2: Improved Baselines with Pyramid Vision Transformer `_.""" def __init__(self, **kwargs): super(PyramidVisionTransformerV2, self).__init__( patch_sizes=[7, 3, 3, 3], paddings=[3, 1, 1, 1], use_abs_pos_embed=False, norm_after_stage=True, use_conv_ffn=True, **kwargs) def pvt_convert(ckpt): new_ckpt = OrderedDict() # Process the concat between q linear weights and kv linear weights use_abs_pos_embed = False use_conv_ffn = False for k in ckpt.keys(): if k.startswith('pos_embed'): use_abs_pos_embed = True if k.find('dwconv') >= 0: use_conv_ffn = True for k, v in ckpt.items(): if k.startswith('head'): continue if k.startswith('norm.'): continue if k.startswith('cls_token'): continue if k.startswith('pos_embed'): stage_i = int(k.replace('pos_embed', '')) new_k = k.replace(f'pos_embed{stage_i}', f'layers.{stage_i - 1}.1.0.pos_embed') if stage_i == 4 and v.size(1) == 50: # 1 (cls token) + 7 * 7 new_v = v[:, 1:, :] # remove cls token else: new_v = v elif k.startswith('patch_embed'): stage_i = int(k.split('.')[0].replace('patch_embed', '')) new_k = k.replace(f'patch_embed{stage_i}', f'layers.{stage_i - 1}.0') new_v = v if 'proj.' in new_k: new_k = new_k.replace('proj.', 'projection.') elif k.startswith('block'): stage_i = int(k.split('.')[0].replace('block', '')) layer_i = int(k.split('.')[1]) new_layer_i = layer_i + use_abs_pos_embed new_k = k.replace(f'block{stage_i}.{layer_i}', f'layers.{stage_i - 1}.1.{new_layer_i}') new_v = v if 'attn.q.' in new_k: sub_item_k = k.replace('q.', 'kv.') new_k = new_k.replace('q.', 'attn.in_proj_') new_v = torch.cat([v, ckpt[sub_item_k]], dim=0) elif 'attn.kv.' in new_k: continue elif 'attn.proj.' in new_k: new_k = new_k.replace('proj.', 'attn.out_proj.') elif 'attn.sr.' in new_k: new_k = new_k.replace('sr.', 'sr.') elif 'mlp.' in new_k: string = f'{new_k}-' new_k = new_k.replace('mlp.', 'ffn.layers.') if 'fc1.weight' in new_k or 'fc2.weight' in new_k: new_v = v.reshape((*v.shape, 1, 1)) new_k = new_k.replace('fc1.', '0.') new_k = new_k.replace('dwconv.dwconv.', '1.') if use_conv_ffn: new_k = new_k.replace('fc2.', '4.') else: new_k = new_k.replace('fc2.', '3.') string += f'{new_k} {v.shape}-{new_v.shape}' elif k.startswith('norm'): stage_i = int(k[4]) new_k = k.replace(f'norm{stage_i}', f'layers.{stage_i - 1}.2') new_v = v else: new_k = k new_v = v new_ckpt[new_k] = new_v return new_ckpt ================================================ FILE: mmdet/models/backbones/regnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings import numpy as np import torch.nn as nn from mmcv.cnn import build_conv_layer, build_norm_layer from mmdet.registry import MODELS from .resnet import ResNet from .resnext import Bottleneck @MODELS.register_module() class RegNet(ResNet): """RegNet backbone. More details can be found in `paper `_ . Args: arch (dict): The parameter of RegNets. - w0 (int): initial width - wa (float): slope of width - wm (float): quantization parameter to quantize the width - depth (int): depth of the backbone - group_w (int): width of group - bot_mul (float): bottleneck ratio, i.e. expansion of bottleneck. strides (Sequence[int]): Strides of the first block of each stage. base_channels (int): Base channels after stem layer. in_channels (int): Number of input image channels. Default: 3. dilations (Sequence[int]): Dilation of each stage. out_indices (Sequence[int]): Output from which stages. style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two layer is the 3x3 conv layer, otherwise the stride-two layer is the first 1x1 conv layer. frozen_stages (int): Stages to be frozen (all param fixed). -1 means not freezing any parameters. norm_cfg (dict): dictionary to construct and config norm layer. norm_eval (bool): Whether to set norm layers to eval mode, namely, freeze running stats (mean and var). Note: Effect on Batch Norm and its variants only. with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. zero_init_residual (bool): whether to use zero init for last norm layer in resblocks to let them behave as identity. pretrained (str, optional): model pretrained path. Default: None init_cfg (dict or list[dict], optional): Initialization config dict. Default: None Example: >>> from mmdet.models import RegNet >>> import torch >>> self = RegNet( arch=dict( w0=88, wa=26.31, wm=2.25, group_w=48, depth=25, bot_mul=1.0)) >>> self.eval() >>> inputs = torch.rand(1, 3, 32, 32) >>> level_outputs = self.forward(inputs) >>> for level_out in level_outputs: ... print(tuple(level_out.shape)) (1, 96, 8, 8) (1, 192, 4, 4) (1, 432, 2, 2) (1, 1008, 1, 1) """ arch_settings = { 'regnetx_400mf': dict(w0=24, wa=24.48, wm=2.54, group_w=16, depth=22, bot_mul=1.0), 'regnetx_800mf': dict(w0=56, wa=35.73, wm=2.28, group_w=16, depth=16, bot_mul=1.0), 'regnetx_1.6gf': dict(w0=80, wa=34.01, wm=2.25, group_w=24, depth=18, bot_mul=1.0), 'regnetx_3.2gf': dict(w0=88, wa=26.31, wm=2.25, group_w=48, depth=25, bot_mul=1.0), 'regnetx_4.0gf': dict(w0=96, wa=38.65, wm=2.43, group_w=40, depth=23, bot_mul=1.0), 'regnetx_6.4gf': dict(w0=184, wa=60.83, wm=2.07, group_w=56, depth=17, bot_mul=1.0), 'regnetx_8.0gf': dict(w0=80, wa=49.56, wm=2.88, group_w=120, depth=23, bot_mul=1.0), 'regnetx_12gf': dict(w0=168, wa=73.36, wm=2.37, group_w=112, depth=19, bot_mul=1.0), } def __init__(self, arch, in_channels=3, stem_channels=32, base_channels=32, strides=(2, 2, 2, 2), dilations=(1, 1, 1, 1), out_indices=(0, 1, 2, 3), style='pytorch', deep_stem=False, avg_down=False, frozen_stages=-1, conv_cfg=None, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, dcn=None, stage_with_dcn=(False, False, False, False), plugins=None, with_cp=False, zero_init_residual=True, pretrained=None, init_cfg=None): super(ResNet, self).__init__(init_cfg) # Generate RegNet parameters first if isinstance(arch, str): assert arch in self.arch_settings, \ f'"arch": "{arch}" is not one of the' \ ' arch_settings' arch = self.arch_settings[arch] elif not isinstance(arch, dict): raise ValueError('Expect "arch" to be either a string ' f'or a dict, got {type(arch)}') widths, num_stages = self.generate_regnet( arch['w0'], arch['wa'], arch['wm'], arch['depth'], ) # Convert to per stage format stage_widths, stage_blocks = self.get_stages_from_blocks(widths) # Generate group widths and bot muls group_widths = [arch['group_w'] for _ in range(num_stages)] self.bottleneck_ratio = [arch['bot_mul'] for _ in range(num_stages)] # Adjust the compatibility of stage_widths and group_widths stage_widths, group_widths = self.adjust_width_group( stage_widths, self.bottleneck_ratio, group_widths) # Group params by stage self.stage_widths = stage_widths self.group_widths = group_widths self.depth = sum(stage_blocks) self.stem_channels = stem_channels self.base_channels = base_channels self.num_stages = num_stages assert num_stages >= 1 and num_stages <= 4 self.strides = strides self.dilations = dilations assert len(strides) == len(dilations) == num_stages self.out_indices = out_indices assert max(out_indices) < num_stages self.style = style self.deep_stem = deep_stem self.avg_down = avg_down self.frozen_stages = frozen_stages self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.with_cp = with_cp self.norm_eval = norm_eval self.dcn = dcn self.stage_with_dcn = stage_with_dcn if dcn is not None: assert len(stage_with_dcn) == num_stages self.plugins = plugins self.zero_init_residual = zero_init_residual self.block = Bottleneck expansion_bak = self.block.expansion self.block.expansion = 1 self.stage_blocks = stage_blocks[:num_stages] self._make_stem_layer(in_channels, stem_channels) block_init_cfg = None assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be specified at the same time' if isinstance(pretrained, str): warnings.warn('DeprecationWarning: pretrained is deprecated, ' 'please use "init_cfg" instead') self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) elif pretrained is None: if init_cfg is None: self.init_cfg = [ dict(type='Kaiming', layer='Conv2d'), dict( type='Constant', val=1, layer=['_BatchNorm', 'GroupNorm']) ] if self.zero_init_residual: block_init_cfg = dict( type='Constant', val=0, override=dict(name='norm3')) else: raise TypeError('pretrained must be a str or None') self.inplanes = stem_channels self.res_layers = [] for i, num_blocks in enumerate(self.stage_blocks): stride = self.strides[i] dilation = self.dilations[i] group_width = self.group_widths[i] width = int(round(self.stage_widths[i] * self.bottleneck_ratio[i])) stage_groups = width // group_width dcn = self.dcn if self.stage_with_dcn[i] else None if self.plugins is not None: stage_plugins = self.make_stage_plugins(self.plugins, i) else: stage_plugins = None res_layer = self.make_res_layer( block=self.block, inplanes=self.inplanes, planes=self.stage_widths[i], num_blocks=num_blocks, stride=stride, dilation=dilation, style=self.style, avg_down=self.avg_down, with_cp=self.with_cp, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, dcn=dcn, plugins=stage_plugins, groups=stage_groups, base_width=group_width, base_channels=self.stage_widths[i], init_cfg=block_init_cfg) self.inplanes = self.stage_widths[i] layer_name = f'layer{i + 1}' self.add_module(layer_name, res_layer) self.res_layers.append(layer_name) self._freeze_stages() self.feat_dim = stage_widths[-1] self.block.expansion = expansion_bak def _make_stem_layer(self, in_channels, base_channels): self.conv1 = build_conv_layer( self.conv_cfg, in_channels, base_channels, kernel_size=3, stride=2, padding=1, bias=False) self.norm1_name, norm1 = build_norm_layer( self.norm_cfg, base_channels, postfix=1) self.add_module(self.norm1_name, norm1) self.relu = nn.ReLU(inplace=True) def generate_regnet(self, initial_width, width_slope, width_parameter, depth, divisor=8): """Generates per block width from RegNet parameters. Args: initial_width ([int]): Initial width of the backbone width_slope ([float]): Slope of the quantized linear function width_parameter ([int]): Parameter used to quantize the width. depth ([int]): Depth of the backbone. divisor (int, optional): The divisor of channels. Defaults to 8. Returns: list, int: return a list of widths of each stage and the number \ of stages """ assert width_slope >= 0 assert initial_width > 0 assert width_parameter > 1 assert initial_width % divisor == 0 widths_cont = np.arange(depth) * width_slope + initial_width ks = np.round( np.log(widths_cont / initial_width) / np.log(width_parameter)) widths = initial_width * np.power(width_parameter, ks) widths = np.round(np.divide(widths, divisor)) * divisor num_stages = len(np.unique(widths)) widths, widths_cont = widths.astype(int).tolist(), widths_cont.tolist() return widths, num_stages @staticmethod def quantize_float(number, divisor): """Converts a float to closest non-zero int divisible by divisor. Args: number (int): Original number to be quantized. divisor (int): Divisor used to quantize the number. Returns: int: quantized number that is divisible by devisor. """ return int(round(number / divisor) * divisor) def adjust_width_group(self, widths, bottleneck_ratio, groups): """Adjusts the compatibility of widths and groups. Args: widths (list[int]): Width of each stage. bottleneck_ratio (float): Bottleneck ratio. groups (int): number of groups in each stage Returns: tuple(list): The adjusted widths and groups of each stage. """ bottleneck_width = [ int(w * b) for w, b in zip(widths, bottleneck_ratio) ] groups = [min(g, w_bot) for g, w_bot in zip(groups, bottleneck_width)] bottleneck_width = [ self.quantize_float(w_bot, g) for w_bot, g in zip(bottleneck_width, groups) ] widths = [ int(w_bot / b) for w_bot, b in zip(bottleneck_width, bottleneck_ratio) ] return widths, groups def get_stages_from_blocks(self, widths): """Gets widths/stage_blocks of network at each stage. Args: widths (list[int]): Width in each stage. Returns: tuple(list): width and depth of each stage """ width_diff = [ width != width_prev for width, width_prev in zip(widths + [0], [0] + widths) ] stage_widths = [ width for width, diff in zip(widths, width_diff[:-1]) if diff ] stage_blocks = np.diff([ depth for depth, diff in zip(range(len(width_diff)), width_diff) if diff ]).tolist() return stage_widths, stage_blocks def forward(self, x): """Forward function.""" x = self.conv1(x) x = self.norm1(x) x = self.relu(x) outs = [] for i, layer_name in enumerate(self.res_layers): res_layer = getattr(self, layer_name) x = res_layer(x) if i in self.out_indices: outs.append(x) return tuple(outs) ================================================ FILE: mmdet/models/backbones/res2net.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math import torch import torch.nn as nn import torch.utils.checkpoint as cp from mmcv.cnn import build_conv_layer, build_norm_layer from mmengine.model import Sequential from mmdet.registry import MODELS from .resnet import Bottleneck as _Bottleneck from .resnet import ResNet class Bottle2neck(_Bottleneck): expansion = 4 def __init__(self, inplanes, planes, scales=4, base_width=26, base_channels=64, stage_type='normal', **kwargs): """Bottle2neck block for Res2Net. If style is "pytorch", the stride-two layer is the 3x3 conv layer, if it is "caffe", the stride-two layer is the first 1x1 conv layer. """ super(Bottle2neck, self).__init__(inplanes, planes, **kwargs) assert scales > 1, 'Res2Net degenerates to ResNet when scales = 1.' width = int(math.floor(self.planes * (base_width / base_channels))) self.norm1_name, norm1 = build_norm_layer( self.norm_cfg, width * scales, postfix=1) self.norm3_name, norm3 = build_norm_layer( self.norm_cfg, self.planes * self.expansion, postfix=3) self.conv1 = build_conv_layer( self.conv_cfg, self.inplanes, width * scales, kernel_size=1, stride=self.conv1_stride, bias=False) self.add_module(self.norm1_name, norm1) if stage_type == 'stage' and self.conv2_stride != 1: self.pool = nn.AvgPool2d( kernel_size=3, stride=self.conv2_stride, padding=1) convs = [] bns = [] fallback_on_stride = False if self.with_dcn: fallback_on_stride = self.dcn.pop('fallback_on_stride', False) if not self.with_dcn or fallback_on_stride: for i in range(scales - 1): convs.append( build_conv_layer( self.conv_cfg, width, width, kernel_size=3, stride=self.conv2_stride, padding=self.dilation, dilation=self.dilation, bias=False)) bns.append( build_norm_layer(self.norm_cfg, width, postfix=i + 1)[1]) self.convs = nn.ModuleList(convs) self.bns = nn.ModuleList(bns) else: assert self.conv_cfg is None, 'conv_cfg must be None for DCN' for i in range(scales - 1): convs.append( build_conv_layer( self.dcn, width, width, kernel_size=3, stride=self.conv2_stride, padding=self.dilation, dilation=self.dilation, bias=False)) bns.append( build_norm_layer(self.norm_cfg, width, postfix=i + 1)[1]) self.convs = nn.ModuleList(convs) self.bns = nn.ModuleList(bns) self.conv3 = build_conv_layer( self.conv_cfg, width * scales, self.planes * self.expansion, kernel_size=1, bias=False) self.add_module(self.norm3_name, norm3) self.stage_type = stage_type self.scales = scales self.width = width delattr(self, 'conv2') delattr(self, self.norm2_name) def forward(self, x): """Forward function.""" def _inner_forward(x): identity = x out = self.conv1(x) out = self.norm1(out) out = self.relu(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv1_plugin_names) spx = torch.split(out, self.width, 1) sp = self.convs[0](spx[0].contiguous()) sp = self.relu(self.bns[0](sp)) out = sp for i in range(1, self.scales - 1): if self.stage_type == 'stage': sp = spx[i] else: sp = sp + spx[i] sp = self.convs[i](sp.contiguous()) sp = self.relu(self.bns[i](sp)) out = torch.cat((out, sp), 1) if self.stage_type == 'normal' or self.conv2_stride == 1: out = torch.cat((out, spx[self.scales - 1]), 1) elif self.stage_type == 'stage': out = torch.cat((out, self.pool(spx[self.scales - 1])), 1) if self.with_plugins: out = self.forward_plugin(out, self.after_conv2_plugin_names) out = self.conv3(out) out = self.norm3(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv3_plugin_names) if self.downsample is not None: identity = self.downsample(x) out += identity return out if self.with_cp and x.requires_grad: out = cp.checkpoint(_inner_forward, x) else: out = _inner_forward(x) out = self.relu(out) return out class Res2Layer(Sequential): """Res2Layer to build Res2Net style backbone. Args: block (nn.Module): block used to build ResLayer. inplanes (int): inplanes of block. planes (int): planes of block. num_blocks (int): number of blocks. stride (int): stride of the first block. Default: 1 avg_down (bool): Use AvgPool instead of stride conv when downsampling in the bottle2neck. Default: False conv_cfg (dict): dictionary to construct and config conv layer. Default: None norm_cfg (dict): dictionary to construct and config norm layer. Default: dict(type='BN') scales (int): Scales used in Res2Net. Default: 4 base_width (int): Basic width of each scale. Default: 26 """ def __init__(self, block, inplanes, planes, num_blocks, stride=1, avg_down=True, conv_cfg=None, norm_cfg=dict(type='BN'), scales=4, base_width=26, **kwargs): self.block = block downsample = None if stride != 1 or inplanes != planes * block.expansion: downsample = nn.Sequential( nn.AvgPool2d( kernel_size=stride, stride=stride, ceil_mode=True, count_include_pad=False), build_conv_layer( conv_cfg, inplanes, planes * block.expansion, kernel_size=1, stride=1, bias=False), build_norm_layer(norm_cfg, planes * block.expansion)[1], ) layers = [] layers.append( block( inplanes=inplanes, planes=planes, stride=stride, downsample=downsample, conv_cfg=conv_cfg, norm_cfg=norm_cfg, scales=scales, base_width=base_width, stage_type='stage', **kwargs)) inplanes = planes * block.expansion for i in range(1, num_blocks): layers.append( block( inplanes=inplanes, planes=planes, stride=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, scales=scales, base_width=base_width, **kwargs)) super(Res2Layer, self).__init__(*layers) @MODELS.register_module() class Res2Net(ResNet): """Res2Net backbone. Args: scales (int): Scales used in Res2Net. Default: 4 base_width (int): Basic width of each scale. Default: 26 depth (int): Depth of res2net, from {50, 101, 152}. in_channels (int): Number of input image channels. Default: 3. num_stages (int): Res2net stages. Default: 4. strides (Sequence[int]): Strides of the first block of each stage. dilations (Sequence[int]): Dilation of each stage. out_indices (Sequence[int]): Output from which stages. style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two layer is the 3x3 conv layer, otherwise the stride-two layer is the first 1x1 conv layer. deep_stem (bool): Replace 7x7 conv in input stem with 3 3x3 conv avg_down (bool): Use AvgPool instead of stride conv when downsampling in the bottle2neck. frozen_stages (int): Stages to be frozen (stop grad and set eval mode). -1 means not freezing any parameters. norm_cfg (dict): Dictionary to construct and config norm layer. norm_eval (bool): Whether to set norm layers to eval mode, namely, freeze running stats (mean and var). Note: Effect on Batch Norm and its variants only. plugins (list[dict]): List of plugins for stages, each dict contains: - cfg (dict, required): Cfg dict to build plugin. - position (str, required): Position inside block to insert plugin, options are 'after_conv1', 'after_conv2', 'after_conv3'. - stages (tuple[bool], optional): Stages to apply plugin, length should be same as 'num_stages'. with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. zero_init_residual (bool): Whether to use zero init for last norm layer in resblocks to let them behave as identity. pretrained (str, optional): model pretrained path. Default: None init_cfg (dict or list[dict], optional): Initialization config dict. Default: None Example: >>> from mmdet.models import Res2Net >>> import torch >>> self = Res2Net(depth=50, scales=4, base_width=26) >>> self.eval() >>> inputs = torch.rand(1, 3, 32, 32) >>> level_outputs = self.forward(inputs) >>> for level_out in level_outputs: ... print(tuple(level_out.shape)) (1, 256, 8, 8) (1, 512, 4, 4) (1, 1024, 2, 2) (1, 2048, 1, 1) """ arch_settings = { 50: (Bottle2neck, (3, 4, 6, 3)), 101: (Bottle2neck, (3, 4, 23, 3)), 152: (Bottle2neck, (3, 8, 36, 3)) } def __init__(self, scales=4, base_width=26, style='pytorch', deep_stem=True, avg_down=True, pretrained=None, init_cfg=None, **kwargs): self.scales = scales self.base_width = base_width super(Res2Net, self).__init__( style='pytorch', deep_stem=True, avg_down=True, pretrained=pretrained, init_cfg=init_cfg, **kwargs) def make_res_layer(self, **kwargs): return Res2Layer( scales=self.scales, base_width=self.base_width, base_channels=self.base_channels, **kwargs) ================================================ FILE: mmdet/models/backbones/resnest.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math import torch import torch.nn as nn import torch.nn.functional as F import torch.utils.checkpoint as cp from mmcv.cnn import build_conv_layer, build_norm_layer from mmengine.model import BaseModule from mmdet.registry import MODELS from ..layers import ResLayer from .resnet import Bottleneck as _Bottleneck from .resnet import ResNetV1d class RSoftmax(nn.Module): """Radix Softmax module in ``SplitAttentionConv2d``. Args: radix (int): Radix of input. groups (int): Groups of input. """ def __init__(self, radix, groups): super().__init__() self.radix = radix self.groups = groups def forward(self, x): batch = x.size(0) if self.radix > 1: x = x.view(batch, self.groups, self.radix, -1).transpose(1, 2) x = F.softmax(x, dim=1) x = x.reshape(batch, -1) else: x = torch.sigmoid(x) return x class SplitAttentionConv2d(BaseModule): """Split-Attention Conv2d in ResNeSt. Args: in_channels (int): Number of channels in the input feature map. channels (int): Number of intermediate channels. kernel_size (int | tuple[int]): Size of the convolution kernel. stride (int | tuple[int]): Stride of the convolution. padding (int | tuple[int]): Zero-padding added to both sides of dilation (int | tuple[int]): Spacing between kernel elements. groups (int): Number of blocked connections from input channels to output channels. groups (int): Same as nn.Conv2d. radix (int): Radix of SpltAtConv2d. Default: 2 reduction_factor (int): Reduction factor of inter_channels. Default: 4. conv_cfg (dict): Config dict for convolution layer. Default: None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Default: None. dcn (dict): Config dict for DCN. Default: None. init_cfg (dict or list[dict], optional): Initialization config dict. Default: None """ def __init__(self, in_channels, channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, radix=2, reduction_factor=4, conv_cfg=None, norm_cfg=dict(type='BN'), dcn=None, init_cfg=None): super(SplitAttentionConv2d, self).__init__(init_cfg) inter_channels = max(in_channels * radix // reduction_factor, 32) self.radix = radix self.groups = groups self.channels = channels self.with_dcn = dcn is not None self.dcn = dcn fallback_on_stride = False if self.with_dcn: fallback_on_stride = self.dcn.pop('fallback_on_stride', False) if self.with_dcn and not fallback_on_stride: assert conv_cfg is None, 'conv_cfg must be None for DCN' conv_cfg = dcn self.conv = build_conv_layer( conv_cfg, in_channels, channels * radix, kernel_size, stride=stride, padding=padding, dilation=dilation, groups=groups * radix, bias=False) # To be consistent with original implementation, starting from 0 self.norm0_name, norm0 = build_norm_layer( norm_cfg, channels * radix, postfix=0) self.add_module(self.norm0_name, norm0) self.relu = nn.ReLU(inplace=True) self.fc1 = build_conv_layer( None, channels, inter_channels, 1, groups=self.groups) self.norm1_name, norm1 = build_norm_layer( norm_cfg, inter_channels, postfix=1) self.add_module(self.norm1_name, norm1) self.fc2 = build_conv_layer( None, inter_channels, channels * radix, 1, groups=self.groups) self.rsoftmax = RSoftmax(radix, groups) @property def norm0(self): """nn.Module: the normalization layer named "norm0" """ return getattr(self, self.norm0_name) @property def norm1(self): """nn.Module: the normalization layer named "norm1" """ return getattr(self, self.norm1_name) def forward(self, x): x = self.conv(x) x = self.norm0(x) x = self.relu(x) batch, rchannel = x.shape[:2] batch = x.size(0) if self.radix > 1: splits = x.view(batch, self.radix, -1, *x.shape[2:]) gap = splits.sum(dim=1) else: gap = x gap = F.adaptive_avg_pool2d(gap, 1) gap = self.fc1(gap) gap = self.norm1(gap) gap = self.relu(gap) atten = self.fc2(gap) atten = self.rsoftmax(atten).view(batch, -1, 1, 1) if self.radix > 1: attens = atten.view(batch, self.radix, -1, *atten.shape[2:]) out = torch.sum(attens * splits, dim=1) else: out = atten * x return out.contiguous() class Bottleneck(_Bottleneck): """Bottleneck block for ResNeSt. Args: inplane (int): Input planes of this block. planes (int): Middle planes of this block. groups (int): Groups of conv2. base_width (int): Base of width in terms of base channels. Default: 4. base_channels (int): Base of channels for calculating width. Default: 64. radix (int): Radix of SpltAtConv2d. Default: 2 reduction_factor (int): Reduction factor of inter_channels in SplitAttentionConv2d. Default: 4. avg_down_stride (bool): Whether to use average pool for stride in Bottleneck. Default: True. kwargs (dict): Key word arguments for base class. """ expansion = 4 def __init__(self, inplanes, planes, groups=1, base_width=4, base_channels=64, radix=2, reduction_factor=4, avg_down_stride=True, **kwargs): """Bottleneck block for ResNeSt.""" super(Bottleneck, self).__init__(inplanes, planes, **kwargs) if groups == 1: width = self.planes else: width = math.floor(self.planes * (base_width / base_channels)) * groups self.avg_down_stride = avg_down_stride and self.conv2_stride > 1 self.norm1_name, norm1 = build_norm_layer( self.norm_cfg, width, postfix=1) self.norm3_name, norm3 = build_norm_layer( self.norm_cfg, self.planes * self.expansion, postfix=3) self.conv1 = build_conv_layer( self.conv_cfg, self.inplanes, width, kernel_size=1, stride=self.conv1_stride, bias=False) self.add_module(self.norm1_name, norm1) self.with_modulated_dcn = False self.conv2 = SplitAttentionConv2d( width, width, kernel_size=3, stride=1 if self.avg_down_stride else self.conv2_stride, padding=self.dilation, dilation=self.dilation, groups=groups, radix=radix, reduction_factor=reduction_factor, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, dcn=self.dcn) delattr(self, self.norm2_name) if self.avg_down_stride: self.avd_layer = nn.AvgPool2d(3, self.conv2_stride, padding=1) self.conv3 = build_conv_layer( self.conv_cfg, width, self.planes * self.expansion, kernel_size=1, bias=False) self.add_module(self.norm3_name, norm3) def forward(self, x): def _inner_forward(x): identity = x out = self.conv1(x) out = self.norm1(out) out = self.relu(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv1_plugin_names) out = self.conv2(out) if self.avg_down_stride: out = self.avd_layer(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv2_plugin_names) out = self.conv3(out) out = self.norm3(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv3_plugin_names) if self.downsample is not None: identity = self.downsample(x) out += identity return out if self.with_cp and x.requires_grad: out = cp.checkpoint(_inner_forward, x) else: out = _inner_forward(x) out = self.relu(out) return out @MODELS.register_module() class ResNeSt(ResNetV1d): """ResNeSt backbone. Args: groups (int): Number of groups of Bottleneck. Default: 1 base_width (int): Base width of Bottleneck. Default: 4 radix (int): Radix of SplitAttentionConv2d. Default: 2 reduction_factor (int): Reduction factor of inter_channels in SplitAttentionConv2d. Default: 4. avg_down_stride (bool): Whether to use average pool for stride in Bottleneck. Default: True. kwargs (dict): Keyword arguments for ResNet. """ arch_settings = { 50: (Bottleneck, (3, 4, 6, 3)), 101: (Bottleneck, (3, 4, 23, 3)), 152: (Bottleneck, (3, 8, 36, 3)), 200: (Bottleneck, (3, 24, 36, 3)) } def __init__(self, groups=1, base_width=4, radix=2, reduction_factor=4, avg_down_stride=True, **kwargs): self.groups = groups self.base_width = base_width self.radix = radix self.reduction_factor = reduction_factor self.avg_down_stride = avg_down_stride super(ResNeSt, self).__init__(**kwargs) def make_res_layer(self, **kwargs): """Pack all blocks in a stage into a ``ResLayer``.""" return ResLayer( groups=self.groups, base_width=self.base_width, base_channels=self.base_channels, radix=self.radix, reduction_factor=self.reduction_factor, avg_down_stride=self.avg_down_stride, **kwargs) ================================================ FILE: mmdet/models/backbones/resnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings import torch.nn as nn import torch.utils.checkpoint as cp from mmcv.cnn import build_conv_layer, build_norm_layer, build_plugin_layer from mmengine.model import BaseModule from torch.nn.modules.batchnorm import _BatchNorm from mmdet.registry import MODELS from ..layers import ResLayer class BasicBlock(BaseModule): expansion = 1 def __init__(self, inplanes, planes, stride=1, dilation=1, downsample=None, style='pytorch', with_cp=False, conv_cfg=None, norm_cfg=dict(type='BN'), dcn=None, plugins=None, init_cfg=None): super(BasicBlock, self).__init__(init_cfg) assert dcn is None, 'Not implemented yet.' assert plugins is None, 'Not implemented yet.' self.norm1_name, norm1 = build_norm_layer(norm_cfg, planes, postfix=1) self.norm2_name, norm2 = build_norm_layer(norm_cfg, planes, postfix=2) self.conv1 = build_conv_layer( conv_cfg, inplanes, planes, 3, stride=stride, padding=dilation, dilation=dilation, bias=False) self.add_module(self.norm1_name, norm1) self.conv2 = build_conv_layer( conv_cfg, planes, planes, 3, padding=1, bias=False) self.add_module(self.norm2_name, norm2) self.relu = nn.ReLU(inplace=True) self.downsample = downsample self.stride = stride self.dilation = dilation self.with_cp = with_cp @property def norm1(self): """nn.Module: normalization layer after the first convolution layer""" return getattr(self, self.norm1_name) @property def norm2(self): """nn.Module: normalization layer after the second convolution layer""" return getattr(self, self.norm2_name) def forward(self, x): """Forward function.""" def _inner_forward(x): identity = x out = self.conv1(x) out = self.norm1(out) out = self.relu(out) out = self.conv2(out) out = self.norm2(out) if self.downsample is not None: identity = self.downsample(x) out += identity return out if self.with_cp and x.requires_grad: out = cp.checkpoint(_inner_forward, x) else: out = _inner_forward(x) out = self.relu(out) return out class Bottleneck(BaseModule): expansion = 4 def __init__(self, inplanes, planes, stride=1, dilation=1, downsample=None, style='pytorch', with_cp=False, conv_cfg=None, norm_cfg=dict(type='BN'), dcn=None, plugins=None, init_cfg=None): """Bottleneck block for ResNet. If style is "pytorch", the stride-two layer is the 3x3 conv layer, if it is "caffe", the stride-two layer is the first 1x1 conv layer. """ super(Bottleneck, self).__init__(init_cfg) assert style in ['pytorch', 'caffe'] assert dcn is None or isinstance(dcn, dict) assert plugins is None or isinstance(plugins, list) if plugins is not None: allowed_position = ['after_conv1', 'after_conv2', 'after_conv3'] assert all(p['position'] in allowed_position for p in plugins) self.inplanes = inplanes self.planes = planes self.stride = stride self.dilation = dilation self.style = style self.with_cp = with_cp self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.dcn = dcn self.with_dcn = dcn is not None self.plugins = plugins self.with_plugins = plugins is not None if self.with_plugins: # collect plugins for conv1/conv2/conv3 self.after_conv1_plugins = [ plugin['cfg'] for plugin in plugins if plugin['position'] == 'after_conv1' ] self.after_conv2_plugins = [ plugin['cfg'] for plugin in plugins if plugin['position'] == 'after_conv2' ] self.after_conv3_plugins = [ plugin['cfg'] for plugin in plugins if plugin['position'] == 'after_conv3' ] if self.style == 'pytorch': self.conv1_stride = 1 self.conv2_stride = stride else: self.conv1_stride = stride self.conv2_stride = 1 self.norm1_name, norm1 = build_norm_layer(norm_cfg, planes, postfix=1) self.norm2_name, norm2 = build_norm_layer(norm_cfg, planes, postfix=2) self.norm3_name, norm3 = build_norm_layer( norm_cfg, planes * self.expansion, postfix=3) self.conv1 = build_conv_layer( conv_cfg, inplanes, planes, kernel_size=1, stride=self.conv1_stride, bias=False) self.add_module(self.norm1_name, norm1) fallback_on_stride = False if self.with_dcn: fallback_on_stride = dcn.pop('fallback_on_stride', False) if not self.with_dcn or fallback_on_stride: self.conv2 = build_conv_layer( conv_cfg, planes, planes, kernel_size=3, stride=self.conv2_stride, padding=dilation, dilation=dilation, bias=False) else: assert self.conv_cfg is None, 'conv_cfg must be None for DCN' self.conv2 = build_conv_layer( dcn, planes, planes, kernel_size=3, stride=self.conv2_stride, padding=dilation, dilation=dilation, bias=False) self.add_module(self.norm2_name, norm2) self.conv3 = build_conv_layer( conv_cfg, planes, planes * self.expansion, kernel_size=1, bias=False) self.add_module(self.norm3_name, norm3) self.relu = nn.ReLU(inplace=True) self.downsample = downsample if self.with_plugins: self.after_conv1_plugin_names = self.make_block_plugins( planes, self.after_conv1_plugins) self.after_conv2_plugin_names = self.make_block_plugins( planes, self.after_conv2_plugins) self.after_conv3_plugin_names = self.make_block_plugins( planes * self.expansion, self.after_conv3_plugins) def make_block_plugins(self, in_channels, plugins): """make plugins for block. Args: in_channels (int): Input channels of plugin. plugins (list[dict]): List of plugins cfg to build. Returns: list[str]: List of the names of plugin. """ assert isinstance(plugins, list) plugin_names = [] for plugin in plugins: plugin = plugin.copy() name, layer = build_plugin_layer( plugin, in_channels=in_channels, postfix=plugin.pop('postfix', '')) assert not hasattr(self, name), f'duplicate plugin {name}' self.add_module(name, layer) plugin_names.append(name) return plugin_names def forward_plugin(self, x, plugin_names): out = x for name in plugin_names: out = getattr(self, name)(out) return out @property def norm1(self): """nn.Module: normalization layer after the first convolution layer""" return getattr(self, self.norm1_name) @property def norm2(self): """nn.Module: normalization layer after the second convolution layer""" return getattr(self, self.norm2_name) @property def norm3(self): """nn.Module: normalization layer after the third convolution layer""" return getattr(self, self.norm3_name) def forward(self, x): """Forward function.""" def _inner_forward(x): identity = x out = self.conv1(x) out = self.norm1(out) out = self.relu(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv1_plugin_names) out = self.conv2(out) out = self.norm2(out) out = self.relu(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv2_plugin_names) out = self.conv3(out) out = self.norm3(out) if self.with_plugins: out = self.forward_plugin(out, self.after_conv3_plugin_names) if self.downsample is not None: identity = self.downsample(x) out += identity return out if self.with_cp and x.requires_grad: out = cp.checkpoint(_inner_forward, x) else: out = _inner_forward(x) out = self.relu(out) return out @MODELS.register_module() class ResNet(BaseModule): """ResNet backbone. Args: depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. stem_channels (int | None): Number of stem channels. If not specified, it will be the same as `base_channels`. Default: None. base_channels (int): Number of base channels of res layer. Default: 64. in_channels (int): Number of input image channels. Default: 3. num_stages (int): Resnet stages. Default: 4. strides (Sequence[int]): Strides of the first block of each stage. dilations (Sequence[int]): Dilation of each stage. out_indices (Sequence[int]): Output from which stages. style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two layer is the 3x3 conv layer, otherwise the stride-two layer is the first 1x1 conv layer. deep_stem (bool): Replace 7x7 conv in input stem with 3 3x3 conv avg_down (bool): Use AvgPool instead of stride conv when downsampling in the bottleneck. frozen_stages (int): Stages to be frozen (stop grad and set eval mode). -1 means not freezing any parameters. norm_cfg (dict): Dictionary to construct and config norm layer. norm_eval (bool): Whether to set norm layers to eval mode, namely, freeze running stats (mean and var). Note: Effect on Batch Norm and its variants only. plugins (list[dict]): List of plugins for stages, each dict contains: - cfg (dict, required): Cfg dict to build plugin. - position (str, required): Position inside block to insert plugin, options are 'after_conv1', 'after_conv2', 'after_conv3'. - stages (tuple[bool], optional): Stages to apply plugin, length should be same as 'num_stages'. with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. zero_init_residual (bool): Whether to use zero init for last norm layer in resblocks to let them behave as identity. pretrained (str, optional): model pretrained path. Default: None init_cfg (dict or list[dict], optional): Initialization config dict. Default: None Example: >>> from mmdet.models import ResNet >>> import torch >>> self = ResNet(depth=18) >>> self.eval() >>> inputs = torch.rand(1, 3, 32, 32) >>> level_outputs = self.forward(inputs) >>> for level_out in level_outputs: ... print(tuple(level_out.shape)) (1, 64, 8, 8) (1, 128, 4, 4) (1, 256, 2, 2) (1, 512, 1, 1) """ arch_settings = { 18: (BasicBlock, (2, 2, 2, 2)), 34: (BasicBlock, (3, 4, 6, 3)), 50: (Bottleneck, (3, 4, 6, 3)), 101: (Bottleneck, (3, 4, 23, 3)), 152: (Bottleneck, (3, 8, 36, 3)) } def __init__(self, depth, in_channels=3, stem_channels=None, base_channels=64, num_stages=4, strides=(1, 2, 2, 2), dilations=(1, 1, 1, 1), out_indices=(0, 1, 2, 3), style='pytorch', deep_stem=False, avg_down=False, frozen_stages=-1, conv_cfg=None, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, dcn=None, stage_with_dcn=(False, False, False, False), plugins=None, with_cp=False, zero_init_residual=True, pretrained=None, init_cfg=None): super(ResNet, self).__init__(init_cfg) self.zero_init_residual = zero_init_residual if depth not in self.arch_settings: raise KeyError(f'invalid depth {depth} for resnet') block_init_cfg = None assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be specified at the same time' if isinstance(pretrained, str): warnings.warn('DeprecationWarning: pretrained is deprecated, ' 'please use "init_cfg" instead') self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) elif pretrained is None: if init_cfg is None: self.init_cfg = [ dict(type='Kaiming', layer='Conv2d'), dict( type='Constant', val=1, layer=['_BatchNorm', 'GroupNorm']) ] block = self.arch_settings[depth][0] if self.zero_init_residual: if block is BasicBlock: block_init_cfg = dict( type='Constant', val=0, override=dict(name='norm2')) elif block is Bottleneck: block_init_cfg = dict( type='Constant', val=0, override=dict(name='norm3')) else: raise TypeError('pretrained must be a str or None') self.depth = depth if stem_channels is None: stem_channels = base_channels self.stem_channels = stem_channels self.base_channels = base_channels self.num_stages = num_stages assert num_stages >= 1 and num_stages <= 4 self.strides = strides self.dilations = dilations assert len(strides) == len(dilations) == num_stages self.out_indices = out_indices assert max(out_indices) < num_stages self.style = style self.deep_stem = deep_stem self.avg_down = avg_down self.frozen_stages = frozen_stages self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.with_cp = with_cp self.norm_eval = norm_eval self.dcn = dcn self.stage_with_dcn = stage_with_dcn if dcn is not None: assert len(stage_with_dcn) == num_stages self.plugins = plugins self.block, stage_blocks = self.arch_settings[depth] self.stage_blocks = stage_blocks[:num_stages] self.inplanes = stem_channels self._make_stem_layer(in_channels, stem_channels) self.res_layers = [] for i, num_blocks in enumerate(self.stage_blocks): stride = strides[i] dilation = dilations[i] dcn = self.dcn if self.stage_with_dcn[i] else None if plugins is not None: stage_plugins = self.make_stage_plugins(plugins, i) else: stage_plugins = None planes = base_channels * 2**i res_layer = self.make_res_layer( block=self.block, inplanes=self.inplanes, planes=planes, num_blocks=num_blocks, stride=stride, dilation=dilation, style=self.style, avg_down=self.avg_down, with_cp=with_cp, conv_cfg=conv_cfg, norm_cfg=norm_cfg, dcn=dcn, plugins=stage_plugins, init_cfg=block_init_cfg) self.inplanes = planes * self.block.expansion layer_name = f'layer{i + 1}' self.add_module(layer_name, res_layer) self.res_layers.append(layer_name) self._freeze_stages() self.feat_dim = self.block.expansion * base_channels * 2**( len(self.stage_blocks) - 1) def make_stage_plugins(self, plugins, stage_idx): """Make plugins for ResNet ``stage_idx`` th stage. Currently we support to insert ``context_block``, ``empirical_attention_block``, ``nonlocal_block`` into the backbone like ResNet/ResNeXt. They could be inserted after conv1/conv2/conv3 of Bottleneck. An example of plugins format could be: Examples: >>> plugins=[ ... dict(cfg=dict(type='xxx', arg1='xxx'), ... stages=(False, True, True, True), ... position='after_conv2'), ... dict(cfg=dict(type='yyy'), ... stages=(True, True, True, True), ... position='after_conv3'), ... dict(cfg=dict(type='zzz', postfix='1'), ... stages=(True, True, True, True), ... position='after_conv3'), ... dict(cfg=dict(type='zzz', postfix='2'), ... stages=(True, True, True, True), ... position='after_conv3') ... ] >>> self = ResNet(depth=18) >>> stage_plugins = self.make_stage_plugins(plugins, 0) >>> assert len(stage_plugins) == 3 Suppose ``stage_idx=0``, the structure of blocks in the stage would be: .. code-block:: none conv1-> conv2->conv3->yyy->zzz1->zzz2 Suppose 'stage_idx=1', the structure of blocks in the stage would be: .. code-block:: none conv1-> conv2->xxx->conv3->yyy->zzz1->zzz2 If stages is missing, the plugin would be applied to all stages. Args: plugins (list[dict]): List of plugins cfg to build. The postfix is required if multiple same type plugins are inserted. stage_idx (int): Index of stage to build Returns: list[dict]: Plugins for current stage """ stage_plugins = [] for plugin in plugins: plugin = plugin.copy() stages = plugin.pop('stages', None) assert stages is None or len(stages) == self.num_stages # whether to insert plugin into current stage if stages is None or stages[stage_idx]: stage_plugins.append(plugin) return stage_plugins def make_res_layer(self, **kwargs): """Pack all blocks in a stage into a ``ResLayer``.""" return ResLayer(**kwargs) @property def norm1(self): """nn.Module: the normalization layer named "norm1" """ return getattr(self, self.norm1_name) def _make_stem_layer(self, in_channels, stem_channels): if self.deep_stem: self.stem = nn.Sequential( build_conv_layer( self.conv_cfg, in_channels, stem_channels // 2, kernel_size=3, stride=2, padding=1, bias=False), build_norm_layer(self.norm_cfg, stem_channels // 2)[1], nn.ReLU(inplace=True), build_conv_layer( self.conv_cfg, stem_channels // 2, stem_channels // 2, kernel_size=3, stride=1, padding=1, bias=False), build_norm_layer(self.norm_cfg, stem_channels // 2)[1], nn.ReLU(inplace=True), build_conv_layer( self.conv_cfg, stem_channels // 2, stem_channels, kernel_size=3, stride=1, padding=1, bias=False), build_norm_layer(self.norm_cfg, stem_channels)[1], nn.ReLU(inplace=True)) else: self.conv1 = build_conv_layer( self.conv_cfg, in_channels, stem_channels, kernel_size=7, stride=2, padding=3, bias=False) self.norm1_name, norm1 = build_norm_layer( self.norm_cfg, stem_channels, postfix=1) self.add_module(self.norm1_name, norm1) self.relu = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) def _freeze_stages(self): if self.frozen_stages >= 0: if self.deep_stem: self.stem.eval() for param in self.stem.parameters(): param.requires_grad = False else: self.norm1.eval() for m in [self.conv1, self.norm1]: for param in m.parameters(): param.requires_grad = False for i in range(1, self.frozen_stages + 1): m = getattr(self, f'layer{i}') m.eval() for param in m.parameters(): param.requires_grad = False def forward(self, x): """Forward function.""" if self.deep_stem: x = self.stem(x) else: x = self.conv1(x) x = self.norm1(x) x = self.relu(x) x = self.maxpool(x) outs = [] for i, layer_name in enumerate(self.res_layers): res_layer = getattr(self, layer_name) x = res_layer(x) if i in self.out_indices: outs.append(x) return tuple(outs) def train(self, mode=True): """Convert the model into training mode while keep normalization layer freezed.""" super(ResNet, self).train(mode) self._freeze_stages() if mode and self.norm_eval: for m in self.modules(): # trick: eval have effect on BatchNorm only if isinstance(m, _BatchNorm): m.eval() @MODELS.register_module() class ResNetV1d(ResNet): r"""ResNetV1d variant described in `Bag of Tricks `_. Compared with default ResNet(ResNetV1b), ResNetV1d replaces the 7x7 conv in the input stem with three 3x3 convs. And in the downsampling block, a 2x2 avg_pool with stride 2 is added before conv, whose stride is changed to 1. """ def __init__(self, **kwargs): super(ResNetV1d, self).__init__( deep_stem=True, avg_down=True, **kwargs) ================================================ FILE: mmdet/models/backbones/resnext.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from mmcv.cnn import build_conv_layer, build_norm_layer from mmdet.registry import MODELS from ..layers import ResLayer from .resnet import Bottleneck as _Bottleneck from .resnet import ResNet class Bottleneck(_Bottleneck): expansion = 4 def __init__(self, inplanes, planes, groups=1, base_width=4, base_channels=64, **kwargs): """Bottleneck block for ResNeXt. If style is "pytorch", the stride-two layer is the 3x3 conv layer, if it is "caffe", the stride-two layer is the first 1x1 conv layer. """ super(Bottleneck, self).__init__(inplanes, planes, **kwargs) if groups == 1: width = self.planes else: width = math.floor(self.planes * (base_width / base_channels)) * groups self.norm1_name, norm1 = build_norm_layer( self.norm_cfg, width, postfix=1) self.norm2_name, norm2 = build_norm_layer( self.norm_cfg, width, postfix=2) self.norm3_name, norm3 = build_norm_layer( self.norm_cfg, self.planes * self.expansion, postfix=3) self.conv1 = build_conv_layer( self.conv_cfg, self.inplanes, width, kernel_size=1, stride=self.conv1_stride, bias=False) self.add_module(self.norm1_name, norm1) fallback_on_stride = False self.with_modulated_dcn = False if self.with_dcn: fallback_on_stride = self.dcn.pop('fallback_on_stride', False) if not self.with_dcn or fallback_on_stride: self.conv2 = build_conv_layer( self.conv_cfg, width, width, kernel_size=3, stride=self.conv2_stride, padding=self.dilation, dilation=self.dilation, groups=groups, bias=False) else: assert self.conv_cfg is None, 'conv_cfg must be None for DCN' self.conv2 = build_conv_layer( self.dcn, width, width, kernel_size=3, stride=self.conv2_stride, padding=self.dilation, dilation=self.dilation, groups=groups, bias=False) self.add_module(self.norm2_name, norm2) self.conv3 = build_conv_layer( self.conv_cfg, width, self.planes * self.expansion, kernel_size=1, bias=False) self.add_module(self.norm3_name, norm3) if self.with_plugins: self._del_block_plugins(self.after_conv1_plugin_names + self.after_conv2_plugin_names + self.after_conv3_plugin_names) self.after_conv1_plugin_names = self.make_block_plugins( width, self.after_conv1_plugins) self.after_conv2_plugin_names = self.make_block_plugins( width, self.after_conv2_plugins) self.after_conv3_plugin_names = self.make_block_plugins( self.planes * self.expansion, self.after_conv3_plugins) def _del_block_plugins(self, plugin_names): """delete plugins for block if exist. Args: plugin_names (list[str]): List of plugins name to delete. """ assert isinstance(plugin_names, list) for plugin_name in plugin_names: del self._modules[plugin_name] @MODELS.register_module() class ResNeXt(ResNet): """ResNeXt backbone. Args: depth (int): Depth of resnet, from {18, 34, 50, 101, 152}. in_channels (int): Number of input image channels. Default: 3. num_stages (int): Resnet stages. Default: 4. groups (int): Group of resnext. base_width (int): Base width of resnext. strides (Sequence[int]): Strides of the first block of each stage. dilations (Sequence[int]): Dilation of each stage. out_indices (Sequence[int]): Output from which stages. style (str): `pytorch` or `caffe`. If set to "pytorch", the stride-two layer is the 3x3 conv layer, otherwise the stride-two layer is the first 1x1 conv layer. frozen_stages (int): Stages to be frozen (all param fixed). -1 means not freezing any parameters. norm_cfg (dict): dictionary to construct and config norm layer. norm_eval (bool): Whether to set norm layers to eval mode, namely, freeze running stats (mean and var). Note: Effect on Batch Norm and its variants only. with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. zero_init_residual (bool): whether to use zero init for last norm layer in resblocks to let them behave as identity. """ arch_settings = { 50: (Bottleneck, (3, 4, 6, 3)), 101: (Bottleneck, (3, 4, 23, 3)), 152: (Bottleneck, (3, 8, 36, 3)) } def __init__(self, groups=1, base_width=4, **kwargs): self.groups = groups self.base_width = base_width super(ResNeXt, self).__init__(**kwargs) def make_res_layer(self, **kwargs): """Pack all blocks in a stage into a ``ResLayer``""" return ResLayer( groups=self.groups, base_width=self.base_width, base_channels=self.base_channels, **kwargs) ================================================ FILE: mmdet/models/backbones/ssd_vgg.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings import torch.nn as nn from mmcv.cnn import VGG from mmengine.model import BaseModule from mmdet.registry import MODELS from ..necks import ssd_neck @MODELS.register_module() class SSDVGG(VGG, BaseModule): """VGG Backbone network for single-shot-detection. Args: depth (int): Depth of vgg, from {11, 13, 16, 19}. with_last_pool (bool): Whether to add a pooling layer at the last of the model ceil_mode (bool): When True, will use `ceil` instead of `floor` to compute the output shape. out_indices (Sequence[int]): Output from which stages. out_feature_indices (Sequence[int]): Output from which feature map. pretrained (str, optional): model pretrained path. Default: None init_cfg (dict or list[dict], optional): Initialization config dict. Default: None input_size (int, optional): Deprecated argumment. Width and height of input, from {300, 512}. l2_norm_scale (float, optional) : Deprecated argumment. L2 normalization layer init scale. Example: >>> self = SSDVGG(input_size=300, depth=11) >>> self.eval() >>> inputs = torch.rand(1, 3, 300, 300) >>> level_outputs = self.forward(inputs) >>> for level_out in level_outputs: ... print(tuple(level_out.shape)) (1, 1024, 19, 19) (1, 512, 10, 10) (1, 256, 5, 5) (1, 256, 3, 3) (1, 256, 1, 1) """ extra_setting = { 300: (256, 'S', 512, 128, 'S', 256, 128, 256, 128, 256), 512: (256, 'S', 512, 128, 'S', 256, 128, 'S', 256, 128, 'S', 256, 128), } def __init__(self, depth, with_last_pool=False, ceil_mode=True, out_indices=(3, 4), out_feature_indices=(22, 34), pretrained=None, init_cfg=None, input_size=None, l2_norm_scale=None): # TODO: in_channels for mmcv.VGG super(SSDVGG, self).__init__( depth, with_last_pool=with_last_pool, ceil_mode=ceil_mode, out_indices=out_indices) self.features.add_module( str(len(self.features)), nn.MaxPool2d(kernel_size=3, stride=1, padding=1)) self.features.add_module( str(len(self.features)), nn.Conv2d(512, 1024, kernel_size=3, padding=6, dilation=6)) self.features.add_module( str(len(self.features)), nn.ReLU(inplace=True)) self.features.add_module( str(len(self.features)), nn.Conv2d(1024, 1024, kernel_size=1)) self.features.add_module( str(len(self.features)), nn.ReLU(inplace=True)) self.out_feature_indices = out_feature_indices assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be specified at the same time' if init_cfg is not None: self.init_cfg = init_cfg elif isinstance(pretrained, str): warnings.warn('DeprecationWarning: pretrained is deprecated, ' 'please use "init_cfg" instead') self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) elif pretrained is None: self.init_cfg = [ dict(type='Kaiming', layer='Conv2d'), dict(type='Constant', val=1, layer='BatchNorm2d'), dict(type='Normal', std=0.01, layer='Linear'), ] else: raise TypeError('pretrained must be a str or None') if input_size is not None: warnings.warn('DeprecationWarning: input_size is deprecated') if l2_norm_scale is not None: warnings.warn('DeprecationWarning: l2_norm_scale in VGG is ' 'deprecated, it has been moved to SSDNeck.') def init_weights(self, pretrained=None): super(VGG, self).init_weights() def forward(self, x): """Forward function.""" outs = [] for i, layer in enumerate(self.features): x = layer(x) if i in self.out_feature_indices: outs.append(x) if len(outs) == 1: return outs[0] else: return tuple(outs) class L2Norm(ssd_neck.L2Norm): def __init__(self, **kwargs): super(L2Norm, self).__init__(**kwargs) warnings.warn('DeprecationWarning: L2Norm in ssd_vgg.py ' 'is deprecated, please use L2Norm in ' 'mmdet/models/necks/ssd_neck.py instead') ================================================ FILE: mmdet/models/backbones/swin.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings from collections import OrderedDict from copy import deepcopy import torch import torch.nn as nn import torch.nn.functional as F import torch.utils.checkpoint as cp from mmcv.cnn import build_norm_layer from mmcv.cnn.bricks.transformer import FFN, build_dropout from mmengine.logging import MMLogger from mmengine.model import BaseModule, ModuleList from mmengine.model.weight_init import (constant_init, trunc_normal_, trunc_normal_init) from mmengine.runner.checkpoint import CheckpointLoader from mmengine.utils import to_2tuple from mmdet.registry import MODELS from ..layers import PatchEmbed, PatchMerging class WindowMSA(BaseModule): """Window based multi-head self-attention (W-MSA) module with relative position bias. Args: embed_dims (int): Number of input channels. num_heads (int): Number of attention heads. window_size (tuple[int]): The height and width of the window. qkv_bias (bool, optional): If True, add a learnable bias to q, k, v. Default: True. qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. Default: None. attn_drop_rate (float, optional): Dropout ratio of attention weight. Default: 0.0 proj_drop_rate (float, optional): Dropout ratio of output. Default: 0. init_cfg (dict | None, optional): The Config for initialization. Default: None. """ def __init__(self, embed_dims, num_heads, window_size, qkv_bias=True, qk_scale=None, attn_drop_rate=0., proj_drop_rate=0., init_cfg=None): super().__init__() self.embed_dims = embed_dims self.window_size = window_size # Wh, Ww self.num_heads = num_heads head_embed_dims = embed_dims // num_heads self.scale = qk_scale or head_embed_dims**-0.5 self.init_cfg = init_cfg # define a parameter table of relative position bias self.relative_position_bias_table = nn.Parameter( torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads)) # 2*Wh-1 * 2*Ww-1, nH # About 2x faster than original impl Wh, Ww = self.window_size rel_index_coords = self.double_step_seq(2 * Ww - 1, Wh, 1, Ww) rel_position_index = rel_index_coords + rel_index_coords.T rel_position_index = rel_position_index.flip(1).contiguous() self.register_buffer('relative_position_index', rel_position_index) self.qkv = nn.Linear(embed_dims, embed_dims * 3, bias=qkv_bias) self.attn_drop = nn.Dropout(attn_drop_rate) self.proj = nn.Linear(embed_dims, embed_dims) self.proj_drop = nn.Dropout(proj_drop_rate) self.softmax = nn.Softmax(dim=-1) def init_weights(self): trunc_normal_(self.relative_position_bias_table, std=0.02) def forward(self, x, mask=None): """ Args: x (tensor): input features with shape of (num_windows*B, N, C) mask (tensor | None, Optional): mask with shape of (num_windows, Wh*Ww, Wh*Ww), value should be between (-inf, 0]. """ B, N, C = x.shape qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) # make torchscript happy (cannot use tensor as tuple) q, k, v = qkv[0], qkv[1], qkv[2] q = q * self.scale attn = (q @ k.transpose(-2, -1)) relative_position_bias = self.relative_position_bias_table[ self.relative_position_index.view(-1)].view( self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH relative_position_bias = relative_position_bias.permute( 2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww attn = attn + relative_position_bias.unsqueeze(0) if mask is not None: nW = mask.shape[0] attn = attn.view(B // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) attn = attn.view(-1, self.num_heads, N, N) attn = self.softmax(attn) attn = self.attn_drop(attn) x = (attn @ v).transpose(1, 2).reshape(B, N, C) x = self.proj(x) x = self.proj_drop(x) return x @staticmethod def double_step_seq(step1, len1, step2, len2): seq1 = torch.arange(0, step1 * len1, step1) seq2 = torch.arange(0, step2 * len2, step2) return (seq1[:, None] + seq2[None, :]).reshape(1, -1) class ShiftWindowMSA(BaseModule): """Shifted Window Multihead Self-Attention Module. Args: embed_dims (int): Number of input channels. num_heads (int): Number of attention heads. window_size (int): The height and width of the window. shift_size (int, optional): The shift step of each window towards right-bottom. If zero, act as regular window-msa. Defaults to 0. qkv_bias (bool, optional): If True, add a learnable bias to q, k, v. Default: True qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. Defaults: None. attn_drop_rate (float, optional): Dropout ratio of attention weight. Defaults: 0. proj_drop_rate (float, optional): Dropout ratio of output. Defaults: 0. dropout_layer (dict, optional): The dropout_layer used before output. Defaults: dict(type='DropPath', drop_prob=0.). init_cfg (dict, optional): The extra config for initialization. Default: None. """ def __init__(self, embed_dims, num_heads, window_size, shift_size=0, qkv_bias=True, qk_scale=None, attn_drop_rate=0, proj_drop_rate=0, dropout_layer=dict(type='DropPath', drop_prob=0.), init_cfg=None): super().__init__(init_cfg) self.window_size = window_size self.shift_size = shift_size assert 0 <= self.shift_size < self.window_size self.w_msa = WindowMSA( embed_dims=embed_dims, num_heads=num_heads, window_size=to_2tuple(window_size), qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop_rate=attn_drop_rate, proj_drop_rate=proj_drop_rate, init_cfg=None) self.drop = build_dropout(dropout_layer) def forward(self, query, hw_shape): B, L, C = query.shape H, W = hw_shape assert L == H * W, 'input feature has wrong size' query = query.view(B, H, W, C) # pad feature maps to multiples of window size pad_r = (self.window_size - W % self.window_size) % self.window_size pad_b = (self.window_size - H % self.window_size) % self.window_size query = F.pad(query, (0, 0, 0, pad_r, 0, pad_b)) H_pad, W_pad = query.shape[1], query.shape[2] # cyclic shift if self.shift_size > 0: shifted_query = torch.roll( query, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) # calculate attention mask for SW-MSA img_mask = torch.zeros((1, H_pad, W_pad, 1), device=query.device) h_slices = (slice(0, -self.window_size), slice(-self.window_size, -self.shift_size), slice(-self.shift_size, None)) w_slices = (slice(0, -self.window_size), slice(-self.window_size, -self.shift_size), slice(-self.shift_size, None)) cnt = 0 for h in h_slices: for w in w_slices: img_mask[:, h, w, :] = cnt cnt += 1 # nW, window_size, window_size, 1 mask_windows = self.window_partition(img_mask) mask_windows = mask_windows.view( -1, self.window_size * self.window_size) attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill( attn_mask == 0, float(0.0)) else: shifted_query = query attn_mask = None # nW*B, window_size, window_size, C query_windows = self.window_partition(shifted_query) # nW*B, window_size*window_size, C query_windows = query_windows.view(-1, self.window_size**2, C) # W-MSA/SW-MSA (nW*B, window_size*window_size, C) attn_windows = self.w_msa(query_windows, mask=attn_mask) # merge windows attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) # B H' W' C shifted_x = self.window_reverse(attn_windows, H_pad, W_pad) # reverse cyclic shift if self.shift_size > 0: x = torch.roll( shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) else: x = shifted_x if pad_r > 0 or pad_b: x = x[:, :H, :W, :].contiguous() x = x.view(B, H * W, C) x = self.drop(x) return x def window_reverse(self, windows, H, W): """ Args: windows: (num_windows*B, window_size, window_size, C) H (int): Height of image W (int): Width of image Returns: x: (B, H, W, C) """ window_size = self.window_size B = int(windows.shape[0] / (H * W / window_size / window_size)) x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) return x def window_partition(self, x): """ Args: x: (B, H, W, C) Returns: windows: (num_windows*B, window_size, window_size, C) """ B, H, W, C = x.shape window_size = self.window_size x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) windows = x.permute(0, 1, 3, 2, 4, 5).contiguous() windows = windows.view(-1, window_size, window_size, C) return windows class SwinBlock(BaseModule): """" Args: embed_dims (int): The feature dimension. num_heads (int): Parallel attention heads. feedforward_channels (int): The hidden dimension for FFNs. window_size (int, optional): The local window scale. Default: 7. shift (bool, optional): whether to shift window or not. Default False. qkv_bias (bool, optional): enable bias for qkv if True. Default: True. qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. Default: None. drop_rate (float, optional): Dropout rate. Default: 0. attn_drop_rate (float, optional): Attention dropout rate. Default: 0. drop_path_rate (float, optional): Stochastic depth rate. Default: 0. act_cfg (dict, optional): The config dict of activation function. Default: dict(type='GELU'). norm_cfg (dict, optional): The config dict of normalization. Default: dict(type='LN'). with_cp (bool, optional): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. Default: False. init_cfg (dict | list | None, optional): The init config. Default: None. """ def __init__(self, embed_dims, num_heads, feedforward_channels, window_size=7, shift=False, qkv_bias=True, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., act_cfg=dict(type='GELU'), norm_cfg=dict(type='LN'), with_cp=False, init_cfg=None): super(SwinBlock, self).__init__() self.init_cfg = init_cfg self.with_cp = with_cp self.norm1 = build_norm_layer(norm_cfg, embed_dims)[1] self.attn = ShiftWindowMSA( embed_dims=embed_dims, num_heads=num_heads, window_size=window_size, shift_size=window_size // 2 if shift else 0, qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop_rate=attn_drop_rate, proj_drop_rate=drop_rate, dropout_layer=dict(type='DropPath', drop_prob=drop_path_rate), init_cfg=None) self.norm2 = build_norm_layer(norm_cfg, embed_dims)[1] self.ffn = FFN( embed_dims=embed_dims, feedforward_channels=feedforward_channels, num_fcs=2, ffn_drop=drop_rate, dropout_layer=dict(type='DropPath', drop_prob=drop_path_rate), act_cfg=act_cfg, add_identity=True, init_cfg=None) def forward(self, x, hw_shape): def _inner_forward(x): identity = x x = self.norm1(x) x = self.attn(x, hw_shape) x = x + identity identity = x x = self.norm2(x) x = self.ffn(x, identity=identity) return x if self.with_cp and x.requires_grad: x = cp.checkpoint(_inner_forward, x) else: x = _inner_forward(x) return x class SwinBlockSequence(BaseModule): """Implements one stage in Swin Transformer. Args: embed_dims (int): The feature dimension. num_heads (int): Parallel attention heads. feedforward_channels (int): The hidden dimension for FFNs. depth (int): The number of blocks in this stage. window_size (int, optional): The local window scale. Default: 7. qkv_bias (bool, optional): enable bias for qkv if True. Default: True. qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. Default: None. drop_rate (float, optional): Dropout rate. Default: 0. attn_drop_rate (float, optional): Attention dropout rate. Default: 0. drop_path_rate (float | list[float], optional): Stochastic depth rate. Default: 0. downsample (BaseModule | None, optional): The downsample operation module. Default: None. act_cfg (dict, optional): The config dict of activation function. Default: dict(type='GELU'). norm_cfg (dict, optional): The config dict of normalization. Default: dict(type='LN'). with_cp (bool, optional): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. Default: False. init_cfg (dict | list | None, optional): The init config. Default: None. """ def __init__(self, embed_dims, num_heads, feedforward_channels, depth, window_size=7, qkv_bias=True, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0., downsample=None, act_cfg=dict(type='GELU'), norm_cfg=dict(type='LN'), with_cp=False, init_cfg=None): super().__init__(init_cfg=init_cfg) if isinstance(drop_path_rate, list): drop_path_rates = drop_path_rate assert len(drop_path_rates) == depth else: drop_path_rates = [deepcopy(drop_path_rate) for _ in range(depth)] self.blocks = ModuleList() for i in range(depth): block = SwinBlock( embed_dims=embed_dims, num_heads=num_heads, feedforward_channels=feedforward_channels, window_size=window_size, shift=False if i % 2 == 0 else True, qkv_bias=qkv_bias, qk_scale=qk_scale, drop_rate=drop_rate, attn_drop_rate=attn_drop_rate, drop_path_rate=drop_path_rates[i], act_cfg=act_cfg, norm_cfg=norm_cfg, with_cp=with_cp, init_cfg=None) self.blocks.append(block) self.downsample = downsample def forward(self, x, hw_shape): for block in self.blocks: x = block(x, hw_shape) if self.downsample: x_down, down_hw_shape = self.downsample(x, hw_shape) return x_down, down_hw_shape, x, hw_shape else: return x, hw_shape, x, hw_shape @MODELS.register_module() class SwinTransformer(BaseModule): """ Swin Transformer A PyTorch implement of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows` - https://arxiv.org/abs/2103.14030 Inspiration from https://github.com/microsoft/Swin-Transformer Args: pretrain_img_size (int | tuple[int]): The size of input image when pretrain. Defaults: 224. in_channels (int): The num of input channels. Defaults: 3. embed_dims (int): The feature dimension. Default: 96. patch_size (int | tuple[int]): Patch size. Default: 4. window_size (int): Window size. Default: 7. mlp_ratio (int): Ratio of mlp hidden dim to embedding dim. Default: 4. depths (tuple[int]): Depths of each Swin Transformer stage. Default: (2, 2, 6, 2). num_heads (tuple[int]): Parallel attention heads of each Swin Transformer stage. Default: (3, 6, 12, 24). strides (tuple[int]): The patch merging or patch embedding stride of each Swin Transformer stage. (In swin, we set kernel size equal to stride.) Default: (4, 2, 2, 2). out_indices (tuple[int]): Output from which stages. Default: (0, 1, 2, 3). qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. Default: None. patch_norm (bool): If add a norm layer for patch embed and patch merging. Default: True. drop_rate (float): Dropout rate. Defaults: 0. attn_drop_rate (float): Attention dropout rate. Default: 0. drop_path_rate (float): Stochastic depth rate. Defaults: 0.1. use_abs_pos_embed (bool): If True, add absolute position embedding to the patch embedding. Defaults: False. act_cfg (dict): Config dict for activation layer. Default: dict(type='GELU'). norm_cfg (dict): Config dict for normalization layer at output of backone. Defaults: dict(type='LN'). with_cp (bool, optional): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. Default: False. pretrained (str, optional): model pretrained path. Default: None. convert_weights (bool): The flag indicates whether the pre-trained model is from the original repo. We may need to convert some keys to make it compatible. Default: False. frozen_stages (int): Stages to be frozen (stop grad and set eval mode). Default: -1 (-1 means not freezing any parameters). init_cfg (dict, optional): The Config for initialization. Defaults to None. """ def __init__(self, pretrain_img_size=224, in_channels=3, embed_dims=96, patch_size=4, window_size=7, mlp_ratio=4, depths=(2, 2, 6, 2), num_heads=(3, 6, 12, 24), strides=(4, 2, 2, 2), out_indices=(0, 1, 2, 3), qkv_bias=True, qk_scale=None, patch_norm=True, drop_rate=0., attn_drop_rate=0., drop_path_rate=0.1, use_abs_pos_embed=False, act_cfg=dict(type='GELU'), norm_cfg=dict(type='LN'), with_cp=False, pretrained=None, convert_weights=False, frozen_stages=-1, init_cfg=None): self.convert_weights = convert_weights self.frozen_stages = frozen_stages if isinstance(pretrain_img_size, int): pretrain_img_size = to_2tuple(pretrain_img_size) elif isinstance(pretrain_img_size, tuple): if len(pretrain_img_size) == 1: pretrain_img_size = to_2tuple(pretrain_img_size[0]) assert len(pretrain_img_size) == 2, \ f'The size of image should have length 1 or 2, ' \ f'but got {len(pretrain_img_size)}' assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be specified at the same time' if isinstance(pretrained, str): warnings.warn('DeprecationWarning: pretrained is deprecated, ' 'please use "init_cfg" instead') self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) elif pretrained is None: self.init_cfg = init_cfg else: raise TypeError('pretrained must be a str or None') super(SwinTransformer, self).__init__(init_cfg=init_cfg) num_layers = len(depths) self.out_indices = out_indices self.use_abs_pos_embed = use_abs_pos_embed assert strides[0] == patch_size, 'Use non-overlapping patch embed.' self.patch_embed = PatchEmbed( in_channels=in_channels, embed_dims=embed_dims, conv_type='Conv2d', kernel_size=patch_size, stride=strides[0], norm_cfg=norm_cfg if patch_norm else None, init_cfg=None) if self.use_abs_pos_embed: patch_row = pretrain_img_size[0] // patch_size patch_col = pretrain_img_size[1] // patch_size num_patches = patch_row * patch_col self.absolute_pos_embed = nn.Parameter( torch.zeros((1, num_patches, embed_dims))) self.drop_after_pos = nn.Dropout(p=drop_rate) # set stochastic depth decay rule total_depth = sum(depths) dpr = [ x.item() for x in torch.linspace(0, drop_path_rate, total_depth) ] self.stages = ModuleList() in_channels = embed_dims for i in range(num_layers): if i < num_layers - 1: downsample = PatchMerging( in_channels=in_channels, out_channels=2 * in_channels, stride=strides[i + 1], norm_cfg=norm_cfg if patch_norm else None, init_cfg=None) else: downsample = None stage = SwinBlockSequence( embed_dims=in_channels, num_heads=num_heads[i], feedforward_channels=mlp_ratio * in_channels, depth=depths[i], window_size=window_size, qkv_bias=qkv_bias, qk_scale=qk_scale, drop_rate=drop_rate, attn_drop_rate=attn_drop_rate, drop_path_rate=dpr[sum(depths[:i]):sum(depths[:i + 1])], downsample=downsample, act_cfg=act_cfg, norm_cfg=norm_cfg, with_cp=with_cp, init_cfg=None) self.stages.append(stage) if downsample: in_channels = downsample.out_channels self.num_features = [int(embed_dims * 2**i) for i in range(num_layers)] # Add a norm layer for each output for i in out_indices: layer = build_norm_layer(norm_cfg, self.num_features[i])[1] layer_name = f'norm{i}' self.add_module(layer_name, layer) def train(self, mode=True): """Convert the model into training mode while keep layers freezed.""" super(SwinTransformer, self).train(mode) self._freeze_stages() def _freeze_stages(self): if self.frozen_stages >= 0: self.patch_embed.eval() for param in self.patch_embed.parameters(): param.requires_grad = False if self.use_abs_pos_embed: self.absolute_pos_embed.requires_grad = False self.drop_after_pos.eval() for i in range(1, self.frozen_stages + 1): if (i - 1) in self.out_indices: norm_layer = getattr(self, f'norm{i-1}') norm_layer.eval() for param in norm_layer.parameters(): param.requires_grad = False m = self.stages[i - 1] m.eval() for param in m.parameters(): param.requires_grad = False def init_weights(self): logger = MMLogger.get_current_instance() if self.init_cfg is None: logger.warn(f'No pre-trained weights for ' f'{self.__class__.__name__}, ' f'training start from scratch') if self.use_abs_pos_embed: trunc_normal_(self.absolute_pos_embed, std=0.02) for m in self.modules(): if isinstance(m, nn.Linear): trunc_normal_init(m, std=.02, bias=0.) elif isinstance(m, nn.LayerNorm): constant_init(m, 1.0) else: assert 'checkpoint' in self.init_cfg, f'Only support ' \ f'specify `Pretrained` in ' \ f'`init_cfg` in ' \ f'{self.__class__.__name__} ' ckpt = CheckpointLoader.load_checkpoint( self.init_cfg.checkpoint, logger=logger, map_location='cpu') if 'state_dict' in ckpt: _state_dict = ckpt['state_dict'] elif 'model' in ckpt: _state_dict = ckpt['model'] else: _state_dict = ckpt if self.convert_weights: # supported loading weight from original repo, _state_dict = swin_converter(_state_dict) state_dict = OrderedDict() for k, v in _state_dict.items(): if k.startswith('backbone.'): state_dict[k[9:]] = v # strip prefix of state_dict if list(state_dict.keys())[0].startswith('module.'): state_dict = {k[7:]: v for k, v in state_dict.items()} # reshape absolute position embedding if state_dict.get('absolute_pos_embed') is not None: absolute_pos_embed = state_dict['absolute_pos_embed'] N1, L, C1 = absolute_pos_embed.size() N2, C2, H, W = self.absolute_pos_embed.size() if N1 != N2 or C1 != C2 or L != H * W: logger.warning('Error in loading absolute_pos_embed, pass') else: state_dict['absolute_pos_embed'] = absolute_pos_embed.view( N2, H, W, C2).permute(0, 3, 1, 2).contiguous() # interpolate position bias table if needed relative_position_bias_table_keys = [ k for k in state_dict.keys() if 'relative_position_bias_table' in k ] for table_key in relative_position_bias_table_keys: table_pretrained = state_dict[table_key] table_current = self.state_dict()[table_key] L1, nH1 = table_pretrained.size() L2, nH2 = table_current.size() if nH1 != nH2: logger.warning(f'Error in loading {table_key}, pass') elif L1 != L2: S1 = int(L1**0.5) S2 = int(L2**0.5) table_pretrained_resized = F.interpolate( table_pretrained.permute(1, 0).reshape(1, nH1, S1, S1), size=(S2, S2), mode='bicubic') state_dict[table_key] = table_pretrained_resized.view( nH2, L2).permute(1, 0).contiguous() # load state_dict self.load_state_dict(state_dict, False) def forward(self, x): x, hw_shape = self.patch_embed(x) if self.use_abs_pos_embed: x = x + self.absolute_pos_embed x = self.drop_after_pos(x) outs = [] for i, stage in enumerate(self.stages): x, hw_shape, out, out_hw_shape = stage(x, hw_shape) if i in self.out_indices: norm_layer = getattr(self, f'norm{i}') out = norm_layer(out) out = out.view(-1, *out_hw_shape, self.num_features[i]).permute(0, 3, 1, 2).contiguous() outs.append(out) return outs def swin_converter(ckpt): new_ckpt = OrderedDict() def correct_unfold_reduction_order(x): out_channel, in_channel = x.shape x = x.reshape(out_channel, 4, in_channel // 4) x = x[:, [0, 2, 1, 3], :].transpose(1, 2).reshape(out_channel, in_channel) return x def correct_unfold_norm_order(x): in_channel = x.shape[0] x = x.reshape(4, in_channel // 4) x = x[[0, 2, 1, 3], :].transpose(0, 1).reshape(in_channel) return x for k, v in ckpt.items(): if k.startswith('head'): continue elif k.startswith('layers'): new_v = v if 'attn.' in k: new_k = k.replace('attn.', 'attn.w_msa.') elif 'mlp.' in k: if 'mlp.fc1.' in k: new_k = k.replace('mlp.fc1.', 'ffn.layers.0.0.') elif 'mlp.fc2.' in k: new_k = k.replace('mlp.fc2.', 'ffn.layers.1.') else: new_k = k.replace('mlp.', 'ffn.') elif 'downsample' in k: new_k = k if 'reduction.' in k: new_v = correct_unfold_reduction_order(v) elif 'norm.' in k: new_v = correct_unfold_norm_order(v) else: new_k = k new_k = new_k.replace('layers', 'stages', 1) elif k.startswith('patch_embed'): new_v = v if 'proj' in k: new_k = k.replace('proj', 'projection') else: new_k = k else: new_v = v new_k = k new_ckpt['backbone.' + new_k] = new_v return new_ckpt ================================================ FILE: mmdet/models/backbones/trident_resnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F import torch.utils.checkpoint as cp from mmcv.cnn import build_conv_layer, build_norm_layer from mmengine.model import BaseModule from torch.nn.modules.utils import _pair from mmdet.models.backbones.resnet import Bottleneck, ResNet from mmdet.registry import MODELS class TridentConv(BaseModule): """Trident Convolution Module. Args: in_channels (int): Number of channels in input. out_channels (int): Number of channels in output. kernel_size (int): Size of convolution kernel. stride (int, optional): Convolution stride. Default: 1. trident_dilations (tuple[int, int, int], optional): Dilations of different trident branch. Default: (1, 2, 3). test_branch_idx (int, optional): In inference, all 3 branches will be used if `test_branch_idx==-1`, otherwise only branch with index `test_branch_idx` will be used. Default: 1. bias (bool, optional): Whether to use bias in convolution or not. Default: False. init_cfg (dict or list[dict], optional): Initialization config dict. Default: None """ def __init__(self, in_channels, out_channels, kernel_size, stride=1, trident_dilations=(1, 2, 3), test_branch_idx=1, bias=False, init_cfg=None): super(TridentConv, self).__init__(init_cfg) self.num_branch = len(trident_dilations) self.with_bias = bias self.test_branch_idx = test_branch_idx self.stride = _pair(stride) self.kernel_size = _pair(kernel_size) self.paddings = _pair(trident_dilations) self.dilations = trident_dilations self.in_channels = in_channels self.out_channels = out_channels self.bias = bias self.weight = nn.Parameter( torch.Tensor(out_channels, in_channels, *self.kernel_size)) if bias: self.bias = nn.Parameter(torch.Tensor(out_channels)) else: self.bias = None def extra_repr(self): tmpstr = f'in_channels={self.in_channels}' tmpstr += f', out_channels={self.out_channels}' tmpstr += f', kernel_size={self.kernel_size}' tmpstr += f', num_branch={self.num_branch}' tmpstr += f', test_branch_idx={self.test_branch_idx}' tmpstr += f', stride={self.stride}' tmpstr += f', paddings={self.paddings}' tmpstr += f', dilations={self.dilations}' tmpstr += f', bias={self.bias}' return tmpstr def forward(self, inputs): if self.training or self.test_branch_idx == -1: outputs = [ F.conv2d(input, self.weight, self.bias, self.stride, padding, dilation) for input, dilation, padding in zip( inputs, self.dilations, self.paddings) ] else: assert len(inputs) == 1 outputs = [ F.conv2d(inputs[0], self.weight, self.bias, self.stride, self.paddings[self.test_branch_idx], self.dilations[self.test_branch_idx]) ] return outputs # Since TridentNet is defined over ResNet50 and ResNet101, here we # only support TridentBottleneckBlock. class TridentBottleneck(Bottleneck): """BottleBlock for TridentResNet. Args: trident_dilations (tuple[int, int, int]): Dilations of different trident branch. test_branch_idx (int): In inference, all 3 branches will be used if `test_branch_idx==-1`, otherwise only branch with index `test_branch_idx` will be used. concat_output (bool): Whether to concat the output list to a Tensor. `True` only in the last Block. """ def __init__(self, trident_dilations, test_branch_idx, concat_output, **kwargs): super(TridentBottleneck, self).__init__(**kwargs) self.trident_dilations = trident_dilations self.num_branch = len(trident_dilations) self.concat_output = concat_output self.test_branch_idx = test_branch_idx self.conv2 = TridentConv( self.planes, self.planes, kernel_size=3, stride=self.conv2_stride, bias=False, trident_dilations=self.trident_dilations, test_branch_idx=test_branch_idx, init_cfg=dict( type='Kaiming', distribution='uniform', mode='fan_in', override=dict(name='conv2'))) def forward(self, x): def _inner_forward(x): num_branch = ( self.num_branch if self.training or self.test_branch_idx == -1 else 1) identity = x if not isinstance(x, list): x = (x, ) * num_branch identity = x if self.downsample is not None: identity = [self.downsample(b) for b in x] out = [self.conv1(b) for b in x] out = [self.norm1(b) for b in out] out = [self.relu(b) for b in out] if self.with_plugins: for k in range(len(out)): out[k] = self.forward_plugin(out[k], self.after_conv1_plugin_names) out = self.conv2(out) out = [self.norm2(b) for b in out] out = [self.relu(b) for b in out] if self.with_plugins: for k in range(len(out)): out[k] = self.forward_plugin(out[k], self.after_conv2_plugin_names) out = [self.conv3(b) for b in out] out = [self.norm3(b) for b in out] if self.with_plugins: for k in range(len(out)): out[k] = self.forward_plugin(out[k], self.after_conv3_plugin_names) out = [ out_b + identity_b for out_b, identity_b in zip(out, identity) ] return out if self.with_cp and x.requires_grad: out = cp.checkpoint(_inner_forward, x) else: out = _inner_forward(x) out = [self.relu(b) for b in out] if self.concat_output: out = torch.cat(out, dim=0) return out def make_trident_res_layer(block, inplanes, planes, num_blocks, stride=1, trident_dilations=(1, 2, 3), style='pytorch', with_cp=False, conv_cfg=None, norm_cfg=dict(type='BN'), dcn=None, plugins=None, test_branch_idx=-1): """Build Trident Res Layers.""" downsample = None if stride != 1 or inplanes != planes * block.expansion: downsample = [] conv_stride = stride downsample.extend([ build_conv_layer( conv_cfg, inplanes, planes * block.expansion, kernel_size=1, stride=conv_stride, bias=False), build_norm_layer(norm_cfg, planes * block.expansion)[1] ]) downsample = nn.Sequential(*downsample) layers = [] for i in range(num_blocks): layers.append( block( inplanes=inplanes, planes=planes, stride=stride if i == 0 else 1, trident_dilations=trident_dilations, downsample=downsample if i == 0 else None, style=style, with_cp=with_cp, conv_cfg=conv_cfg, norm_cfg=norm_cfg, dcn=dcn, plugins=plugins, test_branch_idx=test_branch_idx, concat_output=True if i == num_blocks - 1 else False)) inplanes = planes * block.expansion return nn.Sequential(*layers) @MODELS.register_module() class TridentResNet(ResNet): """The stem layer, stage 1 and stage 2 in Trident ResNet are identical to ResNet, while in stage 3, Trident BottleBlock is utilized to replace the normal BottleBlock to yield trident output. Different branch shares the convolution weight but uses different dilations to achieve multi-scale output. / stage3(b0) \ x - stem - stage1 - stage2 - stage3(b1) - output \ stage3(b2) / Args: depth (int): Depth of resnet, from {50, 101, 152}. num_branch (int): Number of branches in TridentNet. test_branch_idx (int): In inference, all 3 branches will be used if `test_branch_idx==-1`, otherwise only branch with index `test_branch_idx` will be used. trident_dilations (tuple[int]): Dilations of different trident branch. len(trident_dilations) should be equal to num_branch. """ # noqa def __init__(self, depth, num_branch, test_branch_idx, trident_dilations, **kwargs): assert num_branch == len(trident_dilations) assert depth in (50, 101, 152) super(TridentResNet, self).__init__(depth, **kwargs) assert self.num_stages == 3 self.test_branch_idx = test_branch_idx self.num_branch = num_branch last_stage_idx = self.num_stages - 1 stride = self.strides[last_stage_idx] dilation = trident_dilations dcn = self.dcn if self.stage_with_dcn[last_stage_idx] else None if self.plugins is not None: stage_plugins = self.make_stage_plugins(self.plugins, last_stage_idx) else: stage_plugins = None planes = self.base_channels * 2**last_stage_idx res_layer = make_trident_res_layer( TridentBottleneck, inplanes=(self.block.expansion * self.base_channels * 2**(last_stage_idx - 1)), planes=planes, num_blocks=self.stage_blocks[last_stage_idx], stride=stride, trident_dilations=dilation, style=self.style, with_cp=self.with_cp, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, dcn=dcn, plugins=stage_plugins, test_branch_idx=self.test_branch_idx) layer_name = f'layer{last_stage_idx + 1}' self.__setattr__(layer_name, res_layer) self.res_layers.pop(last_stage_idx) self.res_layers.insert(last_stage_idx, layer_name) self._freeze_stages() ================================================ FILE: mmdet/models/data_preprocessors/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .data_preprocessor import (BatchFixedSizePad, BatchResize, BatchSyncRandomResize, BoxInstDataPreprocessor, DetDataPreprocessor, MultiBranchDataPreprocessor) __all__ = [ 'DetDataPreprocessor', 'BatchSyncRandomResize', 'BatchFixedSizePad', 'MultiBranchDataPreprocessor', 'BatchResize', 'BoxInstDataPreprocessor' ] ================================================ FILE: mmdet/models/data_preprocessors/data_preprocessor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import random from numbers import Number from typing import List, Optional, Sequence, Tuple, Union import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from mmengine.dist import barrier, broadcast, get_dist_info from mmengine.logging import MessageHub from mmengine.model import BaseDataPreprocessor, ImgDataPreprocessor from mmengine.structures import PixelData from mmengine.utils import is_seq_of from torch import Tensor from mmdet.models.utils import unfold_wo_center from mmdet.models.utils.misc import samplelist_boxtype2tensor from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.structures.mask import BitmapMasks from mmdet.utils import ConfigType try: import skimage except ImportError: skimage = None @MODELS.register_module() class DetDataPreprocessor(ImgDataPreprocessor): """Image pre-processor for detection tasks. Comparing with the :class:`mmengine.ImgDataPreprocessor`, 1. It supports batch augmentations. 2. It will additionally append batch_input_shape and pad_shape to data_samples considering the object detection task. It provides the data pre-processing as follows - Collate and move data to the target device. - Pad inputs to the maximum size of current batch with defined ``pad_value``. The padding size can be divisible by a defined ``pad_size_divisor`` - Stack inputs to batch_inputs. - Convert inputs from bgr to rgb if the shape of input is (3, H, W). - Normalize image with defined std and mean. - Do batch augmentations during training. Args: mean (Sequence[Number], optional): The pixel mean of R, G, B channels. Defaults to None. std (Sequence[Number], optional): The pixel standard deviation of R, G, B channels. Defaults to None. pad_size_divisor (int): The size of padded image should be divisible by ``pad_size_divisor``. Defaults to 1. pad_value (Number): The padded pixel value. Defaults to 0. pad_mask (bool): Whether to pad instance masks. Defaults to False. mask_pad_value (int): The padded pixel value for instance masks. Defaults to 0. pad_seg (bool): Whether to pad semantic segmentation maps. Defaults to False. seg_pad_value (int): The padded pixel value for semantic segmentation maps. Defaults to 255. bgr_to_rgb (bool): whether to convert image from BGR to RGB. Defaults to False. rgb_to_bgr (bool): whether to convert image from RGB to RGB. Defaults to False. boxtype2tensor (bool): Whether to keep the ``BaseBoxes`` type of bboxes data or not. Defaults to True. non_blocking (bool): Whether block current process when transferring data to device. Defaults to False. batch_augments (list[dict], optional): Batch-level augmentations """ def __init__(self, mean: Sequence[Number] = None, std: Sequence[Number] = None, pad_size_divisor: int = 1, pad_value: Union[float, int] = 0, pad_mask: bool = False, mask_pad_value: int = 0, pad_seg: bool = False, seg_pad_value: int = 255, bgr_to_rgb: bool = False, rgb_to_bgr: bool = False, boxtype2tensor: bool = True, non_blocking: Optional[bool] = False, batch_augments: Optional[List[dict]] = None): super().__init__( mean=mean, std=std, pad_size_divisor=pad_size_divisor, pad_value=pad_value, bgr_to_rgb=bgr_to_rgb, rgb_to_bgr=rgb_to_bgr, non_blocking=non_blocking) if batch_augments is not None: self.batch_augments = nn.ModuleList( [MODELS.build(aug) for aug in batch_augments]) else: self.batch_augments = None self.pad_mask = pad_mask self.mask_pad_value = mask_pad_value self.pad_seg = pad_seg self.seg_pad_value = seg_pad_value self.boxtype2tensor = boxtype2tensor def forward(self, data: dict, training: bool = False) -> dict: """Perform normalization、padding and bgr2rgb conversion based on ``BaseDataPreprocessor``. Args: data (dict): Data sampled from dataloader. training (bool): Whether to enable training time augmentation. Returns: dict: Data in the same format as the model input. """ batch_pad_shape = self._get_pad_shape(data) data = super().forward(data=data, training=training) inputs, data_samples = data['inputs'], data['data_samples'] if data_samples is not None: # NOTE the batched image size information may be useful, e.g. # in DETR, this is needed for the construction of masks, which is # then used for the transformer_head. batch_input_shape = tuple(inputs[0].size()[-2:]) for data_sample, pad_shape in zip(data_samples, batch_pad_shape): data_sample.set_metainfo({ 'batch_input_shape': batch_input_shape, 'pad_shape': pad_shape }) if self.boxtype2tensor: samplelist_boxtype2tensor(data_samples) if self.pad_mask and training: self.pad_gt_masks(data_samples) if self.pad_seg and training: self.pad_gt_sem_seg(data_samples) if training and self.batch_augments is not None: for batch_aug in self.batch_augments: inputs, data_samples = batch_aug(inputs, data_samples) return {'inputs': inputs, 'data_samples': data_samples} def _get_pad_shape(self, data: dict) -> List[tuple]: """Get the pad_shape of each image based on data and pad_size_divisor.""" _batch_inputs = data['inputs'] # Process data with `pseudo_collate`. if is_seq_of(_batch_inputs, torch.Tensor): batch_pad_shape = [] for ori_input in _batch_inputs: pad_h = int( np.ceil(ori_input.shape[1] / self.pad_size_divisor)) * self.pad_size_divisor pad_w = int( np.ceil(ori_input.shape[2] / self.pad_size_divisor)) * self.pad_size_divisor batch_pad_shape.append((pad_h, pad_w)) # Process data with `default_collate`. elif isinstance(_batch_inputs, torch.Tensor): assert _batch_inputs.dim() == 4, ( 'The input of `ImgDataPreprocessor` should be a NCHW tensor ' 'or a list of tensor, but got a tensor with shape: ' f'{_batch_inputs.shape}') pad_h = int( np.ceil(_batch_inputs.shape[1] / self.pad_size_divisor)) * self.pad_size_divisor pad_w = int( np.ceil(_batch_inputs.shape[2] / self.pad_size_divisor)) * self.pad_size_divisor batch_pad_shape = [(pad_h, pad_w)] * _batch_inputs.shape[0] else: raise TypeError('Output of `cast_data` should be a dict ' 'or a tuple with inputs and data_samples, but got' f'{type(data)}： {data}') return batch_pad_shape def pad_gt_masks(self, batch_data_samples: Sequence[DetDataSample]) -> None: """Pad gt_masks to shape of batch_input_shape.""" if 'masks' in batch_data_samples[0].gt_instances: for data_samples in batch_data_samples: masks = data_samples.gt_instances.masks data_samples.gt_instances.masks = masks.pad( data_samples.batch_input_shape, pad_val=self.mask_pad_value) def pad_gt_sem_seg(self, batch_data_samples: Sequence[DetDataSample]) -> None: """Pad gt_sem_seg to shape of batch_input_shape.""" if 'gt_sem_seg' in batch_data_samples[0]: for data_samples in batch_data_samples: gt_sem_seg = data_samples.gt_sem_seg.sem_seg h, w = gt_sem_seg.shape[-2:] pad_h, pad_w = data_samples.batch_input_shape gt_sem_seg = F.pad( gt_sem_seg, pad=(0, max(pad_w - w, 0), 0, max(pad_h - h, 0)), mode='constant', value=self.seg_pad_value) data_samples.gt_sem_seg = PixelData(sem_seg=gt_sem_seg) @MODELS.register_module() class BatchSyncRandomResize(nn.Module): """Batch random resize which synchronizes the random size across ranks. Args: random_size_range (tuple): The multi-scale random range during multi-scale training. interval (int): The iter interval of change image size. Defaults to 10. size_divisor (int): Image size divisible factor. Defaults to 32. """ def __init__(self, random_size_range: Tuple[int, int], interval: int = 10, size_divisor: int = 32) -> None: super().__init__() self.rank, self.world_size = get_dist_info() self._input_size = None self._random_size_range = (round(random_size_range[0] / size_divisor), round(random_size_range[1] / size_divisor)) self._interval = interval self._size_divisor = size_divisor def forward( self, inputs: Tensor, data_samples: List[DetDataSample] ) -> Tuple[Tensor, List[DetDataSample]]: """resize a batch of images and bboxes to shape ``self._input_size``""" h, w = inputs.shape[-2:] if self._input_size is None: self._input_size = (h, w) scale_y = self._input_size[0] / h scale_x = self._input_size[1] / w if scale_x != 1 or scale_y != 1: inputs = F.interpolate( inputs, size=self._input_size, mode='bilinear', align_corners=False) for data_sample in data_samples: img_shape = (int(data_sample.img_shape[0] * scale_y), int(data_sample.img_shape[1] * scale_x)) pad_shape = (int(data_sample.pad_shape[0] * scale_y), int(data_sample.pad_shape[1] * scale_x)) data_sample.set_metainfo({ 'img_shape': img_shape, 'pad_shape': pad_shape, 'batch_input_shape': self._input_size }) data_sample.gt_instances.bboxes[ ..., 0::2] = data_sample.gt_instances.bboxes[..., 0::2] * scale_x data_sample.gt_instances.bboxes[ ..., 1::2] = data_sample.gt_instances.bboxes[..., 1::2] * scale_y if 'ignored_instances' in data_sample: data_sample.ignored_instances.bboxes[ ..., 0::2] = data_sample.ignored_instances.bboxes[ ..., 0::2] * scale_x data_sample.ignored_instances.bboxes[ ..., 1::2] = data_sample.ignored_instances.bboxes[ ..., 1::2] * scale_y message_hub = MessageHub.get_current_instance() if (message_hub.get_info('iter') + 1) % self._interval == 0: self._input_size = self._get_random_size( aspect_ratio=float(w / h), device=inputs.device) return inputs, data_samples def _get_random_size(self, aspect_ratio: float, device: torch.device) -> Tuple[int, int]: """Randomly generate a shape in ``_random_size_range`` and broadcast to all ranks.""" tensor = torch.LongTensor(2).to(device) if self.rank == 0: size = random.randint(*self._random_size_range) size = (self._size_divisor * size, self._size_divisor * int(aspect_ratio * size)) tensor[0] = size[0] tensor[1] = size[1] barrier() broadcast(tensor, 0) input_size = (tensor[0].item(), tensor[1].item()) return input_size @MODELS.register_module() class BatchFixedSizePad(nn.Module): """Fixed size padding for batch images. Args: size (Tuple[int, int]): Fixed padding size. Expected padding shape (h, w). Defaults to None. img_pad_value (int): The padded pixel value for images. Defaults to 0. pad_mask (bool): Whether to pad instance masks. Defaults to False. mask_pad_value (int): The padded pixel value for instance masks. Defaults to 0. pad_seg (bool): Whether to pad semantic segmentation maps. Defaults to False. seg_pad_value (int): The padded pixel value for semantic segmentation maps. Defaults to 255. """ def __init__(self, size: Tuple[int, int], img_pad_value: int = 0, pad_mask: bool = False, mask_pad_value: int = 0, pad_seg: bool = False, seg_pad_value: int = 255) -> None: super().__init__() self.size = size self.pad_mask = pad_mask self.pad_seg = pad_seg self.img_pad_value = img_pad_value self.mask_pad_value = mask_pad_value self.seg_pad_value = seg_pad_value def forward( self, inputs: Tensor, data_samples: Optional[List[dict]] = None ) -> Tuple[Tensor, Optional[List[dict]]]: """Pad image, instance masks, segmantic segmentation maps.""" src_h, src_w = inputs.shape[-2:] dst_h, dst_w = self.size if src_h >= dst_h and src_w >= dst_w: return inputs, data_samples inputs = F.pad( inputs, pad=(0, max(0, dst_w - src_w), 0, max(0, dst_h - src_h)), mode='constant', value=self.img_pad_value) if data_samples is not None: # update batch_input_shape for data_sample in data_samples: data_sample.set_metainfo({ 'batch_input_shape': (dst_h, dst_w), 'pad_shape': (dst_h, dst_w) }) if self.pad_mask: for data_sample in data_samples: masks = data_sample.gt_instances.masks data_sample.gt_instances.masks = masks.pad( (dst_h, dst_w), pad_val=self.mask_pad_value) if self.pad_seg: for data_sample in data_samples: gt_sem_seg = data_sample.gt_sem_seg.sem_seg h, w = gt_sem_seg.shape[-2:] gt_sem_seg = F.pad( gt_sem_seg, pad=(0, max(0, dst_w - w), 0, max(0, dst_h - h)), mode='constant', value=self.seg_pad_value) data_sample.gt_sem_seg = PixelData(sem_seg=gt_sem_seg) return inputs, data_samples @MODELS.register_module() class MultiBranchDataPreprocessor(BaseDataPreprocessor): """DataPreprocessor wrapper for multi-branch data. Take semi-supervised object detection as an example, assume that the ratio of labeled data and unlabeled data in a batch is 1:2, `sup` indicates the branch where the labeled data is augmented, `unsup_teacher` and `unsup_student` indicate the branches where the unlabeled data is augmented by different pipeline. The input format of multi-branch data is shown as below : .. code-block:: none { 'inputs': { 'sup': [Tensor, None, None], 'unsup_teacher': [None, Tensor, Tensor], 'unsup_student': [None, Tensor, Tensor], }, 'data_sample': { 'sup': [DetDataSample, None, None], 'unsup_teacher': [None, DetDataSample, DetDataSample], 'unsup_student': [NOne, DetDataSample, DetDataSample], } } The format of multi-branch data after filtering None is shown as below : .. code-block:: none { 'inputs': { 'sup': [Tensor], 'unsup_teacher': [Tensor, Tensor], 'unsup_student': [Tensor, Tensor], }, 'data_sample': { 'sup': [DetDataSample], 'unsup_teacher': [DetDataSample, DetDataSample], 'unsup_student': [DetDataSample, DetDataSample], } } In order to reuse `DetDataPreprocessor` for the data from different branches, the format of multi-branch data grouped by branch is as below : .. code-block:: none { 'sup': { 'inputs': [Tensor] 'data_sample': [DetDataSample, DetDataSample] }, 'unsup_teacher': { 'inputs': [Tensor, Tensor] 'data_sample': [DetDataSample, DetDataSample] }, 'unsup_student': { 'inputs': [Tensor, Tensor] 'data_sample': [DetDataSample, DetDataSample] }, } After preprocessing data from different branches, the multi-branch data needs to be reformatted as: .. code-block:: none { 'inputs': { 'sup': [Tensor], 'unsup_teacher': [Tensor, Tensor], 'unsup_student': [Tensor, Tensor], }, 'data_sample': { 'sup': [DetDataSample], 'unsup_teacher': [DetDataSample, DetDataSample], 'unsup_student': [DetDataSample, DetDataSample], } } Args: data_preprocessor (:obj:`ConfigDict` or dict): Config of :class:`DetDataPreprocessor` to process the input data. """ def __init__(self, data_preprocessor: ConfigType) -> None: super().__init__() self.data_preprocessor = MODELS.build(data_preprocessor) def forward(self, data: dict, training: bool = False) -> dict: """Perform normalization、padding and bgr2rgb conversion based on ``BaseDataPreprocessor`` for multi-branch data. Args: data (dict): Data sampled from dataloader. training (bool): Whether to enable training time augmentation. Returns: dict: - 'inputs' (Dict[str, obj:`torch.Tensor`]): The forward data of models from different branches. - 'data_sample' (Dict[str, obj:`DetDataSample`]): The annotation info of the sample from different branches. """ if training is False: return self.data_preprocessor(data, training) # Filter out branches with a value of None for key in data.keys(): for branch in data[key].keys(): data[key][branch] = list( filter(lambda x: x is not None, data[key][branch])) # Group data by branch multi_branch_data = {} for key in data.keys(): for branch in data[key].keys(): if multi_branch_data.get(branch, None) is None: multi_branch_data[branch] = {key: data[key][branch]} elif multi_branch_data[branch].get(key, None) is None: multi_branch_data[branch][key] = data[key][branch] else: multi_branch_data[branch][key].append(data[key][branch]) # Preprocess data from different branches for branch, _data in multi_branch_data.items(): multi_branch_data[branch] = self.data_preprocessor(_data, training) # Format data by inputs and data_samples format_data = {} for branch in multi_branch_data.keys(): for key in multi_branch_data[branch].keys(): if format_data.get(key, None) is None: format_data[key] = {branch: multi_branch_data[branch][key]} elif format_data[key].get(branch, None) is None: format_data[key][branch] = multi_branch_data[branch][key] else: format_data[key][branch].append( multi_branch_data[branch][key]) return format_data @property def device(self): return self.data_preprocessor.device def to(self, device: Optional[Union[int, torch.device]], *args, **kwargs) -> nn.Module: """Overrides this method to set the :attr:`device` Args: device (int or torch.device, optional): The desired device of the parameters and buffers in this module. Returns: nn.Module: The model itself. """ return self.data_preprocessor.to(device, *args, **kwargs) def cuda(self, *args, **kwargs) -> nn.Module: """Overrides this method to set the :attr:`device` Returns: nn.Module: The model itself. """ return self.data_preprocessor.cuda(*args, **kwargs) def cpu(self, *args, **kwargs) -> nn.Module: """Overrides this method to set the :attr:`device` Returns: nn.Module: The model itself. """ return self.data_preprocessor.cpu(*args, **kwargs) @MODELS.register_module() class BatchResize(nn.Module): """Batch resize during training. This implementation is modified from https://github.com/Purkialo/CrowdDet/blob/master/lib/data/CrowdHuman.py. It provides the data pre-processing as follows: - A batch of all images will pad to a uniform size and stack them into a torch.Tensor by `DetDataPreprocessor`. - `BatchFixShapeResize` resize all images to the target size. - Padding images to make sure the size of image can be divisible by ``pad_size_divisor``. Args: scale (tuple): Images scales for resizing. pad_size_divisor (int): Image size divisible factor. Defaults to 1. pad_value (Number): The padded pixel value. Defaults to 0. """ def __init__( self, scale: tuple, pad_size_divisor: int = 1, pad_value: Union[float, int] = 0, ) -> None: super().__init__() self.min_size = min(scale) self.max_size = max(scale) self.pad_size_divisor = pad_size_divisor self.pad_value = pad_value def forward( self, inputs: Tensor, data_samples: List[DetDataSample] ) -> Tuple[Tensor, List[DetDataSample]]: """resize a batch of images and bboxes.""" batch_height, batch_width = inputs.shape[-2:] target_height, target_width, scale = self.get_target_size( batch_height, batch_width) inputs = F.interpolate( inputs, size=(target_height, target_width), mode='bilinear', align_corners=False) inputs = self.get_padded_tensor(inputs, self.pad_value) if data_samples is not None: batch_input_shape = tuple(inputs.size()[-2:]) for data_sample in data_samples: img_shape = [ int(scale * _) for _ in list(data_sample.img_shape) ] data_sample.set_metainfo({ 'img_shape': tuple(img_shape), 'batch_input_shape': batch_input_shape, 'pad_shape': batch_input_shape, 'scale_factor': (scale, scale) }) data_sample.gt_instances.bboxes *= scale data_sample.ignored_instances.bboxes *= scale return inputs, data_samples def get_target_size(self, height: int, width: int) -> Tuple[int, int, float]: """Get the target size of a batch of images based on data and scale.""" im_size_min = np.min([height, width]) im_size_max = np.max([height, width]) scale = self.min_size / im_size_min if scale * im_size_max > self.max_size: scale = self.max_size / im_size_max target_height, target_width = int(round(height * scale)), int( round(width * scale)) return target_height, target_width, scale def get_padded_tensor(self, tensor: Tensor, pad_value: int) -> Tensor: """Pad images according to pad_size_divisor.""" assert tensor.ndim == 4 target_height, target_width = tensor.shape[-2], tensor.shape[-1] divisor = self.pad_size_divisor padded_height = (target_height + divisor - 1) // divisor * divisor padded_width = (target_width + divisor - 1) // divisor * divisor padded_tensor = torch.ones([ tensor.shape[0], tensor.shape[1], padded_height, padded_width ]) * pad_value padded_tensor = padded_tensor.type_as(tensor) padded_tensor[:, :, :target_height, :target_width] = tensor return padded_tensor @MODELS.register_module() class BoxInstDataPreprocessor(DetDataPreprocessor): """Pseudo mask pre-processor for BoxInst. Comparing with the :class:`mmdet.DetDataPreprocessor`, 1. It generates masks using box annotations. 2. It computes the images color similarity in LAB color space. Args: mask_stride (int): The mask output stride in boxinst. Defaults to 4. pairwise_size (int): The size of neighborhood for each pixel. Defaults to 3. pairwise_dilation (int): The dilation of neighborhood for each pixel. Defaults to 2. pairwise_color_thresh (float): The thresh of image color similarity. Defaults to 0.3. bottom_pixels_removed (int): The length of removed pixels in bottom. It is caused by the annotation error in coco dataset. Defaults to 10. """ def __init__(self, *arg, mask_stride: int = 4, pairwise_size: int = 3, pairwise_dilation: int = 2, pairwise_color_thresh: float = 0.3, bottom_pixels_removed: int = 10, **kwargs) -> None: super().__init__(*arg, **kwargs) self.mask_stride = mask_stride self.pairwise_size = pairwise_size self.pairwise_dilation = pairwise_dilation self.pairwise_color_thresh = pairwise_color_thresh self.bottom_pixels_removed = bottom_pixels_removed if skimage is None: raise RuntimeError('skimage is not installed,\ please install it by: pip install scikit-image') def get_images_color_similarity(self, inputs: Tensor, image_masks: Tensor) -> Tensor: """Compute the image color similarity in LAB color space.""" assert inputs.dim() == 4 assert inputs.size(0) == 1 unfolded_images = unfold_wo_center( inputs, kernel_size=self.pairwise_size, dilation=self.pairwise_dilation) diff = inputs[:, :, None] - unfolded_images similarity = torch.exp(-torch.norm(diff, dim=1) * 0.5) unfolded_weights = unfold_wo_center( image_masks[None, None], kernel_size=self.pairwise_size, dilation=self.pairwise_dilation) unfolded_weights = torch.max(unfolded_weights, dim=1)[0] return similarity * unfolded_weights def forward(self, data: dict, training: bool = False) -> dict: """Get pseudo mask labels using color similarity.""" det_data = super().forward(data, training) inputs, data_samples = det_data['inputs'], det_data['data_samples'] if training: # get image masks and remove bottom pixels b_img_h, b_img_w = data_samples[0].batch_input_shape img_masks = [] for i in range(inputs.shape[0]): img_h, img_w = data_samples[i].img_shape img_mask = inputs.new_ones((img_h, img_w)) pixels_removed = int(self.bottom_pixels_removed * float(img_h) / float(b_img_h)) if pixels_removed > 0: img_mask[-pixels_removed:, :] = 0 pad_w = b_img_w - img_w pad_h = b_img_h - img_h img_mask = F.pad(img_mask, (0, pad_w, 0, pad_h), 'constant', 0.) img_masks.append(img_mask) img_masks = torch.stack(img_masks, dim=0) start = int(self.mask_stride // 2) img_masks = img_masks[:, start::self.mask_stride, start::self.mask_stride] # Get origin rgb image for color similarity ori_imgs = inputs * self.std + self.mean downsampled_imgs = F.avg_pool2d( ori_imgs.float(), kernel_size=self.mask_stride, stride=self.mask_stride, padding=0) # Compute color similarity for pseudo mask generation for im_i, data_sample in enumerate(data_samples): # TODO: Support rgb2lab in mmengine? images_lab = skimage.color.rgb2lab( downsampled_imgs[im_i].byte().permute(1, 2, 0).cpu().numpy()) images_lab = torch.as_tensor( images_lab, device=ori_imgs.device, dtype=torch.float32) images_lab = images_lab.permute(2, 0, 1)[None] images_color_similarity = self.get_images_color_similarity( images_lab, img_masks[im_i]) pairwise_mask = (images_color_similarity >= self.pairwise_color_thresh).float() per_im_bboxes = data_sample.gt_instances.bboxes if per_im_bboxes.shape[0] > 0: per_im_masks = [] for per_box in per_im_bboxes: mask_full = torch.zeros((b_img_h, b_img_w), device=self.device).float() mask_full[int(per_box[1]):int(per_box[3] + 1), int(per_box[0]):int(per_box[2] + 1)] = 1.0 per_im_masks.append(mask_full) per_im_masks = torch.stack(per_im_masks, dim=0) pairwise_masks = torch.cat( [pairwise_mask for _ in range(per_im_bboxes.shape[0])], dim=0) else: per_im_masks = torch.zeros((0, b_img_h, b_img_w)) pairwise_masks = torch.zeros( (0, self.pairwise_size**2 - 1, b_img_h, b_img_w)) # TODO: Support BitmapMasks with tensor? data_sample.gt_instances.masks = BitmapMasks( per_im_masks.cpu().numpy(), b_img_h, b_img_w) data_sample.gt_instances.pairwise_masks = pairwise_masks return {'inputs': inputs, 'data_samples': data_samples} ================================================ FILE: mmdet/models/dense_heads/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .anchor_free_head import AnchorFreeHead from .anchor_head import AnchorHead from .atss_head import ATSSHead from .autoassign_head import AutoAssignHead from .boxinst_head import BoxInstBboxHead, BoxInstMaskHead from .cascade_rpn_head import CascadeRPNHead, StageCascadeRPNHead from .centernet_head import CenterNetHead from .centernet_update_head import CenterNetUpdateHead from .centripetal_head import CentripetalHead from .condinst_head import CondInstBboxHead, CondInstMaskHead from .conditional_detr_head import ConditionalDETRHead from .corner_head import CornerHead from .dab_detr_head import DABDETRHead from .ddod_head import DDODHead from .deformable_detr_head import DeformableDETRHead from .detr_head import DETRHead from .dino_head import DINOHead from .embedding_rpn_head import EmbeddingRPNHead from .fcos_head import FCOSHead from .fovea_head import FoveaHead from .free_anchor_retina_head import FreeAnchorRetinaHead from .fsaf_head import FSAFHead from .ga_retina_head import GARetinaHead from .ga_rpn_head import GARPNHead from .gfl_head import GFLHead from .guided_anchor_head import FeatureAdaption, GuidedAnchorHead from .lad_head import LADHead from .ld_head import LDHead from .mask2former_head import Mask2FormerHead from .maskformer_head import MaskFormerHead from .nasfcos_head import NASFCOSHead from .paa_head import PAAHead from .pisa_retinanet_head import PISARetinaHead from .pisa_ssd_head import PISASSDHead from .reppoints_head import RepPointsHead from .retina_head import RetinaHead from .retina_sepbn_head import RetinaSepBNHead from .rpn_head import RPNHead from .rtmdet_head import RTMDetHead, RTMDetSepBNHead from .rtmdet_ins_head import RTMDetInsHead, RTMDetInsSepBNHead from .sabl_retina_head import SABLRetinaHead from .solo_head import DecoupledSOLOHead, DecoupledSOLOLightHead, SOLOHead from .solov2_head import SOLOV2Head from .ssd_head import SSDHead from .tood_head import TOODHead from .vfnet_head import VFNetHead from .yolact_head import YOLACTHead, YOLACTProtonet from .yolo_head import YOLOV3Head from .yolof_head import YOLOFHead from .yolox_head import YOLOXHead __all__ = [ 'AnchorFreeHead', 'AnchorHead', 'GuidedAnchorHead', 'FeatureAdaption', 'RPNHead', 'GARPNHead', 'RetinaHead', 'RetinaSepBNHead', 'GARetinaHead', 'SSDHead', 'FCOSHead', 'RepPointsHead', 'FoveaHead', 'FreeAnchorRetinaHead', 'ATSSHead', 'FSAFHead', 'NASFCOSHead', 'PISARetinaHead', 'PISASSDHead', 'GFLHead', 'CornerHead', 'YOLACTHead', 'YOLACTProtonet', 'YOLOV3Head', 'PAAHead', 'SABLRetinaHead', 'CentripetalHead', 'VFNetHead', 'StageCascadeRPNHead', 'CascadeRPNHead', 'EmbeddingRPNHead', 'LDHead', 'AutoAssignHead', 'DETRHead', 'YOLOFHead', 'DeformableDETRHead', 'CenterNetHead', 'YOLOXHead', 'SOLOHead', 'DecoupledSOLOHead', 'DecoupledSOLOLightHead', 'SOLOV2Head', 'LADHead', 'TOODHead', 'MaskFormerHead', 'Mask2FormerHead', 'DDODHead', 'CenterNetUpdateHead', 'RTMDetHead', 'RTMDetSepBNHead', 'CondInstBboxHead', 'CondInstMaskHead', 'RTMDetInsHead', 'RTMDetInsSepBNHead', 'BoxInstBboxHead', 'BoxInstMaskHead', 'ConditionalDETRHead', 'DINOHead', 'DABDETRHead' ] ================================================ FILE: mmdet/models/dense_heads/anchor_free_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import abstractmethod from typing import Any, List, Sequence, Tuple, Union import torch.nn as nn from mmcv.cnn import ConvModule from numpy import ndarray from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.utils import (ConfigType, InstanceList, MultiConfig, OptConfigType, OptInstanceList) from ..task_modules.prior_generators import MlvlPointGenerator from ..utils import multi_apply from .base_dense_head import BaseDenseHead StrideType = Union[Sequence[int], Sequence[Tuple[int, int]]] @MODELS.register_module() class AnchorFreeHead(BaseDenseHead): """Anchor-free head (FCOS, Fovea, RepPoints, etc.). Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. feat_channels (int): Number of hidden channels. Used in child classes. stacked_convs (int): Number of stacking convs of the head. strides (Sequence[int] or Sequence[Tuple[int, int]]): Downsample factor of each feature map. dcn_on_last_conv (bool): If true, use dcn in the last layer of towers. Defaults to False. conv_bias (bool or str): If specified as `auto`, it will be decided by the norm_cfg. Bias of conv will be set as True if `norm_cfg` is None, otherwise False. Default: "auto". loss_cls (:obj:`ConfigDict` or dict): Config of classification loss. loss_bbox (:obj:`ConfigDict` or dict): Config of localization loss. bbox_coder (:obj:`ConfigDict` or dict): Config of bbox coder. Defaults 'DistancePointBBoxCoder'. conv_cfg (:obj:`ConfigDict` or dict, Optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict, Optional): Config dict for normalization layer. Defaults to None. train_cfg (:obj:`ConfigDict` or dict, Optional): Training config of anchor-free head. test_cfg (:obj:`ConfigDict` or dict, Optional): Testing config of anchor-free head. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict]): Initialization config dict. """ # noqa: W605 _version = 1 def __init__( self, num_classes: int, in_channels: int, feat_channels: int = 256, stacked_convs: int = 4, strides: StrideType = (4, 8, 16, 32, 64), dcn_on_last_conv: bool = False, conv_bias: Union[bool, str] = 'auto', loss_cls: ConfigType = dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox: ConfigType = dict(type='IoULoss', loss_weight=1.0), bbox_coder: ConfigType = dict(type='DistancePointBBoxCoder'), conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='conv_cls', std=0.01, bias_prob=0.01)) ) -> None: super().__init__(init_cfg=init_cfg) self.num_classes = num_classes self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) if self.use_sigmoid_cls: self.cls_out_channels = num_classes else: self.cls_out_channels = num_classes + 1 self.in_channels = in_channels self.feat_channels = feat_channels self.stacked_convs = stacked_convs self.strides = strides self.dcn_on_last_conv = dcn_on_last_conv assert conv_bias == 'auto' or isinstance(conv_bias, bool) self.conv_bias = conv_bias self.loss_cls = MODELS.build(loss_cls) self.loss_bbox = MODELS.build(loss_bbox) self.bbox_coder = TASK_UTILS.build(bbox_coder) self.prior_generator = MlvlPointGenerator(strides) # In order to keep a more general interface and be consistent with # anchor_head. We can think of point like one anchor self.num_base_priors = self.prior_generator.num_base_priors[0] self.train_cfg = train_cfg self.test_cfg = test_cfg self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.fp16_enabled = False self._init_layers() def _init_layers(self) -> None: """Initialize layers of the head.""" self._init_cls_convs() self._init_reg_convs() self._init_predictor() def _init_cls_convs(self) -> None: """Initialize classification conv layers of the head.""" self.cls_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels if self.dcn_on_last_conv and i == self.stacked_convs - 1: conv_cfg = dict(type='DCNv2') else: conv_cfg = self.conv_cfg self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=self.norm_cfg, bias=self.conv_bias)) def _init_reg_convs(self) -> None: """Initialize bbox regression conv layers of the head.""" self.reg_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels if self.dcn_on_last_conv and i == self.stacked_convs - 1: conv_cfg = dict(type='DCNv2') else: conv_cfg = self.conv_cfg self.reg_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=self.norm_cfg, bias=self.conv_bias)) def _init_predictor(self) -> None: """Initialize predictor layers of the head.""" self.conv_cls = nn.Conv2d( self.feat_channels, self.cls_out_channels, 3, padding=1) self.conv_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1) def _load_from_state_dict(self, state_dict: dict, prefix: str, local_metadata: dict, strict: bool, missing_keys: Union[List[str], str], unexpected_keys: Union[List[str], str], error_msgs: Union[List[str], str]) -> None: """Hack some keys of the model state dict so that can load checkpoints of previous version.""" version = local_metadata.get('version', None) if version is None: # the key is different in early versions # for example, 'fcos_cls' become 'conv_cls' now bbox_head_keys = [ k for k in state_dict.keys() if k.startswith(prefix) ] ori_predictor_keys = [] new_predictor_keys = [] # e.g. 'fcos_cls' or 'fcos_reg' for key in bbox_head_keys: ori_predictor_keys.append(key) key = key.split('.') if len(key) < 2: conv_name = None elif key[1].endswith('cls'): conv_name = 'conv_cls' elif key[1].endswith('reg'): conv_name = 'conv_reg' elif key[1].endswith('centerness'): conv_name = 'conv_centerness' else: conv_name = None if conv_name is not None: key[1] = conv_name new_predictor_keys.append('.'.join(key)) else: ori_predictor_keys.pop(-1) for i in range(len(new_predictor_keys)): state_dict[new_predictor_keys[i]] = state_dict.pop( ori_predictor_keys[i]) super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) def forward(self, x: Tuple[Tensor]) -> Tuple[List[Tensor], List[Tensor]]: """Forward features from the upstream network. Args: feats (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Usually contain classification scores and bbox predictions. - cls_scores (list[Tensor]): Box scores for each scale level, \ each is a 4D-tensor, the channel number is \ num_points * num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for each scale \ level, each is a 4D-tensor, the channel number is num_points * 4. """ return multi_apply(self.forward_single, x)[:2] def forward_single(self, x: Tensor) -> Tuple[Tensor, ...]: """Forward features of a single scale level. Args: x (Tensor): FPN feature maps of the specified stride. Returns: tuple: Scores for each class, bbox predictions, features after classification and regression conv layers, some models needs these features like FCOS. """ cls_feat = x reg_feat = x for cls_layer in self.cls_convs: cls_feat = cls_layer(cls_feat) cls_score = self.conv_cls(cls_feat) for reg_layer in self.reg_convs: reg_feat = reg_layer(reg_feat) bbox_pred = self.conv_reg(reg_feat) return cls_score, bbox_pred, cls_feat, reg_feat @abstractmethod def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level, each is a 4D-tensor, the channel number is num_points * num_classes. bbox_preds (list[Tensor]): Box energies / deltas for each scale level, each is a 4D-tensor, the channel number is num_points * 4. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. """ raise NotImplementedError @abstractmethod def get_targets(self, points: List[Tensor], batch_gt_instances: InstanceList) -> Any: """Compute regression, classification and centerness targets for points in multiple images. Args: points (list[Tensor]): Points of each fpn level, each has shape (num_points, 2). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. """ raise NotImplementedError # TODO refactor aug_test def aug_test(self, aug_batch_feats: List[Tensor], aug_batch_img_metas: List[List[Tensor]], rescale: bool = False) -> List[ndarray]: """Test function with test time augmentation. Args: aug_batch_feats (list[Tensor]): the outer list indicates test-time augmentations and inner Tensor should have a shape NxCxHxW, which contains features for all images in the batch. aug_batch_img_metas (list[list[dict]]): the outer list indicates test-time augs (multiscale, flip, etc.) and the inner list indicates images in a batch. each dict has image information. rescale (bool, optional): Whether to rescale the results. Defaults to False. Returns: list[ndarray]: bbox results of each class """ return self.aug_test_bboxes( aug_batch_feats, aug_batch_img_metas, rescale=rescale) ================================================ FILE: mmdet/models/dense_heads/anchor_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings from typing import List, Optional, Tuple, Union import torch import torch.nn as nn from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures.bbox import BaseBoxes, cat_boxes, get_box_tensor from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptInstanceList, OptMultiConfig) from ..task_modules.prior_generators import (AnchorGenerator, anchor_inside_flags) from ..task_modules.samplers import PseudoSampler from ..utils import images_to_levels, multi_apply, unmap from .base_dense_head import BaseDenseHead @MODELS.register_module() class AnchorHead(BaseDenseHead): """Anchor-based head (RPN, RetinaNet, SSD, etc.). Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. feat_channels (int): Number of hidden channels. Used in child classes. anchor_generator (dict): Config dict for anchor generator bbox_coder (dict): Config of bounding box coder. reg_decoded_bbox (bool): If true, the regression loss would be applied directly on decoded bounding boxes, converting both the predicted boxes and regression targets to absolute coordinates format. Default False. It should be `True` when using `IoULoss`, `GIoULoss`, or `DIoULoss` in the bbox head. loss_cls (dict): Config of classification loss. loss_bbox (dict): Config of localization loss. train_cfg (dict): Training config of anchor head. test_cfg (dict): Testing config of anchor head. init_cfg (dict or list[dict], optional): Initialization config dict. """ # noqa: W605 def __init__( self, num_classes: int, in_channels: int, feat_channels: int = 256, anchor_generator: ConfigType = dict( type='AnchorGenerator', scales=[8, 16, 32], ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), bbox_coder: ConfigType = dict( type='DeltaXYWHBBoxCoder', clip_border=True, target_means=(.0, .0, .0, .0), target_stds=(1.0, 1.0, 1.0, 1.0)), reg_decoded_bbox: bool = False, loss_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox: ConfigType = dict( type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0), train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: OptMultiConfig = dict( type='Normal', layer='Conv2d', std=0.01) ) -> None: super().__init__(init_cfg=init_cfg) self.in_channels = in_channels self.num_classes = num_classes self.feat_channels = feat_channels self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) if self.use_sigmoid_cls: self.cls_out_channels = num_classes else: self.cls_out_channels = num_classes + 1 if self.cls_out_channels <= 0: raise ValueError(f'num_classes={num_classes} is too small') self.reg_decoded_bbox = reg_decoded_bbox self.bbox_coder = TASK_UTILS.build(bbox_coder) self.loss_cls = MODELS.build(loss_cls) self.loss_bbox = MODELS.build(loss_bbox) self.train_cfg = train_cfg self.test_cfg = test_cfg if self.train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) if train_cfg.get('sampler', None) is not None: self.sampler = TASK_UTILS.build( self.train_cfg['sampler'], default_args=dict(context=self)) else: self.sampler = PseudoSampler(context=self) self.fp16_enabled = False self.prior_generator = TASK_UTILS.build(anchor_generator) # Usually the numbers of anchors for each level are the same # except SSD detectors. So it is an int in the most dense # heads but a list of int in SSDHead self.num_base_priors = self.prior_generator.num_base_priors[0] self._init_layers() @property def num_anchors(self) -> int: warnings.warn('DeprecationWarning: `num_anchors` is deprecated, ' 'for consistency or also use ' '`num_base_priors` instead') return self.prior_generator.num_base_priors[0] @property def anchor_generator(self) -> AnchorGenerator: warnings.warn('DeprecationWarning: anchor_generator is deprecated, ' 'please use "prior_generator" instead') return self.prior_generator def _init_layers(self) -> None: """Initialize layers of the head.""" self.conv_cls = nn.Conv2d(self.in_channels, self.num_base_priors * self.cls_out_channels, 1) reg_dim = self.bbox_coder.encode_size self.conv_reg = nn.Conv2d(self.in_channels, self.num_base_priors * reg_dim, 1) def forward_single(self, x: Tensor) -> Tuple[Tensor, Tensor]: """Forward feature of a single scale level. Args: x (Tensor): Features of a single scale level. Returns: tuple: cls_score (Tensor): Cls scores for a single scale level \ the channels number is num_base_priors * num_classes. bbox_pred (Tensor): Box energies / deltas for a single scale \ level, the channels number is num_base_priors * 4. """ cls_score = self.conv_cls(x) bbox_pred = self.conv_reg(x) return cls_score, bbox_pred def forward(self, x: Tuple[Tensor]) -> Tuple[List[Tensor]]: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of classification scores and bbox prediction. - cls_scores (list[Tensor]): Classification scores for all \ scale levels, each is a 4D-tensor, the channels number \ is num_base_priors * num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for all \ scale levels, each is a 4D-tensor, the channels number \ is num_base_priors * 4. """ return multi_apply(self.forward_single, x) def get_anchors(self, featmap_sizes: List[tuple], batch_img_metas: List[dict], device: Union[torch.device, str] = 'cuda') \ -> Tuple[List[List[Tensor]], List[List[Tensor]]]: """Get anchors according to feature map sizes. Args: featmap_sizes (list[tuple]): Multi-level feature map sizes. batch_img_metas (list[dict]): Image meta info. device (torch.device | str): Device for returned tensors. Defaults to cuda. Returns: tuple: - anchor_list (list[list[Tensor]]): Anchors of each image. - valid_flag_list (list[list[Tensor]]): Valid flags of each image. """ num_imgs = len(batch_img_metas) # since feature map sizes of all images are the same, we only compute # anchors for one time multi_level_anchors = self.prior_generator.grid_priors( featmap_sizes, device=device) anchor_list = [multi_level_anchors for _ in range(num_imgs)] # for each image, we compute valid flags of multi level anchors valid_flag_list = [] for img_id, img_meta in enumerate(batch_img_metas): multi_level_flags = self.prior_generator.valid_flags( featmap_sizes, img_meta['pad_shape'], device) valid_flag_list.append(multi_level_flags) return anchor_list, valid_flag_list def _get_targets_single(self, flat_anchors: Union[Tensor, BaseBoxes], valid_flags: Tensor, gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None, unmap_outputs: bool = True) -> tuple: """Compute regression and classification targets for anchors in a single image. Args: flat_anchors (Tensor or :obj:`BaseBoxes`): Multi-level anchors of the image, which are concatenated into a single tensor or box type of shape (num_anchors, 4) valid_flags (Tensor): Multi level valid flags of the image, which are concatenated into a single tensor of shape (num_anchors, ). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. Returns: tuple: - labels (Tensor): Labels of each level. - label_weights (Tensor): Label weights of each level. - bbox_targets (Tensor): BBox targets of each level. - bbox_weights (Tensor): BBox weights of each level. - pos_inds (Tensor): positive samples indexes. - neg_inds (Tensor): negative samples indexes. - sampling_result (:obj:`SamplingResult`): Sampling results. """ inside_flags = anchor_inside_flags(flat_anchors, valid_flags, img_meta['img_shape'][:2], self.train_cfg['allowed_border']) if not inside_flags.any(): raise ValueError( 'There is no valid anchor inside the image boundary. Please ' 'check the image size and anchor sizes, or set ' '``allowed_border`` to -1 to skip the condition.') # assign gt and sample anchors anchors = flat_anchors[inside_flags] pred_instances = InstanceData(priors=anchors) assign_result = self.assigner.assign(pred_instances, gt_instances, gt_instances_ignore) # No sampling is required except for RPN and # Guided Anchoring algorithms sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) num_valid_anchors = anchors.shape[0] target_dim = gt_instances.bboxes.size(-1) if self.reg_decoded_bbox \ else self.bbox_coder.encode_size bbox_targets = anchors.new_zeros(num_valid_anchors, target_dim) bbox_weights = anchors.new_zeros(num_valid_anchors, target_dim) # TODO: Considering saving memory, is it necessary to be long? labels = anchors.new_full((num_valid_anchors, ), self.num_classes, dtype=torch.long) label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds # `bbox_coder.encode` accepts tensor or box type inputs and generates # tensor targets. If regressing decoded boxes, the code will convert # box type `pos_bbox_targets` to tensor. if len(pos_inds) > 0: if not self.reg_decoded_bbox: pos_bbox_targets = self.bbox_coder.encode( sampling_result.pos_priors, sampling_result.pos_gt_bboxes) else: pos_bbox_targets = sampling_result.pos_gt_bboxes pos_bbox_targets = get_box_tensor(pos_bbox_targets) bbox_targets[pos_inds, :] = pos_bbox_targets bbox_weights[pos_inds, :] = 1.0 labels[pos_inds] = sampling_result.pos_gt_labels if self.train_cfg['pos_weight'] <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = self.train_cfg['pos_weight'] if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 # map up to original set of anchors if unmap_outputs: num_total_anchors = flat_anchors.size(0) labels = unmap( labels, num_total_anchors, inside_flags, fill=self.num_classes) # fill bg label label_weights = unmap(label_weights, num_total_anchors, inside_flags) bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, neg_inds, sampling_result) def get_targets(self, anchor_list: List[List[Tensor]], valid_flag_list: List[List[Tensor]], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs: bool = True, return_sampling_results: bool = False) -> tuple: """Compute regression and classification targets for anchors in multiple images. Args: anchor_list (list[list[Tensor]]): Multi level anchors of each image. The outer list indicates images, and the inner list corresponds to feature levels of the image. Each element of the inner list is a tensor of shape (num_anchors, 4). valid_flag_list (list[list[Tensor]]): Multi level valid flags of each image. The outer list indicates images, and the inner list corresponds to feature levels of the image. Each element of the inner list is a tensor of shape (num_anchors, ) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. return_sampling_results (bool): Whether to return the sampling results. Defaults to False. Returns: tuple: Usually returns a tuple containing learning targets. - labels_list (list[Tensor]): Labels of each level. - label_weights_list (list[Tensor]): Label weights of each level. - bbox_targets_list (list[Tensor]): BBox targets of each level. - bbox_weights_list (list[Tensor]): BBox weights of each level. - avg_factor (int): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using `PseudoSampler`, `avg_factor` is usually equal to the number of positive priors. additional_returns: This function enables user-defined returns from `self._get_targets_single`. These returns are currently refined to properties at each feature map (i.e. having HxW dimension). The results will be concatenated after the end """ num_imgs = len(batch_img_metas) assert len(anchor_list) == len(valid_flag_list) == num_imgs if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None] * num_imgs # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] # concat all level anchors to a single tensor concat_anchor_list = [] concat_valid_flag_list = [] for i in range(num_imgs): assert len(anchor_list[i]) == len(valid_flag_list[i]) concat_anchor_list.append(cat_boxes(anchor_list[i])) concat_valid_flag_list.append(torch.cat(valid_flag_list[i])) # compute targets for each image results = multi_apply( self._get_targets_single, concat_anchor_list, concat_valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs) (all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, pos_inds_list, neg_inds_list, sampling_results_list) = results[:7] rest_results = list(results[7:]) # user-added return values # Get `avg_factor` of all images, which calculate in `SamplingResult`. # When using sampling method, avg_factor is usually the sum of # positive and negative priors. When using `PseudoSampler`, # `avg_factor` is usually equal to the number of positive priors. avg_factor = sum( [results.avg_factor for results in sampling_results_list]) # update `_raw_positive_infos`, which will be used when calling # `get_positive_infos`. self._raw_positive_infos.update(sampling_results=sampling_results_list) # split targets to a list w.r.t. multiple levels labels_list = images_to_levels(all_labels, num_level_anchors) label_weights_list = images_to_levels(all_label_weights, num_level_anchors) bbox_targets_list = images_to_levels(all_bbox_targets, num_level_anchors) bbox_weights_list = images_to_levels(all_bbox_weights, num_level_anchors) res = (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) if return_sampling_results: res = res + (sampling_results_list, ) for i, r in enumerate(rest_results): # user-added return values rest_results[i] = images_to_levels(r, num_level_anchors) return res + tuple(rest_results) def loss_by_feat_single(self, cls_score: Tensor, bbox_pred: Tensor, anchors: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, bbox_weights: Tensor, avg_factor: int) -> tuple: """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: cls_score (Tensor): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W). bbox_pred (Tensor): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). anchors (Tensor): Box reference for each scale level with shape (N, num_total_anchors, 4). labels (Tensor): Labels of each anchors with shape (N, num_total_anchors). label_weights (Tensor): Label weights of each anchor with shape (N, num_total_anchors) bbox_targets (Tensor): BBox regression targets of each anchor weight shape (N, num_total_anchors, 4). bbox_weights (Tensor): BBox regression loss weights of each anchor with shape (N, num_total_anchors, 4). avg_factor (int): Average factor that is used to average the loss. Returns: tuple: loss components. """ # classification loss labels = labels.reshape(-1) label_weights = label_weights.reshape(-1) cls_score = cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) loss_cls = self.loss_cls( cls_score, labels, label_weights, avg_factor=avg_factor) # regression loss target_dim = bbox_targets.size(-1) bbox_targets = bbox_targets.reshape(-1, target_dim) bbox_weights = bbox_weights.reshape(-1, target_dim) bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, self.bbox_coder.encode_size) if self.reg_decoded_bbox: # When the regression loss (e.g. `IouLoss`, `GIouLoss`) # is applied directly on the decoded bounding boxes, it # decodes the already encoded coordinates to absolute format. anchors = anchors.reshape(-1, anchors.size(-1)) bbox_pred = self.bbox_coder.decode(anchors, bbox_pred) bbox_pred = get_box_tensor(bbox_pred) loss_bbox = self.loss_bbox( bbox_pred, bbox_targets, bbox_weights, avg_factor=avg_factor) return loss_cls, loss_bbox def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level has shape (N, num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] # concat all level anchors and flags to a single tensor concat_anchor_list = [] for i in range(len(anchor_list)): concat_anchor_list.append(cat_boxes(anchor_list[i])) all_anchor_list = images_to_levels(concat_anchor_list, num_level_anchors) losses_cls, losses_bbox = multi_apply( self.loss_by_feat_single, cls_scores, bbox_preds, all_anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor=avg_factor) return dict(loss_cls=losses_cls, loss_bbox=losses_bbox) ================================================ FILE: mmdet/models/dense_heads/atss_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Sequence, Tuple import torch import torch.nn as nn from mmcv.cnn import ConvModule, Scale from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import (ConfigType, InstanceList, MultiConfig, OptConfigType, OptInstanceList, reduce_mean) from ..task_modules.prior_generators import anchor_inside_flags from ..utils import images_to_levels, multi_apply, unmap from .anchor_head import AnchorHead @MODELS.register_module() class ATSSHead(AnchorHead): """Detection Head of `ATSS `_. ATSS head structure is similar with FCOS, however ATSS use anchor boxes and assign label by Adaptive Training Sample Selection instead max-iou. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. pred_kernel_size (int): Kernel size of ``nn.Conv2d`` stacked_convs (int): Number of stacking convs of the head. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict): Config dict for normalization layer. Defaults to ``dict(type='GN', num_groups=32, requires_grad=True)``. reg_decoded_bbox (bool): If true, the regression loss would be applied directly on decoded bounding boxes, converting both the predicted boxes and regression targets to absolute coordinates format. Defaults to False. It should be `True` when using `IoULoss`, `GIoULoss`, or `DIoULoss` in the bbox head. loss_centerness (:obj:`ConfigDict` or dict): Config of centerness loss. Defaults to ``dict(type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)``. init_cfg (:obj:`ConfigDict` or dict or list[dict] or list[:obj:`ConfigDict`]): Initialization config dict. """ def __init__(self, num_classes: int, in_channels: int, pred_kernel_size: int = 3, stacked_convs: int = 4, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict( type='GN', num_groups=32, requires_grad=True), reg_decoded_bbox: bool = True, loss_centerness: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='atss_cls', std=0.01, bias_prob=0.01)), **kwargs) -> None: self.pred_kernel_size = pred_kernel_size self.stacked_convs = stacked_convs self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg super().__init__( num_classes=num_classes, in_channels=in_channels, reg_decoded_bbox=reg_decoded_bbox, init_cfg=init_cfg, **kwargs) self.sampling = False self.loss_centerness = MODELS.build(loss_centerness) def _init_layers(self) -> None: """Initialize layers of the head.""" self.relu = nn.ReLU(inplace=True) self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.reg_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) pred_pad_size = self.pred_kernel_size // 2 self.atss_cls = nn.Conv2d( self.feat_channels, self.num_anchors * self.cls_out_channels, self.pred_kernel_size, padding=pred_pad_size) self.atss_reg = nn.Conv2d( self.feat_channels, self.num_base_priors * 4, self.pred_kernel_size, padding=pred_pad_size) self.atss_centerness = nn.Conv2d( self.feat_channels, self.num_base_priors * 1, self.pred_kernel_size, padding=pred_pad_size) self.scales = nn.ModuleList( [Scale(1.0) for _ in self.prior_generator.strides]) def forward(self, x: Tuple[Tensor]) -> Tuple[List[Tensor]]: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Usually a tuple of classification scores and bbox prediction cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_anchors * num_classes. bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_anchors * 4. """ return multi_apply(self.forward_single, x, self.scales) def forward_single(self, x: Tensor, scale: Scale) -> Sequence[Tensor]: """Forward feature of a single scale level. Args: x (Tensor): Features of a single scale level. scale (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize the bbox prediction. Returns: tuple: cls_score (Tensor): Cls scores for a single scale level the channels number is num_anchors * num_classes. bbox_pred (Tensor): Box energies / deltas for a single scale level, the channels number is num_anchors * 4. centerness (Tensor): Centerness for a single scale level, the channel number is (N, num_anchors * 1, H, W). """ cls_feat = x reg_feat = x for cls_conv in self.cls_convs: cls_feat = cls_conv(cls_feat) for reg_conv in self.reg_convs: reg_feat = reg_conv(reg_feat) cls_score = self.atss_cls(cls_feat) # we just follow atss, not apply exp in bbox_pred bbox_pred = scale(self.atss_reg(reg_feat)).float() centerness = self.atss_centerness(reg_feat) return cls_score, bbox_pred, centerness def loss_by_feat_single(self, anchors: Tensor, cls_score: Tensor, bbox_pred: Tensor, centerness: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, avg_factor: float) -> dict: """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: cls_score (Tensor): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W). bbox_pred (Tensor): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). anchors (Tensor): Box reference for each scale level with shape (N, num_total_anchors, 4). labels (Tensor): Labels of each anchors with shape (N, num_total_anchors). label_weights (Tensor): Label weights of each anchor with shape (N, num_total_anchors) bbox_targets (Tensor): BBox regression targets of each anchor weight shape (N, num_total_anchors, 4). avg_factor (float): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using `PseudoSampler`, `avg_factor` is usually equal to the number of positive priors. Returns: dict[str, Tensor]: A dictionary of loss components. """ anchors = anchors.reshape(-1, 4) cls_score = cls_score.permute(0, 2, 3, 1).reshape( -1, self.cls_out_channels).contiguous() bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) centerness = centerness.permute(0, 2, 3, 1).reshape(-1) bbox_targets = bbox_targets.reshape(-1, 4) labels = labels.reshape(-1) label_weights = label_weights.reshape(-1) # classification loss loss_cls = self.loss_cls( cls_score, labels, label_weights, avg_factor=avg_factor) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = self.num_classes pos_inds = ((labels >= 0) & (labels < bg_class_ind)).nonzero().squeeze(1) if len(pos_inds) > 0: pos_bbox_targets = bbox_targets[pos_inds] pos_bbox_pred = bbox_pred[pos_inds] pos_anchors = anchors[pos_inds] pos_centerness = centerness[pos_inds] centerness_targets = self.centerness_target( pos_anchors, pos_bbox_targets) pos_decode_bbox_pred = self.bbox_coder.decode( pos_anchors, pos_bbox_pred) # regression loss loss_bbox = self.loss_bbox( pos_decode_bbox_pred, pos_bbox_targets, weight=centerness_targets, avg_factor=1.0) # centerness loss loss_centerness = self.loss_centerness( pos_centerness, centerness_targets, avg_factor=avg_factor) else: loss_bbox = bbox_pred.sum() * 0 loss_centerness = centerness.sum() * 0 centerness_targets = bbox_targets.new_tensor(0.) return loss_cls, loss_bbox, loss_centerness, centerness_targets.sum() def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], centernesses: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W) centernesses (list[Tensor]): Centerness for each scale level with shape (N, num_anchors * 1, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets avg_factor = reduce_mean( torch.tensor(avg_factor, dtype=torch.float, device=device)).item() losses_cls, losses_bbox, loss_centerness, \ bbox_avg_factor = multi_apply( self.loss_by_feat_single, anchor_list, cls_scores, bbox_preds, centernesses, labels_list, label_weights_list, bbox_targets_list, avg_factor=avg_factor) bbox_avg_factor = sum(bbox_avg_factor) bbox_avg_factor = reduce_mean(bbox_avg_factor).clamp_(min=1).item() losses_bbox = list(map(lambda x: x / bbox_avg_factor, losses_bbox)) return dict( loss_cls=losses_cls, loss_bbox=losses_bbox, loss_centerness=loss_centerness) def centerness_target(self, anchors: Tensor, gts: Tensor) -> Tensor: """Calculate the centerness between anchors and gts. Only calculate pos centerness targets, otherwise there may be nan. Args: anchors (Tensor): Anchors with shape (N, 4), "xyxy" format. gts (Tensor): Ground truth bboxes with shape (N, 4), "xyxy" format. Returns: Tensor: Centerness between anchors and gts. """ anchors_cx = (anchors[:, 2] + anchors[:, 0]) / 2 anchors_cy = (anchors[:, 3] + anchors[:, 1]) / 2 l_ = anchors_cx - gts[:, 0] t_ = anchors_cy - gts[:, 1] r_ = gts[:, 2] - anchors_cx b_ = gts[:, 3] - anchors_cy left_right = torch.stack([l_, r_], dim=1) top_bottom = torch.stack([t_, b_], dim=1) centerness = torch.sqrt( (left_right.min(dim=-1)[0] / left_right.max(dim=-1)[0]) * (top_bottom.min(dim=-1)[0] / top_bottom.max(dim=-1)[0])) assert not torch.isnan(centerness).any() return centerness def get_targets(self, anchor_list: List[List[Tensor]], valid_flag_list: List[List[Tensor]], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs: bool = True) -> tuple: """Get targets for ATSS head. This method is almost the same as `AnchorHead.get_targets()`. Besides returning the targets as the parent method does, it also returns the anchors as the first element of the returned tuple. """ num_imgs = len(batch_img_metas) assert len(anchor_list) == len(valid_flag_list) == num_imgs # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] num_level_anchors_list = [num_level_anchors] * num_imgs # concat all level anchors and flags to a single tensor for i in range(num_imgs): assert len(anchor_list[i]) == len(valid_flag_list[i]) anchor_list[i] = torch.cat(anchor_list[i]) valid_flag_list[i] = torch.cat(valid_flag_list[i]) # compute targets for each image if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None] * num_imgs (all_anchors, all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, pos_inds_list, neg_inds_list, sampling_results_list) = multi_apply( self._get_targets_single, anchor_list, valid_flag_list, num_level_anchors_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs) # Get `avg_factor` of all images, which calculate in `SamplingResult`. # When using sampling method, avg_factor is usually the sum of # positive and negative priors. When using `PseudoSampler`, # `avg_factor` is usually equal to the number of positive priors. avg_factor = sum( [results.avg_factor for results in sampling_results_list]) # split targets to a list w.r.t. multiple levels anchors_list = images_to_levels(all_anchors, num_level_anchors) labels_list = images_to_levels(all_labels, num_level_anchors) label_weights_list = images_to_levels(all_label_weights, num_level_anchors) bbox_targets_list = images_to_levels(all_bbox_targets, num_level_anchors) bbox_weights_list = images_to_levels(all_bbox_weights, num_level_anchors) return (anchors_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) def _get_targets_single(self, flat_anchors: Tensor, valid_flags: Tensor, num_level_anchors: List[int], gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None, unmap_outputs: bool = True) -> tuple: """Compute regression, classification targets for anchors in a single image. Args: flat_anchors (Tensor): Multi-level anchors of the image, which are concatenated into a single tensor of shape (num_anchors ,4) valid_flags (Tensor): Multi level valid flags of the image, which are concatenated into a single tensor of shape (num_anchors,). num_level_anchors (List[int]): Number of anchors of each scale level. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Returns: tuple: N is the number of total anchors in the image. labels (Tensor): Labels of all anchors in the image with shape (N,). label_weights (Tensor): Label weights of all anchor in the image with shape (N,). bbox_targets (Tensor): BBox targets of all anchors in the image with shape (N, 4). bbox_weights (Tensor): BBox weights of all anchors in the image with shape (N, 4) pos_inds (Tensor): Indices of positive anchor with shape (num_pos,). neg_inds (Tensor): Indices of negative anchor with shape (num_neg,). sampling_result (:obj:`SamplingResult`): Sampling results. """ inside_flags = anchor_inside_flags(flat_anchors, valid_flags, img_meta['img_shape'][:2], self.train_cfg['allowed_border']) if not inside_flags.any(): raise ValueError( 'There is no valid anchor inside the image boundary. Please ' 'check the image size and anchor sizes, or set ' '``allowed_border`` to -1 to skip the condition.') # assign gt and sample anchors anchors = flat_anchors[inside_flags, :] num_level_anchors_inside = self.get_num_level_anchors_inside( num_level_anchors, inside_flags) pred_instances = InstanceData(priors=anchors) assign_result = self.assigner.assign(pred_instances, num_level_anchors_inside, gt_instances, gt_instances_ignore) sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) num_valid_anchors = anchors.shape[0] bbox_targets = torch.zeros_like(anchors) bbox_weights = torch.zeros_like(anchors) labels = anchors.new_full((num_valid_anchors, ), self.num_classes, dtype=torch.long) label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: if self.reg_decoded_bbox: pos_bbox_targets = sampling_result.pos_gt_bboxes else: pos_bbox_targets = self.bbox_coder.encode( sampling_result.pos_priors, sampling_result.pos_gt_bboxes) bbox_targets[pos_inds, :] = pos_bbox_targets bbox_weights[pos_inds, :] = 1.0 labels[pos_inds] = sampling_result.pos_gt_labels if self.train_cfg['pos_weight'] <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = self.train_cfg['pos_weight'] if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 # map up to original set of anchors if unmap_outputs: num_total_anchors = flat_anchors.size(0) anchors = unmap(anchors, num_total_anchors, inside_flags) labels = unmap( labels, num_total_anchors, inside_flags, fill=self.num_classes) label_weights = unmap(label_weights, num_total_anchors, inside_flags) bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) return (anchors, labels, label_weights, bbox_targets, bbox_weights, pos_inds, neg_inds, sampling_result) def get_num_level_anchors_inside(self, num_level_anchors, inside_flags): """Get the number of valid anchors in every level.""" split_inside_flags = torch.split(inside_flags, num_level_anchors) num_level_anchors_inside = [ int(flags.sum()) for flags in split_inside_flags ] return num_level_anchors_inside ================================================ FILE: mmdet/models/dense_heads/autoassign_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Sequence, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import Scale from mmengine.model import bias_init_with_prob, normal_init from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures.bbox import bbox_overlaps from mmdet.utils import InstanceList, OptInstanceList, reduce_mean from ..task_modules.prior_generators import MlvlPointGenerator from ..utils import levels_to_images, multi_apply from .fcos_head import FCOSHead EPS = 1e-12 class CenterPrior(nn.Module): """Center Weighting module to adjust the category-specific prior distributions. Args: force_topk (bool): When no point falls into gt_bbox, forcibly select the k points closest to the center to calculate the center prior. Defaults to False. topk (int): The number of points used to calculate the center prior when no point falls in gt_bbox. Only work when force_topk if True. Defaults to 9. num_classes (int): The class number of dataset. Defaults to 80. strides (Sequence[int]): The stride of each input feature map. Defaults to (8, 16, 32, 64, 128). """ def __init__( self, force_topk: bool = False, topk: int = 9, num_classes: int = 80, strides: Sequence[int] = (8, 16, 32, 64, 128) ) -> None: super().__init__() self.mean = nn.Parameter(torch.zeros(num_classes, 2)) self.sigma = nn.Parameter(torch.ones(num_classes, 2)) self.strides = strides self.force_topk = force_topk self.topk = topk def forward(self, anchor_points_list: List[Tensor], gt_instances: InstanceData, inside_gt_bbox_mask: Tensor) -> Tuple[Tensor, Tensor]: """Get the center prior of each point on the feature map for each instance. Args: anchor_points_list (list[Tensor]): list of coordinate of points on feature map. Each with shape (num_points, 2). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. inside_gt_bbox_mask (Tensor): Tensor of bool type, with shape of (num_points, num_gt), each value is used to mark whether this point falls within a certain gt. Returns: tuple[Tensor, Tensor]: - center_prior_weights(Tensor): Float tensor with shape of \ (num_points, num_gt). Each value represents the center \ weighting coefficient. - inside_gt_bbox_mask (Tensor): Tensor of bool type, with shape \ of (num_points, num_gt), each value is used to mark whether this \ point falls within a certain gt or is the topk nearest points for \ a specific gt_bbox. """ gt_bboxes = gt_instances.bboxes labels = gt_instances.labels inside_gt_bbox_mask = inside_gt_bbox_mask.clone() num_gts = len(labels) num_points = sum([len(item) for item in anchor_points_list]) if num_gts == 0: return gt_bboxes.new_zeros(num_points, num_gts), inside_gt_bbox_mask center_prior_list = [] for slvl_points, stride in zip(anchor_points_list, self.strides): # slvl_points: points from single level in FPN, has shape (h*w, 2) # single_level_points has shape (h*w, num_gt, 2) single_level_points = slvl_points[:, None, :].expand( (slvl_points.size(0), len(gt_bboxes), 2)) gt_center_x = ((gt_bboxes[:, 0] + gt_bboxes[:, 2]) / 2) gt_center_y = ((gt_bboxes[:, 1] + gt_bboxes[:, 3]) / 2) gt_center = torch.stack((gt_center_x, gt_center_y), dim=1) gt_center = gt_center[None] # instance_center has shape (1, num_gt, 2) instance_center = self.mean[labels][None] # instance_sigma has shape (1, num_gt, 2) instance_sigma = self.sigma[labels][None] # distance has shape (num_points, num_gt, 2) distance = (((single_level_points - gt_center) / float(stride) - instance_center)**2) center_prior = torch.exp(-distance / (2 * instance_sigma**2)).prod(dim=-1) center_prior_list.append(center_prior) center_prior_weights = torch.cat(center_prior_list, dim=0) if self.force_topk: gt_inds_no_points_inside = torch.nonzero( inside_gt_bbox_mask.sum(0) == 0).reshape(-1) if gt_inds_no_points_inside.numel(): topk_center_index = \ center_prior_weights[:, gt_inds_no_points_inside].topk( self.topk, dim=0)[1] temp_mask = inside_gt_bbox_mask[:, gt_inds_no_points_inside] inside_gt_bbox_mask[:, gt_inds_no_points_inside] = \ torch.scatter(temp_mask, dim=0, index=topk_center_index, src=torch.ones_like( topk_center_index, dtype=torch.bool)) center_prior_weights[~inside_gt_bbox_mask] = 0 return center_prior_weights, inside_gt_bbox_mask @MODELS.register_module() class AutoAssignHead(FCOSHead): """AutoAssignHead head used in AutoAssign. More details can be found in the `paper `_ . Args: force_topk (bool): Used in center prior initialization to handle extremely small gt. Default is False. topk (int): The number of points used to calculate the center prior when no point falls in gt_bbox. Only work when force_topk if True. Defaults to 9. pos_loss_weight (float): The loss weight of positive loss and with default value 0.25. neg_loss_weight (float): The loss weight of negative loss and with default value 0.75. center_loss_weight (float): The loss weight of center prior loss and with default value 0.75. """ def __init__(self, *args, force_topk: bool = False, topk: int = 9, pos_loss_weight: float = 0.25, neg_loss_weight: float = 0.75, center_loss_weight: float = 0.75, **kwargs) -> None: super().__init__(*args, conv_bias=True, **kwargs) self.center_prior = CenterPrior( force_topk=force_topk, topk=topk, num_classes=self.num_classes, strides=self.strides) self.pos_loss_weight = pos_loss_weight self.neg_loss_weight = neg_loss_weight self.center_loss_weight = center_loss_weight self.prior_generator = MlvlPointGenerator(self.strides, offset=0) def init_weights(self) -> None: """Initialize weights of the head. In particular, we have special initialization for classified conv's and regression conv's bias """ super(AutoAssignHead, self).init_weights() bias_cls = bias_init_with_prob(0.02) normal_init(self.conv_cls, std=0.01, bias=bias_cls) normal_init(self.conv_reg, std=0.01, bias=4.0) def forward_single(self, x: Tensor, scale: Scale, stride: int) -> Tuple[Tensor, Tensor, Tensor]: """Forward features of a single scale level. Args: x (Tensor): FPN feature maps of the specified stride. scale (:obj:`mmcv.cnn.Scale`): Learnable scale module to resize the bbox prediction. stride (int): The corresponding stride for feature maps, only used to normalize the bbox prediction when self.norm_on_bbox is True. Returns: tuple[Tensor, Tensor, Tensor]: scores for each class, bbox predictions and centerness predictions of input feature maps. """ cls_score, bbox_pred, cls_feat, reg_feat = super( FCOSHead, self).forward_single(x) centerness = self.conv_centerness(reg_feat) # scale the bbox_pred of different level # float to avoid overflow when enabling FP16 bbox_pred = scale(bbox_pred).float() # bbox_pred needed for gradient computation has been modified # by F.relu(bbox_pred) when run with PyTorch 1.10. So replace # F.relu(bbox_pred) with bbox_pred.clamp(min=0) bbox_pred = bbox_pred.clamp(min=0) bbox_pred *= stride return cls_score, bbox_pred, centerness def get_pos_loss_single(self, cls_score: Tensor, objectness: Tensor, reg_loss: Tensor, gt_instances: InstanceData, center_prior_weights: Tensor) -> Tuple[Tensor]: """Calculate the positive loss of all points in gt_bboxes. Args: cls_score (Tensor): All category scores for each point on the feature map. The shape is (num_points, num_class). objectness (Tensor): Foreground probability of all points, has shape (num_points, 1). reg_loss (Tensor): The regression loss of each gt_bbox and each prediction box, has shape of (num_points, num_gt). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. center_prior_weights (Tensor): Float tensor with shape of (num_points, num_gt). Each value represents the center weighting coefficient. Returns: tuple[Tensor]: - pos_loss (Tensor): The positive loss of all points in the \ gt_bboxes. """ gt_labels = gt_instances.labels # p_loc: localization confidence p_loc = torch.exp(-reg_loss) # p_cls: classification confidence p_cls = (cls_score * objectness)[:, gt_labels] # p_pos: joint confidence indicator p_pos = p_cls * p_loc # 3 is a hyper-parameter to control the contributions of high and # low confidence locations towards positive losses. confidence_weight = torch.exp(p_pos * 3) p_pos_weight = (confidence_weight * center_prior_weights) / ( (confidence_weight * center_prior_weights).sum( 0, keepdim=True)).clamp(min=EPS) reweighted_p_pos = (p_pos * p_pos_weight).sum(0) pos_loss = F.binary_cross_entropy( reweighted_p_pos, torch.ones_like(reweighted_p_pos), reduction='none') pos_loss = pos_loss.sum() * self.pos_loss_weight return pos_loss, def get_neg_loss_single(self, cls_score: Tensor, objectness: Tensor, gt_instances: InstanceData, ious: Tensor, inside_gt_bbox_mask: Tensor) -> Tuple[Tensor]: """Calculate the negative loss of all points in feature map. Args: cls_score (Tensor): All category scores for each point on the feature map. The shape is (num_points, num_class). objectness (Tensor): Foreground probability of all points and is shape of (num_points, 1). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. ious (Tensor): Float tensor with shape of (num_points, num_gt). Each value represent the iou of pred_bbox and gt_bboxes. inside_gt_bbox_mask (Tensor): Tensor of bool type, with shape of (num_points, num_gt), each value is used to mark whether this point falls within a certain gt. Returns: tuple[Tensor]: - neg_loss (Tensor): The negative loss of all points in the \ feature map. """ gt_labels = gt_instances.labels num_gts = len(gt_labels) joint_conf = (cls_score * objectness) p_neg_weight = torch.ones_like(joint_conf) if num_gts > 0: # the order of dinmension would affect the value of # p_neg_weight, we strictly follow the original # implementation. inside_gt_bbox_mask = inside_gt_bbox_mask.permute(1, 0) ious = ious.permute(1, 0) foreground_idxs = torch.nonzero(inside_gt_bbox_mask, as_tuple=True) temp_weight = (1 / (1 - ious[foreground_idxs]).clamp_(EPS)) def normalize(x): return (x - x.min() + EPS) / (x.max() - x.min() + EPS) for instance_idx in range(num_gts): idxs = foreground_idxs[0] == instance_idx if idxs.any(): temp_weight[idxs] = normalize(temp_weight[idxs]) p_neg_weight[foreground_idxs[1], gt_labels[foreground_idxs[0]]] = 1 - temp_weight logits = (joint_conf * p_neg_weight) neg_loss = ( logits**2 * F.binary_cross_entropy( logits, torch.zeros_like(logits), reduction='none')) neg_loss = neg_loss.sum() * self.neg_loss_weight return neg_loss, def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], objectnesses: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level, each is a 4D-tensor, the channel number is num_points * num_classes. bbox_preds (list[Tensor]): Box energies / deltas for each scale level, each is a 4D-tensor, the channel number is num_points * 4. objectnesses (list[Tensor]): objectness for each scale level, each is a 4D-tensor, the channel number is num_points * 1. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert len(cls_scores) == len(bbox_preds) == len(objectnesses) all_num_gt = sum([len(item) for item in batch_gt_instances]) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] all_level_points = self.prior_generator.grid_priors( featmap_sizes, dtype=bbox_preds[0].dtype, device=bbox_preds[0].device) inside_gt_bbox_mask_list, bbox_targets_list = self.get_targets( all_level_points, batch_gt_instances) center_prior_weight_list = [] temp_inside_gt_bbox_mask_list = [] for gt_instances, inside_gt_bbox_mask in zip(batch_gt_instances, inside_gt_bbox_mask_list): center_prior_weight, inside_gt_bbox_mask = \ self.center_prior(all_level_points, gt_instances, inside_gt_bbox_mask) center_prior_weight_list.append(center_prior_weight) temp_inside_gt_bbox_mask_list.append(inside_gt_bbox_mask) inside_gt_bbox_mask_list = temp_inside_gt_bbox_mask_list mlvl_points = torch.cat(all_level_points, dim=0) bbox_preds = levels_to_images(bbox_preds) cls_scores = levels_to_images(cls_scores) objectnesses = levels_to_images(objectnesses) reg_loss_list = [] ious_list = [] num_points = len(mlvl_points) for bbox_pred, encoded_targets, inside_gt_bbox_mask in zip( bbox_preds, bbox_targets_list, inside_gt_bbox_mask_list): temp_num_gt = encoded_targets.size(1) expand_mlvl_points = mlvl_points[:, None, :].expand( num_points, temp_num_gt, 2).reshape(-1, 2) encoded_targets = encoded_targets.reshape(-1, 4) expand_bbox_pred = bbox_pred[:, None, :].expand( num_points, temp_num_gt, 4).reshape(-1, 4) decoded_bbox_preds = self.bbox_coder.decode( expand_mlvl_points, expand_bbox_pred) decoded_target_preds = self.bbox_coder.decode( expand_mlvl_points, encoded_targets) with torch.no_grad(): ious = bbox_overlaps( decoded_bbox_preds, decoded_target_preds, is_aligned=True) ious = ious.reshape(num_points, temp_num_gt) if temp_num_gt: ious = ious.max( dim=-1, keepdim=True).values.repeat(1, temp_num_gt) else: ious = ious.new_zeros(num_points, temp_num_gt) ious[~inside_gt_bbox_mask] = 0 ious_list.append(ious) loss_bbox = self.loss_bbox( decoded_bbox_preds, decoded_target_preds, weight=None, reduction_override='none') reg_loss_list.append(loss_bbox.reshape(num_points, temp_num_gt)) cls_scores = [item.sigmoid() for item in cls_scores] objectnesses = [item.sigmoid() for item in objectnesses] pos_loss_list, = multi_apply(self.get_pos_loss_single, cls_scores, objectnesses, reg_loss_list, batch_gt_instances, center_prior_weight_list) pos_avg_factor = reduce_mean( bbox_pred.new_tensor(all_num_gt)).clamp_(min=1) pos_loss = sum(pos_loss_list) / pos_avg_factor neg_loss_list, = multi_apply(self.get_neg_loss_single, cls_scores, objectnesses, batch_gt_instances, ious_list, inside_gt_bbox_mask_list) neg_avg_factor = sum(item.data.sum() for item in center_prior_weight_list) neg_avg_factor = reduce_mean(neg_avg_factor).clamp_(min=1) neg_loss = sum(neg_loss_list) / neg_avg_factor center_loss = [] for i in range(len(batch_img_metas)): if inside_gt_bbox_mask_list[i].any(): center_loss.append( len(batch_gt_instances[i]) / center_prior_weight_list[i].sum().clamp_(min=EPS)) # when width or height of gt_bbox is smaller than stride of p3 else: center_loss.append(center_prior_weight_list[i].sum() * 0) center_loss = torch.stack(center_loss).mean() * self.center_loss_weight # avoid dead lock in DDP if all_num_gt == 0: pos_loss = bbox_preds[0].sum() * 0 dummy_center_prior_loss = self.center_prior.mean.sum( ) * 0 + self.center_prior.sigma.sum() * 0 center_loss = objectnesses[0].sum() * 0 + dummy_center_prior_loss loss = dict( loss_pos=pos_loss, loss_neg=neg_loss, loss_center=center_loss) return loss def get_targets( self, points: List[Tensor], batch_gt_instances: InstanceList ) -> Tuple[List[Tensor], List[Tensor]]: """Compute regression targets and each point inside or outside gt_bbox in multiple images. Args: points (list[Tensor]): Points of all fpn level, each has shape (num_points, 2). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. Returns: tuple(list[Tensor], list[Tensor]): - inside_gt_bbox_mask_list (list[Tensor]): Each Tensor is with \ bool type and shape of (num_points, num_gt), each value is used \ to mark whether this point falls within a certain gt. - concat_lvl_bbox_targets (list[Tensor]): BBox targets of each \ level. Each tensor has shape (num_points, num_gt, 4). """ concat_points = torch.cat(points, dim=0) # the number of points per img, per lvl inside_gt_bbox_mask_list, bbox_targets_list = multi_apply( self._get_targets_single, batch_gt_instances, points=concat_points) return inside_gt_bbox_mask_list, bbox_targets_list def _get_targets_single(self, gt_instances: InstanceData, points: Tensor) -> Tuple[Tensor, Tensor]: """Compute regression targets and each point inside or outside gt_bbox for a single image. Args: gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. points (Tensor): Points of all fpn level, has shape (num_points, 2). Returns: tuple[Tensor, Tensor]: Containing the following Tensors: - inside_gt_bbox_mask (Tensor): Bool tensor with shape \ (num_points, num_gt), each value is used to mark whether this \ point falls within a certain gt. - bbox_targets (Tensor): BBox targets of each points with each \ gt_bboxes, has shape (num_points, num_gt, 4). """ gt_bboxes = gt_instances.bboxes num_points = points.size(0) num_gts = gt_bboxes.size(0) gt_bboxes = gt_bboxes[None].expand(num_points, num_gts, 4) xs, ys = points[:, 0], points[:, 1] xs = xs[:, None] ys = ys[:, None] left = xs - gt_bboxes[..., 0] right = gt_bboxes[..., 2] - xs top = ys - gt_bboxes[..., 1] bottom = gt_bboxes[..., 3] - ys bbox_targets = torch.stack((left, top, right, bottom), -1) if num_gts: inside_gt_bbox_mask = bbox_targets.min(-1)[0] > 0 else: inside_gt_bbox_mask = bbox_targets.new_zeros((num_points, num_gts), dtype=torch.bool) return inside_gt_bbox_mask, bbox_targets ================================================ FILE: mmdet/models/dense_heads/base_dense_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from abc import ABCMeta, abstractmethod from inspect import signature from typing import List, Optional, Tuple import torch from mmcv.ops import batched_nms from mmengine.config import ConfigDict from mmengine.model import BaseModule, constant_init from mmengine.structures import InstanceData from torch import Tensor from mmdet.structures import SampleList from mmdet.structures.bbox import (cat_boxes, get_box_tensor, get_box_wh, scale_boxes) from mmdet.utils import InstanceList, OptMultiConfig from ..test_time_augs import merge_aug_results from ..utils import (filter_scores_and_topk, select_single_mlvl, unpack_gt_instances) class BaseDenseHead(BaseModule, metaclass=ABCMeta): """Base class for DenseHeads. 1. The ``init_weights`` method is used to initialize densehead's model parameters. After detector initialization, ``init_weights`` is triggered when ``detector.init_weights()`` is called externally. 2. The ``loss`` method is used to calculate the loss of densehead, which includes two steps: (1) the densehead model performs forward propagation to obtain the feature maps (2) The ``loss_by_feat`` method is called based on the feature maps to calculate the loss. .. code:: text loss(): forward() -> loss_by_feat() 3. The ``predict`` method is used to predict detection results, which includes two steps: (1) the densehead model performs forward propagation to obtain the feature maps (2) The ``predict_by_feat`` method is called based on the feature maps to predict detection results including post-processing. .. code:: text predict(): forward() -> predict_by_feat() 4. The ``loss_and_predict`` method is used to return loss and detection results at the same time. It will call densehead's ``forward``, ``loss_by_feat`` and ``predict_by_feat`` methods in order. If one-stage is used as RPN, the densehead needs to return both losses and predictions. This predictions is used as the proposal of roihead. .. code:: text loss_and_predict(): forward() -> loss_by_feat() -> predict_by_feat() """ def __init__(self, init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) # `_raw_positive_infos` will be used in `get_positive_infos`, which # can get positive information. self._raw_positive_infos = dict() def init_weights(self) -> None: """Initialize the weights.""" super().init_weights() # avoid init_cfg overwrite the initialization of `conv_offset` for m in self.modules(): # DeformConv2dPack, ModulatedDeformConv2dPack if hasattr(m, 'conv_offset'): constant_init(m.conv_offset, 0) def get_positive_infos(self) -> InstanceList: """Get positive information from sampling results. Returns: list[:obj:`InstanceData`]: Positive information of each image, usually including positive bboxes, positive labels, positive priors, etc. """ if len(self._raw_positive_infos) == 0: return None sampling_results = self._raw_positive_infos.get( 'sampling_results', None) assert sampling_results is not None positive_infos = [] for sampling_result in enumerate(sampling_results): pos_info = InstanceData() pos_info.bboxes = sampling_result.pos_gt_bboxes pos_info.labels = sampling_result.pos_gt_labels pos_info.priors = sampling_result.pos_priors pos_info.pos_assigned_gt_inds = \ sampling_result.pos_assigned_gt_inds pos_info.pos_inds = sampling_result.pos_inds positive_infos.append(pos_info) return positive_infos def loss(self, x: Tuple[Tensor], batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection head on the features of the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ outs = self(x) outputs = unpack_gt_instances(batch_data_samples) (batch_gt_instances, batch_gt_instances_ignore, batch_img_metas) = outputs loss_inputs = outs + (batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) losses = self.loss_by_feat(*loss_inputs) return losses @abstractmethod def loss_by_feat(self, **kwargs) -> dict: """Calculate the loss based on the features extracted by the detection head.""" pass def loss_and_predict( self, x: Tuple[Tensor], batch_data_samples: SampleList, proposal_cfg: Optional[ConfigDict] = None ) -> Tuple[dict, InstanceList]: """Perform forward propagation of the head, then calculate loss and predictions from the features and data samples. Args: x (tuple[Tensor]): Features from FPN. batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. proposal_cfg (ConfigDict, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. Returns: tuple: the return value is a tuple contains: - losses: (dict[str, Tensor]): A dictionary of loss components. - predictions (list[:obj:`InstanceData`]): Detection results of each image after the post process. """ outputs = unpack_gt_instances(batch_data_samples) (batch_gt_instances, batch_gt_instances_ignore, batch_img_metas) = outputs outs = self(x) loss_inputs = outs + (batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) losses = self.loss_by_feat(*loss_inputs) predictions = self.predict_by_feat( *outs, batch_img_metas=batch_img_metas, cfg=proposal_cfg) return losses, predictions def predict(self, x: Tuple[Tensor], batch_data_samples: SampleList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the detection head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Multi-level features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool, optional): Whether to rescale the results. Defaults to False. Returns: list[obj:`InstanceData`]: Detection results of each image after the post process. """ batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] outs = self(x) predictions = self.predict_by_feat( *outs, batch_img_metas=batch_img_metas, rescale=rescale) return predictions def predict_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], score_factors: Optional[List[Tensor]] = None, batch_img_metas: Optional[List[dict]] = None, cfg: Optional[ConfigDict] = None, rescale: bool = False, with_nms: bool = True) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. Note: When score_factors is not None, the cls_scores are usually multiplied by it then obtain the real score used in NMS, such as CenterNess in FCOS, IoU branch in ATSS. Args: cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * 4, H, W). score_factors (list[Tensor], optional): Score factor for all scale level, each is a 4D-tensor, has shape (batch_size, num_priors * 1, H, W). Defaults to None. batch_img_metas (list[dict], Optional): Batch image meta info. Defaults to None. cfg (ConfigDict, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(cls_scores) == len(bbox_preds) if score_factors is None: # e.g. Retina, FreeAnchor, Foveabox, etc. with_score_factors = False else: # e.g. FCOS, PAA, ATSS, AutoAssign, etc. with_score_factors = True assert len(cls_scores) == len(score_factors) num_levels = len(cls_scores) featmap_sizes = [cls_scores[i].shape[-2:] for i in range(num_levels)] mlvl_priors = self.prior_generator.grid_priors( featmap_sizes, dtype=cls_scores[0].dtype, device=cls_scores[0].device) result_list = [] for img_id in range(len(batch_img_metas)): img_meta = batch_img_metas[img_id] cls_score_list = select_single_mlvl( cls_scores, img_id, detach=True) bbox_pred_list = select_single_mlvl( bbox_preds, img_id, detach=True) if with_score_factors: score_factor_list = select_single_mlvl( score_factors, img_id, detach=True) else: score_factor_list = [None for _ in range(num_levels)] results = self._predict_by_feat_single( cls_score_list=cls_score_list, bbox_pred_list=bbox_pred_list, score_factor_list=score_factor_list, mlvl_priors=mlvl_priors, img_meta=img_meta, cfg=cfg, rescale=rescale, with_nms=with_nms) result_list.append(results) return result_list def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], score_factor_list: List[Tensor], mlvl_priors: List[Tensor], img_meta: dict, cfg: ConfigDict, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: cls_score_list (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). score_factor_list (list[Tensor]): Score factor from all scale levels of a single image, each item has shape (num_priors * 1, H, W). mlvl_priors (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid. In all anchor-based methods, it has shape (num_priors, 4). In all anchor-free methods, it has shape (num_priors, 2) when `with_stride=True`, otherwise it still has shape (num_priors, 4). img_meta (dict): Image meta info. cfg (mmengine.Config): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ if score_factor_list[0] is None: # e.g. Retina, FreeAnchor, etc. with_score_factors = False else: # e.g. FCOS, PAA, ATSS, etc. with_score_factors = True cfg = self.test_cfg if cfg is None else cfg cfg = copy.deepcopy(cfg) img_shape = img_meta['img_shape'] nms_pre = cfg.get('nms_pre', -1) mlvl_bbox_preds = [] mlvl_valid_priors = [] mlvl_scores = [] mlvl_labels = [] if with_score_factors: mlvl_score_factors = [] else: mlvl_score_factors = None for level_idx, (cls_score, bbox_pred, score_factor, priors) in \ enumerate(zip(cls_score_list, bbox_pred_list, score_factor_list, mlvl_priors)): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] dim = self.bbox_coder.encode_size bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, dim) if with_score_factors: score_factor = score_factor.permute(1, 2, 0).reshape(-1).sigmoid() cls_score = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) if self.use_sigmoid_cls: scores = cls_score.sigmoid() else: # remind that we set FG labels to [0, num_class-1] # since mmdet v2.0 # BG cat_id: num_class scores = cls_score.softmax(-1)[:, :-1] # After https://github.com/open-mmlab/mmdetection/pull/6268/, # this operation keeps fewer bboxes under the same `nms_pre`. # There is no difference in performance for most models. If you # find a slight drop in performance, you can set a larger # `nms_pre` than before. score_thr = cfg.get('score_thr', 0) results = filter_scores_and_topk( scores, score_thr, nms_pre, dict(bbox_pred=bbox_pred, priors=priors)) scores, labels, keep_idxs, filtered_results = results bbox_pred = filtered_results['bbox_pred'] priors = filtered_results['priors'] if with_score_factors: score_factor = score_factor[keep_idxs] mlvl_bbox_preds.append(bbox_pred) mlvl_valid_priors.append(priors) mlvl_scores.append(scores) mlvl_labels.append(labels) if with_score_factors: mlvl_score_factors.append(score_factor) bbox_pred = torch.cat(mlvl_bbox_preds) priors = cat_boxes(mlvl_valid_priors) bboxes = self.bbox_coder.decode(priors, bbox_pred, max_shape=img_shape) results = InstanceData() results.bboxes = bboxes results.scores = torch.cat(mlvl_scores) results.labels = torch.cat(mlvl_labels) if with_score_factors: results.score_factors = torch.cat(mlvl_score_factors) return self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta) def _bbox_post_process(self, results: InstanceData, cfg: ConfigDict, rescale: bool = False, with_nms: bool = True, img_meta: Optional[dict] = None) -> InstanceData: """bbox post-processing method. The boxes would be rescaled to the original image scale and do the nms operation. Usually `with_nms` is False is used for aug test. Args: results (:obj:`InstaceData`): Detection instance results, each item has shape (num_bboxes, ). cfg (ConfigDict): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Default to False. with_nms (bool): If True, do nms before return boxes. Default to True. img_meta (dict, optional): Image meta info. Defaults to None. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ if rescale: assert img_meta.get('scale_factor') is not None scale_factor = [1 / s for s in img_meta['scale_factor']] results.bboxes = scale_boxes(results.bboxes, scale_factor) if hasattr(results, 'score_factors'): # TODO： Add sqrt operation in order to be consistent with # the paper. score_factors = results.pop('score_factors') results.scores = results.scores * score_factors # filter small size bboxes if cfg.get('min_bbox_size', -1) >= 0: w, h = get_box_wh(results.bboxes) valid_mask = (w > cfg.min_bbox_size) & (h > cfg.min_bbox_size) if not valid_mask.all(): results = results[valid_mask] # TODO: deal with `with_nms` and `nms_cfg=None` in test_cfg if with_nms and results.bboxes.numel() > 0: bboxes = get_box_tensor(results.bboxes) det_bboxes, keep_idxs = batched_nms(bboxes, results.scores, results.labels, cfg.nms) results = results[keep_idxs] # some nms would reweight the score, such as softnms results.scores = det_bboxes[:, -1] results = results[:cfg.max_per_img] return results def aug_test(self, aug_batch_feats, aug_batch_img_metas, rescale=False, with_ori_nms=False, **kwargs): """Test function with test time augmentation. Args: aug_batch_feats (list[tuple[Tensor]]): The outer list indicates test-time augmentations and inner tuple indicate the multi-level feats from FPN, each Tensor should have a shape (B, C, H, W), aug_batch_img_metas (list[list[dict]]): Meta information of images under the different test-time augs (multiscale, flip, etc.). The outer list indicate the rescale (bool, optional): Whether to rescale the results. Defaults to False. with_ori_nms (bool): Whether execute the nms in original head. Defaults to False. It will be `True` when the head is adopted as `rpn_head`. Returns: list(obj:`InstanceData`): Detection results of the input images. Each item usually contains\ following keys. - scores (Tensor): Classification scores, has a shape (num_instance,) - labels (Tensor): Labels of bboxes, has a shape (num_instances,). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ # TODO: remove this for detr and deformdetr sig_of_get_results = signature(self.get_results) get_results_args = [ p.name for p in sig_of_get_results.parameters.values() ] get_results_single_sig = signature(self._get_results_single) get_results_single_sig_args = [ p.name for p in get_results_single_sig.parameters.values() ] assert ('with_nms' in get_results_args) and \ ('with_nms' in get_results_single_sig_args), \ f'{self.__class__.__name__}' \ 'does not support test-time augmentation ' num_imgs = len(aug_batch_img_metas[0]) aug_batch_results = [] for x, img_metas in zip(aug_batch_feats, aug_batch_img_metas): outs = self.forward(x) batch_instance_results = self.get_results( *outs, img_metas=img_metas, cfg=self.test_cfg, rescale=False, with_nms=with_ori_nms, **kwargs) aug_batch_results.append(batch_instance_results) # after merging, bboxes will be rescaled to the original image batch_results = merge_aug_results(aug_batch_results, aug_batch_img_metas) final_results = [] for img_id in range(num_imgs): results = batch_results[img_id] det_bboxes, keep_idxs = batched_nms(results.bboxes, results.scores, results.labels, self.test_cfg.nms) results = results[keep_idxs] # some nms operation may reweight the score such as softnms results.scores = det_bboxes[:, -1] results = results[:self.test_cfg.max_per_img] if rescale: # all results have been mapped to the original scale # in `merge_aug_results`, so just pass pass else: # map to the first aug image scale scale_factor = results.bboxes.new_tensor( aug_batch_img_metas[0][img_id]['scale_factor']) results.bboxes = \ results.bboxes * scale_factor final_results.append(results) return final_results ================================================ FILE: mmdet/models/dense_heads/base_mask_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from typing import List, Tuple, Union from mmengine.model import BaseModule from torch import Tensor from mmdet.structures import SampleList from mmdet.utils import InstanceList, OptInstanceList, OptMultiConfig from ..utils import unpack_gt_instances class BaseMaskHead(BaseModule, metaclass=ABCMeta): """Base class for mask heads used in One-Stage Instance Segmentation.""" def __init__(self, init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) @abstractmethod def loss_by_feat(self, *args, **kwargs): """Calculate the loss based on the features extracted by the mask head.""" pass @abstractmethod def predict_by_feat(self, *args, **kwargs): """Transform a batch of output features extracted from the head into mask results.""" pass def loss(self, x: Union[List[Tensor], Tuple[Tensor]], batch_data_samples: SampleList, positive_infos: OptInstanceList = None, **kwargs) -> dict: """Perform forward propagation and loss calculation of the mask head on the features of the upstream network. Args: x (list[Tensor] | tuple[Tensor]): Features from FPN. Each has a shape (B, C, H, W). batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. positive_infos (list[:obj:`InstanceData`], optional): Information of positive samples. Used when the label assignment is done outside the MaskHead, e.g., BboxHead in YOLACT or CondInst, etc. When the label assignment is done in MaskHead, it would be None, like SOLO or SOLOv2. All values in it should have shape (num_positive_samples, *). Returns: dict: A dictionary of loss components. """ if positive_infos is None: outs = self(x) else: outs = self(x, positive_infos) assert isinstance(outs, tuple), 'Forward results should be a tuple, ' \ 'even if only one item is returned' outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, batch_img_metas \ = outputs for gt_instances, img_metas in zip(batch_gt_instances, batch_img_metas): img_shape = img_metas['batch_input_shape'] gt_masks = gt_instances.masks.pad(img_shape) gt_instances.masks = gt_masks losses = self.loss_by_feat( *outs, batch_gt_instances=batch_gt_instances, batch_img_metas=batch_img_metas, positive_infos=positive_infos, batch_gt_instances_ignore=batch_gt_instances_ignore, **kwargs) return losses def predict(self, x: Tuple[Tensor], batch_data_samples: SampleList, rescale: bool = False, results_list: OptInstanceList = None, **kwargs) -> InstanceList: """Test function without test-time augmentation. Args: x (tuple[Tensor]): Multi-level features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool, optional): Whether to rescale the results. Defaults to False. results_list (list[obj:`InstanceData`], optional): Detection results of each image after the post process. Only exist if there is a `bbox_head`, like `YOLACT`, `CondInst`, etc. Returns: list[obj:`InstanceData`]: Instance segmentation results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance,) - labels (Tensor): Has a shape (num_instances,). - masks (Tensor): Processed mask results, has a shape (num_instances, h, w). """ batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] if results_list is None: outs = self(x) else: outs = self(x, results_list) results_list = self.predict_by_feat( *outs, batch_img_metas=batch_img_metas, rescale=rescale, results_list=results_list, **kwargs) return results_list ================================================ FILE: mmdet/models/dense_heads/boxinst_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List import torch import torch.nn.functional as F from mmengine import MessageHub from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import InstanceList from ..utils.misc import unfold_wo_center from .condinst_head import CondInstBboxHead, CondInstMaskHead @MODELS.register_module() class BoxInstBboxHead(CondInstBboxHead): """BoxInst box head used in https://arxiv.org/abs/2012.02310.""" def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @MODELS.register_module() class BoxInstMaskHead(CondInstMaskHead): """BoxInst mask head used in https://arxiv.org/abs/2012.02310. This head outputs the mask for BoxInst. Args: pairwise_size (dict): The size of neighborhood for each pixel. Defaults to 3. pairwise_dilation (int): The dilation of neighborhood for each pixel. Defaults to 2. warmup_iters (int): Warmup iterations for pair-wise loss. Defaults to 10000. """ def __init__(self, *arg, pairwise_size: int = 3, pairwise_dilation: int = 2, warmup_iters: int = 10000, **kwargs) -> None: self.pairwise_size = pairwise_size self.pairwise_dilation = pairwise_dilation self.warmup_iters = warmup_iters super().__init__(*arg, **kwargs) def get_pairwise_affinity(self, mask_logits: Tensor) -> Tensor: """Compute the pairwise affinity for each pixel.""" log_fg_prob = F.logsigmoid(mask_logits).unsqueeze(1) log_bg_prob = F.logsigmoid(-mask_logits).unsqueeze(1) log_fg_prob_unfold = unfold_wo_center( log_fg_prob, kernel_size=self.pairwise_size, dilation=self.pairwise_dilation) log_bg_prob_unfold = unfold_wo_center( log_bg_prob, kernel_size=self.pairwise_size, dilation=self.pairwise_dilation) # the probability of making the same prediction: # p_i * p_j + (1 - p_i) * (1 - p_j) # we compute the the probability in log space # to avoid numerical instability log_same_fg_prob = log_fg_prob[:, :, None] + log_fg_prob_unfold log_same_bg_prob = log_bg_prob[:, :, None] + log_bg_prob_unfold # TODO: Figure out the difference between it and directly sum max_ = torch.max(log_same_fg_prob, log_same_bg_prob) log_same_prob = torch.log( torch.exp(log_same_fg_prob - max_) + torch.exp(log_same_bg_prob - max_)) + max_ return -log_same_prob[:, 0] def loss_by_feat(self, mask_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], positive_infos: InstanceList, **kwargs) -> dict: """Calculate the loss based on the features extracted by the mask head. Args: mask_preds (list[Tensor]): List of predicted masks, each has shape (num_classes, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``masks``, and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of multiple images. positive_infos (List[:obj:``InstanceData``]): Information of positive samples of each image that are assigned in detection head. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert positive_infos is not None, \ 'positive_infos should not be None in `BoxInstMaskHead`' losses = dict() loss_mask_project = 0. loss_mask_pairwise = 0. num_imgs = len(mask_preds) total_pos = 0. avg_fatcor = 0. for idx in range(num_imgs): (mask_pred, pos_mask_targets, pos_pairwise_masks, num_pos) = \ self._get_targets_single( mask_preds[idx], batch_gt_instances[idx], positive_infos[idx]) # mask loss total_pos += num_pos if num_pos == 0 or pos_mask_targets is None: loss_project = mask_pred.new_zeros(1).mean() loss_pairwise = mask_pred.new_zeros(1).mean() avg_fatcor += 0. else: # compute the project term loss_project_x = self.loss_mask( mask_pred.max(dim=1, keepdim=True)[0], pos_mask_targets.max(dim=1, keepdim=True)[0], reduction_override='none').sum() loss_project_y = self.loss_mask( mask_pred.max(dim=2, keepdim=True)[0], pos_mask_targets.max(dim=2, keepdim=True)[0], reduction_override='none').sum() loss_project = loss_project_x + loss_project_y # compute the pairwise term pairwise_affinity = self.get_pairwise_affinity(mask_pred) avg_fatcor += pos_pairwise_masks.sum().clamp(min=1.0) loss_pairwise = (pairwise_affinity * pos_pairwise_masks).sum() loss_mask_project += loss_project loss_mask_pairwise += loss_pairwise if total_pos == 0: total_pos += 1 # avoid nan if avg_fatcor == 0: avg_fatcor += 1 # avoid nan loss_mask_project = loss_mask_project / total_pos loss_mask_pairwise = loss_mask_pairwise / avg_fatcor message_hub = MessageHub.get_current_instance() iter = message_hub.get_info('iter') warmup_factor = min(iter / float(self.warmup_iters), 1.0) loss_mask_pairwise *= warmup_factor losses.update( loss_mask_project=loss_mask_project, loss_mask_pairwise=loss_mask_pairwise) return losses def _get_targets_single(self, mask_preds: Tensor, gt_instances: InstanceData, positive_info: InstanceData): """Compute targets for predictions of single image. Args: mask_preds (Tensor): Predicted prototypes with shape (num_classes, H, W). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes``, ``labels``, and ``masks`` attributes. positive_info (:obj:`InstanceData`): Information of positive samples that are assigned in detection head. It usually contains following keys. - pos_assigned_gt_inds (Tensor): Assigner GT indexes of positive proposals, has shape (num_pos, ) - pos_inds (Tensor): Positive index of image, has shape (num_pos, ). - param_pred (Tensor): Positive param preditions with shape (num_pos, num_params). Returns: tuple: Usually returns a tuple containing learning targets. - mask_preds (Tensor): Positive predicted mask with shape (num_pos, mask_h, mask_w). - pos_mask_targets (Tensor): Positive mask targets with shape (num_pos, mask_h, mask_w). - pos_pairwise_masks (Tensor): Positive pairwise masks with shape: (num_pos, num_neighborhood, mask_h, mask_w). - num_pos (int): Positive numbers. """ gt_bboxes = gt_instances.bboxes device = gt_bboxes.device # Note that gt_masks are generated by full box # from BoxInstDataPreprocessor gt_masks = gt_instances.masks.to_tensor( dtype=torch.bool, device=device).float() # Note that pairwise_masks are generated by image color similarity # from BoxInstDataPreprocessor pairwise_masks = gt_instances.pairwise_masks pairwise_masks = pairwise_masks.to(device=device) # process with mask targets pos_assigned_gt_inds = positive_info.get('pos_assigned_gt_inds') scores = positive_info.get('scores') centernesses = positive_info.get('centernesses') num_pos = pos_assigned_gt_inds.size(0) if gt_masks.size(0) == 0 or num_pos == 0: return mask_preds, None, None, 0 # Since we're producing (near) full image masks, # it'd take too much vram to backprop on every single mask. # Thus we select only a subset. if (self.max_masks_to_train != -1) and \ (num_pos > self.max_masks_to_train): perm = torch.randperm(num_pos) select = perm[:self.max_masks_to_train] mask_preds = mask_preds[select] pos_assigned_gt_inds = pos_assigned_gt_inds[select] num_pos = self.max_masks_to_train elif self.topk_masks_per_img != -1: unique_gt_inds = pos_assigned_gt_inds.unique() num_inst_per_gt = max( int(self.topk_masks_per_img / len(unique_gt_inds)), 1) keep_mask_preds = [] keep_pos_assigned_gt_inds = [] for gt_ind in unique_gt_inds: per_inst_pos_inds = (pos_assigned_gt_inds == gt_ind) mask_preds_per_inst = mask_preds[per_inst_pos_inds] gt_inds_per_inst = pos_assigned_gt_inds[per_inst_pos_inds] if sum(per_inst_pos_inds) > num_inst_per_gt: per_inst_scores = scores[per_inst_pos_inds].sigmoid().max( dim=1)[0] per_inst_centerness = centernesses[ per_inst_pos_inds].sigmoid().reshape(-1, ) select = (per_inst_scores * per_inst_centerness).topk( k=num_inst_per_gt, dim=0)[1] mask_preds_per_inst = mask_preds_per_inst[select] gt_inds_per_inst = gt_inds_per_inst[select] keep_mask_preds.append(mask_preds_per_inst) keep_pos_assigned_gt_inds.append(gt_inds_per_inst) mask_preds = torch.cat(keep_mask_preds) pos_assigned_gt_inds = torch.cat(keep_pos_assigned_gt_inds) num_pos = pos_assigned_gt_inds.size(0) # Follow the origin implement start = int(self.mask_out_stride // 2) gt_masks = gt_masks[:, start::self.mask_out_stride, start::self.mask_out_stride] gt_masks = gt_masks.gt(0.5).float() pos_mask_targets = gt_masks[pos_assigned_gt_inds] pos_pairwise_masks = pairwise_masks[pos_assigned_gt_inds] pos_pairwise_masks = pos_pairwise_masks * pos_mask_targets.unsqueeze(1) return (mask_preds, pos_mask_targets, pos_pairwise_masks, num_pos) ================================================ FILE: mmdet/models/dense_heads/cascade_rpn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from __future__ import division import copy from typing import Dict, List, Optional, Tuple, Union import torch import torch.nn as nn from mmcv.ops import DeformConv2d from mmengine.config import ConfigDict from mmengine.model import BaseModule, ModuleList from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures import SampleList from mmdet.utils import (ConfigType, InstanceList, MultiConfig, OptInstanceList, OptMultiConfig) from ..task_modules.assigners import RegionAssigner from ..task_modules.samplers import PseudoSampler from ..utils import (images_to_levels, multi_apply, select_single_mlvl, unpack_gt_instances) from .base_dense_head import BaseDenseHead from .rpn_head import RPNHead class AdaptiveConv(BaseModule): """AdaptiveConv used to adapt the sampling location with the anchors. Args: in_channels (int): Number of channels in the input image. out_channels (int): Number of channels produced by the convolution. kernel_size (int or tuple[int]): Size of the conv kernel. Defaults to 3. stride (int or tuple[int]): Stride of the convolution. Defaults to 1. padding (int or tuple[int]): Zero-padding added to both sides of the input. Defaults to 1. dilation (int or tuple[int]): Spacing between kernel elements. Defaults to 3. groups (int): Number of blocked connections from input channels to output channels. Defaults to 1. bias (bool): If set True, adds a learnable bias to the output. Defaults to False. adapt_type (str): Type of adaptive conv, can be either ``offset`` (arbitrary anchors) or 'dilation' (uniform anchor). Defaults to 'dilation'. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or \ list[dict]): Initialization config dict. """ def __init__( self, in_channels: int, out_channels: int, kernel_size: Union[int, Tuple[int]] = 3, stride: Union[int, Tuple[int]] = 1, padding: Union[int, Tuple[int]] = 1, dilation: Union[int, Tuple[int]] = 3, groups: int = 1, bias: bool = False, adapt_type: str = 'dilation', init_cfg: MultiConfig = dict( type='Normal', std=0.01, override=dict(name='conv')) ) -> None: super().__init__(init_cfg=init_cfg) assert adapt_type in ['offset', 'dilation'] self.adapt_type = adapt_type assert kernel_size == 3, 'Adaptive conv only supports kernels 3' if self.adapt_type == 'offset': assert stride == 1 and padding == 1 and groups == 1, \ 'Adaptive conv offset mode only supports padding: {1}, ' \ f'stride: {1}, groups: {1}' self.conv = DeformConv2d( in_channels, out_channels, kernel_size, padding=padding, stride=stride, groups=groups, bias=bias) else: self.conv = nn.Conv2d( in_channels, out_channels, kernel_size, padding=dilation, dilation=dilation) def forward(self, x: Tensor, offset: Tensor) -> Tensor: """Forward function.""" if self.adapt_type == 'offset': N, _, H, W = x.shape assert offset is not None assert H * W == offset.shape[1] # reshape [N, NA, 18] to (N, 18, H, W) offset = offset.permute(0, 2, 1).reshape(N, -1, H, W) offset = offset.contiguous() x = self.conv(x, offset) else: assert offset is None x = self.conv(x) return x @MODELS.register_module() class StageCascadeRPNHead(RPNHead): """Stage of CascadeRPNHead. Args: in_channels (int): Number of channels in the input feature map. anchor_generator (:obj:`ConfigDict` or dict): anchor generator config. adapt_cfg (:obj:`ConfigDict` or dict): adaptation config. bridged_feature (bool): whether update rpn feature. Defaults to False. with_cls (bool): whether use classification branch. Defaults to True. init_cfg :obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, in_channels: int, anchor_generator: ConfigType = dict( type='AnchorGenerator', scales=[8], ratios=[1.0], strides=[4, 8, 16, 32, 64]), adapt_cfg: ConfigType = dict(type='dilation', dilation=3), bridged_feature: bool = False, with_cls: bool = True, init_cfg: OptMultiConfig = None, **kwargs) -> None: self.with_cls = with_cls self.anchor_strides = anchor_generator['strides'] self.anchor_scales = anchor_generator['scales'] self.bridged_feature = bridged_feature self.adapt_cfg = adapt_cfg super().__init__( in_channels=in_channels, anchor_generator=anchor_generator, init_cfg=init_cfg, **kwargs) # override sampling and sampler if self.train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) # use PseudoSampler when sampling is False if self.train_cfg.get('sampler', None) is not None: self.sampler = TASK_UTILS.build( self.train_cfg['sampler'], default_args=dict(context=self)) else: self.sampler = PseudoSampler(context=self) if init_cfg is None: self.init_cfg = dict( type='Normal', std=0.01, override=[dict(name='rpn_reg')]) if self.with_cls: self.init_cfg['override'].append(dict(name='rpn_cls')) def _init_layers(self) -> None: """Init layers of a CascadeRPN stage.""" adapt_cfg = copy.deepcopy(self.adapt_cfg) adapt_cfg['adapt_type'] = adapt_cfg.pop('type') self.rpn_conv = AdaptiveConv(self.in_channels, self.feat_channels, **adapt_cfg) if self.with_cls: self.rpn_cls = nn.Conv2d(self.feat_channels, self.num_anchors * self.cls_out_channels, 1) self.rpn_reg = nn.Conv2d(self.feat_channels, self.num_anchors * 4, 1) self.relu = nn.ReLU(inplace=True) def forward_single(self, x: Tensor, offset: Tensor) -> Tuple[Tensor]: """Forward function of single scale.""" bridged_x = x x = self.relu(self.rpn_conv(x, offset)) if self.bridged_feature: bridged_x = x # update feature cls_score = self.rpn_cls(x) if self.with_cls else None bbox_pred = self.rpn_reg(x) return bridged_x, cls_score, bbox_pred def forward( self, feats: List[Tensor], offset_list: Optional[List[Tensor]] = None) -> Tuple[List[Tensor]]: """Forward function.""" if offset_list is None: offset_list = [None for _ in range(len(feats))] return multi_apply(self.forward_single, feats, offset_list) def _region_targets_single(self, flat_anchors: Tensor, valid_flags: Tensor, gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: InstanceData, featmap_sizes: List[Tuple[int, int]], num_level_anchors: List[int]) -> tuple: """Get anchor targets based on region for single level. Args: flat_anchors (Tensor): Multi-level anchors of the image, which are concatenated into a single tensor of shape (num_anchors, 4) valid_flags (Tensor): Multi level valid flags of the image, which are concatenated into a single tensor of shape (num_anchors, ). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. featmap_sizes (list[Tuple[int, int]]): Feature map size each level. num_level_anchors (list[int]): The number of anchors in each level. Returns: tuple: - labels (Tensor): Labels of each level. - label_weights (Tensor): Label weights of each level. - bbox_targets (Tensor): BBox targets of each level. - bbox_weights (Tensor): BBox weights of each level. - pos_inds (Tensor): positive samples indexes. - neg_inds (Tensor): negative samples indexes. - sampling_result (:obj:`SamplingResult`): Sampling results. """ pred_instances = InstanceData() pred_instances.priors = flat_anchors pred_instances.valid_flags = valid_flags assign_result = self.assigner.assign( pred_instances, gt_instances, img_meta, featmap_sizes, num_level_anchors, self.anchor_scales[0], self.anchor_strides, gt_instances_ignore=gt_instances_ignore, allowed_border=self.train_cfg['allowed_border']) sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) num_anchors = flat_anchors.shape[0] bbox_targets = torch.zeros_like(flat_anchors) bbox_weights = torch.zeros_like(flat_anchors) labels = flat_anchors.new_zeros(num_anchors, dtype=torch.long) label_weights = flat_anchors.new_zeros(num_anchors, dtype=torch.float) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: if not self.reg_decoded_bbox: pos_bbox_targets = self.bbox_coder.encode( sampling_result.pos_bboxes, sampling_result.pos_gt_bboxes) else: pos_bbox_targets = sampling_result.pos_gt_bboxes bbox_targets[pos_inds, :] = pos_bbox_targets bbox_weights[pos_inds, :] = 1.0 labels[pos_inds] = sampling_result.pos_gt_labels if self.train_cfg['pos_weight'] <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = self.train_cfg['pos_weight'] if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, neg_inds, sampling_result) def region_targets( self, anchor_list: List[List[Tensor]], valid_flag_list: List[List[Tensor]], featmap_sizes: List[Tuple[int, int]], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, return_sampling_results: bool = False, ) -> tuple: """Compute regression and classification targets for anchors when using RegionAssigner. Args: anchor_list (list[list[Tensor]]): Multi level anchors of each image. valid_flag_list (list[list[Tensor]]): Multi level valid flags of each image. featmap_sizes (list[Tuple[int, int]]): Feature map size each level. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: tuple: - labels_list (list[Tensor]): Labels of each level. - label_weights_list (list[Tensor]): Label weights of each level. - bbox_targets_list (list[Tensor]): BBox targets of each level. - bbox_weights_list (list[Tensor]): BBox weights of each level. - avg_factor (int): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using ``PseudoSampler``, ``avg_factor`` is usually equal to the number of positive priors. """ num_imgs = len(batch_img_metas) assert len(anchor_list) == len(valid_flag_list) == num_imgs if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None] * num_imgs # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] # concat all level anchors to a single tensor concat_anchor_list = [] concat_valid_flag_list = [] for i in range(num_imgs): assert len(anchor_list[i]) == len(valid_flag_list[i]) concat_anchor_list.append(torch.cat(anchor_list[i])) concat_valid_flag_list.append(torch.cat(valid_flag_list[i])) # compute targets for each image (all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, pos_inds_list, neg_inds_list, sampling_results_list) = multi_apply( self._region_targets_single, concat_anchor_list, concat_valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, featmap_sizes=featmap_sizes, num_level_anchors=num_level_anchors) # no valid anchors if any([labels is None for labels in all_labels]): return None # sampled anchors of all images avg_factor = sum( [results.avg_factor for results in sampling_results_list]) # split targets to a list w.r.t. multiple levels labels_list = images_to_levels(all_labels, num_level_anchors) label_weights_list = images_to_levels(all_label_weights, num_level_anchors) bbox_targets_list = images_to_levels(all_bbox_targets, num_level_anchors) bbox_weights_list = images_to_levels(all_bbox_weights, num_level_anchors) res = (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) if return_sampling_results: res = res + (sampling_results_list, ) return res def get_targets( self, anchor_list: List[List[Tensor]], valid_flag_list: List[List[Tensor]], featmap_sizes: List[Tuple[int, int]], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, return_sampling_results: bool = False, ) -> tuple: """Compute regression and classification targets for anchors. Args: anchor_list (list[list[Tensor]]): Multi level anchors of each image. valid_flag_list (list[list[Tensor]]): Multi level valid flags of each image. featmap_sizes (list[Tuple[int, int]]): Feature map size each level. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. return_sampling_results (bool): Whether to return the sampling results. Defaults to False. Returns: tuple: - labels_list (list[Tensor]): Labels of each level. - label_weights_list (list[Tensor]): Label weights of each level. - bbox_targets_list (list[Tensor]): BBox targets of each level. - bbox_weights_list (list[Tensor]): BBox weights of each level. - avg_factor (int): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using ``PseudoSampler``, ``avg_factor`` is usually equal to the number of positive priors. """ if isinstance(self.assigner, RegionAssigner): cls_reg_targets = self.region_targets( anchor_list, valid_flag_list, featmap_sizes, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, return_sampling_results=return_sampling_results) else: cls_reg_targets = super().get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, return_sampling_results=return_sampling_results) return cls_reg_targets def anchor_offset(self, anchor_list: List[List[Tensor]], anchor_strides: List[int], featmap_sizes: List[Tuple[int, int]]) -> List[Tensor]: """ Get offset for deformable conv based on anchor shape NOTE: currently support deformable kernel_size=3 and dilation=1 Args: anchor_list (list[list[tensor])): [NI, NLVL, NA, 4] list of multi-level anchors anchor_strides (list[int]): anchor stride of each level Returns: list[tensor]: offset of DeformConv kernel with shapes of [NLVL, NA, 2, 18]. """ def _shape_offset(anchors, stride, ks=3, dilation=1): # currently support kernel_size=3 and dilation=1 assert ks == 3 and dilation == 1 pad = (ks - 1) // 2 idx = torch.arange(-pad, pad + 1, dtype=dtype, device=device) yy, xx = torch.meshgrid(idx, idx) # return order matters xx = xx.reshape(-1) yy = yy.reshape(-1) w = (anchors[:, 2] - anchors[:, 0]) / stride h = (anchors[:, 3] - anchors[:, 1]) / stride w = w / (ks - 1) - dilation h = h / (ks - 1) - dilation offset_x = w[:, None] * xx # (NA, ks**2) offset_y = h[:, None] * yy # (NA, ks**2) return offset_x, offset_y def _ctr_offset(anchors, stride, featmap_size): feat_h, feat_w = featmap_size assert len(anchors) == feat_h * feat_w x = (anchors[:, 0] + anchors[:, 2]) * 0.5 y = (anchors[:, 1] + anchors[:, 3]) * 0.5 # compute centers on feature map x = x / stride y = y / stride # compute predefine centers xx = torch.arange(0, feat_w, device=anchors.device) yy = torch.arange(0, feat_h, device=anchors.device) yy, xx = torch.meshgrid(yy, xx) xx = xx.reshape(-1).type_as(x) yy = yy.reshape(-1).type_as(y) offset_x = x - xx # (NA, ) offset_y = y - yy # (NA, ) return offset_x, offset_y num_imgs = len(anchor_list) num_lvls = len(anchor_list[0]) dtype = anchor_list[0][0].dtype device = anchor_list[0][0].device num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] offset_list = [] for i in range(num_imgs): mlvl_offset = [] for lvl in range(num_lvls): c_offset_x, c_offset_y = _ctr_offset(anchor_list[i][lvl], anchor_strides[lvl], featmap_sizes[lvl]) s_offset_x, s_offset_y = _shape_offset(anchor_list[i][lvl], anchor_strides[lvl]) # offset = ctr_offset + shape_offset offset_x = s_offset_x + c_offset_x[:, None] offset_y = s_offset_y + c_offset_y[:, None] # offset order (y0, x0, y1, x2, .., y8, x8, y9, x9) offset = torch.stack([offset_y, offset_x], dim=-1) offset = offset.reshape(offset.size(0), -1) # [NA, 2*ks**2] mlvl_offset.append(offset) offset_list.append(torch.cat(mlvl_offset)) # [totalNA, 2*ks**2] offset_list = images_to_levels(offset_list, num_level_anchors) return offset_list def loss_by_feat_single(self, cls_score: Tensor, bbox_pred: Tensor, anchors: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, bbox_weights: Tensor, avg_factor: int) -> tuple: """Loss function on single scale.""" # classification loss if self.with_cls: labels = labels.reshape(-1) label_weights = label_weights.reshape(-1) cls_score = cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) loss_cls = self.loss_cls( cls_score, labels, label_weights, avg_factor=avg_factor) # regression loss bbox_targets = bbox_targets.reshape(-1, 4) bbox_weights = bbox_weights.reshape(-1, 4) bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) if self.reg_decoded_bbox: # When the regression loss (e.g. `IouLoss`, `GIouLoss`) # is applied directly on the decoded bounding boxes, it # decodes the already encoded coordinates to absolute format. anchors = anchors.reshape(-1, 4) bbox_pred = self.bbox_coder.decode(anchors, bbox_pred) loss_reg = self.loss_bbox( bbox_pred, bbox_targets, bbox_weights, avg_factor=avg_factor) if self.with_cls: return loss_cls, loss_reg return None, loss_reg def loss_by_feat( self, anchor_list: List[List[Tensor]], valid_flag_list: List[List[Tensor]], cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """Compute losses of the head. Args: anchor_list (list[list[Tensor]]): Multi level anchors of each image. valid_flag_list (list[list[Tensor]]): Multi level valid flags of each image. The outer list indicates images, and the inner list corresponds to feature levels of the image. Each element of the inner list is a tensor of shape (num_anchors, ) cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in bbox_preds] cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, featmap_sizes, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, return_sampling_results=True) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor, sampling_results_list) = cls_reg_targets if not sampling_results_list[0].avg_factor_with_neg: # 200 is hard-coded average factor, # which follows guided anchoring. avg_factor = sum([label.numel() for label in labels_list]) / 200.0 # change per image, per level anchor_list to per_level, per_image mlvl_anchor_list = list(zip(*anchor_list)) # concat mlvl_anchor_list mlvl_anchor_list = [ torch.cat(anchors, dim=0) for anchors in mlvl_anchor_list ] losses = multi_apply( self.loss_by_feat_single, cls_scores, bbox_preds, mlvl_anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor=avg_factor) if self.with_cls: return dict(loss_rpn_cls=losses[0], loss_rpn_reg=losses[1]) return dict(loss_rpn_reg=losses[1]) def predict_by_feat(self, anchor_list: List[List[Tensor]], cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_img_metas: List[dict], cfg: Optional[ConfigDict] = None, rescale: bool = False) -> InstanceList: """Get proposal predict. Overriding to enable input ``anchor_list`` from outside. Args: anchor_list (list[list[Tensor]]): Multi level anchors of each image. cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * 4, H, W). batch_img_metas (list[dict], Optional): Image meta info. cfg (:obj:`ConfigDict`, optional): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(cls_scores) == len(bbox_preds) result_list = [] for img_id in range(len(batch_img_metas)): cls_score_list = select_single_mlvl(cls_scores, img_id) bbox_pred_list = select_single_mlvl(bbox_preds, img_id) proposals = self._predict_by_feat_single( cls_scores=cls_score_list, bbox_preds=bbox_pred_list, mlvl_anchors=anchor_list[img_id], img_meta=batch_img_metas[img_id], cfg=cfg, rescale=rescale) result_list.append(proposals) return result_list def _predict_by_feat_single(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], mlvl_anchors: List[Tensor], img_meta: dict, cfg: ConfigDict, rescale: bool = False) -> InstanceData: """Transform outputs of a single image into bbox predictions. Args: cls_scores (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_anchors * 4, H, W). mlvl_anchors (list[Tensor]): Box reference from all scale levels of a single image, each item has shape (num_total_anchors, 4). img_shape (tuple[int]): Shape of the input image, (height, width, 3). scale_factor (ndarray): Scale factor of the image arange as (w_scale, h_scale, w_scale, h_scale). cfg (:obj:`ConfigDict`): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ cfg = self.test_cfg if cfg is None else cfg cfg = copy.deepcopy(cfg) # bboxes from different level should be independent during NMS, # level_ids are used as labels for batched NMS to separate them level_ids = [] mlvl_scores = [] mlvl_bbox_preds = [] mlvl_valid_anchors = [] nms_pre = cfg.get('nms_pre', -1) for idx in range(len(cls_scores)): rpn_cls_score = cls_scores[idx] rpn_bbox_pred = bbox_preds[idx] assert rpn_cls_score.size()[-2:] == rpn_bbox_pred.size()[-2:] rpn_cls_score = rpn_cls_score.permute(1, 2, 0) if self.use_sigmoid_cls: rpn_cls_score = rpn_cls_score.reshape(-1) scores = rpn_cls_score.sigmoid() else: rpn_cls_score = rpn_cls_score.reshape(-1, 2) # We set FG labels to [0, num_class-1] and BG label to # num_class in RPN head since mmdet v2.5, which is unified to # be consistent with other head since mmdet v2.0. In mmdet v2.0 # to v2.4 we keep BG label as 0 and FG label as 1 in rpn head. scores = rpn_cls_score.softmax(dim=1)[:, 0] rpn_bbox_pred = rpn_bbox_pred.permute(1, 2, 0).reshape(-1, 4) anchors = mlvl_anchors[idx] if 0 < nms_pre < scores.shape[0]: # sort is faster than topk # _, topk_inds = scores.topk(cfg.nms_pre) ranked_scores, rank_inds = scores.sort(descending=True) topk_inds = rank_inds[:nms_pre] scores = ranked_scores[:nms_pre] rpn_bbox_pred = rpn_bbox_pred[topk_inds, :] anchors = anchors[topk_inds, :] mlvl_scores.append(scores) mlvl_bbox_preds.append(rpn_bbox_pred) mlvl_valid_anchors.append(anchors) level_ids.append( scores.new_full((scores.size(0), ), idx, dtype=torch.long)) anchors = torch.cat(mlvl_valid_anchors) rpn_bbox_pred = torch.cat(mlvl_bbox_preds) bboxes = self.bbox_coder.decode( anchors, rpn_bbox_pred, max_shape=img_meta['img_shape']) proposals = InstanceData() proposals.bboxes = bboxes proposals.scores = torch.cat(mlvl_scores) proposals.level_ids = torch.cat(level_ids) return self._bbox_post_process( results=proposals, cfg=cfg, rescale=rescale, img_meta=img_meta) def refine_bboxes(self, anchor_list: List[List[Tensor]], bbox_preds: List[Tensor], img_metas: List[dict]) -> List[List[Tensor]]: """Refine bboxes through stages.""" num_levels = len(bbox_preds) new_anchor_list = [] for img_id in range(len(img_metas)): mlvl_anchors = [] for i in range(num_levels): bbox_pred = bbox_preds[i][img_id].detach() bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) img_shape = img_metas[img_id]['img_shape'] bboxes = self.bbox_coder.decode(anchor_list[img_id][i], bbox_pred, img_shape) mlvl_anchors.append(bboxes) new_anchor_list.append(mlvl_anchors) return new_anchor_list def loss(self, x: Tuple[Tensor], batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection head on the features of the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, _, batch_img_metas = outputs featmap_sizes = [featmap.size()[-2:] for featmap in x] device = x[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) if self.adapt_cfg['type'] == 'offset': offset_list = self.anchor_offset(anchor_list, self.anchor_strides, featmap_sizes) else: offset_list = None x, cls_score, bbox_pred = self(x, offset_list) rpn_loss_inputs = (anchor_list, valid_flag_list, cls_score, bbox_pred, batch_gt_instances, batch_img_metas) losses = self.loss_by_feat(*rpn_loss_inputs) return losses def loss_and_predict( self, x: Tuple[Tensor], batch_data_samples: SampleList, proposal_cfg: Optional[ConfigDict] = None, ) -> Tuple[dict, InstanceList]: """Perform forward propagation of the head, then calculate loss and predictions from the features and data samples. Args: x (tuple[Tensor]): Features from FPN. batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. proposal_cfg (:obj`ConfigDict`, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. Returns: tuple: the return value is a tuple contains: - losses: (dict[str, Tensor]): A dictionary of loss components. - predictions (list[:obj:`InstanceData`]): Detection results of each image after the post process. """ outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, _, batch_img_metas = outputs featmap_sizes = [featmap.size()[-2:] for featmap in x] device = x[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) if self.adapt_cfg['type'] == 'offset': offset_list = self.anchor_offset(anchor_list, self.anchor_strides, featmap_sizes) else: offset_list = None x, cls_score, bbox_pred = self(x, offset_list) rpn_loss_inputs = (anchor_list, valid_flag_list, cls_score, bbox_pred, batch_gt_instances, batch_img_metas) losses = self.loss_by_feat(*rpn_loss_inputs) predictions = self.predict_by_feat( anchor_list, cls_score, bbox_pred, batch_img_metas=batch_img_metas, cfg=proposal_cfg) return losses, predictions def predict(self, x: Tuple[Tensor], batch_data_samples: SampleList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the detection head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Multi-level features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool, optional): Whether to rescale the results. Defaults to False. Returns: list[obj:`InstanceData`]: Detection results of each image after the post process. """ batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] featmap_sizes = [featmap.size()[-2:] for featmap in x] device = x[0].device anchor_list, _ = self.get_anchors( featmap_sizes, batch_img_metas, device=device) if self.adapt_cfg['type'] == 'offset': offset_list = self.anchor_offset(anchor_list, self.anchor_strides, featmap_sizes) else: offset_list = None x, cls_score, bbox_pred = self(x, offset_list) predictions = self.stages[-1].predict_by_feat( anchor_list, cls_score, bbox_pred, batch_img_metas=batch_img_metas, rescale=rescale) return predictions @MODELS.register_module() class CascadeRPNHead(BaseDenseHead): """The CascadeRPNHead will predict more accurate region proposals, which is required for two-stage detectors (such as Fast/Faster R-CNN). CascadeRPN consists of a sequence of RPNStage to progressively improve the accuracy of the detected proposals. More details can be found in ``https://arxiv.org/abs/1909.06720``. Args: num_stages (int): number of CascadeRPN stages. stages (list[:obj:`ConfigDict` or dict]): list of configs to build the stages. train_cfg (list[:obj:`ConfigDict` or dict]): list of configs at training time each stage. test_cfg (:obj:`ConfigDict` or dict): config at testing time. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or \ list[dict]): Initialization config dict. """ def __init__(self, num_classes: int, num_stages: int, stages: List[ConfigType], train_cfg: List[ConfigType], test_cfg: ConfigType, init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) assert num_classes == 1, 'Only support num_classes == 1' assert num_stages == len(stages) self.num_stages = num_stages # Be careful! Pretrained weights cannot be loaded when use # nn.ModuleList self.stages = ModuleList() for i in range(len(stages)): train_cfg_i = train_cfg[i] if train_cfg is not None else None stages[i].update(train_cfg=train_cfg_i) stages[i].update(test_cfg=test_cfg) self.stages.append(MODELS.build(stages[i])) self.train_cfg = train_cfg self.test_cfg = test_cfg def loss_by_feat(self): """loss_by_feat() is implemented in StageCascadeRPNHead.""" pass def predict_by_feat(self): """predict_by_feat() is implemented in StageCascadeRPNHead.""" pass def loss(self, x: Tuple[Tensor], batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection head on the features of the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, _, batch_img_metas = outputs featmap_sizes = [featmap.size()[-2:] for featmap in x] device = x[0].device anchor_list, valid_flag_list = self.stages[0].get_anchors( featmap_sizes, batch_img_metas, device=device) losses = dict() for i in range(self.num_stages): stage = self.stages[i] if stage.adapt_cfg['type'] == 'offset': offset_list = stage.anchor_offset(anchor_list, stage.anchor_strides, featmap_sizes) else: offset_list = None x, cls_score, bbox_pred = stage(x, offset_list) rpn_loss_inputs = (anchor_list, valid_flag_list, cls_score, bbox_pred, batch_gt_instances, batch_img_metas) stage_loss = stage.loss_by_feat(*rpn_loss_inputs) for name, value in stage_loss.items(): losses['s{}.{}'.format(i, name)] = value # refine boxes if i < self.num_stages - 1: anchor_list = stage.refine_bboxes(anchor_list, bbox_pred, batch_img_metas) return losses def loss_and_predict( self, x: Tuple[Tensor], batch_data_samples: SampleList, proposal_cfg: Optional[ConfigDict] = None, ) -> Tuple[dict, InstanceList]: """Perform forward propagation of the head, then calculate loss and predictions from the features and data samples. Args: x (tuple[Tensor]): Features from FPN. batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. proposal_cfg (ConfigDict, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. Returns: tuple: the return value is a tuple contains: - losses: (dict[str, Tensor]): A dictionary of loss components. - predictions (list[:obj:`InstanceData`]): Detection results of each image after the post process. """ outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, _, batch_img_metas = outputs featmap_sizes = [featmap.size()[-2:] for featmap in x] device = x[0].device anchor_list, valid_flag_list = self.stages[0].get_anchors( featmap_sizes, batch_img_metas, device=device) losses = dict() for i in range(self.num_stages): stage = self.stages[i] if stage.adapt_cfg['type'] == 'offset': offset_list = stage.anchor_offset(anchor_list, stage.anchor_strides, featmap_sizes) else: offset_list = None x, cls_score, bbox_pred = stage(x, offset_list) rpn_loss_inputs = (anchor_list, valid_flag_list, cls_score, bbox_pred, batch_gt_instances, batch_img_metas) stage_loss = stage.loss_by_feat(*rpn_loss_inputs) for name, value in stage_loss.items(): losses['s{}.{}'.format(i, name)] = value # refine boxes if i < self.num_stages - 1: anchor_list = stage.refine_bboxes(anchor_list, bbox_pred, batch_img_metas) predictions = self.stages[-1].predict_by_feat( anchor_list, cls_score, bbox_pred, batch_img_metas=batch_img_metas, cfg=proposal_cfg) return losses, predictions def predict(self, x: Tuple[Tensor], batch_data_samples: SampleList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the detection head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Multi-level features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool, optional): Whether to rescale the results. Defaults to False. Returns: list[obj:`InstanceData`]: Detection results of each image after the post process. """ batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] featmap_sizes = [featmap.size()[-2:] for featmap in x] device = x[0].device anchor_list, _ = self.stages[0].get_anchors( featmap_sizes, batch_img_metas, device=device) for i in range(self.num_stages): stage = self.stages[i] if stage.adapt_cfg['type'] == 'offset': offset_list = stage.anchor_offset(anchor_list, stage.anchor_strides, featmap_sizes) else: offset_list = None x, cls_score, bbox_pred = stage(x, offset_list) if i < self.num_stages - 1: anchor_list = stage.refine_bboxes(anchor_list, bbox_pred, batch_img_metas) predictions = self.stages[-1].predict_by_feat( anchor_list, cls_score, bbox_pred, batch_img_metas=batch_img_metas, rescale=rescale) return predictions ================================================ FILE: mmdet/models/dense_heads/centernet_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch import torch.nn as nn from mmcv.ops import batched_nms from mmengine.config import ConfigDict from mmengine.model import bias_init_with_prob, normal_init from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptInstanceList, OptMultiConfig) from ..utils import (gaussian_radius, gen_gaussian_target, get_local_maximum, get_topk_from_heatmap, multi_apply, transpose_and_gather_feat) from .base_dense_head import BaseDenseHead @MODELS.register_module() class CenterNetHead(BaseDenseHead): """Objects as Points Head. CenterHead use center_point to indicate object's position. Paper link Args: in_channels (int): Number of channel in the input feature map. feat_channels (int): Number of channel in the intermediate feature map. num_classes (int): Number of categories excluding the background category. loss_center_heatmap (:obj:`ConfigDict` or dict): Config of center heatmap loss. Defaults to dict(type='GaussianFocalLoss', loss_weight=1.0) loss_wh (:obj:`ConfigDict` or dict): Config of wh loss. Defaults to dict(type='L1Loss', loss_weight=0.1). loss_offset (:obj:`ConfigDict` or dict): Config of offset loss. Defaults to dict(type='L1Loss', loss_weight=1.0). train_cfg (:obj:`ConfigDict` or dict, optional): Training config. Useless in CenterNet, but we keep this variable for SingleStageDetector. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of CenterNet. init_cfg (:obj:`ConfigDict` or dict or list[dict] or list[:obj:`ConfigDict`], optional): Initialization config dict. """ def __init__(self, in_channels: int, feat_channels: int, num_classes: int, loss_center_heatmap: ConfigType = dict( type='GaussianFocalLoss', loss_weight=1.0), loss_wh: ConfigType = dict(type='L1Loss', loss_weight=0.1), loss_offset: ConfigType = dict( type='L1Loss', loss_weight=1.0), train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) self.num_classes = num_classes self.heatmap_head = self._build_head(in_channels, feat_channels, num_classes) self.wh_head = self._build_head(in_channels, feat_channels, 2) self.offset_head = self._build_head(in_channels, feat_channels, 2) self.loss_center_heatmap = MODELS.build(loss_center_heatmap) self.loss_wh = MODELS.build(loss_wh) self.loss_offset = MODELS.build(loss_offset) self.train_cfg = train_cfg self.test_cfg = test_cfg self.fp16_enabled = False def _build_head(self, in_channels: int, feat_channels: int, out_channels: int) -> nn.Sequential: """Build head for each branch.""" layer = nn.Sequential( nn.Conv2d(in_channels, feat_channels, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.Conv2d(feat_channels, out_channels, kernel_size=1)) return layer def init_weights(self) -> None: """Initialize weights of the head.""" bias_init = bias_init_with_prob(0.1) self.heatmap_head[-1].bias.data.fill_(bias_init) for head in [self.wh_head, self.offset_head]: for m in head.modules(): if isinstance(m, nn.Conv2d): normal_init(m, std=0.001) def forward(self, x: Tuple[Tensor, ...]) -> Tuple[List[Tensor]]: """Forward features. Notice CenterNet head does not use FPN. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: center_heatmap_preds (list[Tensor]): center predict heatmaps for all levels, the channels number is num_classes. wh_preds (list[Tensor]): wh predicts for all levels, the channels number is 2. offset_preds (list[Tensor]): offset predicts for all levels, the channels number is 2. """ return multi_apply(self.forward_single, x) def forward_single(self, x: Tensor) -> Tuple[Tensor, ...]: """Forward feature of a single level. Args: x (Tensor): Feature of a single level. Returns: center_heatmap_pred (Tensor): center predict heatmaps, the channels number is num_classes. wh_pred (Tensor): wh predicts, the channels number is 2. offset_pred (Tensor): offset predicts, the channels number is 2. """ center_heatmap_pred = self.heatmap_head(x).sigmoid() wh_pred = self.wh_head(x) offset_pred = self.offset_head(x) return center_heatmap_pred, wh_pred, offset_pred def loss_by_feat( self, center_heatmap_preds: List[Tensor], wh_preds: List[Tensor], offset_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Compute losses of the head. Args: center_heatmap_preds (list[Tensor]): center predict heatmaps for all levels with shape (B, num_classes, H, W). wh_preds (list[Tensor]): wh predicts for all levels with shape (B, 2, H, W). offset_preds (list[Tensor]): offset predicts for all levels with shape (B, 2, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: which has components below: - loss_center_heatmap (Tensor): loss of center heatmap. - loss_wh (Tensor): loss of hw heatmap - loss_offset (Tensor): loss of offset heatmap. """ assert len(center_heatmap_preds) == len(wh_preds) == len( offset_preds) == 1 center_heatmap_pred = center_heatmap_preds[0] wh_pred = wh_preds[0] offset_pred = offset_preds[0] gt_bboxes = [ gt_instances.bboxes for gt_instances in batch_gt_instances ] gt_labels = [ gt_instances.labels for gt_instances in batch_gt_instances ] img_shape = batch_img_metas[0]['batch_input_shape'] target_result, avg_factor = self.get_targets(gt_bboxes, gt_labels, center_heatmap_pred.shape, img_shape) center_heatmap_target = target_result['center_heatmap_target'] wh_target = target_result['wh_target'] offset_target = target_result['offset_target'] wh_offset_target_weight = target_result['wh_offset_target_weight'] # Since the channel of wh_target and offset_target is 2, the avg_factor # of loss_center_heatmap is always 1/2 of loss_wh and loss_offset. loss_center_heatmap = self.loss_center_heatmap( center_heatmap_pred, center_heatmap_target, avg_factor=avg_factor) loss_wh = self.loss_wh( wh_pred, wh_target, wh_offset_target_weight, avg_factor=avg_factor * 2) loss_offset = self.loss_offset( offset_pred, offset_target, wh_offset_target_weight, avg_factor=avg_factor * 2) return dict( loss_center_heatmap=loss_center_heatmap, loss_wh=loss_wh, loss_offset=loss_offset) def get_targets(self, gt_bboxes: List[Tensor], gt_labels: List[Tensor], feat_shape: tuple, img_shape: tuple) -> Tuple[dict, int]: """Compute regression and classification targets in multiple images. Args: gt_bboxes (list[Tensor]): Ground truth bboxes for each image with shape (num_gts, 4) in [tl_x, tl_y, br_x, br_y] format. gt_labels (list[Tensor]): class indices corresponding to each box. feat_shape (tuple): feature map shape with value [B, _, H, W] img_shape (tuple): image shape. Returns: tuple[dict, float]: The float value is mean avg_factor, the dict has components below: - center_heatmap_target (Tensor): targets of center heatmap, \ shape (B, num_classes, H, W). - wh_target (Tensor): targets of wh predict, shape \ (B, 2, H, W). - offset_target (Tensor): targets of offset predict, shape \ (B, 2, H, W). - wh_offset_target_weight (Tensor): weights of wh and offset \ predict, shape (B, 2, H, W). """ img_h, img_w = img_shape[:2] bs, _, feat_h, feat_w = feat_shape width_ratio = float(feat_w / img_w) height_ratio = float(feat_h / img_h) center_heatmap_target = gt_bboxes[-1].new_zeros( [bs, self.num_classes, feat_h, feat_w]) wh_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) offset_target = gt_bboxes[-1].new_zeros([bs, 2, feat_h, feat_w]) wh_offset_target_weight = gt_bboxes[-1].new_zeros( [bs, 2, feat_h, feat_w]) for batch_id in range(bs): gt_bbox = gt_bboxes[batch_id] gt_label = gt_labels[batch_id] center_x = (gt_bbox[:, [0]] + gt_bbox[:, [2]]) * width_ratio / 2 center_y = (gt_bbox[:, [1]] + gt_bbox[:, [3]]) * height_ratio / 2 gt_centers = torch.cat((center_x, center_y), dim=1) for j, ct in enumerate(gt_centers): ctx_int, cty_int = ct.int() ctx, cty = ct scale_box_h = (gt_bbox[j][3] - gt_bbox[j][1]) * height_ratio scale_box_w = (gt_bbox[j][2] - gt_bbox[j][0]) * width_ratio radius = gaussian_radius([scale_box_h, scale_box_w], min_overlap=0.3) radius = max(0, int(radius)) ind = gt_label[j] gen_gaussian_target(center_heatmap_target[batch_id, ind], [ctx_int, cty_int], radius) wh_target[batch_id, 0, cty_int, ctx_int] = scale_box_w wh_target[batch_id, 1, cty_int, ctx_int] = scale_box_h offset_target[batch_id, 0, cty_int, ctx_int] = ctx - ctx_int offset_target[batch_id, 1, cty_int, ctx_int] = cty - cty_int wh_offset_target_weight[batch_id, :, cty_int, ctx_int] = 1 avg_factor = max(1, center_heatmap_target.eq(1).sum()) target_result = dict( center_heatmap_target=center_heatmap_target, wh_target=wh_target, offset_target=offset_target, wh_offset_target_weight=wh_offset_target_weight) return target_result, avg_factor def predict_by_feat(self, center_heatmap_preds: List[Tensor], wh_preds: List[Tensor], offset_preds: List[Tensor], batch_img_metas: Optional[List[dict]] = None, rescale: bool = True, with_nms: bool = False) -> InstanceList: """Transform network output for a batch into bbox predictions. Args: center_heatmap_preds (list[Tensor]): Center predict heatmaps for all levels with shape (B, num_classes, H, W). wh_preds (list[Tensor]): WH predicts for all levels with shape (B, 2, H, W). offset_preds (list[Tensor]): Offset predicts for all levels with shape (B, 2, H, W). batch_img_metas (list[dict], optional): Batch image meta info. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to True. with_nms (bool): If True, do nms before return boxes. Defaults to False. Returns: list[:obj:`InstanceData`]: Instance segmentation results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(center_heatmap_preds) == len(wh_preds) == len( offset_preds) == 1 result_list = [] for img_id in range(len(batch_img_metas)): result_list.append( self._predict_by_feat_single( center_heatmap_preds[0][img_id:img_id + 1, ...], wh_preds[0][img_id:img_id + 1, ...], offset_preds[0][img_id:img_id + 1, ...], batch_img_metas[img_id], rescale=rescale, with_nms=with_nms)) return result_list def _predict_by_feat_single(self, center_heatmap_pred: Tensor, wh_pred: Tensor, offset_pred: Tensor, img_meta: dict, rescale: bool = True, with_nms: bool = False) -> InstanceData: """Transform outputs of a single image into bbox results. Args: center_heatmap_pred (Tensor): Center heatmap for current level with shape (1, num_classes, H, W). wh_pred (Tensor): WH heatmap for current level with shape (1, num_classes, H, W). offset_pred (Tensor): Offset for current level with shape (1, corner_offset_channels, H, W). img_meta (dict): Meta information of current image, e.g., image size, scaling factor, etc. rescale (bool): If True, return boxes in original image space. Defaults to True. with_nms (bool): If True, do nms before return boxes. Defaults to False. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ batch_det_bboxes, batch_labels = self._decode_heatmap( center_heatmap_pred, wh_pred, offset_pred, img_meta['batch_input_shape'], k=self.test_cfg.topk, kernel=self.test_cfg.local_maximum_kernel) det_bboxes = batch_det_bboxes.view([-1, 5]) det_labels = batch_labels.view(-1) batch_border = det_bboxes.new_tensor(img_meta['border'])[..., [2, 0, 2, 0]] det_bboxes[..., :4] -= batch_border if rescale and 'scale_factor' in img_meta: det_bboxes[..., :4] /= det_bboxes.new_tensor( img_meta['scale_factor']).repeat((1, 2)) if with_nms: det_bboxes, det_labels = self._bboxes_nms(det_bboxes, det_labels, self.test_cfg) results = InstanceData() results.bboxes = det_bboxes[..., :4] results.scores = det_bboxes[..., 4] results.labels = det_labels return results def _decode_heatmap(self, center_heatmap_pred: Tensor, wh_pred: Tensor, offset_pred: Tensor, img_shape: tuple, k: int = 100, kernel: int = 3) -> Tuple[Tensor, Tensor]: """Transform outputs into detections raw bbox prediction. Args: center_heatmap_pred (Tensor): center predict heatmap, shape (B, num_classes, H, W). wh_pred (Tensor): wh predict, shape (B, 2, H, W). offset_pred (Tensor): offset predict, shape (B, 2, H, W). img_shape (tuple): image shape in hw format. k (int): Get top k center keypoints from heatmap. Defaults to 100. kernel (int): Max pooling kernel for extract local maximum pixels. Defaults to 3. Returns: tuple[Tensor]: Decoded output of CenterNetHead, containing the following Tensors: - batch_bboxes (Tensor): Coords of each box with shape (B, k, 5) - batch_topk_labels (Tensor): Categories of each box with \ shape (B, k) """ height, width = center_heatmap_pred.shape[2:] inp_h, inp_w = img_shape center_heatmap_pred = get_local_maximum( center_heatmap_pred, kernel=kernel) *batch_dets, topk_ys, topk_xs = get_topk_from_heatmap( center_heatmap_pred, k=k) batch_scores, batch_index, batch_topk_labels = batch_dets wh = transpose_and_gather_feat(wh_pred, batch_index) offset = transpose_and_gather_feat(offset_pred, batch_index) topk_xs = topk_xs + offset[..., 0] topk_ys = topk_ys + offset[..., 1] tl_x = (topk_xs - wh[..., 0] / 2) * (inp_w / width) tl_y = (topk_ys - wh[..., 1] / 2) * (inp_h / height) br_x = (topk_xs + wh[..., 0] / 2) * (inp_w / width) br_y = (topk_ys + wh[..., 1] / 2) * (inp_h / height) batch_bboxes = torch.stack([tl_x, tl_y, br_x, br_y], dim=2) batch_bboxes = torch.cat((batch_bboxes, batch_scores[..., None]), dim=-1) return batch_bboxes, batch_topk_labels def _bboxes_nms(self, bboxes: Tensor, labels: Tensor, cfg: ConfigDict) -> Tuple[Tensor, Tensor]: """bboxes nms.""" if labels.numel() > 0: max_num = cfg.max_per_img bboxes, keep = batched_nms(bboxes[:, :4], bboxes[:, -1].contiguous(), labels, cfg.nms) if max_num > 0: bboxes = bboxes[:max_num] labels = labels[keep][:max_num] return bboxes, labels ================================================ FILE: mmdet/models/dense_heads/centernet_update_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Optional, Sequence, Tuple import torch import torch.nn as nn from mmcv.cnn import Scale from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures.bbox import bbox2distance from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptInstanceList, reduce_mean) from ..utils import multi_apply from .anchor_free_head import AnchorFreeHead INF = 1000000000 RangeType = Sequence[Tuple[int, int]] def _transpose(tensor_list: List[Tensor], num_point_list: list) -> List[Tensor]: """This function is used to transpose image first tensors to level first ones.""" for img_idx in range(len(tensor_list)): tensor_list[img_idx] = torch.split( tensor_list[img_idx], num_point_list, dim=0) tensors_level_first = [] for targets_per_level in zip(*tensor_list): tensors_level_first.append(torch.cat(targets_per_level, dim=0)) return tensors_level_first @MODELS.register_module() class CenterNetUpdateHead(AnchorFreeHead): """CenterNetUpdateHead is an improved version of CenterNet in CenterNet2. Paper link ``_. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channel in the input feature map. regress_ranges (Sequence[Tuple[int, int]]): Regress range of multiple level points. hm_min_radius (int): Heatmap target minimum radius of cls branch. Defaults to 4. hm_min_overlap (float): Heatmap target minimum overlap of cls branch. Defaults to 0.8. more_pos_thresh (float): The filtering threshold when the cls branch adds more positive samples. Defaults to 0.2. more_pos_topk (int): The maximum number of additional positive samples added to each gt. Defaults to 9. soft_weight_on_reg (bool): Whether to use the soft target of the cls branch as the soft weight of the bbox branch. Defaults to False. loss_cls (:obj:`ConfigDict` or dict): Config of cls loss. Defaults to dict(type='GaussianFocalLoss', loss_weight=1.0) loss_bbox (:obj:`ConfigDict` or dict): Config of bbox loss. Defaults to dict(type='GIoULoss', loss_weight=2.0). norm_cfg (:obj:`ConfigDict` or dict, optional): dictionary to construct and config norm layer. Defaults to ``norm_cfg=dict(type='GN', num_groups=32, requires_grad=True)``. train_cfg (:obj:`ConfigDict` or dict, optional): Training config. Unused in CenterNet. Reserved for compatibility with SingleStageDetector. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of CenterNet. """ def __init__(self, num_classes: int, in_channels: int, regress_ranges: RangeType = ((0, 80), (64, 160), (128, 320), (256, 640), (512, INF)), hm_min_radius: int = 4, hm_min_overlap: float = 0.8, more_pos_thresh: float = 0.2, more_pos_topk: int = 9, soft_weight_on_reg: bool = False, loss_cls: ConfigType = dict( type='GaussianFocalLoss', pos_weight=0.25, neg_weight=0.75, loss_weight=1.0), loss_bbox: ConfigType = dict( type='GIoULoss', loss_weight=2.0), norm_cfg: OptConfigType = dict( type='GN', num_groups=32, requires_grad=True), train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, **kwargs) -> None: super().__init__( num_classes=num_classes, in_channels=in_channels, loss_cls=loss_cls, loss_bbox=loss_bbox, norm_cfg=norm_cfg, train_cfg=train_cfg, test_cfg=test_cfg, **kwargs) self.soft_weight_on_reg = soft_weight_on_reg self.hm_min_radius = hm_min_radius self.more_pos_thresh = more_pos_thresh self.more_pos_topk = more_pos_topk self.delta = (1 - hm_min_overlap) / (1 + hm_min_overlap) self.sigmoid_clamp = 0.0001 # GaussianFocalLoss must be sigmoid mode self.use_sigmoid_cls = True self.cls_out_channels = num_classes self.regress_ranges = regress_ranges self.scales = nn.ModuleList([Scale(1.0) for _ in self.strides]) def _init_predictor(self) -> None: """Initialize predictor layers of the head.""" self.conv_cls = nn.Conv2d( self.feat_channels, self.num_classes, 3, padding=1) self.conv_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1) def forward(self, x: Tuple[Tensor]) -> Tuple[List[Tensor], List[Tensor]]: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of each level outputs. - cls_scores (list[Tensor]): Box scores for each scale level, \ each is a 4D-tensor, the channel number is num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for each \ scale level, each is a 4D-tensor, the channel number is 4. """ return multi_apply(self.forward_single, x, self.scales, self.strides) def forward_single(self, x: Tensor, scale: Scale, stride: int) -> Tuple[Tensor, Tensor]: """Forward features of a single scale level. Args: x (Tensor): FPN feature maps of the specified stride. scale (:obj:`mmcv.cnn.Scale`): Learnable scale module to resize the bbox prediction. stride (int): The corresponding stride for feature maps. Returns: tuple: scores for each class, bbox predictions of input feature maps. """ cls_score, bbox_pred, _, _ = super().forward_single(x) # scale the bbox_pred of different level # float to avoid overflow when enabling FP16 bbox_pred = scale(bbox_pred).float() # bbox_pred needed for gradient computation has been modified # by F.relu(bbox_pred) when run with PyTorch 1.10. So replace # F.relu(bbox_pred) with bbox_pred.clamp(min=0) bbox_pred = bbox_pred.clamp(min=0) if not self.training: bbox_pred *= stride return cls_score, bbox_pred def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level, each is a 4D-tensor, the channel number is num_classes. bbox_preds (list[Tensor]): Box energies / deltas for each scale level, each is a 4D-tensor, the channel number is 4. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ num_imgs = cls_scores[0].size(0) assert len(cls_scores) == len(bbox_preds) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] all_level_points = self.prior_generator.grid_priors( featmap_sizes, dtype=bbox_preds[0].dtype, device=bbox_preds[0].device) # 1 flatten outputs flatten_cls_scores = [ cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) for cls_score in cls_scores ] flatten_bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) for bbox_pred in bbox_preds ] flatten_cls_scores = torch.cat(flatten_cls_scores) flatten_bbox_preds = torch.cat(flatten_bbox_preds) # repeat points to align with bbox_preds flatten_points = torch.cat( [points.repeat(num_imgs, 1) for points in all_level_points]) assert (torch.isfinite(flatten_bbox_preds).all().item()) # 2 calc reg and cls branch targets cls_targets, bbox_targets = self.get_targets(all_level_points, batch_gt_instances) # 3 add more pos index for cls branch featmap_sizes = flatten_points.new_tensor(featmap_sizes) pos_inds, cls_labels = self.add_cls_pos_inds(flatten_points, flatten_bbox_preds, featmap_sizes, batch_gt_instances) # 4 calc cls loss if pos_inds is None: # num_gts=0 num_pos_cls = bbox_preds[0].new_tensor(0, dtype=torch.float) else: num_pos_cls = bbox_preds[0].new_tensor( len(pos_inds), dtype=torch.float) num_pos_cls = max(reduce_mean(num_pos_cls), 1.0) flatten_cls_scores = flatten_cls_scores.sigmoid().clamp( min=self.sigmoid_clamp, max=1 - self.sigmoid_clamp) cls_loss = self.loss_cls( flatten_cls_scores, cls_targets, pos_inds=pos_inds, pos_labels=cls_labels, avg_factor=num_pos_cls) # 5 calc reg loss pos_bbox_inds = torch.nonzero( bbox_targets.max(dim=1)[0] >= 0).squeeze(1) pos_bbox_preds = flatten_bbox_preds[pos_bbox_inds] pos_bbox_targets = bbox_targets[pos_bbox_inds] bbox_weight_map = cls_targets.max(dim=1)[0] bbox_weight_map = bbox_weight_map[pos_bbox_inds] bbox_weight_map = bbox_weight_map if self.soft_weight_on_reg \ else torch.ones_like(bbox_weight_map) num_pos_bbox = max(reduce_mean(bbox_weight_map.sum()), 1.0) if len(pos_bbox_inds) > 0: pos_points = flatten_points[pos_bbox_inds] pos_decoded_bbox_preds = self.bbox_coder.decode( pos_points, pos_bbox_preds) pos_decoded_target_preds = self.bbox_coder.decode( pos_points, pos_bbox_targets) bbox_loss = self.loss_bbox( pos_decoded_bbox_preds, pos_decoded_target_preds, weight=bbox_weight_map, avg_factor=num_pos_bbox) else: bbox_loss = flatten_bbox_preds.sum() * 0 return dict(loss_cls=cls_loss, loss_bbox=bbox_loss) def get_targets( self, points: List[Tensor], batch_gt_instances: InstanceList, ) -> Tuple[Tensor, Tensor]: """Compute classification and bbox targets for points in multiple images. Args: points (list[Tensor]): Points of each fpn level, each has shape (num_points, 2). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. Returns: tuple: Targets of each level. - concat_lvl_labels (Tensor): Labels of all level and batch. - concat_lvl_bbox_targets (Tensor): BBox targets of all \ level and batch. """ assert len(points) == len(self.regress_ranges) num_levels = len(points) # the number of points per img, per lvl num_points = [center.size(0) for center in points] # expand regress ranges to align with points expanded_regress_ranges = [ points[i].new_tensor(self.regress_ranges[i])[None].expand_as( points[i]) for i in range(num_levels) ] # concat all levels points and regress ranges concat_regress_ranges = torch.cat(expanded_regress_ranges, dim=0) concat_points = torch.cat(points, dim=0) concat_strides = torch.cat([ concat_points.new_ones(num_points[i]) * self.strides[i] for i in range(num_levels) ]) # get labels and bbox_targets of each image cls_targets_list, bbox_targets_list = multi_apply( self._get_targets_single, batch_gt_instances, points=concat_points, regress_ranges=concat_regress_ranges, strides=concat_strides) bbox_targets_list = _transpose(bbox_targets_list, num_points) cls_targets_list = _transpose(cls_targets_list, num_points) concat_lvl_bbox_targets = torch.cat(bbox_targets_list, 0) concat_lvl_cls_targets = torch.cat(cls_targets_list, dim=0) return concat_lvl_cls_targets, concat_lvl_bbox_targets def _get_targets_single(self, gt_instances: InstanceData, points: Tensor, regress_ranges: Tensor, strides: Tensor) -> Tuple[Tensor, Tensor]: """Compute classification and bbox targets for a single image.""" num_points = points.size(0) num_gts = len(gt_instances) gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels if num_gts == 0: return gt_labels.new_full((num_points, self.num_classes), self.num_classes), \ gt_bboxes.new_full((num_points, 4), -1) # Calculate the regression tblr target corresponding to all points points = points[:, None].expand(num_points, num_gts, 2) gt_bboxes = gt_bboxes[None].expand(num_points, num_gts, 4) strides = strides[:, None, None].expand(num_points, num_gts, 2) bbox_target = bbox2distance(points, gt_bboxes) # M x N x 4 # condition1: inside a gt bbox inside_gt_bbox_mask = bbox_target.min(dim=2)[0] > 0 # M x N # condition2: Calculate the nearest points from # the upper, lower, left and right ranges from # the center of the gt bbox centers = ((gt_bboxes[..., [0, 1]] + gt_bboxes[..., [2, 3]]) / 2) centers_discret = ((centers / strides).int() * strides).float() + \ strides / 2 centers_discret_dist = points - centers_discret dist_x = centers_discret_dist[..., 0].abs() dist_y = centers_discret_dist[..., 1].abs() inside_gt_center3x3_mask = (dist_x <= strides[..., 0]) & \ (dist_y <= strides[..., 0]) # condition3： limit the regression range for each location bbox_target_wh = bbox_target[..., :2] + bbox_target[..., 2:] crit = (bbox_target_wh**2).sum(dim=2)**0.5 / 2 inside_fpn_level_mask = (crit >= regress_ranges[:, [0]]) & \ (crit <= regress_ranges[:, [1]]) bbox_target_mask = inside_gt_bbox_mask & \ inside_gt_center3x3_mask & \ inside_fpn_level_mask # Calculate the distance weight map gt_center_peak_mask = ((centers_discret_dist**2).sum(dim=2) == 0) weighted_dist = ((points - centers)**2).sum(dim=2) # M x N weighted_dist[gt_center_peak_mask] = 0 areas = (gt_bboxes[..., 2] - gt_bboxes[..., 0]) * ( gt_bboxes[..., 3] - gt_bboxes[..., 1]) radius = self.delta**2 * 2 * areas radius = torch.clamp(radius, min=self.hm_min_radius**2) weighted_dist = weighted_dist / radius # Calculate bbox_target bbox_weighted_dist = weighted_dist.clone() bbox_weighted_dist[bbox_target_mask == 0] = INF * 1.0 min_dist, min_inds = bbox_weighted_dist.min(dim=1) bbox_target = bbox_target[range(len(bbox_target)), min_inds] # M x N x 4 --> M x 4 bbox_target[min_dist == INF] = -INF # Convert to feature map scale bbox_target /= strides[:, 0, :].repeat(1, 2) # Calculate cls_target cls_target = self._create_heatmaps_from_dist(weighted_dist, gt_labels) return cls_target, bbox_target @torch.no_grad() def add_cls_pos_inds( self, flatten_points: Tensor, flatten_bbox_preds: Tensor, featmap_sizes: Tensor, batch_gt_instances: InstanceList ) -> Tuple[Optional[Tensor], Optional[Tensor]]: """Provide additional adaptive positive samples to the classification branch. Args: flatten_points (Tensor): The point after flatten, including batch image and all levels. The shape is (N, 2). flatten_bbox_preds (Tensor): The bbox predicts after flatten, including batch image and all levels. The shape is (N, 4). featmap_sizes (Tensor): Feature map size of all layers. The shape is (5, 2). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. Returns: tuple: - pos_inds (Tensor): Adaptively selected positive sample index. - cls_labels (Tensor): Corresponding positive class label. """ outputs = self._get_center3x3_region_index_targets( batch_gt_instances, featmap_sizes) cls_labels, fpn_level_masks, center3x3_inds, \ center3x3_bbox_targets, center3x3_masks = outputs num_gts, total_level, K = cls_labels.shape[0], len( self.strides), center3x3_masks.shape[-1] if num_gts == 0: return None, None # The out-of-bounds index is forcibly set to 0 # to prevent loss calculation errors center3x3_inds[center3x3_masks == 0] = 0 reg_pred_center3x3 = flatten_bbox_preds[center3x3_inds] center3x3_points = flatten_points[center3x3_inds].view(-1, 2) center3x3_bbox_targets_expand = center3x3_bbox_targets.view( -1, 4).clamp(min=0) pos_decoded_bbox_preds = self.bbox_coder.decode( center3x3_points, reg_pred_center3x3.view(-1, 4)) pos_decoded_target_preds = self.bbox_coder.decode( center3x3_points, center3x3_bbox_targets_expand) center3x3_bbox_loss = self.loss_bbox( pos_decoded_bbox_preds, pos_decoded_target_preds, None, reduction_override='none').view(num_gts, total_level, K) / self.loss_bbox.loss_weight # Invalid index Loss set to infinity center3x3_bbox_loss[center3x3_masks == 0] = INF # 4 is the center point of the sampled 9 points, the center point # of gt bbox after discretization. # The center point of gt bbox after discretization # must be a positive sample, so we force its loss to be set to 0. center3x3_bbox_loss.view(-1, K)[fpn_level_masks.view(-1), 4] = 0 center3x3_bbox_loss = center3x3_bbox_loss.view(num_gts, -1) loss_thr = torch.kthvalue( center3x3_bbox_loss, self.more_pos_topk, dim=1)[0] loss_thr[loss_thr > self.more_pos_thresh] = self.more_pos_thresh new_pos = center3x3_bbox_loss < loss_thr.view(num_gts, 1) pos_inds = center3x3_inds.view(num_gts, -1)[new_pos] cls_labels = cls_labels.view(num_gts, 1).expand(num_gts, total_level * K)[new_pos] return pos_inds, cls_labels def _create_heatmaps_from_dist(self, weighted_dist: Tensor, cls_labels: Tensor) -> Tensor: """Generate heatmaps of classification branch based on weighted distance map.""" heatmaps = weighted_dist.new_zeros( (weighted_dist.shape[0], self.num_classes)) for c in range(self.num_classes): inds = (cls_labels == c) # N if inds.int().sum() == 0: continue heatmaps[:, c] = torch.exp(-weighted_dist[:, inds].min(dim=1)[0]) zeros = heatmaps[:, c] < 1e-4 heatmaps[zeros, c] = 0 return heatmaps def _get_center3x3_region_index_targets(self, bacth_gt_instances: InstanceList, shapes_per_level: Tensor) -> tuple: """Get the center (and the 3x3 region near center) locations and target of each objects.""" cls_labels = [] inside_fpn_level_masks = [] center3x3_inds = [] center3x3_masks = [] center3x3_bbox_targets = [] total_levels = len(self.strides) batch = len(bacth_gt_instances) shapes_per_level = shapes_per_level.long() area_per_level = (shapes_per_level[:, 0] * shapes_per_level[:, 1]) # Select a total of 9 positions of 3x3 in the center of the gt bbox # as candidate positive samples K = 9 dx = shapes_per_level.new_tensor([-1, 0, 1, -1, 0, 1, -1, 0, 1]).view(1, 1, K) dy = shapes_per_level.new_tensor([-1, -1, -1, 0, 0, 0, 1, 1, 1]).view(1, 1, K) regress_ranges = shapes_per_level.new_tensor(self.regress_ranges).view( len(self.regress_ranges), 2) # L x 2 strides = shapes_per_level.new_tensor(self.strides) start_coord_pre_level = [] _start = 0 for level in range(total_levels): start_coord_pre_level.append(_start) _start = _start + batch * area_per_level[level] start_coord_pre_level = shapes_per_level.new_tensor( start_coord_pre_level).view(1, total_levels, 1) area_per_level = area_per_level.view(1, total_levels, 1) for im_i in range(batch): gt_instance = bacth_gt_instances[im_i] gt_bboxes = gt_instance.bboxes gt_labels = gt_instance.labels num_gts = gt_bboxes.shape[0] if num_gts == 0: continue cls_labels.append(gt_labels) gt_bboxes = gt_bboxes[:, None].expand(num_gts, total_levels, 4) expanded_strides = strides[None, :, None].expand(num_gts, total_levels, 2) expanded_regress_ranges = regress_ranges[None].expand( num_gts, total_levels, 2) expanded_shapes_per_level = shapes_per_level[None].expand( num_gts, total_levels, 2) # calc reg_target centers = ((gt_bboxes[..., [0, 1]] + gt_bboxes[..., [2, 3]]) / 2) centers_inds = (centers / expanded_strides).long() centers_discret = centers_inds * expanded_strides \ + expanded_strides // 2 bbox_target = bbox2distance(centers_discret, gt_bboxes) # M x N x 4 # calc inside_fpn_level_mask bbox_target_wh = bbox_target[..., :2] + bbox_target[..., 2:] crit = (bbox_target_wh**2).sum(dim=2)**0.5 / 2 inside_fpn_level_mask = \ (crit >= expanded_regress_ranges[..., 0]) & \ (crit <= expanded_regress_ranges[..., 1]) inside_gt_bbox_mask = bbox_target.min(dim=2)[0] >= 0 inside_fpn_level_mask = inside_gt_bbox_mask & inside_fpn_level_mask inside_fpn_level_masks.append(inside_fpn_level_mask) # calc center3x3_ind and mask expand_ws = expanded_shapes_per_level[..., 1:2].expand( num_gts, total_levels, K) expand_hs = expanded_shapes_per_level[..., 0:1].expand( num_gts, total_levels, K) centers_inds_x = centers_inds[..., 0:1] centers_inds_y = centers_inds[..., 1:2] center3x3_idx = start_coord_pre_level + \ im_i * area_per_level + \ (centers_inds_y + dy) * expand_ws + \ (centers_inds_x + dx) center3x3_mask = \ ((centers_inds_y + dy) < expand_hs) & \ ((centers_inds_y + dy) >= 0) & \ ((centers_inds_x + dx) < expand_ws) & \ ((centers_inds_x + dx) >= 0) # recalc center3x3 region reg target bbox_target = bbox_target / expanded_strides.repeat(1, 1, 2) center3x3_bbox_target = bbox_target[..., None, :].expand( num_gts, total_levels, K, 4).clone() center3x3_bbox_target[..., 0] += dx center3x3_bbox_target[..., 1] += dy center3x3_bbox_target[..., 2] -= dx center3x3_bbox_target[..., 3] -= dy # update center3x3_mask center3x3_mask = center3x3_mask & ( center3x3_bbox_target.min(dim=3)[0] >= 0) # n x L x K center3x3_inds.append(center3x3_idx) center3x3_masks.append(center3x3_mask) center3x3_bbox_targets.append(center3x3_bbox_target) if len(inside_fpn_level_masks) > 0: cls_labels = torch.cat(cls_labels, dim=0) inside_fpn_level_masks = torch.cat(inside_fpn_level_masks, dim=0) center3x3_inds = torch.cat(center3x3_inds, dim=0).long() center3x3_bbox_targets = torch.cat(center3x3_bbox_targets, dim=0) center3x3_masks = torch.cat(center3x3_masks, dim=0) else: cls_labels = shapes_per_level.new_zeros(0).long() inside_fpn_level_masks = shapes_per_level.new_zeros( (0, total_levels)).bool() center3x3_inds = shapes_per_level.new_zeros( (0, total_levels, K)).long() center3x3_bbox_targets = shapes_per_level.new_zeros( (0, total_levels, K, 4)).float() center3x3_masks = shapes_per_level.new_zeros( (0, total_levels, K)).bool() return cls_labels, inside_fpn_level_masks, center3x3_inds, \ center3x3_bbox_targets, center3x3_masks ================================================ FILE: mmdet/models/dense_heads/centripetal_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch.nn as nn from mmcv.cnn import ConvModule from mmcv.ops import DeformConv2d from mmengine.model import normal_init from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import (ConfigType, InstanceList, OptInstanceList, OptMultiConfig) from ..utils import multi_apply from .corner_head import CornerHead @MODELS.register_module() class CentripetalHead(CornerHead): """Head of CentripetalNet: Pursuing High-quality Keypoint Pairs for Object Detection. CentripetalHead inherits from :class:`CornerHead`. It removes the embedding branch and adds guiding shift and centripetal shift branches. More details can be found in the `paper `_ . Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. num_feat_levels (int): Levels of feature from the previous module. 2 for HourglassNet-104 and 1 for HourglassNet-52. HourglassNet-104 outputs the final feature and intermediate supervision feature and HourglassNet-52 only outputs the final feature. Defaults to 2. corner_emb_channels (int): Channel of embedding vector. Defaults to 1. train_cfg (:obj:`ConfigDict` or dict, optional): Training config. Useless in CornerHead, but we keep this variable for SingleStageDetector. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of CornerHead. loss_heatmap (:obj:`ConfigDict` or dict): Config of corner heatmap loss. Defaults to GaussianFocalLoss. loss_embedding (:obj:`ConfigDict` or dict): Config of corner embedding loss. Defaults to AssociativeEmbeddingLoss. loss_offset (:obj:`ConfigDict` or dict): Config of corner offset loss. Defaults to SmoothL1Loss. loss_guiding_shift (:obj:`ConfigDict` or dict): Config of guiding shift loss. Defaults to SmoothL1Loss. loss_centripetal_shift (:obj:`ConfigDict` or dict): Config of centripetal shift loss. Defaults to SmoothL1Loss. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. """ def __init__(self, *args, centripetal_shift_channels: int = 2, guiding_shift_channels: int = 2, feat_adaption_conv_kernel: int = 3, loss_guiding_shift: ConfigType = dict( type='SmoothL1Loss', beta=1.0, loss_weight=0.05), loss_centripetal_shift: ConfigType = dict( type='SmoothL1Loss', beta=1.0, loss_weight=1), init_cfg: OptMultiConfig = None, **kwargs) -> None: assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' assert centripetal_shift_channels == 2, ( 'CentripetalHead only support centripetal_shift_channels == 2') self.centripetal_shift_channels = centripetal_shift_channels assert guiding_shift_channels == 2, ( 'CentripetalHead only support guiding_shift_channels == 2') self.guiding_shift_channels = guiding_shift_channels self.feat_adaption_conv_kernel = feat_adaption_conv_kernel super().__init__(*args, init_cfg=init_cfg, **kwargs) self.loss_guiding_shift = MODELS.build(loss_guiding_shift) self.loss_centripetal_shift = MODELS.build(loss_centripetal_shift) def _init_centripetal_layers(self) -> None: """Initialize centripetal layers. Including feature adaption deform convs (feat_adaption), deform offset prediction convs (dcn_off), guiding shift (guiding_shift) and centripetal shift ( centripetal_shift). Each branch has two parts: prefix `tl_` for top-left and `br_` for bottom-right. """ self.tl_feat_adaption = nn.ModuleList() self.br_feat_adaption = nn.ModuleList() self.tl_dcn_offset = nn.ModuleList() self.br_dcn_offset = nn.ModuleList() self.tl_guiding_shift = nn.ModuleList() self.br_guiding_shift = nn.ModuleList() self.tl_centripetal_shift = nn.ModuleList() self.br_centripetal_shift = nn.ModuleList() for _ in range(self.num_feat_levels): self.tl_feat_adaption.append( DeformConv2d(self.in_channels, self.in_channels, self.feat_adaption_conv_kernel, 1, 1)) self.br_feat_adaption.append( DeformConv2d(self.in_channels, self.in_channels, self.feat_adaption_conv_kernel, 1, 1)) self.tl_guiding_shift.append( self._make_layers( out_channels=self.guiding_shift_channels, in_channels=self.in_channels)) self.br_guiding_shift.append( self._make_layers( out_channels=self.guiding_shift_channels, in_channels=self.in_channels)) self.tl_dcn_offset.append( ConvModule( self.guiding_shift_channels, self.feat_adaption_conv_kernel**2 * self.guiding_shift_channels, 1, bias=False, act_cfg=None)) self.br_dcn_offset.append( ConvModule( self.guiding_shift_channels, self.feat_adaption_conv_kernel**2 * self.guiding_shift_channels, 1, bias=False, act_cfg=None)) self.tl_centripetal_shift.append( self._make_layers( out_channels=self.centripetal_shift_channels, in_channels=self.in_channels)) self.br_centripetal_shift.append( self._make_layers( out_channels=self.centripetal_shift_channels, in_channels=self.in_channels)) def _init_layers(self) -> None: """Initialize layers for CentripetalHead. Including two parts: CornerHead layers and CentripetalHead layers """ super()._init_layers() # using _init_layers in CornerHead self._init_centripetal_layers() def init_weights(self) -> None: super().init_weights() for i in range(self.num_feat_levels): normal_init(self.tl_feat_adaption[i], std=0.01) normal_init(self.br_feat_adaption[i], std=0.01) normal_init(self.tl_dcn_offset[i].conv, std=0.1) normal_init(self.br_dcn_offset[i].conv, std=0.1) _ = [x.conv.reset_parameters() for x in self.tl_guiding_shift[i]] _ = [x.conv.reset_parameters() for x in self.br_guiding_shift[i]] _ = [ x.conv.reset_parameters() for x in self.tl_centripetal_shift[i] ] _ = [ x.conv.reset_parameters() for x in self.br_centripetal_shift[i] ] def forward_single(self, x: Tensor, lvl_ind: int) -> List[Tensor]: """Forward feature of a single level. Args: x (Tensor): Feature of a single level. lvl_ind (int): Level index of current feature. Returns: tuple[Tensor]: A tuple of CentripetalHead's output for current feature level. Containing the following Tensors: - tl_heat (Tensor): Predicted top-left corner heatmap. - br_heat (Tensor): Predicted bottom-right corner heatmap. - tl_off (Tensor): Predicted top-left offset heatmap. - br_off (Tensor): Predicted bottom-right offset heatmap. - tl_guiding_shift (Tensor): Predicted top-left guiding shift heatmap. - br_guiding_shift (Tensor): Predicted bottom-right guiding shift heatmap. - tl_centripetal_shift (Tensor): Predicted top-left centripetal shift heatmap. - br_centripetal_shift (Tensor): Predicted bottom-right centripetal shift heatmap. """ tl_heat, br_heat, _, _, tl_off, br_off, tl_pool, br_pool = super( ).forward_single( x, lvl_ind, return_pool=True) tl_guiding_shift = self.tl_guiding_shift[lvl_ind](tl_pool) br_guiding_shift = self.br_guiding_shift[lvl_ind](br_pool) tl_dcn_offset = self.tl_dcn_offset[lvl_ind](tl_guiding_shift.detach()) br_dcn_offset = self.br_dcn_offset[lvl_ind](br_guiding_shift.detach()) tl_feat_adaption = self.tl_feat_adaption[lvl_ind](tl_pool, tl_dcn_offset) br_feat_adaption = self.br_feat_adaption[lvl_ind](br_pool, br_dcn_offset) tl_centripetal_shift = self.tl_centripetal_shift[lvl_ind]( tl_feat_adaption) br_centripetal_shift = self.br_centripetal_shift[lvl_ind]( br_feat_adaption) result_list = [ tl_heat, br_heat, tl_off, br_off, tl_guiding_shift, br_guiding_shift, tl_centripetal_shift, br_centripetal_shift ] return result_list def loss_by_feat( self, tl_heats: List[Tensor], br_heats: List[Tensor], tl_offs: List[Tensor], br_offs: List[Tensor], tl_guiding_shifts: List[Tensor], br_guiding_shifts: List[Tensor], tl_centripetal_shifts: List[Tensor], br_centripetal_shifts: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: tl_heats (list[Tensor]): Top-left corner heatmaps for each level with shape (N, num_classes, H, W). br_heats (list[Tensor]): Bottom-right corner heatmaps for each level with shape (N, num_classes, H, W). tl_offs (list[Tensor]): Top-left corner offsets for each level with shape (N, corner_offset_channels, H, W). br_offs (list[Tensor]): Bottom-right corner offsets for each level with shape (N, corner_offset_channels, H, W). tl_guiding_shifts (list[Tensor]): Top-left guiding shifts for each level with shape (N, guiding_shift_channels, H, W). br_guiding_shifts (list[Tensor]): Bottom-right guiding shifts for each level with shape (N, guiding_shift_channels, H, W). tl_centripetal_shifts (list[Tensor]): Top-left centripetal shifts for each level with shape (N, centripetal_shift_channels, H, W). br_centripetal_shifts (list[Tensor]): Bottom-right centripetal shifts for each level with shape (N, centripetal_shift_channels, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Specify which bounding boxes can be ignored when computing the loss. Returns: dict[str, Tensor]: A dictionary of loss components. Containing the following losses: - det_loss (list[Tensor]): Corner keypoint losses of all feature levels. - off_loss (list[Tensor]): Corner offset losses of all feature levels. - guiding_loss (list[Tensor]): Guiding shift losses of all feature levels. - centripetal_loss (list[Tensor]): Centripetal shift losses of all feature levels. """ gt_bboxes = [ gt_instances.bboxes for gt_instances in batch_gt_instances ] gt_labels = [ gt_instances.labels for gt_instances in batch_gt_instances ] targets = self.get_targets( gt_bboxes, gt_labels, tl_heats[-1].shape, batch_img_metas[0]['batch_input_shape'], with_corner_emb=self.with_corner_emb, with_guiding_shift=True, with_centripetal_shift=True) mlvl_targets = [targets for _ in range(self.num_feat_levels)] [det_losses, off_losses, guiding_losses, centripetal_losses ] = multi_apply(self.loss_by_feat_single, tl_heats, br_heats, tl_offs, br_offs, tl_guiding_shifts, br_guiding_shifts, tl_centripetal_shifts, br_centripetal_shifts, mlvl_targets) loss_dict = dict( det_loss=det_losses, off_loss=off_losses, guiding_loss=guiding_losses, centripetal_loss=centripetal_losses) return loss_dict def loss_by_feat_single(self, tl_hmp: Tensor, br_hmp: Tensor, tl_off: Tensor, br_off: Tensor, tl_guiding_shift: Tensor, br_guiding_shift: Tensor, tl_centripetal_shift: Tensor, br_centripetal_shift: Tensor, targets: dict) -> Tuple[Tensor, ...]: """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: tl_hmp (Tensor): Top-left corner heatmap for current level with shape (N, num_classes, H, W). br_hmp (Tensor): Bottom-right corner heatmap for current level with shape (N, num_classes, H, W). tl_off (Tensor): Top-left corner offset for current level with shape (N, corner_offset_channels, H, W). br_off (Tensor): Bottom-right corner offset for current level with shape (N, corner_offset_channels, H, W). tl_guiding_shift (Tensor): Top-left guiding shift for current level with shape (N, guiding_shift_channels, H, W). br_guiding_shift (Tensor): Bottom-right guiding shift for current level with shape (N, guiding_shift_channels, H, W). tl_centripetal_shift (Tensor): Top-left centripetal shift for current level with shape (N, centripetal_shift_channels, H, W). br_centripetal_shift (Tensor): Bottom-right centripetal shift for current level with shape (N, centripetal_shift_channels, H, W). targets (dict): Corner target generated by `get_targets`. Returns: tuple[torch.Tensor]: Losses of the head's different branches containing the following losses: - det_loss (Tensor): Corner keypoint loss. - off_loss (Tensor): Corner offset loss. - guiding_loss (Tensor): Guiding shift loss. - centripetal_loss (Tensor): Centripetal shift loss. """ targets['corner_embedding'] = None det_loss, _, _, off_loss = super().loss_by_feat_single( tl_hmp, br_hmp, None, None, tl_off, br_off, targets) gt_tl_guiding_shift = targets['topleft_guiding_shift'] gt_br_guiding_shift = targets['bottomright_guiding_shift'] gt_tl_centripetal_shift = targets['topleft_centripetal_shift'] gt_br_centripetal_shift = targets['bottomright_centripetal_shift'] gt_tl_heatmap = targets['topleft_heatmap'] gt_br_heatmap = targets['bottomright_heatmap'] # We only compute the offset loss at the real corner position. # The value of real corner would be 1 in heatmap ground truth. # The mask is computed in class agnostic mode and its shape is # batch * 1 * width * height. tl_mask = gt_tl_heatmap.eq(1).sum(1).gt(0).unsqueeze(1).type_as( gt_tl_heatmap) br_mask = gt_br_heatmap.eq(1).sum(1).gt(0).unsqueeze(1).type_as( gt_br_heatmap) # Guiding shift loss tl_guiding_loss = self.loss_guiding_shift( tl_guiding_shift, gt_tl_guiding_shift, tl_mask, avg_factor=tl_mask.sum()) br_guiding_loss = self.loss_guiding_shift( br_guiding_shift, gt_br_guiding_shift, br_mask, avg_factor=br_mask.sum()) guiding_loss = (tl_guiding_loss + br_guiding_loss) / 2.0 # Centripetal shift loss tl_centripetal_loss = self.loss_centripetal_shift( tl_centripetal_shift, gt_tl_centripetal_shift, tl_mask, avg_factor=tl_mask.sum()) br_centripetal_loss = self.loss_centripetal_shift( br_centripetal_shift, gt_br_centripetal_shift, br_mask, avg_factor=br_mask.sum()) centripetal_loss = (tl_centripetal_loss + br_centripetal_loss) / 2.0 return det_loss, off_loss, guiding_loss, centripetal_loss def predict_by_feat(self, tl_heats: List[Tensor], br_heats: List[Tensor], tl_offs: List[Tensor], br_offs: List[Tensor], tl_guiding_shifts: List[Tensor], br_guiding_shifts: List[Tensor], tl_centripetal_shifts: List[Tensor], br_centripetal_shifts: List[Tensor], batch_img_metas: Optional[List[dict]] = None, rescale: bool = False, with_nms: bool = True) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. Args: tl_heats (list[Tensor]): Top-left corner heatmaps for each level with shape (N, num_classes, H, W). br_heats (list[Tensor]): Bottom-right corner heatmaps for each level with shape (N, num_classes, H, W). tl_offs (list[Tensor]): Top-left corner offsets for each level with shape (N, corner_offset_channels, H, W). br_offs (list[Tensor]): Bottom-right corner offsets for each level with shape (N, corner_offset_channels, H, W). tl_guiding_shifts (list[Tensor]): Top-left guiding shifts for each level with shape (N, guiding_shift_channels, H, W). Useless in this function, we keep this arg because it's the raw output from CentripetalHead. br_guiding_shifts (list[Tensor]): Bottom-right guiding shifts for each level with shape (N, guiding_shift_channels, H, W). Useless in this function, we keep this arg because it's the raw output from CentripetalHead. tl_centripetal_shifts (list[Tensor]): Top-left centripetal shifts for each level with shape (N, centripetal_shift_channels, H, W). br_centripetal_shifts (list[Tensor]): Bottom-right centripetal shifts for each level with shape (N, centripetal_shift_channels, H, W). batch_img_metas (list[dict], optional): Batch image meta info. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert tl_heats[-1].shape[0] == br_heats[-1].shape[0] == len( batch_img_metas) result_list = [] for img_id in range(len(batch_img_metas)): result_list.append( self._predict_by_feat_single( tl_heats[-1][img_id:img_id + 1, :], br_heats[-1][img_id:img_id + 1, :], tl_offs[-1][img_id:img_id + 1, :], br_offs[-1][img_id:img_id + 1, :], batch_img_metas[img_id], tl_emb=None, br_emb=None, tl_centripetal_shift=tl_centripetal_shifts[-1][ img_id:img_id + 1, :], br_centripetal_shift=br_centripetal_shifts[-1][ img_id:img_id + 1, :], rescale=rescale, with_nms=with_nms)) return result_list ================================================ FILE: mmdet/models/dense_heads/condinst_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import Dict, List, Optional, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule, Scale from mmengine.config import ConfigDict from mmengine.model import BaseModule, kaiming_init from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures.bbox import cat_boxes from mmdet.utils import (ConfigType, InstanceList, MultiConfig, OptConfigType, OptInstanceList, reduce_mean) from ..task_modules.prior_generators import MlvlPointGenerator from ..utils import (aligned_bilinear, filter_scores_and_topk, multi_apply, relative_coordinate_maps, select_single_mlvl) from ..utils.misc import empty_instances from .base_mask_head import BaseMaskHead from .fcos_head import FCOSHead INF = 1e8 @MODELS.register_module() class CondInstBboxHead(FCOSHead): """CondInst box head used in https://arxiv.org/abs/1904.02689. Note that CondInst Bbox Head is a extension of FCOS head. Two differences are described as follows: 1. CondInst box head predicts a set of params for each instance. 2. CondInst box head return the pos_gt_inds and pos_inds. Args: num_params (int): Number of params for instance segmentation. """ def __init__(self, *args, num_params: int = 169, **kwargs) -> None: self.num_params = num_params super().__init__(*args, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" super()._init_layers() self.controller = nn.Conv2d( self.feat_channels, self.num_params, 3, padding=1) def forward_single(self, x: Tensor, scale: Scale, stride: int) -> Tuple[Tensor, Tensor, Tensor, Tensor]: """Forward features of a single scale level. Args: x (Tensor): FPN feature maps of the specified stride. scale (:obj:`mmcv.cnn.Scale`): Learnable scale module to resize the bbox prediction. stride (int): The corresponding stride for feature maps, only used to normalize the bbox prediction when self.norm_on_bbox is True. Returns: tuple: scores for each class, bbox predictions, centerness predictions and param predictions of input feature maps. """ cls_score, bbox_pred, cls_feat, reg_feat = \ super(FCOSHead, self).forward_single(x) if self.centerness_on_reg: centerness = self.conv_centerness(reg_feat) else: centerness = self.conv_centerness(cls_feat) # scale the bbox_pred of different level # float to avoid overflow when enabling FP16 bbox_pred = scale(bbox_pred).float() if self.norm_on_bbox: # bbox_pred needed for gradient computation has been modified # by F.relu(bbox_pred) when run with PyTorch 1.10. So replace # F.relu(bbox_pred) with bbox_pred.clamp(min=0) bbox_pred = bbox_pred.clamp(min=0) if not self.training: bbox_pred *= stride else: bbox_pred = bbox_pred.exp() param_pred = self.controller(reg_feat) return cls_score, bbox_pred, centerness, param_pred def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], centernesses: List[Tensor], param_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level, each is a 4D-tensor, the channel number is num_points * num_classes. bbox_preds (list[Tensor]): Box energies / deltas for each scale level, each is a 4D-tensor, the channel number is num_points * 4. centernesses (list[Tensor]): centerness for each scale level, each is a 4D-tensor, the channel number is num_points * 1. param_preds (List[Tensor]): param_pred for each scale level, each is a 4D-tensor, the channel number is num_params. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert len(cls_scores) == len(bbox_preds) == len(centernesses) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] # Need stride for rel coord compute all_level_points_strides = self.prior_generator.grid_priors( featmap_sizes, dtype=bbox_preds[0].dtype, device=bbox_preds[0].device, with_stride=True) all_level_points = [i[:, :2] for i in all_level_points_strides] all_level_strides = [i[:, 2] for i in all_level_points_strides] labels, bbox_targets, pos_inds_list, pos_gt_inds_list = \ self.get_targets(all_level_points, batch_gt_instances) num_imgs = cls_scores[0].size(0) # flatten cls_scores, bbox_preds and centerness flatten_cls_scores = [ cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) for cls_score in cls_scores ] flatten_bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) for bbox_pred in bbox_preds ] flatten_centerness = [ centerness.permute(0, 2, 3, 1).reshape(-1) for centerness in centernesses ] flatten_cls_scores = torch.cat(flatten_cls_scores) flatten_bbox_preds = torch.cat(flatten_bbox_preds) flatten_centerness = torch.cat(flatten_centerness) flatten_labels = torch.cat(labels) flatten_bbox_targets = torch.cat(bbox_targets) # repeat points to align with bbox_preds flatten_points = torch.cat( [points.repeat(num_imgs, 1) for points in all_level_points]) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = self.num_classes pos_inds = ((flatten_labels >= 0) & (flatten_labels < bg_class_ind)).nonzero().reshape(-1) num_pos = torch.tensor( len(pos_inds), dtype=torch.float, device=bbox_preds[0].device) num_pos = max(reduce_mean(num_pos), 1.0) loss_cls = self.loss_cls( flatten_cls_scores, flatten_labels, avg_factor=num_pos) pos_bbox_preds = flatten_bbox_preds[pos_inds] pos_centerness = flatten_centerness[pos_inds] pos_bbox_targets = flatten_bbox_targets[pos_inds] pos_centerness_targets = self.centerness_target(pos_bbox_targets) # centerness weighted iou loss centerness_denorm = max( reduce_mean(pos_centerness_targets.sum().detach()), 1e-6) if len(pos_inds) > 0: pos_points = flatten_points[pos_inds] pos_decoded_bbox_preds = self.bbox_coder.decode( pos_points, pos_bbox_preds) pos_decoded_target_preds = self.bbox_coder.decode( pos_points, pos_bbox_targets) loss_bbox = self.loss_bbox( pos_decoded_bbox_preds, pos_decoded_target_preds, weight=pos_centerness_targets, avg_factor=centerness_denorm) loss_centerness = self.loss_centerness( pos_centerness, pos_centerness_targets, avg_factor=num_pos) else: loss_bbox = pos_bbox_preds.sum() loss_centerness = pos_centerness.sum() self._raw_positive_infos.update(cls_scores=cls_scores) self._raw_positive_infos.update(centernesses=centernesses) self._raw_positive_infos.update(param_preds=param_preds) self._raw_positive_infos.update(all_level_points=all_level_points) self._raw_positive_infos.update(all_level_strides=all_level_strides) self._raw_positive_infos.update(pos_gt_inds_list=pos_gt_inds_list) self._raw_positive_infos.update(pos_inds_list=pos_inds_list) return dict( loss_cls=loss_cls, loss_bbox=loss_bbox, loss_centerness=loss_centerness) def get_targets( self, points: List[Tensor], batch_gt_instances: InstanceList ) -> Tuple[List[Tensor], List[Tensor], List[Tensor], List[Tensor]]: """Compute regression, classification and centerness targets for points in multiple images. Args: points (list[Tensor]): Points of each fpn level, each has shape (num_points, 2). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. Returns: tuple: Targets of each level. - concat_lvl_labels (list[Tensor]): Labels of each level. - concat_lvl_bbox_targets (list[Tensor]): BBox targets of each \ level. - pos_inds_list (list[Tensor]): pos_inds of each image. - pos_gt_inds_list (List[Tensor]): pos_gt_inds of each image. """ assert len(points) == len(self.regress_ranges) num_levels = len(points) # expand regress ranges to align with points expanded_regress_ranges = [ points[i].new_tensor(self.regress_ranges[i])[None].expand_as( points[i]) for i in range(num_levels) ] # concat all levels points and regress ranges concat_regress_ranges = torch.cat(expanded_regress_ranges, dim=0) concat_points = torch.cat(points, dim=0) # the number of points per img, per lvl num_points = [center.size(0) for center in points] # get labels and bbox_targets of each image labels_list, bbox_targets_list, pos_inds_list, pos_gt_inds_list = \ multi_apply( self._get_targets_single, batch_gt_instances, points=concat_points, regress_ranges=concat_regress_ranges, num_points_per_lvl=num_points) # split to per img, per level labels_list = [labels.split(num_points, 0) for labels in labels_list] bbox_targets_list = [ bbox_targets.split(num_points, 0) for bbox_targets in bbox_targets_list ] # concat per level image concat_lvl_labels = [] concat_lvl_bbox_targets = [] for i in range(num_levels): concat_lvl_labels.append( torch.cat([labels[i] for labels in labels_list])) bbox_targets = torch.cat( [bbox_targets[i] for bbox_targets in bbox_targets_list]) if self.norm_on_bbox: bbox_targets = bbox_targets / self.strides[i] concat_lvl_bbox_targets.append(bbox_targets) return (concat_lvl_labels, concat_lvl_bbox_targets, pos_inds_list, pos_gt_inds_list) def _get_targets_single( self, gt_instances: InstanceData, points: Tensor, regress_ranges: Tensor, num_points_per_lvl: List[int] ) -> Tuple[Tensor, Tensor, Tensor, Tensor]: """Compute regression and classification targets for a single image.""" num_points = points.size(0) num_gts = len(gt_instances) gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels gt_masks = gt_instances.get('masks', None) if num_gts == 0: return gt_labels.new_full((num_points,), self.num_classes), \ gt_bboxes.new_zeros((num_points, 4)), \ gt_bboxes.new_zeros((0,), dtype=torch.int64), \ gt_bboxes.new_zeros((0,), dtype=torch.int64) areas = (gt_bboxes[:, 2] - gt_bboxes[:, 0]) * ( gt_bboxes[:, 3] - gt_bboxes[:, 1]) # TODO: figure out why these two are different # areas = areas[None].expand(num_points, num_gts) areas = areas[None].repeat(num_points, 1) regress_ranges = regress_ranges[:, None, :].expand( num_points, num_gts, 2) gt_bboxes = gt_bboxes[None].expand(num_points, num_gts, 4) xs, ys = points[:, 0], points[:, 1] xs = xs[:, None].expand(num_points, num_gts) ys = ys[:, None].expand(num_points, num_gts) left = xs - gt_bboxes[..., 0] right = gt_bboxes[..., 2] - xs top = ys - gt_bboxes[..., 1] bottom = gt_bboxes[..., 3] - ys bbox_targets = torch.stack((left, top, right, bottom), -1) if self.center_sampling: # condition1: inside a `center bbox` radius = self.center_sample_radius # if gt_mask not None, use gt mask's centroid to determine # the center region rather than gt_bbox center if gt_masks is None: center_xs = (gt_bboxes[..., 0] + gt_bboxes[..., 2]) / 2 center_ys = (gt_bboxes[..., 1] + gt_bboxes[..., 3]) / 2 else: h, w = gt_masks.height, gt_masks.width masks = gt_masks.to_tensor( dtype=torch.bool, device=gt_bboxes.device) yys = torch.arange( 0, h, dtype=torch.float32, device=masks.device) xxs = torch.arange( 0, w, dtype=torch.float32, device=masks.device) # m00/m10/m01 represent the moments of a contour # centroid is computed by m00/m10 and m00/m01 m00 = masks.sum(dim=-1).sum(dim=-1).clamp(min=1e-6) m10 = (masks * xxs).sum(dim=-1).sum(dim=-1) m01 = (masks * yys[:, None]).sum(dim=-1).sum(dim=-1) center_xs = m10 / m00 center_ys = m01 / m00 center_xs = center_xs[None].expand(num_points, num_gts) center_ys = center_ys[None].expand(num_points, num_gts) center_gts = torch.zeros_like(gt_bboxes) stride = center_xs.new_zeros(center_xs.shape) # project the points on current lvl back to the `original` sizes lvl_begin = 0 for lvl_idx, num_points_lvl in enumerate(num_points_per_lvl): lvl_end = lvl_begin + num_points_lvl stride[lvl_begin:lvl_end] = self.strides[lvl_idx] * radius lvl_begin = lvl_end x_mins = center_xs - stride y_mins = center_ys - stride x_maxs = center_xs + stride y_maxs = center_ys + stride center_gts[..., 0] = torch.where(x_mins > gt_bboxes[..., 0], x_mins, gt_bboxes[..., 0]) center_gts[..., 1] = torch.where(y_mins > gt_bboxes[..., 1], y_mins, gt_bboxes[..., 1]) center_gts[..., 2] = torch.where(x_maxs > gt_bboxes[..., 2], gt_bboxes[..., 2], x_maxs) center_gts[..., 3] = torch.where(y_maxs > gt_bboxes[..., 3], gt_bboxes[..., 3], y_maxs) cb_dist_left = xs - center_gts[..., 0] cb_dist_right = center_gts[..., 2] - xs cb_dist_top = ys - center_gts[..., 1] cb_dist_bottom = center_gts[..., 3] - ys center_bbox = torch.stack( (cb_dist_left, cb_dist_top, cb_dist_right, cb_dist_bottom), -1) inside_gt_bbox_mask = center_bbox.min(-1)[0] > 0 else: # condition1: inside a gt bbox inside_gt_bbox_mask = bbox_targets.min(-1)[0] > 0 # condition2: limit the regression range for each location max_regress_distance = bbox_targets.max(-1)[0] inside_regress_range = ( (max_regress_distance >= regress_ranges[..., 0]) & (max_regress_distance <= regress_ranges[..., 1])) # if there are still more than one objects for a location, # we choose the one with minimal area areas[inside_gt_bbox_mask == 0] = INF areas[inside_regress_range == 0] = INF min_area, min_area_inds = areas.min(dim=1) labels = gt_labels[min_area_inds] labels[min_area == INF] = self.num_classes # set as BG bbox_targets = bbox_targets[range(num_points), min_area_inds] # return pos_inds & pos_gt_inds bg_class_ind = self.num_classes pos_inds = ((labels >= 0) & (labels < bg_class_ind)).nonzero().reshape(-1) pos_gt_inds = min_area_inds[labels < self.num_classes] return labels, bbox_targets, pos_inds, pos_gt_inds def get_positive_infos(self) -> InstanceList: """Get positive information from sampling results. Returns: list[:obj:`InstanceData`]: Positive information of each image, usually including positive bboxes, positive labels, positive priors, etc. """ assert len(self._raw_positive_infos) > 0 pos_gt_inds_list = self._raw_positive_infos['pos_gt_inds_list'] pos_inds_list = self._raw_positive_infos['pos_inds_list'] num_imgs = len(pos_gt_inds_list) cls_score_list = [] centerness_list = [] param_pred_list = [] point_list = [] stride_list = [] for cls_score_per_lvl, centerness_per_lvl, param_pred_per_lvl,\ point_per_lvl, stride_per_lvl in \ zip(self._raw_positive_infos['cls_scores'], self._raw_positive_infos['centernesses'], self._raw_positive_infos['param_preds'], self._raw_positive_infos['all_level_points'], self._raw_positive_infos['all_level_strides']): cls_score_per_lvl = \ cls_score_per_lvl.permute( 0, 2, 3, 1).reshape(num_imgs, -1, self.num_classes) centerness_per_lvl = \ centerness_per_lvl.permute( 0, 2, 3, 1).reshape(num_imgs, -1, 1) param_pred_per_lvl = \ param_pred_per_lvl.permute( 0, 2, 3, 1).reshape(num_imgs, -1, self.num_params) point_per_lvl = point_per_lvl.unsqueeze(0).repeat(num_imgs, 1, 1) stride_per_lvl = stride_per_lvl.unsqueeze(0).repeat(num_imgs, 1) cls_score_list.append(cls_score_per_lvl) centerness_list.append(centerness_per_lvl) param_pred_list.append(param_pred_per_lvl) point_list.append(point_per_lvl) stride_list.append(stride_per_lvl) cls_scores = torch.cat(cls_score_list, dim=1) centernesses = torch.cat(centerness_list, dim=1) param_preds = torch.cat(param_pred_list, dim=1) all_points = torch.cat(point_list, dim=1) all_strides = torch.cat(stride_list, dim=1) positive_infos = [] for i, (pos_gt_inds, pos_inds) in enumerate(zip(pos_gt_inds_list, pos_inds_list)): pos_info = InstanceData() pos_info.points = all_points[i][pos_inds] pos_info.strides = all_strides[i][pos_inds] pos_info.scores = cls_scores[i][pos_inds] pos_info.centernesses = centernesses[i][pos_inds] pos_info.param_preds = param_preds[i][pos_inds] pos_info.pos_assigned_gt_inds = pos_gt_inds pos_info.pos_inds = pos_inds positive_infos.append(pos_info) return positive_infos def predict_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], score_factors: Optional[List[Tensor]] = None, param_preds: Optional[List[Tensor]] = None, batch_img_metas: Optional[List[dict]] = None, cfg: Optional[ConfigDict] = None, rescale: bool = False, with_nms: bool = True) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. Note: When score_factors is not None, the cls_scores are usually multiplied by it then obtain the real score used in NMS, such as CenterNess in FCOS, IoU branch in ATSS. Args: cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * 4, H, W). score_factors (list[Tensor], optional): Score factor for all scale level, each is a 4D-tensor, has shape (batch_size, num_priors * 1, H, W). Defaults to None. param_preds (list[Tensor], optional): Params for all scale level, each is a 4D-tensor, has shape (batch_size, num_priors * num_params, H, W) batch_img_metas (list[dict], Optional): Batch image meta info. Defaults to None. cfg (ConfigDict, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(cls_scores) == len(bbox_preds) if score_factors is None: # e.g. Retina, FreeAnchor, Foveabox, etc. with_score_factors = False else: # e.g. FCOS, PAA, ATSS, AutoAssign, etc. with_score_factors = True assert len(cls_scores) == len(score_factors) num_levels = len(cls_scores) featmap_sizes = [cls_scores[i].shape[-2:] for i in range(num_levels)] all_level_points_strides = self.prior_generator.grid_priors( featmap_sizes, dtype=bbox_preds[0].dtype, device=bbox_preds[0].device, with_stride=True) all_level_points = [i[:, :2] for i in all_level_points_strides] all_level_strides = [i[:, 2] for i in all_level_points_strides] result_list = [] for img_id in range(len(batch_img_metas)): img_meta = batch_img_metas[img_id] cls_score_list = select_single_mlvl( cls_scores, img_id, detach=True) bbox_pred_list = select_single_mlvl( bbox_preds, img_id, detach=True) if with_score_factors: score_factor_list = select_single_mlvl( score_factors, img_id, detach=True) else: score_factor_list = [None for _ in range(num_levels)] param_pred_list = select_single_mlvl( param_preds, img_id, detach=True) results = self._predict_by_feat_single( cls_score_list=cls_score_list, bbox_pred_list=bbox_pred_list, score_factor_list=score_factor_list, param_pred_list=param_pred_list, mlvl_points=all_level_points, mlvl_strides=all_level_strides, img_meta=img_meta, cfg=cfg, rescale=rescale, with_nms=with_nms) result_list.append(results) return result_list def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], score_factor_list: List[Tensor], param_pred_list: List[Tensor], mlvl_points: List[Tensor], mlvl_strides: List[Tensor], img_meta: dict, cfg: ConfigDict, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: cls_score_list (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). score_factor_list (list[Tensor]): Score factor from all scale levels of a single image, each item has shape (num_priors * 1, H, W). param_pred_list (List[Tensor]): Param predition from all scale levels of a single image, each item has shape (num_priors * num_params, H, W). mlvl_points (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid. It has shape (num_priors, 2) mlvl_strides (List[Tensor]): Each element in the list is the stride of a single level in feature pyramid. It has shape (num_priors, 1) img_meta (dict): Image meta info. cfg (mmengine.Config): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ if score_factor_list[0] is None: # e.g. Retina, FreeAnchor, etc. with_score_factors = False else: # e.g. FCOS, PAA, ATSS, etc. with_score_factors = True cfg = self.test_cfg if cfg is None else cfg cfg = copy.deepcopy(cfg) img_shape = img_meta['img_shape'] nms_pre = cfg.get('nms_pre', -1) mlvl_bbox_preds = [] mlvl_param_preds = [] mlvl_valid_points = [] mlvl_valid_strides = [] mlvl_scores = [] mlvl_labels = [] if with_score_factors: mlvl_score_factors = [] else: mlvl_score_factors = None for level_idx, (cls_score, bbox_pred, score_factor, param_pred, points, strides) in \ enumerate(zip(cls_score_list, bbox_pred_list, score_factor_list, param_pred_list, mlvl_points, mlvl_strides)): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] dim = self.bbox_coder.encode_size bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, dim) if with_score_factors: score_factor = score_factor.permute(1, 2, 0).reshape(-1).sigmoid() cls_score = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) if self.use_sigmoid_cls: scores = cls_score.sigmoid() else: # remind that we set FG labels to [0, num_class-1] # since mmdet v2.0 # BG cat_id: num_class scores = cls_score.softmax(-1)[:, :-1] param_pred = param_pred.permute(1, 2, 0).reshape(-1, self.num_params) # After https://github.com/open-mmlab/mmdetection/pull/6268/, # this operation keeps fewer bboxes under the same `nms_pre`. # There is no difference in performance for most models. If you # find a slight drop in performance, you can set a larger # `nms_pre` than before. score_thr = cfg.get('score_thr', 0) results = filter_scores_and_topk( scores, score_thr, nms_pre, dict( bbox_pred=bbox_pred, param_pred=param_pred, points=points, strides=strides)) scores, labels, keep_idxs, filtered_results = results bbox_pred = filtered_results['bbox_pred'] param_pred = filtered_results['param_pred'] points = filtered_results['points'] strides = filtered_results['strides'] if with_score_factors: score_factor = score_factor[keep_idxs] mlvl_bbox_preds.append(bbox_pred) mlvl_param_preds.append(param_pred) mlvl_valid_points.append(points) mlvl_valid_strides.append(strides) mlvl_scores.append(scores) mlvl_labels.append(labels) if with_score_factors: mlvl_score_factors.append(score_factor) bbox_pred = torch.cat(mlvl_bbox_preds) priors = cat_boxes(mlvl_valid_points) bboxes = self.bbox_coder.decode(priors, bbox_pred, max_shape=img_shape) results = InstanceData() results.bboxes = bboxes results.scores = torch.cat(mlvl_scores) results.labels = torch.cat(mlvl_labels) results.param_preds = torch.cat(mlvl_param_preds) results.points = torch.cat(mlvl_valid_points) results.strides = torch.cat(mlvl_valid_strides) if with_score_factors: results.score_factors = torch.cat(mlvl_score_factors) return self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta) class MaskFeatModule(BaseModule): """CondInst mask feature map branch used in \ https://arxiv.org/abs/1904.02689. Args: in_channels (int): Number of channels in the input feature map. feat_channels (int): Number of hidden channels of the mask feature map branch. start_level (int): The starting feature map level from RPN that will be used to predict the mask feature map. end_level (int): The ending feature map level from rpn that will be used to predict the mask feature map. out_channels (int): Number of output channels of the mask feature map branch. This is the channel count of the mask feature map that to be dynamically convolved with the predicted kernel. mask_stride (int): Downsample factor of the mask feature map output. Defaults to 4. num_stacked_convs (int): Number of convs in mask feature branch. conv_cfg (dict): Config dict for convolution layer. Default: None. norm_cfg (dict): Config dict for normalization layer. Default: None. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, in_channels: int, feat_channels: int, start_level: int, end_level: int, out_channels: int, mask_stride: int = 4, num_stacked_convs: int = 4, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, init_cfg: MultiConfig = [ dict(type='Normal', layer='Conv2d', std=0.01) ], **kwargs) -> None: super().__init__(init_cfg=init_cfg) self.in_channels = in_channels self.feat_channels = feat_channels self.start_level = start_level self.end_level = end_level self.mask_stride = mask_stride self.num_stacked_convs = num_stacked_convs assert start_level >= 0 and end_level >= start_level self.out_channels = out_channels self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self._init_layers() def _init_layers(self) -> None: """Initialize layers of the head.""" self.convs_all_levels = nn.ModuleList() for i in range(self.start_level, self.end_level + 1): convs_per_level = nn.Sequential() convs_per_level.add_module( f'conv{i}', ConvModule( self.in_channels, self.feat_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, inplace=False, bias=False)) self.convs_all_levels.append(convs_per_level) conv_branch = [] for _ in range(self.num_stacked_convs): conv_branch.append( ConvModule( self.feat_channels, self.feat_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, bias=False)) self.conv_branch = nn.Sequential(*conv_branch) self.conv_pred = nn.Conv2d( self.feat_channels, self.out_channels, 1, stride=1) def init_weights(self) -> None: """Initialize weights of the head.""" super().init_weights() kaiming_init(self.convs_all_levels, a=1, distribution='uniform') kaiming_init(self.conv_branch, a=1, distribution='uniform') kaiming_init(self.conv_pred, a=1, distribution='uniform') def forward(self, x: Tuple[Tensor]) -> Tensor: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: Tensor: The predicted mask feature map. """ inputs = x[self.start_level:self.end_level + 1] assert len(inputs) == (self.end_level - self.start_level + 1) feature_add_all_level = self.convs_all_levels[0](inputs[0]) target_h, target_w = feature_add_all_level.size()[2:] for i in range(1, len(inputs)): input_p = inputs[i] x_p = self.convs_all_levels[i](input_p) h, w = x_p.size()[2:] factor_h = target_h // h factor_w = target_w // w assert factor_h == factor_w feature_per_level = aligned_bilinear(x_p, factor_h) feature_add_all_level = feature_add_all_level + \ feature_per_level feature_add_all_level = self.conv_branch(feature_add_all_level) feature_pred = self.conv_pred(feature_add_all_level) return feature_pred @MODELS.register_module() class CondInstMaskHead(BaseMaskHead): """CondInst mask head used in https://arxiv.org/abs/1904.02689. This head outputs the mask for CondInst. Args: mask_feature_head (dict): Config of CondInstMaskFeatHead. num_layers (int): Number of dynamic conv layers. feat_channels (int): Number of channels in the dynamic conv. mask_out_stride (int): The stride of the mask feat. size_of_interest (int): The size of the region used in rel coord. max_masks_to_train (int): Maximum number of masks to train for each image. loss_segm (:obj:`ConfigDict` or dict, optional): Config of segmentation loss. train_cfg (:obj:`ConfigDict` or dict, optional): Training config of head. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of head. """ def __init__(self, mask_feature_head: ConfigType, num_layers: int = 3, feat_channels: int = 8, mask_out_stride: int = 4, size_of_interest: int = 8, max_masks_to_train: int = -1, topk_masks_per_img: int = -1, loss_mask: ConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None) -> None: super().__init__() self.mask_feature_head = MaskFeatModule(**mask_feature_head) self.mask_feat_stride = self.mask_feature_head.mask_stride self.in_channels = self.mask_feature_head.out_channels self.num_layers = num_layers self.feat_channels = feat_channels self.size_of_interest = size_of_interest self.mask_out_stride = mask_out_stride self.max_masks_to_train = max_masks_to_train self.topk_masks_per_img = topk_masks_per_img self.prior_generator = MlvlPointGenerator([self.mask_feat_stride]) self.train_cfg = train_cfg self.test_cfg = test_cfg self.loss_mask = MODELS.build(loss_mask) self._init_layers() def _init_layers(self) -> None: """Initialize layers of the head.""" weight_nums, bias_nums = [], [] for i in range(self.num_layers): if i == 0: weight_nums.append((self.in_channels + 2) * self.feat_channels) bias_nums.append(self.feat_channels) elif i == self.num_layers - 1: weight_nums.append(self.feat_channels * 1) bias_nums.append(1) else: weight_nums.append(self.feat_channels * self.feat_channels) bias_nums.append(self.feat_channels) self.weight_nums = weight_nums self.bias_nums = bias_nums self.num_params = sum(weight_nums) + sum(bias_nums) def parse_dynamic_params( self, params: Tensor) -> Tuple[List[Tensor], List[Tensor]]: """parse the dynamic params for dynamic conv.""" num_insts = params.size(0) params_splits = list( torch.split_with_sizes( params, self.weight_nums + self.bias_nums, dim=1)) weight_splits = params_splits[:self.num_layers] bias_splits = params_splits[self.num_layers:] for i in range(self.num_layers): if i < self.num_layers - 1: weight_splits[i] = weight_splits[i].reshape( num_insts * self.in_channels, -1, 1, 1) bias_splits[i] = bias_splits[i].reshape(num_insts * self.in_channels) else: # out_channels x in_channels x 1 x 1 weight_splits[i] = weight_splits[i].reshape( num_insts * 1, -1, 1, 1) bias_splits[i] = bias_splits[i].reshape(num_insts) return weight_splits, bias_splits def dynamic_conv_forward(self, features: Tensor, weights: List[Tensor], biases: List[Tensor], num_insts: int) -> Tensor: """dynamic forward, each layer follow a relu.""" n_layers = len(weights) x = features for i, (w, b) in enumerate(zip(weights, biases)): x = F.conv2d(x, w, bias=b, stride=1, padding=0, groups=num_insts) if i < n_layers - 1: x = F.relu(x) return x def forward(self, x: tuple, positive_infos: InstanceList) -> tuple: """Forward feature from the upstream network to get prototypes and linearly combine the prototypes, using masks coefficients, into instance masks. Finally, crop the instance masks with given bboxes. Args: x (Tuple[Tensor]): Feature from the upstream network, which is a 4D-tensor. positive_infos (List[:obj:``InstanceData``]): Positive information that calculate from detect head. Returns: tuple: Predicted instance segmentation masks """ mask_feats = self.mask_feature_head(x) return multi_apply(self.forward_single, mask_feats, positive_infos) def forward_single(self, mask_feat: Tensor, positive_info: InstanceData) -> Tensor: """Forward features of a each image.""" pos_param_preds = positive_info.get('param_preds') pos_points = positive_info.get('points') pos_strides = positive_info.get('strides') num_inst = pos_param_preds.shape[0] mask_feat = mask_feat[None].repeat(num_inst, 1, 1, 1) _, _, H, W = mask_feat.size() if num_inst == 0: return (pos_param_preds.new_zeros((0, 1, H, W)), ) locations = self.prior_generator.single_level_grid_priors( mask_feat.size()[2:], 0, device=mask_feat.device) rel_coords = relative_coordinate_maps(locations, pos_points, pos_strides, self.size_of_interest, mask_feat.size()[2:]) mask_head_inputs = torch.cat([rel_coords, mask_feat], dim=1) mask_head_inputs = mask_head_inputs.reshape(1, -1, H, W) weights, biases = self.parse_dynamic_params(pos_param_preds) mask_preds = self.dynamic_conv_forward(mask_head_inputs, weights, biases, num_inst) mask_preds = mask_preds.reshape(-1, H, W) mask_preds = aligned_bilinear( mask_preds.unsqueeze(0), int(self.mask_feat_stride / self.mask_out_stride)).squeeze(0) return (mask_preds, ) def loss_by_feat(self, mask_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], positive_infos: InstanceList, **kwargs) -> dict: """Calculate the loss based on the features extracted by the mask head. Args: mask_preds (list[Tensor]): List of predicted masks, each has shape (num_classes, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``masks``, and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of multiple images. positive_infos (List[:obj:``InstanceData``]): Information of positive samples of each image that are assigned in detection head. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert positive_infos is not None, \ 'positive_infos should not be None in `CondInstMaskHead`' losses = dict() loss_mask = 0. num_imgs = len(mask_preds) total_pos = 0 for idx in range(num_imgs): (mask_pred, pos_mask_targets, num_pos) = \ self._get_targets_single( mask_preds[idx], batch_gt_instances[idx], positive_infos[idx]) # mask loss total_pos += num_pos if num_pos == 0 or pos_mask_targets is None: loss = mask_pred.new_zeros(1).mean() else: loss = self.loss_mask( mask_pred, pos_mask_targets, reduction_override='none').sum() loss_mask += loss if total_pos == 0: total_pos += 1 # avoid nan loss_mask = loss_mask / total_pos losses.update(loss_mask=loss_mask) return losses def _get_targets_single(self, mask_preds: Tensor, gt_instances: InstanceData, positive_info: InstanceData): """Compute targets for predictions of single image. Args: mask_preds (Tensor): Predicted prototypes with shape (num_classes, H, W). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes``, ``labels``, and ``masks`` attributes. positive_info (:obj:`InstanceData`): Information of positive samples that are assigned in detection head. It usually contains following keys. - pos_assigned_gt_inds (Tensor): Assigner GT indexes of positive proposals, has shape (num_pos, ) - pos_inds (Tensor): Positive index of image, has shape (num_pos, ). - param_pred (Tensor): Positive param preditions with shape (num_pos, num_params). Returns: tuple: Usually returns a tuple containing learning targets. - mask_preds (Tensor): Positive predicted mask with shape (num_pos, mask_h, mask_w). - pos_mask_targets (Tensor): Positive mask targets with shape (num_pos, mask_h, mask_w). - num_pos (int): Positive numbers. """ gt_bboxes = gt_instances.bboxes device = gt_bboxes.device gt_masks = gt_instances.masks.to_tensor( dtype=torch.bool, device=device).float() # process with mask targets pos_assigned_gt_inds = positive_info.get('pos_assigned_gt_inds') scores = positive_info.get('scores') centernesses = positive_info.get('centernesses') num_pos = pos_assigned_gt_inds.size(0) if gt_masks.size(0) == 0 or num_pos == 0: return mask_preds, None, 0 # Since we're producing (near) full image masks, # it'd take too much vram to backprop on every single mask. # Thus we select only a subset. if (self.max_masks_to_train != -1) and \ (num_pos > self.max_masks_to_train): perm = torch.randperm(num_pos) select = perm[:self.max_masks_to_train] mask_preds = mask_preds[select] pos_assigned_gt_inds = pos_assigned_gt_inds[select] num_pos = self.max_masks_to_train elif self.topk_masks_per_img != -1: unique_gt_inds = pos_assigned_gt_inds.unique() num_inst_per_gt = max( int(self.topk_masks_per_img / len(unique_gt_inds)), 1) keep_mask_preds = [] keep_pos_assigned_gt_inds = [] for gt_ind in unique_gt_inds: per_inst_pos_inds = (pos_assigned_gt_inds == gt_ind) mask_preds_per_inst = mask_preds[per_inst_pos_inds] gt_inds_per_inst = pos_assigned_gt_inds[per_inst_pos_inds] if sum(per_inst_pos_inds) > num_inst_per_gt: per_inst_scores = scores[per_inst_pos_inds].sigmoid().max( dim=1)[0] per_inst_centerness = centernesses[ per_inst_pos_inds].sigmoid().reshape(-1, ) select = (per_inst_scores * per_inst_centerness).topk( k=num_inst_per_gt, dim=0)[1] mask_preds_per_inst = mask_preds_per_inst[select] gt_inds_per_inst = gt_inds_per_inst[select] keep_mask_preds.append(mask_preds_per_inst) keep_pos_assigned_gt_inds.append(gt_inds_per_inst) mask_preds = torch.cat(keep_mask_preds) pos_assigned_gt_inds = torch.cat(keep_pos_assigned_gt_inds) num_pos = pos_assigned_gt_inds.size(0) # Follow the origin implement start = int(self.mask_out_stride // 2) gt_masks = gt_masks[:, start::self.mask_out_stride, start::self.mask_out_stride] gt_masks = gt_masks.gt(0.5).float() pos_mask_targets = gt_masks[pos_assigned_gt_inds] return (mask_preds, pos_mask_targets, num_pos) def predict_by_feat(self, mask_preds: List[Tensor], results_list: InstanceList, batch_img_metas: List[dict], rescale: bool = True, **kwargs) -> InstanceList: """Transform a batch of output features extracted from the head into mask results. Args: mask_preds (list[Tensor]): Predicted prototypes with shape (num_classes, H, W). results_list (List[:obj:``InstanceData``]): BBoxHead results. batch_img_metas (list[dict]): Meta information of all images. rescale (bool, optional): Whether to rescale the results. Defaults to False. Returns: list[:obj:`InstanceData`]: Processed results of multiple images.Each :obj:`InstanceData` usually contains following keys. - scores (Tensor): Classification scores, has shape (num_instance,). - labels (Tensor): Has shape (num_instances,). - masks (Tensor): Processed mask results, has shape (num_instances, h, w). """ assert len(mask_preds) == len(results_list) == len(batch_img_metas) for img_id in range(len(batch_img_metas)): img_meta = batch_img_metas[img_id] results = results_list[img_id] bboxes = results.bboxes mask_pred = mask_preds[img_id] if bboxes.shape[0] == 0 or mask_pred.shape[0] == 0: results_list[img_id] = empty_instances( [img_meta], bboxes.device, task_type='mask', instance_results=[results])[0] else: im_mask = self._predict_by_feat_single( mask_preds=mask_pred, bboxes=bboxes, img_meta=img_meta, rescale=rescale) results.masks = im_mask return results_list def _predict_by_feat_single(self, mask_preds: Tensor, bboxes: Tensor, img_meta: dict, rescale: bool, cfg: OptConfigType = None): """Transform a single image's features extracted from the head into mask results. Args: mask_preds (Tensor): Predicted prototypes, has shape [H, W, N]. img_meta (dict): Meta information of each image, e.g., image size, scaling factor, etc. rescale (bool): If rescale is False, then returned masks will fit the scale of imgs[0]. cfg (dict, optional): Config used in test phase. Defaults to None. Returns: :obj:`InstanceData`: Processed results of single image. it usually contains following keys. - scores (Tensor): Classification scores, has shape (num_instance,). - labels (Tensor): Has shape (num_instances,). - masks (Tensor): Processed mask results, has shape (num_instances, h, w). """ cfg = self.test_cfg if cfg is None else cfg scale_factor = bboxes.new_tensor(img_meta['scale_factor']).repeat( (1, 2)) img_h, img_w = img_meta['img_shape'][:2] ori_h, ori_w = img_meta['ori_shape'][:2] mask_preds = mask_preds.sigmoid().unsqueeze(0) mask_preds = aligned_bilinear(mask_preds, self.mask_out_stride) mask_preds = mask_preds[:, :, :img_h, :img_w] if rescale: # in-placed rescale the bboxes scale_factor = bboxes.new_tensor(img_meta['scale_factor']).repeat( (1, 2)) bboxes /= scale_factor masks = F.interpolate( mask_preds, (ori_h, ori_w), mode='bilinear', align_corners=False).squeeze(0) > cfg.mask_thr else: masks = mask_preds.squeeze(0) > cfg.mask_thr return masks ================================================ FILE: mmdet/models/dense_heads/conditional_detr_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple import torch import torch.nn as nn from mmengine.model import bias_init_with_prob from torch import Tensor from mmdet.models.layers.transformer import inverse_sigmoid from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import InstanceList from .detr_head import DETRHead @MODELS.register_module() class ConditionalDETRHead(DETRHead): """Head of Conditional DETR. Conditional DETR: Conditional DETR for Fast Training Convergence. More details can be found in the `paper. `_ . """ def init_weights(self): """Initialize weights of the transformer head.""" super().init_weights() # The initialization below for transformer head is very # important as we use Focal_loss for loss_cls if self.loss_cls.use_sigmoid: bias_init = bias_init_with_prob(0.01) nn.init.constant_(self.fc_cls.bias, bias_init) def forward(self, hidden_states: Tensor, references: Tensor) -> Tuple[Tensor, Tensor]: """"Forward function. Args: hidden_states (Tensor): Features from transformer decoder. If `return_intermediate_dec` is True output has shape (num_decoder_layers, bs, num_queries, dim), else has shape (1, bs, num_queries, dim) which only contains the last layer outputs. references (Tensor): References from transformer decoder, has shape (bs, num_queries, 2). Returns: tuple[Tensor]: results of head containing the following tensor. - layers_cls_scores (Tensor): Outputs from the classification head, shape (num_decoder_layers, bs, num_queries, cls_out_channels). Note cls_out_channels should include background. - layers_bbox_preds (Tensor): Sigmoid outputs from the regression head with normalized coordinate format (cx, cy, w, h), has shape (num_decoder_layers, bs, num_queries, 4). """ references_unsigmoid = inverse_sigmoid(references) layers_bbox_preds = [] for layer_id in range(hidden_states.shape[0]): tmp_reg_preds = self.fc_reg( self.activate(self.reg_ffn(hidden_states[layer_id]))) tmp_reg_preds[..., :2] += references_unsigmoid outputs_coord = tmp_reg_preds.sigmoid() layers_bbox_preds.append(outputs_coord) layers_bbox_preds = torch.stack(layers_bbox_preds) layers_cls_scores = self.fc_cls(hidden_states) return layers_cls_scores, layers_bbox_preds def loss(self, hidden_states: Tensor, references: Tensor, batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection head on the features of the upstream network. Args: hidden_states (Tensor): Features from the transformer decoder, has shape (num_decoder_layers, bs, num_queries, dim). references (Tensor): References from the transformer decoder, has shape (num_decoder_layers, bs, num_queries, 2). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ batch_gt_instances = [] batch_img_metas = [] for data_sample in batch_data_samples: batch_img_metas.append(data_sample.metainfo) batch_gt_instances.append(data_sample.gt_instances) outs = self(hidden_states, references) loss_inputs = outs + (batch_gt_instances, batch_img_metas) losses = self.loss_by_feat(*loss_inputs) return losses def loss_and_predict( self, hidden_states: Tensor, references: Tensor, batch_data_samples: SampleList) -> Tuple[dict, InstanceList]: """Perform forward propagation of the head, then calculate loss and predictions from the features and data samples. Over-write because img_metas are needed as inputs for bbox_head. Args: hidden_states (Tensor): Features from the transformer decoder, has shape (num_decoder_layers, bs, num_queries, dim). references (Tensor): References from the transformer decoder, has shape (num_decoder_layers, bs, num_queries, 2). batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. Returns: tuple: The return value is a tuple contains: - losses: (dict[str, Tensor]): A dictionary of loss components. - predictions (list[:obj:`InstanceData`]): Detection results of each image after the post process. """ batch_gt_instances = [] batch_img_metas = [] for data_sample in batch_data_samples: batch_img_metas.append(data_sample.metainfo) batch_gt_instances.append(data_sample.gt_instances) outs = self(hidden_states, references) loss_inputs = outs + (batch_gt_instances, batch_img_metas) losses = self.loss_by_feat(*loss_inputs) predictions = self.predict_by_feat( *outs, batch_img_metas=batch_img_metas) return losses, predictions def predict(self, hidden_states: Tensor, references: Tensor, batch_data_samples: SampleList, rescale: bool = True) -> InstanceList: """Perform forward propagation of the detection head and predict detection results on the features of the upstream network. Over-write because img_metas are needed as inputs for bbox_head. Args: hidden_states (Tensor): Features from the transformer decoder, has shape (num_decoder_layers, bs, num_queries, dim). references (Tensor): References from the transformer decoder, has shape (num_decoder_layers, bs, num_queries, 2). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool, optional): Whether to rescale the results. Defaults to True. Returns: list[obj:`InstanceData`]: Detection results of each image after the post process. """ batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] last_layer_hidden_state = hidden_states[-1].unsqueeze(0) outs = self(last_layer_hidden_state, references) predictions = self.predict_by_feat( *outs, batch_img_metas=batch_img_metas, rescale=rescale) return predictions ================================================ FILE: mmdet/models/dense_heads/corner_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from logging import warning from math import ceil, log from typing import List, Optional, Sequence, Tuple import torch import torch.nn as nn from mmcv.cnn import ConvModule from mmcv.ops import CornerPool, batched_nms from mmengine.config import ConfigDict from mmengine.model import BaseModule, bias_init_with_prob from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptInstanceList, OptMultiConfig) from ..utils import (gather_feat, gaussian_radius, gen_gaussian_target, get_local_maximum, get_topk_from_heatmap, multi_apply, transpose_and_gather_feat) from .base_dense_head import BaseDenseHead class BiCornerPool(BaseModule): """Bidirectional Corner Pooling Module (TopLeft, BottomRight, etc.) Args: in_channels (int): Input channels of module. directions (list[str]): Directions of two CornerPools. out_channels (int): Output channels of module. feat_channels (int): Feature channels of module. norm_cfg (:obj:`ConfigDict` or dict): Dictionary to construct and config norm layer. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. """ def __init__(self, in_channels: int, directions: List[int], feat_channels: int = 128, out_channels: int = 128, norm_cfg: ConfigType = dict(type='BN', requires_grad=True), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg) self.direction1_conv = ConvModule( in_channels, feat_channels, 3, padding=1, norm_cfg=norm_cfg) self.direction2_conv = ConvModule( in_channels, feat_channels, 3, padding=1, norm_cfg=norm_cfg) self.aftpool_conv = ConvModule( feat_channels, out_channels, 3, padding=1, norm_cfg=norm_cfg, act_cfg=None) self.conv1 = ConvModule( in_channels, out_channels, 1, norm_cfg=norm_cfg, act_cfg=None) self.conv2 = ConvModule( in_channels, out_channels, 3, padding=1, norm_cfg=norm_cfg) self.direction1_pool = CornerPool(directions[0]) self.direction2_pool = CornerPool(directions[1]) self.relu = nn.ReLU(inplace=True) def forward(self, x: Tensor) -> Tensor: """Forward features from the upstream network. Args: x (tensor): Input feature of BiCornerPool. Returns: conv2 (tensor): Output feature of BiCornerPool. """ direction1_conv = self.direction1_conv(x) direction2_conv = self.direction2_conv(x) direction1_feat = self.direction1_pool(direction1_conv) direction2_feat = self.direction2_pool(direction2_conv) aftpool_conv = self.aftpool_conv(direction1_feat + direction2_feat) conv1 = self.conv1(x) relu = self.relu(aftpool_conv + conv1) conv2 = self.conv2(relu) return conv2 @MODELS.register_module() class CornerHead(BaseDenseHead): """Head of CornerNet: Detecting Objects as Paired Keypoints. Code is modified from the `official github repo `_ . More details can be found in the `paper `_ . Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. num_feat_levels (int): Levels of feature from the previous module. 2 for HourglassNet-104 and 1 for HourglassNet-52. Because HourglassNet-104 outputs the final feature and intermediate supervision feature and HourglassNet-52 only outputs the final feature. Defaults to 2. corner_emb_channels (int): Channel of embedding vector. Defaults to 1. train_cfg (:obj:`ConfigDict` or dict, optional): Training config. Useless in CornerHead, but we keep this variable for SingleStageDetector. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of CornerHead. loss_heatmap (:obj:`ConfigDict` or dict): Config of corner heatmap loss. Defaults to GaussianFocalLoss. loss_embedding (:obj:`ConfigDict` or dict): Config of corner embedding loss. Defaults to AssociativeEmbeddingLoss. loss_offset (:obj:`ConfigDict` or dict): Config of corner offset loss. Defaults to SmoothL1Loss. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. """ def __init__(self, num_classes: int, in_channels: int, num_feat_levels: int = 2, corner_emb_channels: int = 1, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, loss_heatmap: ConfigType = dict( type='GaussianFocalLoss', alpha=2.0, gamma=4.0, loss_weight=1), loss_embedding: ConfigType = dict( type='AssociativeEmbeddingLoss', pull_weight=0.25, push_weight=0.25), loss_offset: ConfigType = dict( type='SmoothL1Loss', beta=1.0, loss_weight=1), init_cfg: OptMultiConfig = None) -> None: assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' super().__init__(init_cfg=init_cfg) self.num_classes = num_classes self.in_channels = in_channels self.corner_emb_channels = corner_emb_channels self.with_corner_emb = self.corner_emb_channels > 0 self.corner_offset_channels = 2 self.num_feat_levels = num_feat_levels self.loss_heatmap = MODELS.build( loss_heatmap) if loss_heatmap is not None else None self.loss_embedding = MODELS.build( loss_embedding) if loss_embedding is not None else None self.loss_offset = MODELS.build( loss_offset) if loss_offset is not None else None self.train_cfg = train_cfg self.test_cfg = test_cfg self._init_layers() def _make_layers(self, out_channels: int, in_channels: int = 256, feat_channels: int = 256) -> nn.Sequential: """Initialize conv sequential for CornerHead.""" return nn.Sequential( ConvModule(in_channels, feat_channels, 3, padding=1), ConvModule( feat_channels, out_channels, 1, norm_cfg=None, act_cfg=None)) def _init_corner_kpt_layers(self) -> None: """Initialize corner keypoint layers. Including corner heatmap branch and corner offset branch. Each branch has two parts: prefix `tl_` for top-left and `br_` for bottom-right. """ self.tl_pool, self.br_pool = nn.ModuleList(), nn.ModuleList() self.tl_heat, self.br_heat = nn.ModuleList(), nn.ModuleList() self.tl_off, self.br_off = nn.ModuleList(), nn.ModuleList() for _ in range(self.num_feat_levels): self.tl_pool.append( BiCornerPool( self.in_channels, ['top', 'left'], out_channels=self.in_channels)) self.br_pool.append( BiCornerPool( self.in_channels, ['bottom', 'right'], out_channels=self.in_channels)) self.tl_heat.append( self._make_layers( out_channels=self.num_classes, in_channels=self.in_channels)) self.br_heat.append( self._make_layers( out_channels=self.num_classes, in_channels=self.in_channels)) self.tl_off.append( self._make_layers( out_channels=self.corner_offset_channels, in_channels=self.in_channels)) self.br_off.append( self._make_layers( out_channels=self.corner_offset_channels, in_channels=self.in_channels)) def _init_corner_emb_layers(self) -> None: """Initialize corner embedding layers. Only include corner embedding branch with two parts: prefix `tl_` for top-left and `br_` for bottom-right. """ self.tl_emb, self.br_emb = nn.ModuleList(), nn.ModuleList() for _ in range(self.num_feat_levels): self.tl_emb.append( self._make_layers( out_channels=self.corner_emb_channels, in_channels=self.in_channels)) self.br_emb.append( self._make_layers( out_channels=self.corner_emb_channels, in_channels=self.in_channels)) def _init_layers(self) -> None: """Initialize layers for CornerHead. Including two parts: corner keypoint layers and corner embedding layers """ self._init_corner_kpt_layers() if self.with_corner_emb: self._init_corner_emb_layers() def init_weights(self) -> None: super().init_weights() bias_init = bias_init_with_prob(0.1) for i in range(self.num_feat_levels): # The initialization of parameters are different between # nn.Conv2d and ConvModule. Our experiments show that # using the original initialization of nn.Conv2d increases # the final mAP by about 0.2% self.tl_heat[i][-1].conv.reset_parameters() self.tl_heat[i][-1].conv.bias.data.fill_(bias_init) self.br_heat[i][-1].conv.reset_parameters() self.br_heat[i][-1].conv.bias.data.fill_(bias_init) self.tl_off[i][-1].conv.reset_parameters() self.br_off[i][-1].conv.reset_parameters() if self.with_corner_emb: self.tl_emb[i][-1].conv.reset_parameters() self.br_emb[i][-1].conv.reset_parameters() def forward(self, feats: Tuple[Tensor]) -> tuple: """Forward features from the upstream network. Args: feats (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Usually a tuple of corner heatmaps, offset heatmaps and embedding heatmaps. - tl_heats (list[Tensor]): Top-left corner heatmaps for all levels, each is a 4D-tensor, the channels number is num_classes. - br_heats (list[Tensor]): Bottom-right corner heatmaps for all levels, each is a 4D-tensor, the channels number is num_classes. - tl_embs (list[Tensor] | list[None]): Top-left embedding heatmaps for all levels, each is a 4D-tensor or None. If not None, the channels number is corner_emb_channels. - br_embs (list[Tensor] | list[None]): Bottom-right embedding heatmaps for all levels, each is a 4D-tensor or None. If not None, the channels number is corner_emb_channels. - tl_offs (list[Tensor]): Top-left offset heatmaps for all levels, each is a 4D-tensor. The channels number is corner_offset_channels. - br_offs (list[Tensor]): Bottom-right offset heatmaps for all levels, each is a 4D-tensor. The channels number is corner_offset_channels. """ lvl_ind = list(range(self.num_feat_levels)) return multi_apply(self.forward_single, feats, lvl_ind) def forward_single(self, x: Tensor, lvl_ind: int, return_pool: bool = False) -> List[Tensor]: """Forward feature of a single level. Args: x (Tensor): Feature of a single level. lvl_ind (int): Level index of current feature. return_pool (bool): Return corner pool feature or not. Defaults to False. Returns: tuple[Tensor]: A tuple of CornerHead's output for current feature level. Containing the following Tensors: - tl_heat (Tensor): Predicted top-left corner heatmap. - br_heat (Tensor): Predicted bottom-right corner heatmap. - tl_emb (Tensor | None): Predicted top-left embedding heatmap. None for `self.with_corner_emb == False`. - br_emb (Tensor | None): Predicted bottom-right embedding heatmap. None for `self.with_corner_emb == False`. - tl_off (Tensor): Predicted top-left offset heatmap. - br_off (Tensor): Predicted bottom-right offset heatmap. - tl_pool (Tensor): Top-left corner pool feature. Not must have. - br_pool (Tensor): Bottom-right corner pool feature. Not must have. """ tl_pool = self.tl_pool[lvl_ind](x) tl_heat = self.tl_heat[lvl_ind](tl_pool) br_pool = self.br_pool[lvl_ind](x) br_heat = self.br_heat[lvl_ind](br_pool) tl_emb, br_emb = None, None if self.with_corner_emb: tl_emb = self.tl_emb[lvl_ind](tl_pool) br_emb = self.br_emb[lvl_ind](br_pool) tl_off = self.tl_off[lvl_ind](tl_pool) br_off = self.br_off[lvl_ind](br_pool) result_list = [tl_heat, br_heat, tl_emb, br_emb, tl_off, br_off] if return_pool: result_list.append(tl_pool) result_list.append(br_pool) return result_list def get_targets(self, gt_bboxes: List[Tensor], gt_labels: List[Tensor], feat_shape: Sequence[int], img_shape: Sequence[int], with_corner_emb: bool = False, with_guiding_shift: bool = False, with_centripetal_shift: bool = False) -> dict: """Generate corner targets. Including corner heatmap, corner offset. Optional: corner embedding, corner guiding shift, centripetal shift. For CornerNet, we generate corner heatmap, corner offset and corner embedding from this function. For CentripetalNet, we generate corner heatmap, corner offset, guiding shift and centripetal shift from this function. Args: gt_bboxes (list[Tensor]): Ground truth bboxes of each image, each has shape (num_gt, 4). gt_labels (list[Tensor]): Ground truth labels of each box, each has shape (num_gt, ). feat_shape (Sequence[int]): Shape of output feature, [batch, channel, height, width]. img_shape (Sequence[int]): Shape of input image, [height, width, channel]. with_corner_emb (bool): Generate corner embedding target or not. Defaults to False. with_guiding_shift (bool): Generate guiding shift target or not. Defaults to False. with_centripetal_shift (bool): Generate centripetal shift target or not. Defaults to False. Returns: dict: Ground truth of corner heatmap, corner offset, corner embedding, guiding shift and centripetal shift. Containing the following keys: - topleft_heatmap (Tensor): Ground truth top-left corner heatmap. - bottomright_heatmap (Tensor): Ground truth bottom-right corner heatmap. - topleft_offset (Tensor): Ground truth top-left corner offset. - bottomright_offset (Tensor): Ground truth bottom-right corner offset. - corner_embedding (list[list[list[int]]]): Ground truth corner embedding. Not must have. - topleft_guiding_shift (Tensor): Ground truth top-left corner guiding shift. Not must have. - bottomright_guiding_shift (Tensor): Ground truth bottom-right corner guiding shift. Not must have. - topleft_centripetal_shift (Tensor): Ground truth top-left corner centripetal shift. Not must have. - bottomright_centripetal_shift (Tensor): Ground truth bottom-right corner centripetal shift. Not must have. """ batch_size, _, height, width = feat_shape img_h, img_w = img_shape[:2] width_ratio = float(width / img_w) height_ratio = float(height / img_h) gt_tl_heatmap = gt_bboxes[-1].new_zeros( [batch_size, self.num_classes, height, width]) gt_br_heatmap = gt_bboxes[-1].new_zeros( [batch_size, self.num_classes, height, width]) gt_tl_offset = gt_bboxes[-1].new_zeros([batch_size, 2, height, width]) gt_br_offset = gt_bboxes[-1].new_zeros([batch_size, 2, height, width]) if with_corner_emb: match = [] # Guiding shift is a kind of offset, from center to corner if with_guiding_shift: gt_tl_guiding_shift = gt_bboxes[-1].new_zeros( [batch_size, 2, height, width]) gt_br_guiding_shift = gt_bboxes[-1].new_zeros( [batch_size, 2, height, width]) # Centripetal shift is also a kind of offset, from center to corner # and normalized by log. if with_centripetal_shift: gt_tl_centripetal_shift = gt_bboxes[-1].new_zeros( [batch_size, 2, height, width]) gt_br_centripetal_shift = gt_bboxes[-1].new_zeros( [batch_size, 2, height, width]) for batch_id in range(batch_size): # Ground truth of corner embedding per image is a list of coord set corner_match = [] for box_id in range(len(gt_labels[batch_id])): left, top, right, bottom = gt_bboxes[batch_id][box_id] center_x = (left + right) / 2.0 center_y = (top + bottom) / 2.0 label = gt_labels[batch_id][box_id] # Use coords in the feature level to generate ground truth scale_left = left * width_ratio scale_right = right * width_ratio scale_top = top * height_ratio scale_bottom = bottom * height_ratio scale_center_x = center_x * width_ratio scale_center_y = center_y * height_ratio # Int coords on feature map/ground truth tensor left_idx = int(min(scale_left, width - 1)) right_idx = int(min(scale_right, width - 1)) top_idx = int(min(scale_top, height - 1)) bottom_idx = int(min(scale_bottom, height - 1)) # Generate gaussian heatmap scale_box_width = ceil(scale_right - scale_left) scale_box_height = ceil(scale_bottom - scale_top) radius = gaussian_radius((scale_box_height, scale_box_width), min_overlap=0.3) radius = max(0, int(radius)) gt_tl_heatmap[batch_id, label] = gen_gaussian_target( gt_tl_heatmap[batch_id, label], [left_idx, top_idx], radius) gt_br_heatmap[batch_id, label] = gen_gaussian_target( gt_br_heatmap[batch_id, label], [right_idx, bottom_idx], radius) # Generate corner offset left_offset = scale_left - left_idx top_offset = scale_top - top_idx right_offset = scale_right - right_idx bottom_offset = scale_bottom - bottom_idx gt_tl_offset[batch_id, 0, top_idx, left_idx] = left_offset gt_tl_offset[batch_id, 1, top_idx, left_idx] = top_offset gt_br_offset[batch_id, 0, bottom_idx, right_idx] = right_offset gt_br_offset[batch_id, 1, bottom_idx, right_idx] = bottom_offset # Generate corner embedding if with_corner_emb: corner_match.append([[top_idx, left_idx], [bottom_idx, right_idx]]) # Generate guiding shift if with_guiding_shift: gt_tl_guiding_shift[batch_id, 0, top_idx, left_idx] = scale_center_x - left_idx gt_tl_guiding_shift[batch_id, 1, top_idx, left_idx] = scale_center_y - top_idx gt_br_guiding_shift[batch_id, 0, bottom_idx, right_idx] = right_idx - scale_center_x gt_br_guiding_shift[ batch_id, 1, bottom_idx, right_idx] = bottom_idx - scale_center_y # Generate centripetal shift if with_centripetal_shift: gt_tl_centripetal_shift[batch_id, 0, top_idx, left_idx] = log(scale_center_x - scale_left) gt_tl_centripetal_shift[batch_id, 1, top_idx, left_idx] = log(scale_center_y - scale_top) gt_br_centripetal_shift[batch_id, 0, bottom_idx, right_idx] = log(scale_right - scale_center_x) gt_br_centripetal_shift[batch_id, 1, bottom_idx, right_idx] = log(scale_bottom - scale_center_y) if with_corner_emb: match.append(corner_match) target_result = dict( topleft_heatmap=gt_tl_heatmap, topleft_offset=gt_tl_offset, bottomright_heatmap=gt_br_heatmap, bottomright_offset=gt_br_offset) if with_corner_emb: target_result.update(corner_embedding=match) if with_guiding_shift: target_result.update( topleft_guiding_shift=gt_tl_guiding_shift, bottomright_guiding_shift=gt_br_guiding_shift) if with_centripetal_shift: target_result.update( topleft_centripetal_shift=gt_tl_centripetal_shift, bottomright_centripetal_shift=gt_br_centripetal_shift) return target_result def loss_by_feat( self, tl_heats: List[Tensor], br_heats: List[Tensor], tl_embs: List[Tensor], br_embs: List[Tensor], tl_offs: List[Tensor], br_offs: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: tl_heats (list[Tensor]): Top-left corner heatmaps for each level with shape (N, num_classes, H, W). br_heats (list[Tensor]): Bottom-right corner heatmaps for each level with shape (N, num_classes, H, W). tl_embs (list[Tensor]): Top-left corner embeddings for each level with shape (N, corner_emb_channels, H, W). br_embs (list[Tensor]): Bottom-right corner embeddings for each level with shape (N, corner_emb_channels, H, W). tl_offs (list[Tensor]): Top-left corner offsets for each level with shape (N, corner_offset_channels, H, W). br_offs (list[Tensor]): Bottom-right corner offsets for each level with shape (N, corner_offset_channels, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Specify which bounding boxes can be ignored when computing the loss. Returns: dict[str, Tensor]: A dictionary of loss components. Containing the following losses: - det_loss (list[Tensor]): Corner keypoint losses of all feature levels. - pull_loss (list[Tensor]): Part one of AssociativeEmbedding losses of all feature levels. - push_loss (list[Tensor]): Part two of AssociativeEmbedding losses of all feature levels. - off_loss (list[Tensor]): Corner offset losses of all feature levels. """ gt_bboxes = [ gt_instances.bboxes for gt_instances in batch_gt_instances ] gt_labels = [ gt_instances.labels for gt_instances in batch_gt_instances ] targets = self.get_targets( gt_bboxes, gt_labels, tl_heats[-1].shape, batch_img_metas[0]['batch_input_shape'], with_corner_emb=self.with_corner_emb) mlvl_targets = [targets for _ in range(self.num_feat_levels)] det_losses, pull_losses, push_losses, off_losses = multi_apply( self.loss_by_feat_single, tl_heats, br_heats, tl_embs, br_embs, tl_offs, br_offs, mlvl_targets) loss_dict = dict(det_loss=det_losses, off_loss=off_losses) if self.with_corner_emb: loss_dict.update(pull_loss=pull_losses, push_loss=push_losses) return loss_dict def loss_by_feat_single(self, tl_hmp: Tensor, br_hmp: Tensor, tl_emb: Optional[Tensor], br_emb: Optional[Tensor], tl_off: Tensor, br_off: Tensor, targets: dict) -> Tuple[Tensor, ...]: """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: tl_hmp (Tensor): Top-left corner heatmap for current level with shape (N, num_classes, H, W). br_hmp (Tensor): Bottom-right corner heatmap for current level with shape (N, num_classes, H, W). tl_emb (Tensor, optional): Top-left corner embedding for current level with shape (N, corner_emb_channels, H, W). br_emb (Tensor, optional): Bottom-right corner embedding for current level with shape (N, corner_emb_channels, H, W). tl_off (Tensor): Top-left corner offset for current level with shape (N, corner_offset_channels, H, W). br_off (Tensor): Bottom-right corner offset for current level with shape (N, corner_offset_channels, H, W). targets (dict): Corner target generated by `get_targets`. Returns: tuple[torch.Tensor]: Losses of the head's different branches containing the following losses: - det_loss (Tensor): Corner keypoint loss. - pull_loss (Tensor): Part one of AssociativeEmbedding loss. - push_loss (Tensor): Part two of AssociativeEmbedding loss. - off_loss (Tensor): Corner offset loss. """ gt_tl_hmp = targets['topleft_heatmap'] gt_br_hmp = targets['bottomright_heatmap'] gt_tl_off = targets['topleft_offset'] gt_br_off = targets['bottomright_offset'] gt_embedding = targets['corner_embedding'] # Detection loss tl_det_loss = self.loss_heatmap( tl_hmp.sigmoid(), gt_tl_hmp, avg_factor=max(1, gt_tl_hmp.eq(1).sum())) br_det_loss = self.loss_heatmap( br_hmp.sigmoid(), gt_br_hmp, avg_factor=max(1, gt_br_hmp.eq(1).sum())) det_loss = (tl_det_loss + br_det_loss) / 2.0 # AssociativeEmbedding loss if self.with_corner_emb and self.loss_embedding is not None: pull_loss, push_loss = self.loss_embedding(tl_emb, br_emb, gt_embedding) else: pull_loss, push_loss = None, None # Offset loss # We only compute the offset loss at the real corner position. # The value of real corner would be 1 in heatmap ground truth. # The mask is computed in class agnostic mode and its shape is # batch * 1 * width * height. tl_off_mask = gt_tl_hmp.eq(1).sum(1).gt(0).unsqueeze(1).type_as( gt_tl_hmp) br_off_mask = gt_br_hmp.eq(1).sum(1).gt(0).unsqueeze(1).type_as( gt_br_hmp) tl_off_loss = self.loss_offset( tl_off, gt_tl_off, tl_off_mask, avg_factor=max(1, tl_off_mask.sum())) br_off_loss = self.loss_offset( br_off, gt_br_off, br_off_mask, avg_factor=max(1, br_off_mask.sum())) off_loss = (tl_off_loss + br_off_loss) / 2.0 return det_loss, pull_loss, push_loss, off_loss def predict_by_feat(self, tl_heats: List[Tensor], br_heats: List[Tensor], tl_embs: List[Tensor], br_embs: List[Tensor], tl_offs: List[Tensor], br_offs: List[Tensor], batch_img_metas: Optional[List[dict]] = None, rescale: bool = False, with_nms: bool = True) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. Args: tl_heats (list[Tensor]): Top-left corner heatmaps for each level with shape (N, num_classes, H, W). br_heats (list[Tensor]): Bottom-right corner heatmaps for each level with shape (N, num_classes, H, W). tl_embs (list[Tensor]): Top-left corner embeddings for each level with shape (N, corner_emb_channels, H, W). br_embs (list[Tensor]): Bottom-right corner embeddings for each level with shape (N, corner_emb_channels, H, W). tl_offs (list[Tensor]): Top-left corner offsets for each level with shape (N, corner_offset_channels, H, W). br_offs (list[Tensor]): Bottom-right corner offsets for each level with shape (N, corner_offset_channels, H, W). batch_img_metas (list[dict], optional): Batch image meta info. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert tl_heats[-1].shape[0] == br_heats[-1].shape[0] == len( batch_img_metas) result_list = [] for img_id in range(len(batch_img_metas)): result_list.append( self._predict_by_feat_single( tl_heats[-1][img_id:img_id + 1, :], br_heats[-1][img_id:img_id + 1, :], tl_offs[-1][img_id:img_id + 1, :], br_offs[-1][img_id:img_id + 1, :], batch_img_metas[img_id], tl_emb=tl_embs[-1][img_id:img_id + 1, :], br_emb=br_embs[-1][img_id:img_id + 1, :], rescale=rescale, with_nms=with_nms)) return result_list def _predict_by_feat_single(self, tl_heat: Tensor, br_heat: Tensor, tl_off: Tensor, br_off: Tensor, img_meta: dict, tl_emb: Optional[Tensor] = None, br_emb: Optional[Tensor] = None, tl_centripetal_shift: Optional[Tensor] = None, br_centripetal_shift: Optional[Tensor] = None, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: tl_heat (Tensor): Top-left corner heatmap for current level with shape (N, num_classes, H, W). br_heat (Tensor): Bottom-right corner heatmap for current level with shape (N, num_classes, H, W). tl_off (Tensor): Top-left corner offset for current level with shape (N, corner_offset_channels, H, W). br_off (Tensor): Bottom-right corner offset for current level with shape (N, corner_offset_channels, H, W). img_meta (dict): Meta information of current image, e.g., image size, scaling factor, etc. tl_emb (Tensor): Top-left corner embedding for current level with shape (N, corner_emb_channels, H, W). br_emb (Tensor): Bottom-right corner embedding for current level with shape (N, corner_emb_channels, H, W). tl_centripetal_shift: Top-left corner's centripetal shift for current level with shape (N, 2, H, W). br_centripetal_shift: Bottom-right corner's centripetal shift for current level with shape (N, 2, H, W). rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ if isinstance(img_meta, (list, tuple)): img_meta = img_meta[0] batch_bboxes, batch_scores, batch_clses = self._decode_heatmap( tl_heat=tl_heat.sigmoid(), br_heat=br_heat.sigmoid(), tl_off=tl_off, br_off=br_off, tl_emb=tl_emb, br_emb=br_emb, tl_centripetal_shift=tl_centripetal_shift, br_centripetal_shift=br_centripetal_shift, img_meta=img_meta, k=self.test_cfg.corner_topk, kernel=self.test_cfg.local_maximum_kernel, distance_threshold=self.test_cfg.distance_threshold) if rescale and 'scale_factor' in img_meta: batch_bboxes /= batch_bboxes.new_tensor( img_meta['scale_factor']).repeat((1, 2)) bboxes = batch_bboxes.view([-1, 4]) scores = batch_scores.view(-1) clses = batch_clses.view(-1) det_bboxes = torch.cat([bboxes, scores.unsqueeze(-1)], -1) keepinds = (det_bboxes[:, -1] > -0.1) det_bboxes = det_bboxes[keepinds] det_labels = clses[keepinds] if with_nms: det_bboxes, det_labels = self._bboxes_nms(det_bboxes, det_labels, self.test_cfg) results = InstanceData() results.bboxes = det_bboxes[..., :4] results.scores = det_bboxes[..., 4] results.labels = det_labels return results def _bboxes_nms(self, bboxes: Tensor, labels: Tensor, cfg: ConfigDict) -> Tuple[Tensor, Tensor]: """bboxes nms.""" if 'nms_cfg' in cfg: warning.warn('nms_cfg in test_cfg will be deprecated. ' 'Please rename it as nms') if 'nms' not in cfg: cfg.nms = cfg.nms_cfg if labels.numel() > 0: max_num = cfg.max_per_img bboxes, keep = batched_nms(bboxes[:, :4], bboxes[:, -1].contiguous(), labels, cfg.nms) if max_num > 0: bboxes = bboxes[:max_num] labels = labels[keep][:max_num] return bboxes, labels def _decode_heatmap(self, tl_heat: Tensor, br_heat: Tensor, tl_off: Tensor, br_off: Tensor, tl_emb: Optional[Tensor] = None, br_emb: Optional[Tensor] = None, tl_centripetal_shift: Optional[Tensor] = None, br_centripetal_shift: Optional[Tensor] = None, img_meta: Optional[dict] = None, k: int = 100, kernel: int = 3, distance_threshold: float = 0.5, num_dets: int = 1000) -> Tuple[Tensor, Tensor, Tensor]: """Transform outputs into detections raw bbox prediction. Args: tl_heat (Tensor): Top-left corner heatmap for current level with shape (N, num_classes, H, W). br_heat (Tensor): Bottom-right corner heatmap for current level with shape (N, num_classes, H, W). tl_off (Tensor): Top-left corner offset for current level with shape (N, corner_offset_channels, H, W). br_off (Tensor): Bottom-right corner offset for current level with shape (N, corner_offset_channels, H, W). tl_emb (Tensor, Optional): Top-left corner embedding for current level with shape (N, corner_emb_channels, H, W). br_emb (Tensor, Optional): Bottom-right corner embedding for current level with shape (N, corner_emb_channels, H, W). tl_centripetal_shift (Tensor, Optional): Top-left centripetal shift for current level with shape (N, 2, H, W). br_centripetal_shift (Tensor, Optional): Bottom-right centripetal shift for current level with shape (N, 2, H, W). img_meta (dict): Meta information of current image, e.g., image size, scaling factor, etc. k (int): Get top k corner keypoints from heatmap. kernel (int): Max pooling kernel for extract local maximum pixels. distance_threshold (float): Distance threshold. Top-left and bottom-right corner keypoints with feature distance less than the threshold will be regarded as keypoints from same object. num_dets (int): Num of raw boxes before doing nms. Returns: tuple[torch.Tensor]: Decoded output of CornerHead, containing the following Tensors: - bboxes (Tensor): Coords of each box. - scores (Tensor): Scores of each box. - clses (Tensor): Categories of each box. """ with_embedding = tl_emb is not None and br_emb is not None with_centripetal_shift = ( tl_centripetal_shift is not None and br_centripetal_shift is not None) assert with_embedding + with_centripetal_shift == 1 batch, _, height, width = tl_heat.size() if torch.onnx.is_in_onnx_export(): inp_h, inp_w = img_meta['pad_shape_for_onnx'][:2] else: inp_h, inp_w = img_meta['batch_input_shape'][:2] # perform nms on heatmaps tl_heat = get_local_maximum(tl_heat, kernel=kernel) br_heat = get_local_maximum(br_heat, kernel=kernel) tl_scores, tl_inds, tl_clses, tl_ys, tl_xs = get_topk_from_heatmap( tl_heat, k=k) br_scores, br_inds, br_clses, br_ys, br_xs = get_topk_from_heatmap( br_heat, k=k) # We use repeat instead of expand here because expand is a # shallow-copy function. Thus it could cause unexpected testing result # sometimes. Using expand will decrease about 10% mAP during testing # compared to repeat. tl_ys = tl_ys.view(batch, k, 1).repeat(1, 1, k) tl_xs = tl_xs.view(batch, k, 1).repeat(1, 1, k) br_ys = br_ys.view(batch, 1, k).repeat(1, k, 1) br_xs = br_xs.view(batch, 1, k).repeat(1, k, 1) tl_off = transpose_and_gather_feat(tl_off, tl_inds) tl_off = tl_off.view(batch, k, 1, 2) br_off = transpose_and_gather_feat(br_off, br_inds) br_off = br_off.view(batch, 1, k, 2) tl_xs = tl_xs + tl_off[..., 0] tl_ys = tl_ys + tl_off[..., 1] br_xs = br_xs + br_off[..., 0] br_ys = br_ys + br_off[..., 1] if with_centripetal_shift: tl_centripetal_shift = transpose_and_gather_feat( tl_centripetal_shift, tl_inds).view(batch, k, 1, 2).exp() br_centripetal_shift = transpose_and_gather_feat( br_centripetal_shift, br_inds).view(batch, 1, k, 2).exp() tl_ctxs = tl_xs + tl_centripetal_shift[..., 0] tl_ctys = tl_ys + tl_centripetal_shift[..., 1] br_ctxs = br_xs - br_centripetal_shift[..., 0] br_ctys = br_ys - br_centripetal_shift[..., 1] # all possible boxes based on top k corners (ignoring class) tl_xs *= (inp_w / width) tl_ys *= (inp_h / height) br_xs *= (inp_w / width) br_ys *= (inp_h / height) if with_centripetal_shift: tl_ctxs *= (inp_w / width) tl_ctys *= (inp_h / height) br_ctxs *= (inp_w / width) br_ctys *= (inp_h / height) x_off, y_off = 0, 0 # no crop if not torch.onnx.is_in_onnx_export(): # since `RandomCenterCropPad` is done on CPU with numpy and it's # not dynamic traceable when exporting to ONNX, thus 'border' # does not appears as key in 'img_meta'. As a tmp solution, # we move this 'border' handle part to the postprocess after # finished exporting to ONNX, which is handle in # `mmdet/core/export/model_wrappers.py`. Though difference between # pytorch and exported onnx model, it might be ignored since # comparable performance is achieved between them (e.g. 40.4 vs # 40.6 on COCO val2017, for CornerNet without test-time flip) if 'border' in img_meta: x_off = img_meta['border'][2] y_off = img_meta['border'][0] tl_xs -= x_off tl_ys -= y_off br_xs -= x_off br_ys -= y_off zeros = tl_xs.new_zeros(*tl_xs.size()) tl_xs = torch.where(tl_xs > 0.0, tl_xs, zeros) tl_ys = torch.where(tl_ys > 0.0, tl_ys, zeros) br_xs = torch.where(br_xs > 0.0, br_xs, zeros) br_ys = torch.where(br_ys > 0.0, br_ys, zeros) bboxes = torch.stack((tl_xs, tl_ys, br_xs, br_ys), dim=3) area_bboxes = ((br_xs - tl_xs) * (br_ys - tl_ys)).abs() if with_centripetal_shift: tl_ctxs -= x_off tl_ctys -= y_off br_ctxs -= x_off br_ctys -= y_off tl_ctxs *= tl_ctxs.gt(0.0).type_as(tl_ctxs) tl_ctys *= tl_ctys.gt(0.0).type_as(tl_ctys) br_ctxs *= br_ctxs.gt(0.0).type_as(br_ctxs) br_ctys *= br_ctys.gt(0.0).type_as(br_ctys) ct_bboxes = torch.stack((tl_ctxs, tl_ctys, br_ctxs, br_ctys), dim=3) area_ct_bboxes = ((br_ctxs - tl_ctxs) * (br_ctys - tl_ctys)).abs() rcentral = torch.zeros_like(ct_bboxes) # magic nums from paper section 4.1 mu = torch.ones_like(area_bboxes) / 2.4 mu[area_bboxes > 3500] = 1 / 2.1 # large bbox have smaller mu bboxes_center_x = (bboxes[..., 0] + bboxes[..., 2]) / 2 bboxes_center_y = (bboxes[..., 1] + bboxes[..., 3]) / 2 rcentral[..., 0] = bboxes_center_x - mu * (bboxes[..., 2] - bboxes[..., 0]) / 2 rcentral[..., 1] = bboxes_center_y - mu * (bboxes[..., 3] - bboxes[..., 1]) / 2 rcentral[..., 2] = bboxes_center_x + mu * (bboxes[..., 2] - bboxes[..., 0]) / 2 rcentral[..., 3] = bboxes_center_y + mu * (bboxes[..., 3] - bboxes[..., 1]) / 2 area_rcentral = ((rcentral[..., 2] - rcentral[..., 0]) * (rcentral[..., 3] - rcentral[..., 1])).abs() dists = area_ct_bboxes / area_rcentral tl_ctx_inds = (ct_bboxes[..., 0] <= rcentral[..., 0]) | ( ct_bboxes[..., 0] >= rcentral[..., 2]) tl_cty_inds = (ct_bboxes[..., 1] <= rcentral[..., 1]) | ( ct_bboxes[..., 1] >= rcentral[..., 3]) br_ctx_inds = (ct_bboxes[..., 2] <= rcentral[..., 0]) | ( ct_bboxes[..., 2] >= rcentral[..., 2]) br_cty_inds = (ct_bboxes[..., 3] <= rcentral[..., 1]) | ( ct_bboxes[..., 3] >= rcentral[..., 3]) if with_embedding: tl_emb = transpose_and_gather_feat(tl_emb, tl_inds) tl_emb = tl_emb.view(batch, k, 1) br_emb = transpose_and_gather_feat(br_emb, br_inds) br_emb = br_emb.view(batch, 1, k) dists = torch.abs(tl_emb - br_emb) tl_scores = tl_scores.view(batch, k, 1).repeat(1, 1, k) br_scores = br_scores.view(batch, 1, k).repeat(1, k, 1) scores = (tl_scores + br_scores) / 2 # scores for all possible boxes # tl and br should have same class tl_clses = tl_clses.view(batch, k, 1).repeat(1, 1, k) br_clses = br_clses.view(batch, 1, k).repeat(1, k, 1) cls_inds = (tl_clses != br_clses) # reject boxes based on distances dist_inds = dists > distance_threshold # reject boxes based on widths and heights width_inds = (br_xs <= tl_xs) height_inds = (br_ys <= tl_ys) # No use `scores[cls_inds]`, instead we use `torch.where` here. # Since only 1-D indices with type 'tensor(bool)' are supported # when exporting to ONNX, any other bool indices with more dimensions # (e.g. 2-D bool tensor) as input parameter in node is invalid negative_scores = -1 * torch.ones_like(scores) scores = torch.where(cls_inds, negative_scores, scores) scores = torch.where(width_inds, negative_scores, scores) scores = torch.where(height_inds, negative_scores, scores) scores = torch.where(dist_inds, negative_scores, scores) if with_centripetal_shift: scores[tl_ctx_inds] = -1 scores[tl_cty_inds] = -1 scores[br_ctx_inds] = -1 scores[br_cty_inds] = -1 scores = scores.view(batch, -1) scores, inds = torch.topk(scores, num_dets) scores = scores.unsqueeze(2) bboxes = bboxes.view(batch, -1, 4) bboxes = gather_feat(bboxes, inds) clses = tl_clses.contiguous().view(batch, -1, 1) clses = gather_feat(clses, inds) return bboxes, scores, clses ================================================ FILE: mmdet/models/dense_heads/dab_detr_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple import torch.nn as nn from mmcv.cnn import Linear from mmengine.model import bias_init_with_prob, constant_init from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import InstanceList from ..layers import MLP, inverse_sigmoid from .conditional_detr_head import ConditionalDETRHead @MODELS.register_module() class DABDETRHead(ConditionalDETRHead): """Head of DAB-DETR. DAB-DETR: Dynamic Anchor Boxes are Better Queries for DETR. More details can be found in the `paper `_ . """ def _init_layers(self) -> None: """Initialize layers of the transformer head.""" # cls branch self.fc_cls = Linear(self.embed_dims, self.cls_out_channels) # reg branch self.fc_reg = MLP(self.embed_dims, self.embed_dims, 4, 3) def init_weights(self) -> None: """initialize weights.""" if self.loss_cls.use_sigmoid: bias_init = bias_init_with_prob(0.01) nn.init.constant_(self.fc_cls.bias, bias_init) constant_init(self.fc_reg.layers[-1], 0., bias=0.) def forward(self, hidden_states: Tensor, references: Tensor) -> Tuple[Tensor, Tensor]: """"Forward function. Args: hidden_states (Tensor): Features from transformer decoder. If `return_intermediate_dec` is True output has shape (num_decoder_layers, bs, num_queries, dim), else has shape (1, bs, num_queries, dim) which only contains the last layer outputs. references (Tensor): References from transformer decoder. If `return_intermediate_dec` is True output has shape (num_decoder_layers, bs, num_queries, 2/4), else has shape (1, bs, num_queries, 2/4) which only contains the last layer reference. Returns: tuple[Tensor]: results of head containing the following tensor. - layers_cls_scores (Tensor): Outputs from the classification head, shape (num_decoder_layers, bs, num_queries, cls_out_channels). Note cls_out_channels should include background. - layers_bbox_preds (Tensor): Sigmoid outputs from the regression head with normalized coordinate format (cx, cy, w, h), has shape (num_decoder_layers, bs, num_queries, 4). """ layers_cls_scores = self.fc_cls(hidden_states) references_before_sigmoid = inverse_sigmoid(references, eps=1e-3) tmp_reg_preds = self.fc_reg(hidden_states) tmp_reg_preds[..., :references_before_sigmoid. size(-1)] += references_before_sigmoid layers_bbox_preds = tmp_reg_preds.sigmoid() return layers_cls_scores, layers_bbox_preds def predict(self, hidden_states: Tensor, references: Tensor, batch_data_samples: SampleList, rescale: bool = True) -> InstanceList: """Perform forward propagation of the detection head and predict detection results on the features of the upstream network. Over-write because img_metas are needed as inputs for bbox_head. Args: hidden_states (Tensor): Feature from the transformer decoder, has shape (num_decoder_layers, bs, num_queries, dim). references (Tensor): references from the transformer decoder, has shape (num_decoder_layers, bs, num_queries, 2/4). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool, optional): Whether to rescale the results. Defaults to True. Returns: list[obj:`InstanceData`]: Detection results of each image after the post process. """ batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] last_layer_hidden_state = hidden_states[-1].unsqueeze(0) last_layer_reference = references[-1].unsqueeze(0) outs = self(last_layer_hidden_state, last_layer_reference) predictions = self.predict_by_feat( *outs, batch_img_metas=batch_img_metas, rescale=rescale) return predictions ================================================ FILE: mmdet/models/dense_heads/ddod_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Sequence, Tuple import torch import torch.nn as nn from mmcv.cnn import ConvModule, Scale from mmengine.model import bias_init_with_prob, normal_init from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures.bbox import bbox_overlaps from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptInstanceList, reduce_mean) from ..task_modules.prior_generators import anchor_inside_flags from ..utils import images_to_levels, multi_apply, unmap from .anchor_head import AnchorHead EPS = 1e-12 @MODELS.register_module() class DDODHead(AnchorHead): """Detection Head of `DDOD `_. DDOD head decomposes conjunctions lying in most current one-stage detectors via label assignment disentanglement, spatial feature disentanglement, and pyramid supervision disentanglement. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. stacked_convs (int): The number of stacked Conv. Defaults to 4. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. use_dcn (bool): Use dcn, Same as ATSS when False. Defaults to True. norm_cfg (:obj:`ConfigDict` or dict): Normal config of ddod head. Defaults to dict(type='GN', num_groups=32, requires_grad=True). loss_iou (:obj:`ConfigDict` or dict): Config of IoU loss. Defaults to dict(type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0). """ def __init__(self, num_classes: int, in_channels: int, stacked_convs: int = 4, conv_cfg: OptConfigType = None, use_dcn: bool = True, norm_cfg: ConfigType = dict( type='GN', num_groups=32, requires_grad=True), loss_iou: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), **kwargs) -> None: self.stacked_convs = stacked_convs self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.use_dcn = use_dcn super().__init__(num_classes, in_channels, **kwargs) if self.train_cfg: self.cls_assigner = TASK_UTILS.build(self.train_cfg['assigner']) self.reg_assigner = TASK_UTILS.build( self.train_cfg['reg_assigner']) self.loss_iou = MODELS.build(loss_iou) def _init_layers(self) -> None: """Initialize layers of the head.""" self.relu = nn.ReLU(inplace=True) self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=dict(type='DCN', deform_groups=1) if i == 0 and self.use_dcn else self.conv_cfg, norm_cfg=self.norm_cfg)) self.reg_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=dict(type='DCN', deform_groups=1) if i == 0 and self.use_dcn else self.conv_cfg, norm_cfg=self.norm_cfg)) self.atss_cls = nn.Conv2d( self.feat_channels, self.num_base_priors * self.cls_out_channels, 3, padding=1) self.atss_reg = nn.Conv2d( self.feat_channels, self.num_base_priors * 4, 3, padding=1) self.atss_iou = nn.Conv2d( self.feat_channels, self.num_base_priors * 1, 3, padding=1) self.scales = nn.ModuleList( [Scale(1.0) for _ in self.prior_generator.strides]) # we use the global list in loss self.cls_num_pos_samples_per_level = [ 0. for _ in range(len(self.prior_generator.strides)) ] self.reg_num_pos_samples_per_level = [ 0. for _ in range(len(self.prior_generator.strides)) ] def init_weights(self) -> None: """Initialize weights of the head.""" for m in self.cls_convs: normal_init(m.conv, std=0.01) for m in self.reg_convs: normal_init(m.conv, std=0.01) normal_init(self.atss_reg, std=0.01) normal_init(self.atss_iou, std=0.01) bias_cls = bias_init_with_prob(0.01) normal_init(self.atss_cls, std=0.01, bias=bias_cls) def forward(self, x: Tuple[Tensor]) -> Tuple[List[Tensor]]: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of classification scores, bbox predictions, and iou predictions. - cls_scores (list[Tensor]): Classification scores for all \ scale levels, each is a 4D-tensor, the channels number is \ num_base_priors * num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for all \ scale levels, each is a 4D-tensor, the channels number is \ num_base_priors * 4. - iou_preds (list[Tensor]): IoU scores for all scale levels, \ each is a 4D-tensor, the channels number is num_base_priors * 1. """ return multi_apply(self.forward_single, x, self.scales) def forward_single(self, x: Tensor, scale: Scale) -> Sequence[Tensor]: """Forward feature of a single scale level. Args: x (Tensor): Features of a single scale level. scale (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize the bbox prediction. Returns: tuple: - cls_score (Tensor): Cls scores for a single scale level \ the channels number is num_base_priors * num_classes. - bbox_pred (Tensor): Box energies / deltas for a single \ scale level, the channels number is num_base_priors * 4. - iou_pred (Tensor): Iou for a single scale level, the \ channel number is (N, num_base_priors * 1, H, W). """ cls_feat = x reg_feat = x for cls_conv in self.cls_convs: cls_feat = cls_conv(cls_feat) for reg_conv in self.reg_convs: reg_feat = reg_conv(reg_feat) cls_score = self.atss_cls(cls_feat) # we just follow atss, not apply exp in bbox_pred bbox_pred = scale(self.atss_reg(reg_feat)).float() iou_pred = self.atss_iou(reg_feat) return cls_score, bbox_pred, iou_pred def loss_cls_by_feat_single(self, cls_score: Tensor, labels: Tensor, label_weights: Tensor, reweight_factor: List[float], avg_factor: float) -> Tuple[Tensor]: """Compute cls loss of a single scale level. Args: cls_score (Tensor): Box scores for each scale level Has shape (N, num_base_priors * num_classes, H, W). labels (Tensor): Labels of each anchors with shape (N, num_total_anchors). label_weights (Tensor): Label weights of each anchor with shape (N, num_total_anchors) reweight_factor (List[float]): Reweight factor for cls and reg loss. avg_factor (float): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using `PseudoSampler`, `avg_factor` is usually equal to the number of positive priors. Returns: Tuple[Tensor]: A tuple of loss components. """ cls_score = cls_score.permute(0, 2, 3, 1).reshape( -1, self.cls_out_channels).contiguous() labels = labels.reshape(-1) label_weights = label_weights.reshape(-1) loss_cls = self.loss_cls( cls_score, labels, label_weights, avg_factor=avg_factor) return reweight_factor * loss_cls, def loss_reg_by_feat_single(self, anchors: Tensor, bbox_pred: Tensor, iou_pred: Tensor, labels, label_weights: Tensor, bbox_targets: Tensor, bbox_weights: Tensor, reweight_factor: List[float], avg_factor: float) -> Tuple[Tensor, Tensor]: """Compute reg loss of a single scale level based on the features extracted by the detection head. Args: anchors (Tensor): Box reference for each scale level with shape (N, num_total_anchors, 4). bbox_pred (Tensor): Box energies / deltas for each scale level with shape (N, num_base_priors * 4, H, W). iou_pred (Tensor): Iou for a single scale level, the channel number is (N, num_base_priors * 1, H, W). labels (Tensor): Labels of each anchors with shape (N, num_total_anchors). label_weights (Tensor): Label weights of each anchor with shape (N, num_total_anchors) bbox_targets (Tensor): BBox regression targets of each anchor weight shape (N, num_total_anchors, 4). bbox_weights (Tensor): BBox weights of all anchors in the image with shape (N, 4) reweight_factor (List[float]): Reweight factor for cls and reg loss. avg_factor (float): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using `PseudoSampler`, `avg_factor` is usually equal to the number of positive priors. Returns: Tuple[Tensor, Tensor]: A tuple of loss components. """ anchors = anchors.reshape(-1, 4) bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) iou_pred = iou_pred.permute(0, 2, 3, 1).reshape(-1, ) bbox_targets = bbox_targets.reshape(-1, 4) bbox_weights = bbox_weights.reshape(-1, 4) labels = labels.reshape(-1) label_weights = label_weights.reshape(-1) iou_targets = label_weights.new_zeros(labels.shape) iou_weights = label_weights.new_zeros(labels.shape) iou_weights[(bbox_weights.sum(axis=1) > 0).nonzero( as_tuple=False)] = 1. # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = self.num_classes pos_inds = ((labels >= 0) & (labels < bg_class_ind)).nonzero(as_tuple=False).squeeze(1) if len(pos_inds) > 0: pos_bbox_targets = bbox_targets[pos_inds] pos_bbox_pred = bbox_pred[pos_inds] pos_anchors = anchors[pos_inds] pos_decode_bbox_pred = self.bbox_coder.decode( pos_anchors, pos_bbox_pred) pos_decode_bbox_targets = self.bbox_coder.decode( pos_anchors, pos_bbox_targets) # regression loss loss_bbox = self.loss_bbox( pos_decode_bbox_pred, pos_decode_bbox_targets, avg_factor=avg_factor) iou_targets[pos_inds] = bbox_overlaps( pos_decode_bbox_pred.detach(), pos_decode_bbox_targets, is_aligned=True) loss_iou = self.loss_iou( iou_pred, iou_targets, iou_weights, avg_factor=avg_factor) else: loss_bbox = bbox_pred.sum() * 0 loss_iou = iou_pred.sum() * 0 return reweight_factor * loss_bbox, reweight_factor * loss_iou def calc_reweight_factor(self, labels_list: List[Tensor]) -> List[float]: """Compute reweight_factor for regression and classification loss.""" # get pos samples for each level bg_class_ind = self.num_classes for ii, each_level_label in enumerate(labels_list): pos_inds = ((each_level_label >= 0) & (each_level_label < bg_class_ind)).nonzero( as_tuple=False).squeeze(1) self.cls_num_pos_samples_per_level[ii] += len(pos_inds) # get reweight factor from 1 ~ 2 with bilinear interpolation min_pos_samples = min(self.cls_num_pos_samples_per_level) max_pos_samples = max(self.cls_num_pos_samples_per_level) interval = 1. / (max_pos_samples - min_pos_samples + 1e-10) reweight_factor_per_level = [] for pos_samples in self.cls_num_pos_samples_per_level: factor = 2. - (pos_samples - min_pos_samples) * interval reweight_factor_per_level.append(factor) return reweight_factor_per_level def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], iou_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_base_priors * num_classes, H, W) bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_base_priors * 4, H, W) iou_preds (list[Tensor]): Score factor for all scale level, each is a 4D-tensor, has shape (batch_size, 1, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) # calculate common vars for cls and reg assigners at once targets_com = self.process_predictions_and_anchors( anchor_list, valid_flag_list, cls_scores, bbox_preds, batch_img_metas, batch_gt_instances_ignore) (anchor_list, valid_flag_list, num_level_anchors_list, cls_score_list, bbox_pred_list, batch_gt_instances_ignore) = targets_com # classification branch assigner cls_targets = self.get_cls_targets( anchor_list, valid_flag_list, num_level_anchors_list, cls_score_list, bbox_pred_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (cls_anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_targets avg_factor = reduce_mean( torch.tensor(avg_factor, dtype=torch.float, device=device)).item() avg_factor = max(avg_factor, 1.0) reweight_factor_per_level = self.calc_reweight_factor(labels_list) cls_losses_cls, = multi_apply( self.loss_cls_by_feat_single, cls_scores, labels_list, label_weights_list, reweight_factor_per_level, avg_factor=avg_factor) # regression branch assigner reg_targets = self.get_reg_targets( anchor_list, valid_flag_list, num_level_anchors_list, cls_score_list, bbox_pred_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (reg_anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = reg_targets avg_factor = reduce_mean( torch.tensor(avg_factor, dtype=torch.float, device=device)).item() avg_factor = max(avg_factor, 1.0) reweight_factor_per_level = self.calc_reweight_factor(labels_list) reg_losses_bbox, reg_losses_iou = multi_apply( self.loss_reg_by_feat_single, reg_anchor_list, bbox_preds, iou_preds, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, reweight_factor_per_level, avg_factor=avg_factor) return dict( loss_cls=cls_losses_cls, loss_bbox=reg_losses_bbox, loss_iou=reg_losses_iou) def process_predictions_and_anchors( self, anchor_list: List[List[Tensor]], valid_flag_list: List[List[Tensor]], cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> tuple: """Compute common vars for regression and classification targets. Args: anchor_list (List[List[Tensor]]): anchors of each image. valid_flag_list (List[List[Tensor]]): Valid flags of each image. cls_scores (List[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * num_classes. bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * 4. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Return: tuple[Tensor]: A tuple of common loss vars. """ num_imgs = len(batch_img_metas) assert len(anchor_list) == len(valid_flag_list) == num_imgs # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] num_level_anchors_list = [num_level_anchors] * num_imgs anchor_list_ = [] valid_flag_list_ = [] # concat all level anchors and flags to a single tensor for i in range(num_imgs): assert len(anchor_list[i]) == len(valid_flag_list[i]) anchor_list_.append(torch.cat(anchor_list[i])) valid_flag_list_.append(torch.cat(valid_flag_list[i])) # compute targets for each image if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None for _ in range(num_imgs)] num_levels = len(cls_scores) cls_score_list = [] bbox_pred_list = [] mlvl_cls_score_list = [ cls_score.permute(0, 2, 3, 1).reshape( num_imgs, -1, self.num_base_priors * self.cls_out_channels) for cls_score in cls_scores ] mlvl_bbox_pred_list = [ bbox_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, self.num_base_priors * 4) for bbox_pred in bbox_preds ] for i in range(num_imgs): mlvl_cls_tensor_list = [ mlvl_cls_score_list[j][i] for j in range(num_levels) ] mlvl_bbox_tensor_list = [ mlvl_bbox_pred_list[j][i] for j in range(num_levels) ] cat_mlvl_cls_score = torch.cat(mlvl_cls_tensor_list, dim=0) cat_mlvl_bbox_pred = torch.cat(mlvl_bbox_tensor_list, dim=0) cls_score_list.append(cat_mlvl_cls_score) bbox_pred_list.append(cat_mlvl_bbox_pred) return (anchor_list_, valid_flag_list_, num_level_anchors_list, cls_score_list, bbox_pred_list, batch_gt_instances_ignore) def get_cls_targets(self, anchor_list: List[Tensor], valid_flag_list: List[Tensor], num_level_anchors_list: List[int], cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs: bool = True) -> tuple: """Get cls targets for DDOD head. This method is almost the same as `AnchorHead.get_targets()`. Besides returning the targets as the parent method does, it also returns the anchors as the first element of the returned tuple. Args: anchor_list (list[Tensor]): anchors of each image. valid_flag_list (list[Tensor]): Valid flags of each image. num_level_anchors_list (list[Tensor]): Number of anchors of each scale level of all image. cls_score_list (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * num_classes. bbox_pred_list (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * 4. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Return: tuple[Tensor]: A tuple of cls targets components. """ (all_anchors, all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, pos_inds_list, neg_inds_list, sampling_results_list) = multi_apply( self._get_targets_single, anchor_list, valid_flag_list, cls_score_list, bbox_pred_list, num_level_anchors_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs, is_cls_assigner=True) # Get `avg_factor` of all images, which calculate in `SamplingResult`. # When using sampling method, avg_factor is usually the sum of # positive and negative priors. When using `PseudoSampler`, # `avg_factor` is usually equal to the number of positive priors. avg_factor = sum( [results.avg_factor for results in sampling_results_list]) # split targets to a list w.r.t. multiple levels anchors_list = images_to_levels(all_anchors, num_level_anchors_list[0]) labels_list = images_to_levels(all_labels, num_level_anchors_list[0]) label_weights_list = images_to_levels(all_label_weights, num_level_anchors_list[0]) bbox_targets_list = images_to_levels(all_bbox_targets, num_level_anchors_list[0]) bbox_weights_list = images_to_levels(all_bbox_weights, num_level_anchors_list[0]) return (anchors_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) def get_reg_targets(self, anchor_list: List[Tensor], valid_flag_list: List[Tensor], num_level_anchors_list: List[int], cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs: bool = True) -> tuple: """Get reg targets for DDOD head. This method is almost the same as `AnchorHead.get_targets()` when is_cls_assigner is False. Besides returning the targets as the parent method does, it also returns the anchors as the first element of the returned tuple. Args: anchor_list (list[Tensor]): anchors of each image. valid_flag_list (list[Tensor]): Valid flags of each image. num_level_anchors_list (list[Tensor]): Number of anchors of each scale level of all image. cls_score_list (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * num_classes. bbox_pred_list (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * 4. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Return: tuple[Tensor]: A tuple of reg targets components. """ (all_anchors, all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, pos_inds_list, neg_inds_list, sampling_results_list) = multi_apply( self._get_targets_single, anchor_list, valid_flag_list, cls_score_list, bbox_pred_list, num_level_anchors_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs, is_cls_assigner=False) # Get `avg_factor` of all images, which calculate in `SamplingResult`. # When using sampling method, avg_factor is usually the sum of # positive and negative priors. When using `PseudoSampler`, # `avg_factor` is usually equal to the number of positive priors. avg_factor = sum( [results.avg_factor for results in sampling_results_list]) # split targets to a list w.r.t. multiple levels anchors_list = images_to_levels(all_anchors, num_level_anchors_list[0]) labels_list = images_to_levels(all_labels, num_level_anchors_list[0]) label_weights_list = images_to_levels(all_label_weights, num_level_anchors_list[0]) bbox_targets_list = images_to_levels(all_bbox_targets, num_level_anchors_list[0]) bbox_weights_list = images_to_levels(all_bbox_weights, num_level_anchors_list[0]) return (anchors_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) def _get_targets_single(self, flat_anchors: Tensor, valid_flags: Tensor, cls_scores: Tensor, bbox_preds: Tensor, num_level_anchors: List[int], gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None, unmap_outputs: bool = True, is_cls_assigner: bool = True) -> tuple: """Compute regression, classification targets for anchors in a single image. Args: flat_anchors (Tensor): Multi-level anchors of the image, which are concatenated into a single tensor of shape (num_base_priors, 4). valid_flags (Tensor): Multi level valid flags of the image, which are concatenated into a single tensor of shape (num_base_priors,). cls_scores (Tensor): Classification scores for all scale levels of the image. bbox_preds (Tensor): Box energies / deltas for all scale levels of the image. num_level_anchors (List[int]): Number of anchors of each scale level. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. is_cls_assigner (bool): Classification or regression. Defaults to True. Returns: tuple: N is the number of total anchors in the image. - anchors (Tensor): all anchors in the image with shape (N, 4). - labels (Tensor): Labels of all anchors in the image with \ shape (N, ). - label_weights (Tensor): Label weights of all anchor in the \ image with shape (N, ). - bbox_targets (Tensor): BBox targets of all anchors in the \ image with shape (N, 4). - bbox_weights (Tensor): BBox weights of all anchors in the \ image with shape (N, 4) - pos_inds (Tensor): Indices of positive anchor with shape \ (num_pos, ). - neg_inds (Tensor): Indices of negative anchor with shape \ (num_neg, ). - sampling_result (:obj:`SamplingResult`): Sampling results. """ inside_flags = anchor_inside_flags(flat_anchors, valid_flags, img_meta['img_shape'][:2], self.train_cfg['allowed_border']) if not inside_flags.any(): raise ValueError( 'There is no valid anchor inside the image boundary. Please ' 'check the image size and anchor sizes, or set ' '``allowed_border`` to -1 to skip the condition.') # assign gt and sample anchors anchors = flat_anchors[inside_flags, :] num_level_anchors_inside = self.get_num_level_anchors_inside( num_level_anchors, inside_flags) bbox_preds_valid = bbox_preds[inside_flags, :] cls_scores_valid = cls_scores[inside_flags, :] assigner = self.cls_assigner if is_cls_assigner else self.reg_assigner # decode prediction out of assigner bbox_preds_valid = self.bbox_coder.decode(anchors, bbox_preds_valid) pred_instances = InstanceData( priors=anchors, bboxes=bbox_preds_valid, scores=cls_scores_valid) assign_result = assigner.assign( pred_instances=pred_instances, num_level_priors=num_level_anchors_inside, gt_instances=gt_instances, gt_instances_ignore=gt_instances_ignore) sampling_result = self.sampler.sample( assign_result=assign_result, pred_instances=pred_instances, gt_instances=gt_instances) num_valid_anchors = anchors.shape[0] bbox_targets = torch.zeros_like(anchors) bbox_weights = torch.zeros_like(anchors) labels = anchors.new_full((num_valid_anchors, ), self.num_classes, dtype=torch.long) label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: pos_bbox_targets = self.bbox_coder.encode( sampling_result.pos_bboxes, sampling_result.pos_gt_bboxes) bbox_targets[pos_inds, :] = pos_bbox_targets bbox_weights[pos_inds, :] = 1.0 labels[pos_inds] = sampling_result.pos_gt_labels if self.train_cfg['pos_weight'] <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = self.train_cfg['pos_weight'] if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 # map up to original set of anchors if unmap_outputs: num_total_anchors = flat_anchors.size(0) anchors = unmap(anchors, num_total_anchors, inside_flags) labels = unmap( labels, num_total_anchors, inside_flags, fill=self.num_classes) label_weights = unmap(label_weights, num_total_anchors, inside_flags) bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) return (anchors, labels, label_weights, bbox_targets, bbox_weights, pos_inds, neg_inds, sampling_result) def get_num_level_anchors_inside(self, num_level_anchors: List[int], inside_flags: Tensor) -> List[int]: """Get the anchors of each scale level inside. Args: num_level_anchors (list[int]): Number of anchors of each scale level. inside_flags (Tensor): Multi level inside flags of the image, which are concatenated into a single tensor of shape (num_base_priors,). Returns: list[int]: Number of anchors of each scale level inside. """ split_inside_flags = torch.split(inside_flags, num_level_anchors) num_level_anchors_inside = [ int(flags.sum()) for flags in split_inside_flags ] return num_level_anchors_inside ================================================ FILE: mmdet/models/dense_heads/deformable_detr_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import Dict, List, Tuple import torch import torch.nn as nn from mmcv.cnn import Linear from mmengine.model import bias_init_with_prob, constant_init from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import InstanceList, OptInstanceList from ..layers import inverse_sigmoid from .detr_head import DETRHead @MODELS.register_module() class DeformableDETRHead(DETRHead): r"""Head of DeformDETR: Deformable DETR: Deformable Transformers for End-to-End Object Detection. Code is modified from the `official github repo `_. More details can be found in the `paper `_ . Args: share_pred_layer (bool): Whether to share parameters for all the prediction layers. Defaults to `False`. num_pred_layer (int): The number of the prediction layers. Defaults to 6. as_two_stage (bool, optional): Whether to generate the proposal from the outputs of encoder. Defaults to `False`. """ def __init__(self, *args, share_pred_layer: bool = False, num_pred_layer: int = 6, as_two_stage: bool = False, **kwargs) -> None: self.share_pred_layer = share_pred_layer self.num_pred_layer = num_pred_layer self.as_two_stage = as_two_stage super().__init__(*args, **kwargs) def _init_layers(self) -> None: """Initialize classification branch and regression branch of head.""" fc_cls = Linear(self.embed_dims, self.cls_out_channels) reg_branch = [] for _ in range(self.num_reg_fcs): reg_branch.append(Linear(self.embed_dims, self.embed_dims)) reg_branch.append(nn.ReLU()) reg_branch.append(Linear(self.embed_dims, 4)) reg_branch = nn.Sequential(*reg_branch) if self.share_pred_layer: self.cls_branches = nn.ModuleList( [fc_cls for _ in range(self.num_pred_layer)]) self.reg_branches = nn.ModuleList( [reg_branch for _ in range(self.num_pred_layer)]) else: self.cls_branches = nn.ModuleList( [copy.deepcopy(fc_cls) for _ in range(self.num_pred_layer)]) self.reg_branches = nn.ModuleList([ copy.deepcopy(reg_branch) for _ in range(self.num_pred_layer) ]) def init_weights(self) -> None: """Initialize weights of the Deformable DETR head.""" if self.loss_cls.use_sigmoid: bias_init = bias_init_with_prob(0.01) for m in self.cls_branches: nn.init.constant_(m.bias, bias_init) for m in self.reg_branches: constant_init(m[-1], 0, bias=0) nn.init.constant_(self.reg_branches[0][-1].bias.data[2:], -2.0) if self.as_two_stage: for m in self.reg_branches: nn.init.constant_(m[-1].bias.data[2:], 0.0) def forward(self, hidden_states: Tensor, references: List[Tensor]) -> Tuple[Tensor]: """Forward function. Args: hidden_states (Tensor): Hidden states output from each decoder layer, has shape (num_decoder_layers, bs, num_queries, dim). references (list[Tensor]): List of the reference from the decoder. The first reference is the `init_reference` (initial) and the other num_decoder_layers(6) references are `inter_references` (intermediate). The `init_reference` has shape (bs, num_queries, 4) when `as_two_stage` of the detector is `True`, otherwise (bs, num_queries, 2). Each `inter_reference` has shape (bs, num_queries, 4) when `with_box_refine` of the detector is `True`, otherwise (bs, num_queries, 2). The coordinates are arranged as (cx, cy) when the last dimension is 2, and (cx, cy, w, h) when it is 4. Returns: tuple[Tensor]: results of head containing the following tensor. - all_layers_outputs_classes (Tensor): Outputs from the classification head, has shape (num_decoder_layers, bs, num_queries, cls_out_channels). - all_layers_outputs_coords (Tensor): Sigmoid outputs from the regression head with normalized coordinate format (cx, cy, w, h), has shape (num_decoder_layers, bs, num_queries, 4) with the last dimension arranged as (cx, cy, w, h). """ all_layers_outputs_classes = [] all_layers_outputs_coords = [] for layer_id in range(hidden_states.shape[0]): reference = inverse_sigmoid(references[layer_id]) # NOTE The last reference will not be used. hidden_state = hidden_states[layer_id] outputs_class = self.cls_branches[layer_id](hidden_state) tmp_reg_preds = self.reg_branches[layer_id](hidden_state) if reference.shape[-1] == 4: # When `layer` is 0 and `as_two_stage` of the detector # is `True`, or when `layer` is greater than 0 and # `with_box_refine` of the detector is `True`. tmp_reg_preds += reference else: # When `layer` is 0 and `as_two_stage` of the detector # is `False`, or when `layer` is greater than 0 and # `with_box_refine` of the detector is `False`. assert reference.shape[-1] == 2 tmp_reg_preds[..., :2] += reference outputs_coord = tmp_reg_preds.sigmoid() all_layers_outputs_classes.append(outputs_class) all_layers_outputs_coords.append(outputs_coord) all_layers_outputs_classes = torch.stack(all_layers_outputs_classes) all_layers_outputs_coords = torch.stack(all_layers_outputs_coords) return all_layers_outputs_classes, all_layers_outputs_coords def loss(self, hidden_states: Tensor, references: List[Tensor], enc_outputs_class: Tensor, enc_outputs_coord: Tensor, batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection head on the queries of the upstream network. Args: hidden_states (Tensor): Hidden states output from each decoder layer, has shape (num_decoder_layers, num_queries, bs, dim). references (list[Tensor]): List of the reference from the decoder. The first reference is the `init_reference` (initial) and the other num_decoder_layers(6) references are `inter_references` (intermediate). The `init_reference` has shape (bs, num_queries, 4) when `as_two_stage` of the detector is `True`, otherwise (bs, num_queries, 2). Each `inter_reference` has shape (bs, num_queries, 4) when `with_box_refine` of the detector is `True`, otherwise (bs, num_queries, 2). The coordinates are arranged as (cx, cy) when the last dimension is 2, and (cx, cy, w, h) when it is 4. enc_outputs_class (Tensor): The score of each point on encode feature map, has shape (bs, num_feat_points, cls_out_channels). Only when `as_two_stage` is `True` it would be passed in, otherwise it would be `None`. enc_outputs_coord (Tensor): The proposal generate from the encode feature map, has shape (bs, num_feat_points, 4) with the last dimension arranged as (cx, cy, w, h). Only when `as_two_stage` is `True` it would be passed in, otherwise it would be `None`. batch_data_samples (list[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ batch_gt_instances = [] batch_img_metas = [] for data_sample in batch_data_samples: batch_img_metas.append(data_sample.metainfo) batch_gt_instances.append(data_sample.gt_instances) outs = self(hidden_states, references) loss_inputs = outs + (enc_outputs_class, enc_outputs_coord, batch_gt_instances, batch_img_metas) losses = self.loss_by_feat(*loss_inputs) return losses def loss_by_feat( self, all_layers_cls_scores: Tensor, all_layers_bbox_preds: Tensor, enc_cls_scores: Tensor, enc_bbox_preds: Tensor, batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """Loss function. Args: all_layers_cls_scores (Tensor): Classification scores of all decoder layers, has shape (num_decoder_layers, bs, num_queries, cls_out_channels). all_layers_bbox_preds (Tensor): Regression outputs of all decoder layers. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and has shape (num_decoder_layers, bs, num_queries, 4) with the last dimension arranged as (cx, cy, w, h). enc_cls_scores (Tensor): The score of each point on encode feature map, has shape (bs, num_feat_points, cls_out_channels). Only when `as_two_stage` is `True` it would be passes in, otherwise, it would be `None`. enc_bbox_preds (Tensor): The proposal generate from the encode feature map, has shape (bs, num_feat_points, 4) with the last dimension arranged as (cx, cy, w, h). Only when `as_two_stage` is `True` it would be passed in, otherwise it would be `None`. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ loss_dict = super().loss_by_feat(all_layers_cls_scores, all_layers_bbox_preds, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) # loss of proposal generated from encode feature map. if enc_cls_scores is not None: proposal_gt_instances = copy.deepcopy(batch_gt_instances) for i in range(len(proposal_gt_instances)): proposal_gt_instances[i].labels = torch.zeros_like( proposal_gt_instances[i].labels) enc_loss_cls, enc_losses_bbox, enc_losses_iou = \ self.loss_by_feat_single( enc_cls_scores, enc_bbox_preds, batch_gt_instances=proposal_gt_instances, batch_img_metas=batch_img_metas) loss_dict['enc_loss_cls'] = enc_loss_cls loss_dict['enc_loss_bbox'] = enc_losses_bbox loss_dict['enc_loss_iou'] = enc_losses_iou return loss_dict def predict(self, hidden_states: Tensor, references: List[Tensor], batch_data_samples: SampleList, rescale: bool = True) -> InstanceList: """Perform forward propagation and loss calculation of the detection head on the queries of the upstream network. Args: hidden_states (Tensor): Hidden states output from each decoder layer, has shape (num_decoder_layers, num_queries, bs, dim). references (list[Tensor]): List of the reference from the decoder. The first reference is the `init_reference` (initial) and the other num_decoder_layers(6) references are `inter_references` (intermediate). The `init_reference` has shape (bs, num_queries, 4) when `as_two_stage` of the detector is `True`, otherwise (bs, num_queries, 2). Each `inter_reference` has shape (bs, num_queries, 4) when `with_box_refine` of the detector is `True`, otherwise (bs, num_queries, 2). The coordinates are arranged as (cx, cy) when the last dimension is 2, and (cx, cy, w, h) when it is 4. batch_data_samples (list[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool, optional): If `True`, return boxes in original image space. Defaults to `True`. Returns: list[obj:`InstanceData`]: Detection results of each image after the post process. """ batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] outs = self(hidden_states, references) predictions = self.predict_by_feat( *outs, batch_img_metas=batch_img_metas, rescale=rescale) return predictions def predict_by_feat(self, all_layers_cls_scores: Tensor, all_layers_bbox_preds: Tensor, batch_img_metas: List[Dict], rescale: bool = False) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. Args: all_layers_cls_scores (Tensor): Classification scores of all decoder layers, has shape (num_decoder_layers, bs, num_queries, cls_out_channels). all_layers_bbox_preds (Tensor): Regression outputs of all decoder layers. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and shape (num_decoder_layers, bs, num_queries, 4) with the last dimension arranged as (cx, cy, w, h). batch_img_metas (list[dict]): Meta information of each image. rescale (bool, optional): If `True`, return boxes in original image space. Default `False`. Returns: list[obj:`InstanceData`]: Detection results of each image after the post process. """ cls_scores = all_layers_cls_scores[-1] bbox_preds = all_layers_bbox_preds[-1] result_list = [] for img_id in range(len(batch_img_metas)): cls_score = cls_scores[img_id] bbox_pred = bbox_preds[img_id] img_meta = batch_img_metas[img_id] results = self._predict_by_feat_single(cls_score, bbox_pred, img_meta, rescale) result_list.append(results) return result_list ================================================ FILE: mmdet/models/dense_heads/dense_test_mixins.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import sys import warnings from inspect import signature import torch from mmcv.ops import batched_nms from mmengine.structures import InstanceData from mmdet.structures.bbox import bbox_mapping_back from ..test_time_augs import merge_aug_proposals if sys.version_info >= (3, 7): from mmdet.utils.contextmanagers import completed class BBoxTestMixin(object): """Mixin class for testing det bboxes via DenseHead.""" def simple_test_bboxes(self, feats, img_metas, rescale=False): """Test det bboxes without test-time augmentation, can be applied in DenseHead except for ``RPNHead`` and its variants, e.g., ``GARPNHead``, etc. Args: feats (tuple[torch.Tensor]): Multi-level features from the upstream network, each is a 4D-tensor. img_metas (list[dict]): List of image information. rescale (bool, optional): Whether to rescale the results. Defaults to False. Returns: list[obj:`InstanceData`]: Detection results of each image after the post process. \ Each item usually contains following keys. \ - scores (Tensor): Classification scores, has a shape (num_instance,) - labels (Tensor): Labels of bboxes, has a shape (num_instances,). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ warnings.warn('You are calling `simple_test_bboxes` in ' '`dense_test_mixins`, but the `dense_test_mixins`' 'will be deprecated soon. Please use ' '`simple_test` instead.') outs = self.forward(feats) results_list = self.get_results( *outs, img_metas=img_metas, rescale=rescale) return results_list def aug_test_bboxes(self, feats, img_metas, rescale=False): """Test det bboxes with test time augmentation, can be applied in DenseHead except for ``RPNHead`` and its variants, e.g., ``GARPNHead``, etc. Args: feats (list[Tensor]): the outer list indicates test-time augmentations and inner Tensor should have a shape NxCxHxW, which contains features for all images in the batch. img_metas (list[list[dict]]): the outer list indicates test-time augs (multiscale, flip, etc.) and the inner list indicates images in a batch. each dict has image information. rescale (bool, optional): Whether to rescale the results. Defaults to False. Returns: list[tuple[Tensor, Tensor]]: Each item in result_list is 2-tuple. The first item is ``bboxes`` with shape (n, 5), where 5 represent (tl_x, tl_y, br_x, br_y, score). The shape of the second tensor in the tuple is ``labels`` with shape (n,). The length of list should always be 1. """ warnings.warn('You are calling `aug_test_bboxes` in ' '`dense_test_mixins`, but the `dense_test_mixins`' 'will be deprecated soon. Please use ' '`aug_test` instead.') # check with_nms argument gb_sig = signature(self.get_results) gb_args = [p.name for p in gb_sig.parameters.values()] gbs_sig = signature(self._get_results_single) gbs_args = [p.name for p in gbs_sig.parameters.values()] assert ('with_nms' in gb_args) and ('with_nms' in gbs_args), \ f'{self.__class__.__name__}' \ ' does not support test-time augmentation' aug_bboxes = [] aug_scores = [] aug_labels = [] for x, img_meta in zip(feats, img_metas): # only one image in the batch outs = self.forward(x) bbox_outputs = self.get_results( *outs, img_metas=img_meta, cfg=self.test_cfg, rescale=False, with_nms=False)[0] aug_bboxes.append(bbox_outputs.bboxes) aug_scores.append(bbox_outputs.scores) if len(bbox_outputs) >= 3: aug_labels.append(bbox_outputs.labels) # after merging, bboxes will be rescaled to the original image size merged_bboxes, merged_scores = self.merge_aug_bboxes( aug_bboxes, aug_scores, img_metas) merged_labels = torch.cat(aug_labels, dim=0) if aug_labels else None if merged_bboxes.numel() == 0: det_bboxes = torch.cat([merged_bboxes, merged_scores[:, None]], -1) return [ (det_bboxes, merged_labels), ] det_bboxes, keep_idxs = batched_nms(merged_bboxes, merged_scores, merged_labels, self.test_cfg.nms) det_bboxes = det_bboxes[:self.test_cfg.max_per_img] det_labels = merged_labels[keep_idxs][:self.test_cfg.max_per_img] if rescale: _det_bboxes = det_bboxes else: _det_bboxes = det_bboxes.clone() _det_bboxes[:, :4] *= det_bboxes.new_tensor( img_metas[0][0]['scale_factor']) results = InstanceData() results.bboxes = _det_bboxes[:, :4] results.scores = _det_bboxes[:, 4] results.labels = det_labels return [results] def aug_test_rpn(self, feats, img_metas): """Test with augmentation for only for ``RPNHead`` and its variants, e.g., ``GARPNHead``, etc. Args: feats (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. img_metas (list[dict]): Meta info of each image. Returns: list[Tensor]: Proposals of each image, each item has shape (n, 5), where 5 represent (tl_x, tl_y, br_x, br_y, score). """ samples_per_gpu = len(img_metas[0]) aug_proposals = [[] for _ in range(samples_per_gpu)] for x, img_meta in zip(feats, img_metas): results_list = self.simple_test_rpn(x, img_meta) for i, results in enumerate(results_list): proposals = torch.cat( [results.bboxes, results.scores[:, None]], dim=-1) aug_proposals[i].append(proposals) # reorganize the order of 'img_metas' to match the dimensions # of 'aug_proposals' aug_img_metas = [] for i in range(samples_per_gpu): aug_img_meta = [] for j in range(len(img_metas)): aug_img_meta.append(img_metas[j][i]) aug_img_metas.append(aug_img_meta) # after merging, proposals will be rescaled to the original image size merged_proposals = [] for proposals, aug_img_meta in zip(aug_proposals, aug_img_metas): merged_proposal = merge_aug_proposals(proposals, aug_img_meta, self.test_cfg) results = InstanceData() results.bboxes = merged_proposal[:, :4] results.scores = merged_proposal[:, 4] merged_proposals.append(results) return merged_proposals if sys.version_info >= (3, 7): async def async_simple_test_rpn(self, x, img_metas): sleep_interval = self.test_cfg.pop('async_sleep_interval', 0.025) async with completed( __name__, 'rpn_head_forward', sleep_interval=sleep_interval): rpn_outs = self(x) proposal_list = self.get_results(*rpn_outs, img_metas=img_metas) return proposal_list def merge_aug_bboxes(self, aug_bboxes, aug_scores, img_metas): """Merge augmented detection bboxes and scores. Args: aug_bboxes (list[Tensor]): shape (n, 4*#class) aug_scores (list[Tensor] or None): shape (n, #class) img_shapes (list[Tensor]): shape (3, ). Returns: tuple[Tensor]: ``bboxes`` with shape (n,4), where 4 represent (tl_x, tl_y, br_x, br_y) and ``scores`` with shape (n,). """ recovered_bboxes = [] for bboxes, img_info in zip(aug_bboxes, img_metas): img_shape = img_info[0]['img_shape'] scale_factor = img_info[0]['scale_factor'] flip = img_info[0]['flip'] flip_direction = img_info[0]['flip_direction'] bboxes = bbox_mapping_back(bboxes, img_shape, scale_factor, flip, flip_direction) recovered_bboxes.append(bboxes) bboxes = torch.cat(recovered_bboxes, dim=0) if aug_scores is None: return bboxes else: scores = torch.cat(aug_scores, dim=0) return bboxes, scores ================================================ FILE: mmdet/models/dense_heads/detr_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import Linear from mmcv.cnn.bricks.transformer import FFN from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox_cxcywh_to_xyxy, bbox_xyxy_to_cxcywh from mmdet.utils import (ConfigType, InstanceList, OptInstanceList, OptMultiConfig, reduce_mean) from ..utils import multi_apply @MODELS.register_module() class DETRHead(BaseModule): r"""Head of DETR. DETR:End-to-End Object Detection with Transformers. More details can be found in the `paper `_ . Args: num_classes (int): Number of categories excluding the background. embed_dims (int): The dims of Transformer embedding. num_reg_fcs (int): Number of fully-connected layers used in `FFN`, which is then used for the regression head. Defaults to 2. sync_cls_avg_factor (bool): Whether to sync the `avg_factor` of all ranks. Default to `False`. loss_cls (:obj:`ConfigDict` or dict): Config of the classification loss. Defaults to `CrossEntropyLoss`. loss_bbox (:obj:`ConfigDict` or dict): Config of the regression bbox loss. Defaults to `L1Loss`. loss_iou (:obj:`ConfigDict` or dict): Config of the regression iou loss. Defaults to `GIoULoss`. train_cfg (:obj:`ConfigDict` or dict): Training config of transformer head. test_cfg (:obj:`ConfigDict` or dict): Testing config of transformer head. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ _version = 2 def __init__( self, num_classes: int, embed_dims: int = 256, num_reg_fcs: int = 2, sync_cls_avg_factor: bool = False, loss_cls: ConfigType = dict( type='CrossEntropyLoss', bg_cls_weight=0.1, use_sigmoid=False, loss_weight=1.0, class_weight=1.0), loss_bbox: ConfigType = dict(type='L1Loss', loss_weight=5.0), loss_iou: ConfigType = dict(type='GIoULoss', loss_weight=2.0), train_cfg: ConfigType = dict( assigner=dict( type='HungarianAssigner', match_costs=[ dict(type='ClassificationCost', weight=1.), dict(type='BBoxL1Cost', weight=5.0, box_format='xywh'), dict(type='IoUCost', iou_mode='giou', weight=2.0) ])), test_cfg: ConfigType = dict(max_per_img=100), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) self.bg_cls_weight = 0 self.sync_cls_avg_factor = sync_cls_avg_factor class_weight = loss_cls.get('class_weight', None) if class_weight is not None and (self.__class__ is DETRHead): assert isinstance(class_weight, float), 'Expected ' \ 'class_weight to have type float. Found ' \ f'{type(class_weight)}.' # NOTE following the official DETR repo, bg_cls_weight means # relative classification weight of the no-object class. bg_cls_weight = loss_cls.get('bg_cls_weight', class_weight) assert isinstance(bg_cls_weight, float), 'Expected ' \ 'bg_cls_weight to have type float. Found ' \ f'{type(bg_cls_weight)}.' class_weight = torch.ones(num_classes + 1) * class_weight # set background class as the last indice class_weight[num_classes] = bg_cls_weight loss_cls.update({'class_weight': class_weight}) if 'bg_cls_weight' in loss_cls: loss_cls.pop('bg_cls_weight') self.bg_cls_weight = bg_cls_weight if train_cfg: assert 'assigner' in train_cfg, 'assigner should be provided ' \ 'when train_cfg is set.' assigner = train_cfg['assigner'] self.assigner = TASK_UTILS.build(assigner) if train_cfg.get('sampler', None) is not None: raise RuntimeError('DETR do not build sampler.') self.num_classes = num_classes self.embed_dims = embed_dims self.num_reg_fcs = num_reg_fcs self.train_cfg = train_cfg self.test_cfg = test_cfg self.loss_cls = MODELS.build(loss_cls) self.loss_bbox = MODELS.build(loss_bbox) self.loss_iou = MODELS.build(loss_iou) if self.loss_cls.use_sigmoid: self.cls_out_channels = num_classes else: self.cls_out_channels = num_classes + 1 self._init_layers() def _init_layers(self) -> None: """Initialize layers of the transformer head.""" # cls branch self.fc_cls = Linear(self.embed_dims, self.cls_out_channels) # reg branch self.activate = nn.ReLU() self.reg_ffn = FFN( self.embed_dims, self.embed_dims, self.num_reg_fcs, dict(type='ReLU', inplace=True), dropout=0.0, add_residual=False) # NOTE the activations of reg_branch here is the same as # those in transformer, but they are actually different # in DAB-DETR (prelu in transformer and relu in reg_branch) self.fc_reg = Linear(self.embed_dims, 4) def forward(self, hidden_states: Tensor) -> Tuple[Tensor]: """"Forward function. Args: hidden_states (Tensor): Features from transformer decoder. If `return_intermediate_dec` in detr.py is True output has shape (num_decoder_layers, bs, num_queries, dim), else has shape (1, bs, num_queries, dim) which only contains the last layer outputs. Returns: tuple[Tensor]: results of head containing the following tensor. - layers_cls_scores (Tensor): Outputs from the classification head, shape (num_decoder_layers, bs, num_queries, cls_out_channels). Note cls_out_channels should include background. - layers_bbox_preds (Tensor): Sigmoid outputs from the regression head with normalized coordinate format (cx, cy, w, h), has shape (num_decoder_layers, bs, num_queries, 4). """ layers_cls_scores = self.fc_cls(hidden_states) layers_bbox_preds = self.fc_reg( self.activate(self.reg_ffn(hidden_states))).sigmoid() return layers_cls_scores, layers_bbox_preds def loss(self, hidden_states: Tensor, batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection head on the features of the upstream network. Args: hidden_states (Tensor): Feature from the transformer decoder, has shape (num_decoder_layers, bs, num_queries, cls_out_channels) or (num_decoder_layers, num_queries, bs, cls_out_channels). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ batch_gt_instances = [] batch_img_metas = [] for data_sample in batch_data_samples: batch_img_metas.append(data_sample.metainfo) batch_gt_instances.append(data_sample.gt_instances) outs = self(hidden_states) loss_inputs = outs + (batch_gt_instances, batch_img_metas) losses = self.loss_by_feat(*loss_inputs) return losses def loss_by_feat( self, all_layers_cls_scores: Tensor, all_layers_bbox_preds: Tensor, batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """"Loss function. Only outputs from the last feature level are used for computing losses by default. Args: all_layers_cls_scores (Tensor): Classification outputs of each decoder layers. Each is a 4D-tensor, has shape (num_decoder_layers, bs, num_queries, cls_out_channels). all_layers_bbox_preds (Tensor): Sigmoid regression outputs of each decoder layers. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and shape (num_decoder_layers, bs, num_queries, 4). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert batch_gt_instances_ignore is None, \ f'{self.__class__.__name__} only supports ' \ 'for batch_gt_instances_ignore setting to None.' losses_cls, losses_bbox, losses_iou = multi_apply( self.loss_by_feat_single, all_layers_cls_scores, all_layers_bbox_preds, batch_gt_instances=batch_gt_instances, batch_img_metas=batch_img_metas) loss_dict = dict() # loss from the last decoder layer loss_dict['loss_cls'] = losses_cls[-1] loss_dict['loss_bbox'] = losses_bbox[-1] loss_dict['loss_iou'] = losses_iou[-1] # loss from other decoder layers num_dec_layer = 0 for loss_cls_i, loss_bbox_i, loss_iou_i in \ zip(losses_cls[:-1], losses_bbox[:-1], losses_iou[:-1]): loss_dict[f'd{num_dec_layer}.loss_cls'] = loss_cls_i loss_dict[f'd{num_dec_layer}.loss_bbox'] = loss_bbox_i loss_dict[f'd{num_dec_layer}.loss_iou'] = loss_iou_i num_dec_layer += 1 return loss_dict def loss_by_feat_single(self, cls_scores: Tensor, bbox_preds: Tensor, batch_gt_instances: InstanceList, batch_img_metas: List[dict]) -> Tuple[Tensor]: """Loss function for outputs from a single decoder layer of a single feature level. Args: cls_scores (Tensor): Box score logits from a single decoder layer for all images, has shape (bs, num_queries, cls_out_channels). bbox_preds (Tensor): Sigmoid outputs from a single decoder layer for all images, with normalized coordinate (cx, cy, w, h) and shape (bs, num_queries, 4). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. Returns: Tuple[Tensor]: A tuple including `loss_cls`, `loss_box` and `loss_iou`. """ num_imgs = cls_scores.size(0) cls_scores_list = [cls_scores[i] for i in range(num_imgs)] bbox_preds_list = [bbox_preds[i] for i in range(num_imgs)] cls_reg_targets = self.get_targets(cls_scores_list, bbox_preds_list, batch_gt_instances, batch_img_metas) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, num_total_pos, num_total_neg) = cls_reg_targets labels = torch.cat(labels_list, 0) label_weights = torch.cat(label_weights_list, 0) bbox_targets = torch.cat(bbox_targets_list, 0) bbox_weights = torch.cat(bbox_weights_list, 0) # classification loss cls_scores = cls_scores.reshape(-1, self.cls_out_channels) # construct weighted avg_factor to match with the official DETR repo cls_avg_factor = num_total_pos * 1.0 + \ num_total_neg * self.bg_cls_weight if self.sync_cls_avg_factor: cls_avg_factor = reduce_mean( cls_scores.new_tensor([cls_avg_factor])) cls_avg_factor = max(cls_avg_factor, 1) loss_cls = self.loss_cls( cls_scores, labels, label_weights, avg_factor=cls_avg_factor) # Compute the average number of gt boxes across all gpus, for # normalization purposes num_total_pos = loss_cls.new_tensor([num_total_pos]) num_total_pos = torch.clamp(reduce_mean(num_total_pos), min=1).item() # construct factors used for rescale bboxes factors = [] for img_meta, bbox_pred in zip(batch_img_metas, bbox_preds): img_h, img_w, = img_meta['img_shape'] factor = bbox_pred.new_tensor([img_w, img_h, img_w, img_h]).unsqueeze(0).repeat( bbox_pred.size(0), 1) factors.append(factor) factors = torch.cat(factors, 0) # DETR regress the relative position of boxes (cxcywh) in the image, # thus the learning target is normalized by the image size. So here # we need to re-scale them for calculating IoU loss bbox_preds = bbox_preds.reshape(-1, 4) bboxes = bbox_cxcywh_to_xyxy(bbox_preds) * factors bboxes_gt = bbox_cxcywh_to_xyxy(bbox_targets) * factors # regression IoU loss, defaultly GIoU loss loss_iou = self.loss_iou( bboxes, bboxes_gt, bbox_weights, avg_factor=num_total_pos) # regression L1 loss loss_bbox = self.loss_bbox( bbox_preds, bbox_targets, bbox_weights, avg_factor=num_total_pos) return loss_cls, loss_bbox, loss_iou def get_targets(self, cls_scores_list: List[Tensor], bbox_preds_list: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict]) -> tuple: """Compute regression and classification targets for a batch image. Outputs from a single decoder layer of a single feature level are used. Args: cls_scores_list (list[Tensor]): Box score logits from a single decoder layer for each image, has shape [num_queries, cls_out_channels]. bbox_preds_list (list[Tensor]): Sigmoid outputs from a single decoder layer for each image, with normalized coordinate (cx, cy, w, h) and shape [num_queries, 4]. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. Returns: tuple: a tuple containing the following targets. - labels_list (list[Tensor]): Labels for all images. - label_weights_list (list[Tensor]): Label weights for all images. - bbox_targets_list (list[Tensor]): BBox targets for all images. - bbox_weights_list (list[Tensor]): BBox weights for all images. - num_total_pos (int): Number of positive samples in all images. - num_total_neg (int): Number of negative samples in all images. """ (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, pos_inds_list, neg_inds_list) = multi_apply(self._get_targets_single, cls_scores_list, bbox_preds_list, batch_gt_instances, batch_img_metas) num_total_pos = sum((inds.numel() for inds in pos_inds_list)) num_total_neg = sum((inds.numel() for inds in neg_inds_list)) return (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, num_total_pos, num_total_neg) def _get_targets_single(self, cls_score: Tensor, bbox_pred: Tensor, gt_instances: InstanceData, img_meta: dict) -> tuple: """Compute regression and classification targets for one image. Outputs from a single decoder layer of a single feature level are used. Args: cls_score (Tensor): Box score logits from a single decoder layer for one image. Shape [num_queries, cls_out_channels]. bbox_pred (Tensor): Sigmoid outputs from a single decoder layer for one image, with normalized coordinate (cx, cy, w, h) and shape [num_queries, 4]. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for one image. Returns: tuple[Tensor]: a tuple containing the following for one image. - labels (Tensor): Labels of each image. - label_weights (Tensor]): Label weights of each image. - bbox_targets (Tensor): BBox targets of each image. - bbox_weights (Tensor): BBox weights of each image. - pos_inds (Tensor): Sampled positive indices for each image. - neg_inds (Tensor): Sampled negative indices for each image. """ img_h, img_w = img_meta['img_shape'] factor = bbox_pred.new_tensor([img_w, img_h, img_w, img_h]).unsqueeze(0) num_bboxes = bbox_pred.size(0) # convert bbox_pred from xywh, normalized to xyxy, unnormalized bbox_pred = bbox_cxcywh_to_xyxy(bbox_pred) bbox_pred = bbox_pred * factor pred_instances = InstanceData(scores=cls_score, bboxes=bbox_pred) # assigner and sampler assign_result = self.assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances, img_meta=img_meta) gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels pos_inds = torch.nonzero( assign_result.gt_inds > 0, as_tuple=False).squeeze(-1).unique() neg_inds = torch.nonzero( assign_result.gt_inds == 0, as_tuple=False).squeeze(-1).unique() pos_assigned_gt_inds = assign_result.gt_inds[pos_inds] - 1 pos_gt_bboxes = gt_bboxes[pos_assigned_gt_inds.long(), :] # label targets labels = gt_bboxes.new_full((num_bboxes, ), self.num_classes, dtype=torch.long) labels[pos_inds] = gt_labels[pos_assigned_gt_inds] label_weights = gt_bboxes.new_ones(num_bboxes) # bbox targets bbox_targets = torch.zeros_like(bbox_pred) bbox_weights = torch.zeros_like(bbox_pred) bbox_weights[pos_inds] = 1.0 # DETR regress the relative position of boxes (cxcywh) in the image. # Thus the learning target should be normalized by the image size, also # the box format should be converted from defaultly x1y1x2y2 to cxcywh. pos_gt_bboxes_normalized = pos_gt_bboxes / factor pos_gt_bboxes_targets = bbox_xyxy_to_cxcywh(pos_gt_bboxes_normalized) bbox_targets[pos_inds] = pos_gt_bboxes_targets return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, neg_inds) def loss_and_predict( self, hidden_states: Tuple[Tensor], batch_data_samples: SampleList) -> Tuple[dict, InstanceList]: """Perform forward propagation of the head, then calculate loss and predictions from the features and data samples. Over-write because img_metas are needed as inputs for bbox_head. Args: hidden_states (tuple[Tensor]): Feature from the transformer decoder, has shape (num_decoder_layers, bs, num_queries, dim). batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. Returns: tuple: the return value is a tuple contains: - losses: (dict[str, Tensor]): A dictionary of loss components. - predictions (list[:obj:`InstanceData`]): Detection results of each image after the post process. """ batch_gt_instances = [] batch_img_metas = [] for data_sample in batch_data_samples: batch_img_metas.append(data_sample.metainfo) batch_gt_instances.append(data_sample.gt_instances) outs = self(hidden_states) loss_inputs = outs + (batch_gt_instances, batch_img_metas) losses = self.loss_by_feat(*loss_inputs) predictions = self.predict_by_feat( *outs, batch_img_metas=batch_img_metas) return losses, predictions def predict(self, hidden_states: Tuple[Tensor], batch_data_samples: SampleList, rescale: bool = True) -> InstanceList: """Perform forward propagation of the detection head and predict detection results on the features of the upstream network. Over-write because img_metas are needed as inputs for bbox_head. Args: hidden_states (tuple[Tensor]): Multi-level features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool, optional): Whether to rescale the results. Defaults to True. Returns: list[obj:`InstanceData`]: Detection results of each image after the post process. """ batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] last_layer_hidden_state = hidden_states[-1].unsqueeze(0) outs = self(last_layer_hidden_state) predictions = self.predict_by_feat( *outs, batch_img_metas=batch_img_metas, rescale=rescale) return predictions def predict_by_feat(self, layer_cls_scores: Tensor, layer_bbox_preds: Tensor, batch_img_metas: List[dict], rescale: bool = True) -> InstanceList: """Transform network outputs for a batch into bbox predictions. Args: layer_cls_scores (Tensor): Classification outputs of the last or all decoder layer. Each is a 4D-tensor, has shape (num_decoder_layers, bs, num_queries, cls_out_channels). layer_bbox_preds (Tensor): Sigmoid regression outputs of the last or all decoder layer. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and shape (num_decoder_layers, bs, num_queries, 4). batch_img_metas (list[dict]): Meta information of each image. rescale (bool, optional): If `True`, return boxes in original image space. Defaults to `True`. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ # NOTE only using outputs from the last feature level, # and only the outputs from the last decoder layer is used. cls_scores = layer_cls_scores[-1] bbox_preds = layer_bbox_preds[-1] result_list = [] for img_id in range(len(batch_img_metas)): cls_score = cls_scores[img_id] bbox_pred = bbox_preds[img_id] img_meta = batch_img_metas[img_id] results = self._predict_by_feat_single(cls_score, bbox_pred, img_meta, rescale) result_list.append(results) return result_list def _predict_by_feat_single(self, cls_score: Tensor, bbox_pred: Tensor, img_meta: dict, rescale: bool = True) -> InstanceData: """Transform outputs from the last decoder layer into bbox predictions for each image. Args: cls_score (Tensor): Box score logits from the last decoder layer for each image. Shape [num_queries, cls_out_channels]. bbox_pred (Tensor): Sigmoid outputs from the last decoder layer for each image, with coordinate format (cx, cy, w, h) and shape [num_queries, 4]. img_meta (dict): Image meta info. rescale (bool): If True, return boxes in original image space. Default True. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(cls_score) == len(bbox_pred) # num_queries max_per_img = self.test_cfg.get('max_per_img', len(cls_score)) img_shape = img_meta['img_shape'] # exclude background if self.loss_cls.use_sigmoid: cls_score = cls_score.sigmoid() scores, indexes = cls_score.view(-1).topk(max_per_img) det_labels = indexes % self.num_classes bbox_index = indexes // self.num_classes bbox_pred = bbox_pred[bbox_index] else: scores, det_labels = F.softmax(cls_score, dim=-1)[..., :-1].max(-1) scores, bbox_index = scores.topk(max_per_img) bbox_pred = bbox_pred[bbox_index] det_labels = det_labels[bbox_index] det_bboxes = bbox_cxcywh_to_xyxy(bbox_pred) det_bboxes[:, 0::2] = det_bboxes[:, 0::2] * img_shape[1] det_bboxes[:, 1::2] = det_bboxes[:, 1::2] * img_shape[0] det_bboxes[:, 0::2].clamp_(min=0, max=img_shape[1]) det_bboxes[:, 1::2].clamp_(min=0, max=img_shape[0]) if rescale: assert img_meta.get('scale_factor') is not None det_bboxes /= det_bboxes.new_tensor( img_meta['scale_factor']).repeat((1, 2)) results = InstanceData() results.bboxes = det_bboxes results.scores = scores results.labels = det_labels return results ================================================ FILE: mmdet/models/dense_heads/dino_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Tuple import torch from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox_cxcywh_to_xyxy, bbox_xyxy_to_cxcywh from mmdet.utils import InstanceList, OptInstanceList, reduce_mean from ..utils import multi_apply from .deformable_detr_head import DeformableDETRHead @MODELS.register_module() class DINOHead(DeformableDETRHead): r"""Head of the DINO: DETR with Improved DeNoising Anchor Boxes for End-to-End Object Detection Code is modified from the `official github repo `_. More details can be found in the `paper `_ . """ def loss(self, hidden_states: Tensor, references: List[Tensor], enc_outputs_class: Tensor, enc_outputs_coord: Tensor, batch_data_samples: SampleList, dn_meta: Dict[str, int]) -> dict: """Perform forward propagation and loss calculation of the detection head on the queries of the upstream network. Args: hidden_states (Tensor): Hidden states output from each decoder layer, has shape (num_decoder_layers, bs, num_queries_total, dim), where `num_queries_total` is the sum of `num_denoising_queries` and `num_matching_queries` when `self.training` is `True`, else `num_matching_queries`. references (list[Tensor]): List of the reference from the decoder. The first reference is the `init_reference` (initial) and the other num_decoder_layers(6) references are `inter_references` (intermediate). The `init_reference` has shape (bs, num_queries_total, 4) and each `inter_reference` has shape (bs, num_queries, 4) with the last dimension arranged as (cx, cy, w, h). enc_outputs_class (Tensor): The score of each point on encode feature map, has shape (bs, num_feat_points, cls_out_channels). enc_outputs_coord (Tensor): The proposal generate from the encode feature map, has shape (bs, num_feat_points, 4) with the last dimension arranged as (cx, cy, w, h). batch_data_samples (list[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. dn_meta (Dict[str, int]): The dictionary saves information about group collation, including 'num_denoising_queries' and 'num_denoising_groups'. It will be used for split outputs of denoising and matching parts and loss calculation. Returns: dict: A dictionary of loss components. """ batch_gt_instances = [] batch_img_metas = [] for data_sample in batch_data_samples: batch_img_metas.append(data_sample.metainfo) batch_gt_instances.append(data_sample.gt_instances) outs = self(hidden_states, references) loss_inputs = outs + (enc_outputs_class, enc_outputs_coord, batch_gt_instances, batch_img_metas, dn_meta) losses = self.loss_by_feat(*loss_inputs) return losses def loss_by_feat( self, all_layers_cls_scores: Tensor, all_layers_bbox_preds: Tensor, enc_cls_scores: Tensor, enc_bbox_preds: Tensor, batch_gt_instances: InstanceList, batch_img_metas: List[dict], dn_meta: Dict[str, int], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """Loss function. Args: all_layers_cls_scores (Tensor): Classification scores of all decoder layers, has shape (num_decoder_layers, bs, num_queries_total, cls_out_channels), where `num_queries_total` is the sum of `num_denoising_queries` and `num_matching_queries`. all_layers_bbox_preds (Tensor): Regression outputs of all decoder layers. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and has shape (num_decoder_layers, bs, num_queries_total, 4). enc_cls_scores (Tensor): The score of each point on encode feature map, has shape (bs, num_feat_points, cls_out_channels). enc_bbox_preds (Tensor): The proposal generate from the encode feature map, has shape (bs, num_feat_points, 4) with the last dimension arranged as (cx, cy, w, h). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. dn_meta (Dict[str, int]): The dictionary saves information about group collation, including 'num_denoising_queries' and 'num_denoising_groups'. It will be used for split outputs of denoising and matching parts and loss calculation. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ # extract denoising and matching part of outputs (all_layers_matching_cls_scores, all_layers_matching_bbox_preds, all_layers_denoising_cls_scores, all_layers_denoising_bbox_preds) = \ self.split_outputs( all_layers_cls_scores, all_layers_bbox_preds, dn_meta) loss_dict = super(DeformableDETRHead, self).loss_by_feat( all_layers_matching_cls_scores, all_layers_matching_bbox_preds, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) # NOTE DETRHead.loss_by_feat but not DeformableDETRHead.loss_by_feat # is called, because the encoder loss calculations are different # between DINO and DeformableDETR. # loss of proposal generated from encode feature map. if enc_cls_scores is not None: # NOTE The enc_loss calculation of the DINO is # different from that of Deformable DETR. enc_loss_cls, enc_losses_bbox, enc_losses_iou = \ self.loss_by_feat_single( enc_cls_scores, enc_bbox_preds, batch_gt_instances=batch_gt_instances, batch_img_metas=batch_img_metas) loss_dict['enc_loss_cls'] = enc_loss_cls loss_dict['enc_loss_bbox'] = enc_losses_bbox loss_dict['enc_loss_iou'] = enc_losses_iou if all_layers_denoising_cls_scores is not None: # calculate denoising loss from all decoder layers dn_losses_cls, dn_losses_bbox, dn_losses_iou = self.loss_dn( all_layers_denoising_cls_scores, all_layers_denoising_bbox_preds, batch_gt_instances=batch_gt_instances, batch_img_metas=batch_img_metas, dn_meta=dn_meta) # collate denoising loss loss_dict['dn_loss_cls'] = dn_losses_cls[-1] loss_dict['dn_loss_bbox'] = dn_losses_bbox[-1] loss_dict['dn_loss_iou'] = dn_losses_iou[-1] for num_dec_layer, (loss_cls_i, loss_bbox_i, loss_iou_i) in \ enumerate(zip(dn_losses_cls[:-1], dn_losses_bbox[:-1], dn_losses_iou[:-1])): loss_dict[f'd{num_dec_layer}.dn_loss_cls'] = loss_cls_i loss_dict[f'd{num_dec_layer}.dn_loss_bbox'] = loss_bbox_i loss_dict[f'd{num_dec_layer}.dn_loss_iou'] = loss_iou_i return loss_dict def loss_dn(self, all_layers_denoising_cls_scores: Tensor, all_layers_denoising_bbox_preds: Tensor, batch_gt_instances: InstanceList, batch_img_metas: List[dict], dn_meta: Dict[str, int]) -> Tuple[List[Tensor]]: """Calculate denoising loss. Args: all_layers_denoising_cls_scores (Tensor): Classification scores of all decoder layers in denoising part, has shape ( num_decoder_layers, bs, num_denoising_queries, cls_out_channels). all_layers_denoising_bbox_preds (Tensor): Regression outputs of all decoder layers in denoising part. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and has shape (num_decoder_layers, bs, num_denoising_queries, 4). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. dn_meta (Dict[str, int]): The dictionary saves information about group collation, including 'num_denoising_queries' and 'num_denoising_groups'. It will be used for split outputs of denoising and matching parts and loss calculation. Returns: Tuple[List[Tensor]]: The loss_dn_cls, loss_dn_bbox, and loss_dn_iou of each decoder layers. """ return multi_apply( self._loss_dn_single, all_layers_denoising_cls_scores, all_layers_denoising_bbox_preds, batch_gt_instances=batch_gt_instances, batch_img_metas=batch_img_metas, dn_meta=dn_meta) def _loss_dn_single(self, dn_cls_scores: Tensor, dn_bbox_preds: Tensor, batch_gt_instances: InstanceList, batch_img_metas: List[dict], dn_meta: Dict[str, int]) -> Tuple[Tensor]: """Denoising loss for outputs from a single decoder layer. Args: dn_cls_scores (Tensor): Classification scores of a single decoder layer in denoising part, has shape (bs, num_denoising_queries, cls_out_channels). dn_bbox_preds (Tensor): Regression outputs of a single decoder layer in denoising part. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and has shape (bs, num_denoising_queries, 4). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. dn_meta (Dict[str, int]): The dictionary saves information about group collation, including 'num_denoising_queries' and 'num_denoising_groups'. It will be used for split outputs of denoising and matching parts and loss calculation. Returns: Tuple[Tensor]: A tuple including `loss_cls`, `loss_box` and `loss_iou`. """ cls_reg_targets = self.get_dn_targets(batch_gt_instances, batch_img_metas, dn_meta) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, num_total_pos, num_total_neg) = cls_reg_targets labels = torch.cat(labels_list, 0) label_weights = torch.cat(label_weights_list, 0) bbox_targets = torch.cat(bbox_targets_list, 0) bbox_weights = torch.cat(bbox_weights_list, 0) # classification loss cls_scores = dn_cls_scores.reshape(-1, self.cls_out_channels) # construct weighted avg_factor to match with the official DETR repo cls_avg_factor = \ num_total_pos * 1.0 + num_total_neg * self.bg_cls_weight if self.sync_cls_avg_factor: cls_avg_factor = reduce_mean( cls_scores.new_tensor([cls_avg_factor])) cls_avg_factor = max(cls_avg_factor, 1) if len(cls_scores) > 0: loss_cls = self.loss_cls( cls_scores, labels, label_weights, avg_factor=cls_avg_factor) else: loss_cls = torch.zeros( 1, dtype=cls_scores.dtype, device=cls_scores.device) # Compute the average number of gt boxes across all gpus, for # normalization purposes num_total_pos = loss_cls.new_tensor([num_total_pos]) num_total_pos = torch.clamp(reduce_mean(num_total_pos), min=1).item() # construct factors used for rescale bboxes factors = [] for img_meta, bbox_pred in zip(batch_img_metas, dn_bbox_preds): img_h, img_w = img_meta['img_shape'] factor = bbox_pred.new_tensor([img_w, img_h, img_w, img_h]).unsqueeze(0).repeat( bbox_pred.size(0), 1) factors.append(factor) factors = torch.cat(factors) # DETR regress the relative position of boxes (cxcywh) in the image, # thus the learning target is normalized by the image size. So here # we need to re-scale them for calculating IoU loss bbox_preds = dn_bbox_preds.reshape(-1, 4) bboxes = bbox_cxcywh_to_xyxy(bbox_preds) * factors bboxes_gt = bbox_cxcywh_to_xyxy(bbox_targets) * factors # regression IoU loss, defaultly GIoU loss loss_iou = self.loss_iou( bboxes, bboxes_gt, bbox_weights, avg_factor=num_total_pos) # regression L1 loss loss_bbox = self.loss_bbox( bbox_preds, bbox_targets, bbox_weights, avg_factor=num_total_pos) return loss_cls, loss_bbox, loss_iou def get_dn_targets(self, batch_gt_instances: InstanceList, batch_img_metas: dict, dn_meta: Dict[str, int]) -> tuple: """Get targets in denoising part for a batch of images. Args: batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. dn_meta (Dict[str, int]): The dictionary saves information about group collation, including 'num_denoising_queries' and 'num_denoising_groups'. It will be used for split outputs of denoising and matching parts and loss calculation. Returns: tuple: a tuple containing the following targets. - labels_list (list[Tensor]): Labels for all images. - label_weights_list (list[Tensor]): Label weights for all images. - bbox_targets_list (list[Tensor]): BBox targets for all images. - bbox_weights_list (list[Tensor]): BBox weights for all images. - num_total_pos (int): Number of positive samples in all images. - num_total_neg (int): Number of negative samples in all images. """ (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, pos_inds_list, neg_inds_list) = multi_apply( self._get_dn_targets_single, batch_gt_instances, batch_img_metas, dn_meta=dn_meta) num_total_pos = sum((inds.numel() for inds in pos_inds_list)) num_total_neg = sum((inds.numel() for inds in neg_inds_list)) return (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, num_total_pos, num_total_neg) def _get_dn_targets_single(self, gt_instances: InstanceData, img_meta: dict, dn_meta: Dict[str, int]) -> tuple: """Get targets in denoising part for one image. Args: gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for one image. dn_meta (Dict[str, int]): The dictionary saves information about group collation, including 'num_denoising_queries' and 'num_denoising_groups'. It will be used for split outputs of denoising and matching parts and loss calculation. Returns: tuple[Tensor]: a tuple containing the following for one image. - labels (Tensor): Labels of each image. - label_weights (Tensor]): Label weights of each image. - bbox_targets (Tensor): BBox targets of each image. - bbox_weights (Tensor): BBox weights of each image. - pos_inds (Tensor): Sampled positive indices for each image. - neg_inds (Tensor): Sampled negative indices for each image. """ gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels num_groups = dn_meta['num_denoising_groups'] num_denoising_queries = dn_meta['num_denoising_queries'] num_queries_each_group = int(num_denoising_queries / num_groups) device = gt_bboxes.device if len(gt_labels) > 0: t = torch.arange(len(gt_labels), dtype=torch.long, device=device) t = t.unsqueeze(0).repeat(num_groups, 1) pos_assigned_gt_inds = t.flatten() pos_inds = torch.arange( num_groups, dtype=torch.long, device=device) pos_inds = pos_inds.unsqueeze(1) * num_queries_each_group + t pos_inds = pos_inds.flatten() else: pos_inds = pos_assigned_gt_inds = \ gt_bboxes.new_tensor([], dtype=torch.long) neg_inds = pos_inds + num_queries_each_group // 2 # label targets labels = gt_bboxes.new_full((num_denoising_queries, ), self.num_classes, dtype=torch.long) labels[pos_inds] = gt_labels[pos_assigned_gt_inds] label_weights = gt_bboxes.new_ones(num_denoising_queries) # bbox targets bbox_targets = torch.zeros(num_denoising_queries, 4, device=device) bbox_weights = torch.zeros(num_denoising_queries, 4, device=device) bbox_weights[pos_inds] = 1.0 img_h, img_w = img_meta['img_shape'] # DETR regress the relative position of boxes (cxcywh) in the image. # Thus the learning target should be normalized by the image size, also # the box format should be converted from defaultly x1y1x2y2 to cxcywh. factor = gt_bboxes.new_tensor([img_w, img_h, img_w, img_h]).unsqueeze(0) gt_bboxes_normalized = gt_bboxes / factor gt_bboxes_targets = bbox_xyxy_to_cxcywh(gt_bboxes_normalized) bbox_targets[pos_inds] = gt_bboxes_targets.repeat([num_groups, 1]) return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, neg_inds) @staticmethod def split_outputs(all_layers_cls_scores: Tensor, all_layers_bbox_preds: Tensor, dn_meta: Dict[str, int]) -> Tuple[Tensor]: """Split outputs of the denoising part and the matching part. For the total outputs of `num_queries_total` length, the former `num_denoising_queries` outputs are from denoising queries, and the rest `num_matching_queries` ones are from matching queries, where `num_queries_total` is the sum of `num_denoising_queries` and `num_matching_queries`. Args: all_layers_cls_scores (Tensor): Classification scores of all decoder layers, has shape (num_decoder_layers, bs, num_queries_total, cls_out_channels). all_layers_bbox_preds (Tensor): Regression outputs of all decoder layers. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and has shape (num_decoder_layers, bs, num_queries_total, 4). dn_meta (Dict[str, int]): The dictionary saves information about group collation, including 'num_denoising_queries' and 'num_denoising_groups'. Returns: Tuple[Tensor]: a tuple containing the following outputs. - all_layers_matching_cls_scores (Tensor): Classification scores of all decoder layers in matching part, has shape (num_decoder_layers, bs, num_matching_queries, cls_out_channels). - all_layers_matching_bbox_preds (Tensor): Regression outputs of all decoder layers in matching part. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and has shape (num_decoder_layers, bs, num_matching_queries, 4). - all_layers_denoising_cls_scores (Tensor): Classification scores of all decoder layers in denoising part, has shape (num_decoder_layers, bs, num_denoising_queries, cls_out_channels). - all_layers_denoising_bbox_preds (Tensor): Regression outputs of all decoder layers in denoising part. Each is a 4D-tensor with normalized coordinate format (cx, cy, w, h) and has shape (num_decoder_layers, bs, num_denoising_queries, 4). """ num_denoising_queries = dn_meta['num_denoising_queries'] if dn_meta is not None: all_layers_denoising_cls_scores = \ all_layers_cls_scores[:, :, : num_denoising_queries, :] all_layers_denoising_bbox_preds = \ all_layers_bbox_preds[:, :, : num_denoising_queries, :] all_layers_matching_cls_scores = \ all_layers_cls_scores[:, :, num_denoising_queries:, :] all_layers_matching_bbox_preds = \ all_layers_bbox_preds[:, :, num_denoising_queries:, :] else: all_layers_denoising_cls_scores = None all_layers_denoising_bbox_preds = None all_layers_matching_cls_scores = all_layers_cls_scores all_layers_matching_bbox_preds = all_layers_bbox_preds return (all_layers_matching_cls_scores, all_layers_matching_bbox_preds, all_layers_denoising_cls_scores, all_layers_denoising_bbox_preds) ================================================ FILE: mmdet/models/dense_heads/embedding_rpn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List import torch import torch.nn as nn from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures.bbox import bbox_cxcywh_to_xyxy from mmdet.structures.det_data_sample import SampleList from mmdet.utils import InstanceList, OptConfigType @MODELS.register_module() class EmbeddingRPNHead(BaseModule): """RPNHead in the `Sparse R-CNN `_ . Unlike traditional RPNHead, this module does not need FPN input, but just decode `init_proposal_bboxes` and expand the first dimension of `init_proposal_bboxes` and `init_proposal_features` to the batch_size. Args: num_proposals (int): Number of init_proposals. Defaults to 100. proposal_feature_channel (int): Channel number of init_proposal_feature. Defaults to 256. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict]): Initialization config dict. Defaults to None. """ def __init__(self, num_proposals: int = 100, proposal_feature_channel: int = 256, init_cfg: OptConfigType = None, **kwargs) -> None: # `**kwargs` is necessary to avoid some potential error. assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' super().__init__(init_cfg=init_cfg) self.num_proposals = num_proposals self.proposal_feature_channel = proposal_feature_channel self._init_layers() def _init_layers(self) -> None: """Initialize a sparse set of proposal boxes and proposal features.""" self.init_proposal_bboxes = nn.Embedding(self.num_proposals, 4) self.init_proposal_features = nn.Embedding( self.num_proposals, self.proposal_feature_channel) def init_weights(self) -> None: """Initialize the init_proposal_bboxes as normalized. [c_x, c_y, w, h], and we initialize it to the size of the entire image. """ super().init_weights() nn.init.constant_(self.init_proposal_bboxes.weight[:, :2], 0.5) nn.init.constant_(self.init_proposal_bboxes.weight[:, 2:], 1) def _decode_init_proposals(self, x: List[Tensor], batch_data_samples: SampleList) -> InstanceList: """Decode init_proposal_bboxes according to the size of images and expand dimension of init_proposal_features to batch_size. Args: x (list[Tensor]): List of FPN features. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: List[:obj:`InstanceData`:] Detection results of each image. Each item usually contains following keys. - proposals: Decoded proposal bboxes, has shape (num_proposals, 4). - features: init_proposal_features, expanded proposal features, has shape (num_proposals, proposal_feature_channel). - imgs_whwh: Tensor with shape (num_proposals, 4), the dimension means [img_width, img_height, img_width, img_height]. """ batch_img_metas = [] for data_sample in batch_data_samples: batch_img_metas.append(data_sample.metainfo) proposals = self.init_proposal_bboxes.weight.clone() proposals = bbox_cxcywh_to_xyxy(proposals) imgs_whwh = [] for meta in batch_img_metas: h, w = meta['img_shape'][:2] imgs_whwh.append(x[0].new_tensor([[w, h, w, h]])) imgs_whwh = torch.cat(imgs_whwh, dim=0) imgs_whwh = imgs_whwh[:, None, :] proposals = proposals * imgs_whwh rpn_results_list = [] for idx in range(len(batch_img_metas)): rpn_results = InstanceData() rpn_results.bboxes = proposals[idx] rpn_results.imgs_whwh = imgs_whwh[idx].repeat( self.num_proposals, 1) rpn_results.features = self.init_proposal_features.weight.clone() rpn_results_list.append(rpn_results) return rpn_results_list def loss(self, *args, **kwargs): """Perform forward propagation and loss calculation of the detection head on the features of the upstream network.""" raise NotImplementedError( 'EmbeddingRPNHead does not have `loss`, please use ' '`predict` or `loss_and_predict` instead.') def predict(self, x: List[Tensor], batch_data_samples: SampleList, **kwargs) -> InstanceList: """Perform forward propagation of the detection head and predict detection results on the features of the upstream network.""" # `**kwargs` is necessary to avoid some potential error. return self._decode_init_proposals( x=x, batch_data_samples=batch_data_samples) def loss_and_predict(self, x: List[Tensor], batch_data_samples: SampleList, **kwargs) -> tuple: """Perform forward propagation of the head, then calculate loss and predictions from the features and data samples.""" # `**kwargs` is necessary to avoid some potential error. predictions = self._decode_init_proposals( x=x, batch_data_samples=batch_data_samples) return dict(), predictions ================================================ FILE: mmdet/models/dense_heads/fcos_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Tuple import torch import torch.nn as nn from mmcv.cnn import Scale from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import (ConfigType, InstanceList, MultiConfig, OptInstanceList, RangeType, reduce_mean) from ..utils import multi_apply from .anchor_free_head import AnchorFreeHead INF = 1e8 @MODELS.register_module() class FCOSHead(AnchorFreeHead): """Anchor-free head used in `FCOS `_. The FCOS head does not use anchor boxes. Instead bounding boxes are predicted at each pixel and a centerness measure is used to suppress low-quality predictions. Here norm_on_bbox, centerness_on_reg, dcn_on_last_conv are training tricks used in official repo, which will bring remarkable mAP gains of up to 4.9. Please see https://github.com/tianzhi0549/FCOS for more detail. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. strides (Sequence[int] or Sequence[Tuple[int, int]]): Strides of points in multiple feature levels. Defaults to (4, 8, 16, 32, 64). regress_ranges (Sequence[Tuple[int, int]]): Regress range of multiple level points. center_sampling (bool): If true, use center sampling. Defaults to False. center_sample_radius (float): Radius of center sampling. Defaults to 1.5. norm_on_bbox (bool): If true, normalize the regression targets with FPN strides. Defaults to False. centerness_on_reg (bool): If true, position centerness on the regress branch. Please refer to https://github.com/tianzhi0549/FCOS/issues/89#issuecomment-516877042. Defaults to False. conv_bias (bool or str): If specified as `auto`, it will be decided by the norm_cfg. Bias of conv will be set as True if `norm_cfg` is None, otherwise False. Defaults to "auto". loss_cls (:obj:`ConfigDict` or dict): Config of classification loss. loss_bbox (:obj:`ConfigDict` or dict): Config of localization loss. loss_centerness (:obj:`ConfigDict`, or dict): Config of centerness loss. norm_cfg (:obj:`ConfigDict` or dict): dictionary to construct and config norm layer. Defaults to ``norm_cfg=dict(type='GN', num_groups=32, requires_grad=True)``. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict]): Initialization config dict. Example: >>> self = FCOSHead(11, 7) >>> feats = [torch.rand(1, 7, s, s) for s in [4, 8, 16, 32, 64]] >>> cls_score, bbox_pred, centerness = self.forward(feats) >>> assert len(cls_score) == len(self.scales) """ # noqa: E501 def __init__(self, num_classes: int, in_channels: int, regress_ranges: RangeType = ((-1, 64), (64, 128), (128, 256), (256, 512), (512, INF)), center_sampling: bool = False, center_sample_radius: float = 1.5, norm_on_bbox: bool = False, centerness_on_reg: bool = False, loss_cls: ConfigType = dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox: ConfigType = dict(type='IoULoss', loss_weight=1.0), loss_centerness: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), norm_cfg: ConfigType = dict( type='GN', num_groups=32, requires_grad=True), init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='conv_cls', std=0.01, bias_prob=0.01)), **kwargs) -> None: self.regress_ranges = regress_ranges self.center_sampling = center_sampling self.center_sample_radius = center_sample_radius self.norm_on_bbox = norm_on_bbox self.centerness_on_reg = centerness_on_reg super().__init__( num_classes=num_classes, in_channels=in_channels, loss_cls=loss_cls, loss_bbox=loss_bbox, norm_cfg=norm_cfg, init_cfg=init_cfg, **kwargs) self.loss_centerness = MODELS.build(loss_centerness) def _init_layers(self) -> None: """Initialize layers of the head.""" super()._init_layers() self.conv_centerness = nn.Conv2d(self.feat_channels, 1, 3, padding=1) self.scales = nn.ModuleList([Scale(1.0) for _ in self.strides]) def forward( self, x: Tuple[Tensor] ) -> Tuple[List[Tensor], List[Tensor], List[Tensor]]: """Forward features from the upstream network. Args: feats (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of each level outputs. - cls_scores (list[Tensor]): Box scores for each scale level, \ each is a 4D-tensor, the channel number is \ num_points * num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for each \ scale level, each is a 4D-tensor, the channel number is \ num_points * 4. - centernesses (list[Tensor]): centerness for each scale level, \ each is a 4D-tensor, the channel number is num_points * 1. """ return multi_apply(self.forward_single, x, self.scales, self.strides) def forward_single(self, x: Tensor, scale: Scale, stride: int) -> Tuple[Tensor, Tensor, Tensor]: """Forward features of a single scale level. Args: x (Tensor): FPN feature maps of the specified stride. scale (:obj:`mmcv.cnn.Scale`): Learnable scale module to resize the bbox prediction. stride (int): The corresponding stride for feature maps, only used to normalize the bbox prediction when self.norm_on_bbox is True. Returns: tuple: scores for each class, bbox predictions and centerness predictions of input feature maps. """ cls_score, bbox_pred, cls_feat, reg_feat = super().forward_single(x) if self.centerness_on_reg: centerness = self.conv_centerness(reg_feat) else: centerness = self.conv_centerness(cls_feat) # scale the bbox_pred of different level # float to avoid overflow when enabling FP16 bbox_pred = scale(bbox_pred).float() if self.norm_on_bbox: # bbox_pred needed for gradient computation has been modified # by F.relu(bbox_pred) when run with PyTorch 1.10. So replace # F.relu(bbox_pred) with bbox_pred.clamp(min=0) bbox_pred = bbox_pred.clamp(min=0) if not self.training: bbox_pred *= stride else: bbox_pred = bbox_pred.exp() return cls_score, bbox_pred, centerness def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], centernesses: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level, each is a 4D-tensor, the channel number is num_points * num_classes. bbox_preds (list[Tensor]): Box energies / deltas for each scale level, each is a 4D-tensor, the channel number is num_points * 4. centernesses (list[Tensor]): centerness for each scale level, each is a 4D-tensor, the channel number is num_points * 1. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert len(cls_scores) == len(bbox_preds) == len(centernesses) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] all_level_points = self.prior_generator.grid_priors( featmap_sizes, dtype=bbox_preds[0].dtype, device=bbox_preds[0].device) labels, bbox_targets = self.get_targets(all_level_points, batch_gt_instances) num_imgs = cls_scores[0].size(0) # flatten cls_scores, bbox_preds and centerness flatten_cls_scores = [ cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) for cls_score in cls_scores ] flatten_bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) for bbox_pred in bbox_preds ] flatten_centerness = [ centerness.permute(0, 2, 3, 1).reshape(-1) for centerness in centernesses ] flatten_cls_scores = torch.cat(flatten_cls_scores) flatten_bbox_preds = torch.cat(flatten_bbox_preds) flatten_centerness = torch.cat(flatten_centerness) flatten_labels = torch.cat(labels) flatten_bbox_targets = torch.cat(bbox_targets) # repeat points to align with bbox_preds flatten_points = torch.cat( [points.repeat(num_imgs, 1) for points in all_level_points]) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = self.num_classes pos_inds = ((flatten_labels >= 0) & (flatten_labels < bg_class_ind)).nonzero().reshape(-1) num_pos = torch.tensor( len(pos_inds), dtype=torch.float, device=bbox_preds[0].device) num_pos = max(reduce_mean(num_pos), 1.0) loss_cls = self.loss_cls( flatten_cls_scores, flatten_labels, avg_factor=num_pos) pos_bbox_preds = flatten_bbox_preds[pos_inds] pos_centerness = flatten_centerness[pos_inds] pos_bbox_targets = flatten_bbox_targets[pos_inds] pos_centerness_targets = self.centerness_target(pos_bbox_targets) # centerness weighted iou loss centerness_denorm = max( reduce_mean(pos_centerness_targets.sum().detach()), 1e-6) if len(pos_inds) > 0: pos_points = flatten_points[pos_inds] pos_decoded_bbox_preds = self.bbox_coder.decode( pos_points, pos_bbox_preds) pos_decoded_target_preds = self.bbox_coder.decode( pos_points, pos_bbox_targets) loss_bbox = self.loss_bbox( pos_decoded_bbox_preds, pos_decoded_target_preds, weight=pos_centerness_targets, avg_factor=centerness_denorm) loss_centerness = self.loss_centerness( pos_centerness, pos_centerness_targets, avg_factor=num_pos) else: loss_bbox = pos_bbox_preds.sum() loss_centerness = pos_centerness.sum() return dict( loss_cls=loss_cls, loss_bbox=loss_bbox, loss_centerness=loss_centerness) def get_targets( self, points: List[Tensor], batch_gt_instances: InstanceList ) -> Tuple[List[Tensor], List[Tensor]]: """Compute regression, classification and centerness targets for points in multiple images. Args: points (list[Tensor]): Points of each fpn level, each has shape (num_points, 2). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. Returns: tuple: Targets of each level. - concat_lvl_labels (list[Tensor]): Labels of each level. - concat_lvl_bbox_targets (list[Tensor]): BBox targets of each \ level. """ assert len(points) == len(self.regress_ranges) num_levels = len(points) # expand regress ranges to align with points expanded_regress_ranges = [ points[i].new_tensor(self.regress_ranges[i])[None].expand_as( points[i]) for i in range(num_levels) ] # concat all levels points and regress ranges concat_regress_ranges = torch.cat(expanded_regress_ranges, dim=0) concat_points = torch.cat(points, dim=0) # the number of points per img, per lvl num_points = [center.size(0) for center in points] # get labels and bbox_targets of each image labels_list, bbox_targets_list = multi_apply( self._get_targets_single, batch_gt_instances, points=concat_points, regress_ranges=concat_regress_ranges, num_points_per_lvl=num_points) # split to per img, per level labels_list = [labels.split(num_points, 0) for labels in labels_list] bbox_targets_list = [ bbox_targets.split(num_points, 0) for bbox_targets in bbox_targets_list ] # concat per level image concat_lvl_labels = [] concat_lvl_bbox_targets = [] for i in range(num_levels): concat_lvl_labels.append( torch.cat([labels[i] for labels in labels_list])) bbox_targets = torch.cat( [bbox_targets[i] for bbox_targets in bbox_targets_list]) if self.norm_on_bbox: bbox_targets = bbox_targets / self.strides[i] concat_lvl_bbox_targets.append(bbox_targets) return concat_lvl_labels, concat_lvl_bbox_targets def _get_targets_single( self, gt_instances: InstanceData, points: Tensor, regress_ranges: Tensor, num_points_per_lvl: List[int]) -> Tuple[Tensor, Tensor]: """Compute regression and classification targets for a single image.""" num_points = points.size(0) num_gts = len(gt_instances) gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels if num_gts == 0: return gt_labels.new_full((num_points,), self.num_classes), \ gt_bboxes.new_zeros((num_points, 4)) areas = (gt_bboxes[:, 2] - gt_bboxes[:, 0]) * ( gt_bboxes[:, 3] - gt_bboxes[:, 1]) # TODO: figure out why these two are different # areas = areas[None].expand(num_points, num_gts) areas = areas[None].repeat(num_points, 1) regress_ranges = regress_ranges[:, None, :].expand( num_points, num_gts, 2) gt_bboxes = gt_bboxes[None].expand(num_points, num_gts, 4) xs, ys = points[:, 0], points[:, 1] xs = xs[:, None].expand(num_points, num_gts) ys = ys[:, None].expand(num_points, num_gts) left = xs - gt_bboxes[..., 0] right = gt_bboxes[..., 2] - xs top = ys - gt_bboxes[..., 1] bottom = gt_bboxes[..., 3] - ys bbox_targets = torch.stack((left, top, right, bottom), -1) if self.center_sampling: # condition1: inside a `center bbox` radius = self.center_sample_radius center_xs = (gt_bboxes[..., 0] + gt_bboxes[..., 2]) / 2 center_ys = (gt_bboxes[..., 1] + gt_bboxes[..., 3]) / 2 center_gts = torch.zeros_like(gt_bboxes) stride = center_xs.new_zeros(center_xs.shape) # project the points on current lvl back to the `original` sizes lvl_begin = 0 for lvl_idx, num_points_lvl in enumerate(num_points_per_lvl): lvl_end = lvl_begin + num_points_lvl stride[lvl_begin:lvl_end] = self.strides[lvl_idx] * radius lvl_begin = lvl_end x_mins = center_xs - stride y_mins = center_ys - stride x_maxs = center_xs + stride y_maxs = center_ys + stride center_gts[..., 0] = torch.where(x_mins > gt_bboxes[..., 0], x_mins, gt_bboxes[..., 0]) center_gts[..., 1] = torch.where(y_mins > gt_bboxes[..., 1], y_mins, gt_bboxes[..., 1]) center_gts[..., 2] = torch.where(x_maxs > gt_bboxes[..., 2], gt_bboxes[..., 2], x_maxs) center_gts[..., 3] = torch.where(y_maxs > gt_bboxes[..., 3], gt_bboxes[..., 3], y_maxs) cb_dist_left = xs - center_gts[..., 0] cb_dist_right = center_gts[..., 2] - xs cb_dist_top = ys - center_gts[..., 1] cb_dist_bottom = center_gts[..., 3] - ys center_bbox = torch.stack( (cb_dist_left, cb_dist_top, cb_dist_right, cb_dist_bottom), -1) inside_gt_bbox_mask = center_bbox.min(-1)[0] > 0 else: # condition1: inside a gt bbox inside_gt_bbox_mask = bbox_targets.min(-1)[0] > 0 # condition2: limit the regression range for each location max_regress_distance = bbox_targets.max(-1)[0] inside_regress_range = ( (max_regress_distance >= regress_ranges[..., 0]) & (max_regress_distance <= regress_ranges[..., 1])) # if there are still more than one objects for a location, # we choose the one with minimal area areas[inside_gt_bbox_mask == 0] = INF areas[inside_regress_range == 0] = INF min_area, min_area_inds = areas.min(dim=1) labels = gt_labels[min_area_inds] labels[min_area == INF] = self.num_classes # set as BG bbox_targets = bbox_targets[range(num_points), min_area_inds] return labels, bbox_targets def centerness_target(self, pos_bbox_targets: Tensor) -> Tensor: """Compute centerness targets. Args: pos_bbox_targets (Tensor): BBox targets of positive bboxes in shape (num_pos, 4) Returns: Tensor: Centerness target. """ # only calculate pos centerness targets, otherwise there may be nan left_right = pos_bbox_targets[:, [0, 2]] top_bottom = pos_bbox_targets[:, [1, 3]] if len(left_right) == 0: centerness_targets = left_right[..., 0] else: centerness_targets = ( left_right.min(dim=-1)[0] / left_right.max(dim=-1)[0]) * ( top_bottom.min(dim=-1)[0] / top_bottom.max(dim=-1)[0]) return torch.sqrt(centerness_targets) ================================================ FILE: mmdet/models/dense_heads/fovea_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Optional, Tuple import torch import torch.nn as nn from mmcv.cnn import ConvModule from mmcv.ops import DeformConv2d from mmengine.config import ConfigDict from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import InstanceList, OptInstanceList, OptMultiConfig from ..utils import filter_scores_and_topk, multi_apply from .anchor_free_head import AnchorFreeHead INF = 1e8 class FeatureAlign(BaseModule): """Feature Align Module. Feature Align Module is implemented based on DCN v1. It uses anchor shape prediction rather than feature map to predict offsets of deform conv layer. Args: in_channels (int): Number of channels in the input feature map. out_channels (int): Number of channels in the output feature map. kernel_size (int): Size of the convolution kernel. ``norm_cfg=dict(type='GN', num_groups=32, requires_grad=True)``. deform_groups: (int): Group number of DCN in FeatureAdaption module. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. """ def __init__( self, in_channels: int, out_channels: int, kernel_size: int = 3, deform_groups: int = 4, init_cfg: OptMultiConfig = dict( type='Normal', layer='Conv2d', std=0.1, override=dict(type='Normal', name='conv_adaption', std=0.01)) ) -> None: super().__init__(init_cfg=init_cfg) offset_channels = kernel_size * kernel_size * 2 self.conv_offset = nn.Conv2d( 4, deform_groups * offset_channels, 1, bias=False) self.conv_adaption = DeformConv2d( in_channels, out_channels, kernel_size=kernel_size, padding=(kernel_size - 1) // 2, deform_groups=deform_groups) self.relu = nn.ReLU(inplace=True) def forward(self, x: Tensor, shape: Tensor) -> Tensor: """Forward function of feature align module. Args: x (Tensor): Features from the upstream network. shape (Tensor): Exponential of bbox predictions. Returns: x (Tensor): The aligned features. """ offset = self.conv_offset(shape) x = self.relu(self.conv_adaption(x, offset)) return x @MODELS.register_module() class FoveaHead(AnchorFreeHead): """Detection Head of `FoveaBox: Beyond Anchor-based Object Detector. `_. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. base_edge_list (list[int]): List of edges. scale_ranges (list[tuple]): Range of scales. sigma (float): Super parameter of ``FoveaHead``. with_deform (bool): Whether use deform conv. deform_groups (int): Deformable conv group size. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. """ def __init__(self, num_classes: int, in_channels: int, base_edge_list: List[int] = (16, 32, 64, 128, 256), scale_ranges: List[tuple] = ((8, 32), (16, 64), (32, 128), (64, 256), (128, 512)), sigma: float = 0.4, with_deform: bool = False, deform_groups: int = 4, init_cfg: OptMultiConfig = dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='conv_cls', std=0.01, bias_prob=0.01)), **kwargs) -> None: self.base_edge_list = base_edge_list self.scale_ranges = scale_ranges self.sigma = sigma self.with_deform = with_deform self.deform_groups = deform_groups super().__init__( num_classes=num_classes, in_channels=in_channels, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" # box branch super()._init_reg_convs() self.conv_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1) # cls branch if not self.with_deform: super()._init_cls_convs() self.conv_cls = nn.Conv2d( self.feat_channels, self.cls_out_channels, 3, padding=1) else: self.cls_convs = nn.ModuleList() self.cls_convs.append( ConvModule( self.feat_channels, (self.feat_channels * 4), 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, bias=self.norm_cfg is None)) self.cls_convs.append( ConvModule((self.feat_channels * 4), (self.feat_channels * 4), 1, stride=1, padding=0, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, bias=self.norm_cfg is None)) self.feature_adaption = FeatureAlign( self.feat_channels, self.feat_channels, kernel_size=3, deform_groups=self.deform_groups) self.conv_cls = nn.Conv2d( int(self.feat_channels * 4), self.cls_out_channels, 3, padding=1) def forward_single(self, x: Tensor) -> Tuple[Tensor, Tensor]: """Forward features of a single scale level. Args: x (Tensor): FPN feature maps of the specified stride. Returns: tuple: scores for each class and bbox predictions of input feature maps. """ cls_feat = x reg_feat = x for reg_layer in self.reg_convs: reg_feat = reg_layer(reg_feat) bbox_pred = self.conv_reg(reg_feat) if self.with_deform: cls_feat = self.feature_adaption(cls_feat, bbox_pred.exp()) for cls_layer in self.cls_convs: cls_feat = cls_layer(cls_feat) cls_score = self.conv_cls(cls_feat) return cls_score, bbox_pred def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level, each is a 4D-tensor, the channel number is num_priors * num_classes. bbox_preds (list[Tensor]): Box energies / deltas for each scale level, each is a 4D-tensor, the channel number is num_priors * 4. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert len(cls_scores) == len(bbox_preds) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] priors = self.prior_generator.grid_priors( featmap_sizes, dtype=bbox_preds[0].dtype, device=bbox_preds[0].device) num_imgs = cls_scores[0].size(0) flatten_cls_scores = [ cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) for cls_score in cls_scores ] flatten_bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) for bbox_pred in bbox_preds ] flatten_cls_scores = torch.cat(flatten_cls_scores) flatten_bbox_preds = torch.cat(flatten_bbox_preds) flatten_labels, flatten_bbox_targets = self.get_targets( batch_gt_instances, featmap_sizes, priors) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes pos_inds = ((flatten_labels >= 0) & (flatten_labels < self.num_classes)).nonzero().view(-1) num_pos = len(pos_inds) loss_cls = self.loss_cls( flatten_cls_scores, flatten_labels, avg_factor=num_pos + num_imgs) if num_pos > 0: pos_bbox_preds = flatten_bbox_preds[pos_inds] pos_bbox_targets = flatten_bbox_targets[pos_inds] pos_weights = pos_bbox_targets.new_ones(pos_bbox_targets.size()) loss_bbox = self.loss_bbox( pos_bbox_preds, pos_bbox_targets, pos_weights, avg_factor=num_pos) else: loss_bbox = torch.tensor( 0, dtype=flatten_bbox_preds.dtype, device=flatten_bbox_preds.device) return dict(loss_cls=loss_cls, loss_bbox=loss_bbox) def get_targets( self, batch_gt_instances: InstanceList, featmap_sizes: List[tuple], priors_list: List[Tensor]) -> Tuple[List[Tensor], List[Tensor]]: """Compute regression and classification for priors in multiple images. Args: batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. featmap_sizes (list[tuple]): Size tuple of feature maps. priors_list (list[Tensor]): Priors list of each fpn level, each has shape (num_priors, 2). Returns: tuple: Targets of each level. - flatten_labels (list[Tensor]): Labels of each level. - flatten_bbox_targets (list[Tensor]): BBox targets of each level. """ label_list, bbox_target_list = multi_apply( self._get_targets_single, batch_gt_instances, featmap_size_list=featmap_sizes, priors_list=priors_list) flatten_labels = [ torch.cat([ labels_level_img.flatten() for labels_level_img in labels_level ]) for labels_level in zip(*label_list) ] flatten_bbox_targets = [ torch.cat([ bbox_targets_level_img.reshape(-1, 4) for bbox_targets_level_img in bbox_targets_level ]) for bbox_targets_level in zip(*bbox_target_list) ] flatten_labels = torch.cat(flatten_labels) flatten_bbox_targets = torch.cat(flatten_bbox_targets) return flatten_labels, flatten_bbox_targets def _get_targets_single(self, gt_instances: InstanceData, featmap_size_list: List[tuple] = None, priors_list: List[Tensor] = None) -> tuple: """Compute regression and classification targets for a single image. Args: gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. featmap_size_list (list[tuple]): Size tuple of feature maps. priors_list (list[Tensor]): Priors of each fpn level, each has shape (num_priors, 2). Returns: tuple: - label_list (list[Tensor]): Labels of all anchors in the image. - box_target_list (list[Tensor]): BBox targets of all anchors in the image. """ gt_bboxes_raw = gt_instances.bboxes gt_labels_raw = gt_instances.labels gt_areas = torch.sqrt((gt_bboxes_raw[:, 2] - gt_bboxes_raw[:, 0]) * (gt_bboxes_raw[:, 3] - gt_bboxes_raw[:, 1])) label_list = [] bbox_target_list = [] # for each pyramid, find the cls and box target for base_len, (lower_bound, upper_bound), stride, featmap_size, \ priors in zip(self.base_edge_list, self.scale_ranges, self.strides, featmap_size_list, priors_list): # FG cat_id: [0, num_classes -1], BG cat_id: num_classes priors = priors.view(*featmap_size, 2) x, y = priors[..., 0], priors[..., 1] labels = gt_labels_raw.new_full(featmap_size, self.num_classes) bbox_targets = gt_bboxes_raw.new_ones(featmap_size[0], featmap_size[1], 4) # scale assignment hit_indices = ((gt_areas >= lower_bound) & (gt_areas <= upper_bound)).nonzero().flatten() if len(hit_indices) == 0: label_list.append(labels) bbox_target_list.append(torch.log(bbox_targets)) continue _, hit_index_order = torch.sort(-gt_areas[hit_indices]) hit_indices = hit_indices[hit_index_order] gt_bboxes = gt_bboxes_raw[hit_indices, :] / stride gt_labels = gt_labels_raw[hit_indices] half_w = 0.5 * (gt_bboxes[:, 2] - gt_bboxes[:, 0]) half_h = 0.5 * (gt_bboxes[:, 3] - gt_bboxes[:, 1]) # valid fovea area: left, right, top, down pos_left = torch.ceil( gt_bboxes[:, 0] + (1 - self.sigma) * half_w - 0.5).long(). \ clamp(0, featmap_size[1] - 1) pos_right = torch.floor( gt_bboxes[:, 0] + (1 + self.sigma) * half_w - 0.5).long(). \ clamp(0, featmap_size[1] - 1) pos_top = torch.ceil( gt_bboxes[:, 1] + (1 - self.sigma) * half_h - 0.5).long(). \ clamp(0, featmap_size[0] - 1) pos_down = torch.floor( gt_bboxes[:, 1] + (1 + self.sigma) * half_h - 0.5).long(). \ clamp(0, featmap_size[0] - 1) for px1, py1, px2, py2, label, (gt_x1, gt_y1, gt_x2, gt_y2) in \ zip(pos_left, pos_top, pos_right, pos_down, gt_labels, gt_bboxes_raw[hit_indices, :]): labels[py1:py2 + 1, px1:px2 + 1] = label bbox_targets[py1:py2 + 1, px1:px2 + 1, 0] = \ (x[py1:py2 + 1, px1:px2 + 1] - gt_x1) / base_len bbox_targets[py1:py2 + 1, px1:px2 + 1, 1] = \ (y[py1:py2 + 1, px1:px2 + 1] - gt_y1) / base_len bbox_targets[py1:py2 + 1, px1:px2 + 1, 2] = \ (gt_x2 - x[py1:py2 + 1, px1:px2 + 1]) / base_len bbox_targets[py1:py2 + 1, px1:px2 + 1, 3] = \ (gt_y2 - y[py1:py2 + 1, px1:px2 + 1]) / base_len bbox_targets = bbox_targets.clamp(min=1. / 16, max=16.) label_list.append(labels) bbox_target_list.append(torch.log(bbox_targets)) return label_list, bbox_target_list # Same as base_dense_head/_predict_by_feat_single except self._bbox_decode def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], score_factor_list: List[Tensor], mlvl_priors: List[Tensor], img_meta: dict, cfg: Optional[ConfigDict] = None, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: cls_score_list (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). score_factor_list (list[Tensor]): Score factor from all scale levels of a single image, each item has shape (num_priors * 1, H, W). mlvl_priors (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid, has shape (num_priors, 2). img_meta (dict): Image meta info. cfg (ConfigDict, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ cfg = self.test_cfg if cfg is None else cfg assert len(cls_score_list) == len(bbox_pred_list) img_shape = img_meta['img_shape'] nms_pre = cfg.get('nms_pre', -1) mlvl_bboxes = [] mlvl_scores = [] mlvl_labels = [] for level_idx, (cls_score, bbox_pred, stride, base_len, priors) in \ enumerate(zip(cls_score_list, bbox_pred_list, self.strides, self.base_edge_list, mlvl_priors)): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) scores = cls_score.permute(1, 2, 0).reshape( -1, self.cls_out_channels).sigmoid() # After https://github.com/open-mmlab/mmdetection/pull/6268/, # this operation keeps fewer bboxes under the same `nms_pre`. # There is no difference in performance for most models. If you # find a slight drop in performance, you can set a larger # `nms_pre` than before. results = filter_scores_and_topk( scores, cfg.score_thr, nms_pre, dict(bbox_pred=bbox_pred, priors=priors)) scores, labels, _, filtered_results = results bbox_pred = filtered_results['bbox_pred'] priors = filtered_results['priors'] bboxes = self._bbox_decode(priors, bbox_pred, base_len, img_shape) mlvl_bboxes.append(bboxes) mlvl_scores.append(scores) mlvl_labels.append(labels) results = InstanceData() results.bboxes = torch.cat(mlvl_bboxes) results.scores = torch.cat(mlvl_scores) results.labels = torch.cat(mlvl_labels) return self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta) def _bbox_decode(self, priors: Tensor, bbox_pred: Tensor, base_len: int, max_shape: int) -> Tensor: """Function to decode bbox. Args: priors (Tensor): Center proiors of an image, has shape (num_instances, 2). bbox_preds (Tensor): Box energies / deltas for all instances, has shape (batch_size, num_instances, 4). base_len (int): The base length. max_shape (int): The max shape of bbox. Returns: Tensor: Decoded bboxes in (tl_x, tl_y, br_x, br_y) format. Has shape (batch_size, num_instances, 4). """ bbox_pred = bbox_pred.exp() y = priors[:, 1] x = priors[:, 0] x1 = (x - base_len * bbox_pred[:, 0]). \ clamp(min=0, max=max_shape[1] - 1) y1 = (y - base_len * bbox_pred[:, 1]). \ clamp(min=0, max=max_shape[0] - 1) x2 = (x + base_len * bbox_pred[:, 2]). \ clamp(min=0, max=max_shape[1] - 1) y2 = (y + base_len * bbox_pred[:, 3]). \ clamp(min=0, max=max_shape[0] - 1) decoded_bboxes = torch.stack([x1, y1, x2, y2], -1) return decoded_bboxes ================================================ FILE: mmdet/models/dense_heads/free_anchor_retina_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List import torch import torch.nn.functional as F from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures.bbox import bbox_overlaps from mmdet.utils import InstanceList, OptConfigType, OptInstanceList from ..utils import multi_apply from .retina_head import RetinaHead EPS = 1e-12 @MODELS.register_module() class FreeAnchorRetinaHead(RetinaHead): """FreeAnchor RetinaHead used in https://arxiv.org/abs/1909.02466. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. stacked_convs (int): Number of conv layers in cls and reg tower. Defaults to 4. conv_cfg (:obj:`ConfigDict` or dict, optional): dictionary to construct and config conv layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict, optional): dictionary to construct and config norm layer. Defaults to norm_cfg=dict(type='GN', num_groups=32, requires_grad=True). pre_anchor_topk (int): Number of boxes that be token in each bag. Defaults to 50 bbox_thr (float): The threshold of the saturated linear function. It is usually the same with the IoU threshold used in NMS. Defaults to 0.6. gamma (float): Gamma parameter in focal loss. Defaults to 2.0. alpha (float): Alpha parameter in focal loss. Defaults to 0.5. """ def __init__(self, num_classes: int, in_channels: int, stacked_convs: int = 4, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, pre_anchor_topk: int = 50, bbox_thr: float = 0.6, gamma: float = 2.0, alpha: float = 0.5, **kwargs) -> None: super().__init__( num_classes=num_classes, in_channels=in_channels, stacked_convs=stacked_convs, conv_cfg=conv_cfg, norm_cfg=norm_cfg, **kwargs) self.pre_anchor_topk = pre_anchor_topk self.bbox_thr = bbox_thr self.gamma = gamma self.alpha = alpha def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level has shape (N, num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, _ = self.get_anchors( featmap_sizes=featmap_sizes, batch_img_metas=batch_img_metas, device=device) concat_anchor_list = [torch.cat(anchor) for anchor in anchor_list] # concatenate each level cls_scores = [ cls.permute(0, 2, 3, 1).reshape(cls.size(0), -1, self.cls_out_channels) for cls in cls_scores ] bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(bbox_pred.size(0), -1, 4) for bbox_pred in bbox_preds ] cls_scores = torch.cat(cls_scores, dim=1) cls_probs = torch.sigmoid(cls_scores) bbox_preds = torch.cat(bbox_preds, dim=1) box_probs, positive_losses, num_pos_list = multi_apply( self.positive_loss_single, cls_probs, bbox_preds, concat_anchor_list, batch_gt_instances) num_pos = sum(num_pos_list) positive_loss = torch.cat(positive_losses).sum() / max(1, num_pos) # box_prob: P{a_{j} \in A_{+}} box_probs = torch.stack(box_probs, dim=0) # negative_loss: # \sum_{j}{ FL((1 - P{a_{j} \in A_{+}}) * (1 - P_{j}^{bg})) } / n||B|| negative_loss = self.negative_bag_loss(cls_probs, box_probs).sum() / \ max(1, num_pos * self.pre_anchor_topk) # avoid the absence of gradients in regression subnet # when no ground-truth in a batch if num_pos == 0: positive_loss = bbox_preds.sum() * 0 losses = { 'positive_bag_loss': positive_loss, 'negative_bag_loss': negative_loss } return losses def positive_loss_single(self, cls_prob: Tensor, bbox_pred: Tensor, flat_anchors: Tensor, gt_instances: InstanceData) -> tuple: """Compute positive loss. Args: cls_prob (Tensor): Classification probability of shape (num_anchors, num_classes). bbox_pred (Tensor): Box probability of shape (num_anchors, 4). flat_anchors (Tensor): Multi-level anchors of the image, which are concatenated into a single tensor of shape (num_anchors, 4) gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. Returns: tuple: - box_prob (Tensor): Box probability of shape (num_anchors, 4). - positive_loss (Tensor): Positive loss of shape (num_pos, ). - num_pos (int): positive samples indexes. """ gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels with torch.no_grad(): if len(gt_bboxes) == 0: image_box_prob = torch.zeros( flat_anchors.size(0), self.cls_out_channels).type_as(bbox_pred) else: # box_localization: a_{j}^{loc}, shape: [j, 4] pred_boxes = self.bbox_coder.decode(flat_anchors, bbox_pred) # object_box_iou: IoU_{ij}^{loc}, shape: [i, j] object_box_iou = bbox_overlaps(gt_bboxes, pred_boxes) # object_box_prob: P{a_{j} -> b_{i}}, shape: [i, j] t1 = self.bbox_thr t2 = object_box_iou.max( dim=1, keepdim=True).values.clamp(min=t1 + 1e-12) object_box_prob = ((object_box_iou - t1) / (t2 - t1)).clamp( min=0, max=1) # object_cls_box_prob: P{a_{j} -> b_{i}}, shape: [i, c, j] num_obj = gt_labels.size(0) indices = torch.stack( [torch.arange(num_obj).type_as(gt_labels), gt_labels], dim=0) object_cls_box_prob = torch.sparse_coo_tensor( indices, object_box_prob) # image_box_iou: P{a_{j} \in A_{+}}, shape: [c, j] """ from "start" to "end" implement: image_box_iou = torch.sparse.max(object_cls_box_prob, dim=0).t() """ # start box_cls_prob = torch.sparse.sum( object_cls_box_prob, dim=0).to_dense() indices = torch.nonzero(box_cls_prob, as_tuple=False).t_() if indices.numel() == 0: image_box_prob = torch.zeros( flat_anchors.size(0), self.cls_out_channels).type_as(object_box_prob) else: nonzero_box_prob = torch.where( (gt_labels.unsqueeze(dim=-1) == indices[0]), object_box_prob[:, indices[1]], torch.tensor( [0]).type_as(object_box_prob)).max(dim=0).values # upmap to shape [j, c] image_box_prob = torch.sparse_coo_tensor( indices.flip([0]), nonzero_box_prob, size=(flat_anchors.size(0), self.cls_out_channels)).to_dense() # end box_prob = image_box_prob # construct bags for objects match_quality_matrix = bbox_overlaps(gt_bboxes, flat_anchors) _, matched = torch.topk( match_quality_matrix, self.pre_anchor_topk, dim=1, sorted=False) del match_quality_matrix # matched_cls_prob: P_{ij}^{cls} matched_cls_prob = torch.gather( cls_prob[matched], 2, gt_labels.view(-1, 1, 1).repeat(1, self.pre_anchor_topk, 1)).squeeze(2) # matched_box_prob: P_{ij}^{loc} matched_anchors = flat_anchors[matched] matched_object_targets = self.bbox_coder.encode( matched_anchors, gt_bboxes.unsqueeze(dim=1).expand_as(matched_anchors)) loss_bbox = self.loss_bbox( bbox_pred[matched], matched_object_targets, reduction_override='none').sum(-1) matched_box_prob = torch.exp(-loss_bbox) # positive_losses: {-log( Mean-max(P_{ij}^{cls} * P_{ij}^{loc}) )} num_pos = len(gt_bboxes) positive_loss = self.positive_bag_loss(matched_cls_prob, matched_box_prob) return box_prob, positive_loss, num_pos def positive_bag_loss(self, matched_cls_prob: Tensor, matched_box_prob: Tensor) -> Tensor: """Compute positive bag loss. :math:`-log( Mean-max(P_{ij}^{cls} * P_{ij}^{loc}) )`. :math:`P_{ij}^{cls}`: matched_cls_prob, classification probability of matched samples. :math:`P_{ij}^{loc}`: matched_box_prob, box probability of matched samples. Args: matched_cls_prob (Tensor): Classification probability of matched samples in shape (num_gt, pre_anchor_topk). matched_box_prob (Tensor): BBox probability of matched samples, in shape (num_gt, pre_anchor_topk). Returns: Tensor: Positive bag loss in shape (num_gt,). """ # noqa: E501, W605 # bag_prob = Mean-max(matched_prob) matched_prob = matched_cls_prob * matched_box_prob weight = 1 / torch.clamp(1 - matched_prob, 1e-12, None) weight /= weight.sum(dim=1).unsqueeze(dim=-1) bag_prob = (weight * matched_prob).sum(dim=1) # positive_bag_loss = -self.alpha * log(bag_prob) return self.alpha * F.binary_cross_entropy( bag_prob, torch.ones_like(bag_prob), reduction='none') def negative_bag_loss(self, cls_prob: Tensor, box_prob: Tensor) -> Tensor: """Compute negative bag loss. :math:`FL((1 - P_{a_{j} \in A_{+}}) * (1 - P_{j}^{bg}))`. :math:`P_{a_{j} \in A_{+}}`: Box_probability of matched samples. :math:`P_{j}^{bg}`: Classification probability of negative samples. Args: cls_prob (Tensor): Classification probability, in shape (num_img, num_anchors, num_classes). box_prob (Tensor): Box probability, in shape (num_img, num_anchors, num_classes). Returns: Tensor: Negative bag loss in shape (num_img, num_anchors, num_classes). """ # noqa: E501, W605 prob = cls_prob * (1 - box_prob) # There are some cases when neg_prob = 0. # This will cause the neg_prob.log() to be inf without clamp. prob = prob.clamp(min=EPS, max=1 - EPS) negative_bag_loss = prob**self.gamma * F.binary_cross_entropy( prob, torch.zeros_like(prob), reduction='none') return (1 - self.alpha) * negative_bag_loss ================================================ FILE: mmdet/models/dense_heads/fsaf_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Optional, Tuple import numpy as np import torch from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import InstanceList, OptInstanceList, OptMultiConfig from ..losses.accuracy import accuracy from ..losses.utils import weight_reduce_loss from ..task_modules.prior_generators import anchor_inside_flags from ..utils import images_to_levels, multi_apply, unmap from .retina_head import RetinaHead @MODELS.register_module() class FSAFHead(RetinaHead): """Anchor-free head used in `FSAF `_. The head contains two subnetworks. The first classifies anchor boxes and the second regresses deltas for the anchors (num_anchors is 1 for anchor- free methods) Args: *args: Same as its base class in :class:`RetinaHead` score_threshold (float, optional): The score_threshold to calculate positive recall. If given, prediction scores lower than this value is counted as incorrect prediction. Defaults to None. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict]): Initialization config dict. **kwargs: Same as its base class in :class:`RetinaHead` Example: >>> import torch >>> self = FSAFHead(11, 7) >>> x = torch.rand(1, 7, 32, 32) >>> cls_score, bbox_pred = self.forward_single(x) >>> # Each anchor predicts a score for each class except background >>> cls_per_anchor = cls_score.shape[1] / self.num_anchors >>> box_per_anchor = bbox_pred.shape[1] / self.num_anchors >>> assert cls_per_anchor == self.num_classes >>> assert box_per_anchor == 4 """ def __init__(self, *args, score_threshold: Optional[float] = None, init_cfg: OptMultiConfig = None, **kwargs) -> None: # The positive bias in self.retina_reg conv is to prevent predicted \ # bbox with 0 area if init_cfg is None: init_cfg = dict( type='Normal', layer='Conv2d', std=0.01, override=[ dict( type='Normal', name='retina_cls', std=0.01, bias_prob=0.01), dict( type='Normal', name='retina_reg', std=0.01, bias=0.25) ]) super().__init__(*args, init_cfg=init_cfg, **kwargs) self.score_threshold = score_threshold def forward_single(self, x: Tensor) -> Tuple[Tensor, Tensor]: """Forward feature map of a single scale level. Args: x (Tensor): Feature map of a single scale level. Returns: tuple[Tensor, Tensor]: - cls_score (Tensor): Box scores for each scale level Has \ shape (N, num_points * num_classes, H, W). - bbox_pred (Tensor): Box energies / deltas for each scale \ level with shape (N, num_points * 4, H, W). """ cls_score, bbox_pred = super().forward_single(x) # relu: TBLR encoder only accepts positive bbox_pred return cls_score, self.relu(bbox_pred) def _get_targets_single(self, flat_anchors: Tensor, valid_flags: Tensor, gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None, unmap_outputs: bool = True) -> tuple: """Compute regression and classification targets for anchors in a single image. Most of the codes are the same with the base class :obj: `AnchorHead`, except that it also collects and returns the matched gt index in the image (from 0 to num_gt-1). If the anchor bbox is not matched to any gt, the corresponding value in pos_gt_inds is -1. Args: flat_anchors (Tensor): Multi-level anchors of the image, which are concatenated into a single tensor of shape (num_anchors, 4) valid_flags (Tensor): Multi level valid flags of the image, which are concatenated into a single tensor of shape (num_anchors, ). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. """ inside_flags = anchor_inside_flags(flat_anchors, valid_flags, img_meta['img_shape'][:2], self.train_cfg['allowed_border']) if not inside_flags.any(): raise ValueError( 'There is no valid anchor inside the image boundary. Please ' 'check the image size and anchor sizes, or set ' '``allowed_border`` to -1 to skip the condition.') # Assign gt and sample anchors anchors = flat_anchors[inside_flags.type(torch.bool), :] pred_instances = InstanceData(priors=anchors) assign_result = self.assigner.assign(pred_instances, gt_instances, gt_instances_ignore) sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) num_valid_anchors = anchors.shape[0] bbox_targets = torch.zeros_like(anchors) bbox_weights = torch.zeros_like(anchors) labels = anchors.new_full((num_valid_anchors, ), self.num_classes, dtype=torch.long) label_weights = anchors.new_zeros( (num_valid_anchors, self.cls_out_channels), dtype=torch.float) pos_gt_inds = anchors.new_full((num_valid_anchors, ), -1, dtype=torch.long) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: if not self.reg_decoded_bbox: pos_bbox_targets = self.bbox_coder.encode( sampling_result.pos_bboxes, sampling_result.pos_gt_bboxes) else: # When the regression loss (e.g. `IouLoss`, `GIouLoss`) # is applied directly on the decoded bounding boxes, both # the predicted boxes and regression targets should be with # absolute coordinate format. pos_bbox_targets = sampling_result.pos_gt_bboxes bbox_targets[pos_inds, :] = pos_bbox_targets bbox_weights[pos_inds, :] = 1.0 # The assigned gt_index for each anchor. (0-based) pos_gt_inds[pos_inds] = sampling_result.pos_assigned_gt_inds labels[pos_inds] = sampling_result.pos_gt_labels if self.train_cfg['pos_weight'] <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = self.train_cfg['pos_weight'] if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 # shadowed_labels is a tensor composed of tuples # (anchor_inds, class_label) that indicate those anchors lying in the # outer region of a gt or overlapped by another gt with a smaller # area. # # Therefore, only the shadowed labels are ignored for loss calculation. # the key `shadowed_labels` is defined in :obj:`CenterRegionAssigner` shadowed_labels = assign_result.get_extra_property('shadowed_labels') if shadowed_labels is not None and shadowed_labels.numel(): if len(shadowed_labels.shape) == 2: idx_, label_ = shadowed_labels[:, 0], shadowed_labels[:, 1] assert (labels[idx_] != label_).all(), \ 'One label cannot be both positive and ignored' label_weights[idx_, label_] = 0 else: label_weights[shadowed_labels] = 0 # map up to original set of anchors if unmap_outputs: num_total_anchors = flat_anchors.size(0) labels = unmap( labels, num_total_anchors, inside_flags, fill=self.num_classes) # fill bg label label_weights = unmap(label_weights, num_total_anchors, inside_flags) bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) pos_gt_inds = unmap( pos_gt_inds, num_total_anchors, inside_flags, fill=-1) return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, neg_inds, sampling_result, pos_gt_inds) def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """Compute loss of the head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_points * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_points * 4, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ for i in range(len(bbox_preds)): # loop over fpn level # avoid 0 area of the predicted bbox bbox_preds[i] = bbox_preds[i].clamp(min=1e-4) # TODO: It may directly use the base-class loss function. featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels batch_size = len(batch_img_metas) device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, return_sampling_results=True) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor, sampling_results_list, pos_assigned_gt_inds_list) = cls_reg_targets num_gts = np.array(list(map(len, batch_gt_instances))) # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] # concat all level anchors and flags to a single tensor concat_anchor_list = [] for i in range(len(anchor_list)): concat_anchor_list.append(torch.cat(anchor_list[i])) all_anchor_list = images_to_levels(concat_anchor_list, num_level_anchors) losses_cls, losses_bbox = multi_apply( self.loss_by_feat_single, cls_scores, bbox_preds, all_anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor=avg_factor) # `pos_assigned_gt_inds_list` (length: fpn_levels) stores the assigned # gt index of each anchor bbox in each fpn level. cum_num_gts = list(np.cumsum(num_gts)) # length of batch_size for i, assign in enumerate(pos_assigned_gt_inds_list): # loop over fpn levels for j in range(1, batch_size): # loop over batch size # Convert gt indices in each img to those in the batch assign[j][assign[j] >= 0] += int(cum_num_gts[j - 1]) pos_assigned_gt_inds_list[i] = assign.flatten() labels_list[i] = labels_list[i].flatten() num_gts = num_gts.sum() # total number of gt in the batch # The unique label index of each gt in the batch label_sequence = torch.arange(num_gts, device=device) # Collect the average loss of each gt in each level with torch.no_grad(): loss_levels, = multi_apply( self.collect_loss_level_single, losses_cls, losses_bbox, pos_assigned_gt_inds_list, labels_seq=label_sequence) # Shape: (fpn_levels, num_gts). Loss of each gt at each fpn level loss_levels = torch.stack(loss_levels, dim=0) # Locate the best fpn level for loss back-propagation if loss_levels.numel() == 0: # zero gt argmin = loss_levels.new_empty((num_gts, ), dtype=torch.long) else: _, argmin = loss_levels.min(dim=0) # Reweight the loss of each (anchor, label) pair, so that only those # at the best gt level are back-propagated. losses_cls, losses_bbox, pos_inds = multi_apply( self.reweight_loss_single, losses_cls, losses_bbox, pos_assigned_gt_inds_list, labels_list, list(range(len(losses_cls))), min_levels=argmin) num_pos = torch.cat(pos_inds, 0).sum().float() pos_recall = self.calculate_pos_recall(cls_scores, labels_list, pos_inds) if num_pos == 0: # No gt num_total_neg = sum( [results.num_neg for results in sampling_results_list]) avg_factor = num_pos + num_total_neg else: avg_factor = num_pos for i in range(len(losses_cls)): losses_cls[i] /= avg_factor losses_bbox[i] /= avg_factor return dict( loss_cls=losses_cls, loss_bbox=losses_bbox, num_pos=num_pos / batch_size, pos_recall=pos_recall) def calculate_pos_recall(self, cls_scores: List[Tensor], labels_list: List[Tensor], pos_inds: List[Tensor]) -> Tensor: """Calculate positive recall with score threshold. Args: cls_scores (list[Tensor]): Classification scores at all fpn levels. Each tensor is in shape (N, num_classes * num_anchors, H, W) labels_list (list[Tensor]): The label that each anchor is assigned to. Shape (N * H * W * num_anchors, ) pos_inds (list[Tensor]): List of bool tensors indicating whether the anchor is assigned to a positive label. Shape (N * H * W * num_anchors, ) Returns: Tensor: A single float number indicating the positive recall. """ with torch.no_grad(): num_class = self.num_classes scores = [ cls.permute(0, 2, 3, 1).reshape(-1, num_class)[pos] for cls, pos in zip(cls_scores, pos_inds) ] labels = [ label.reshape(-1)[pos] for label, pos in zip(labels_list, pos_inds) ] scores = torch.cat(scores, dim=0) labels = torch.cat(labels, dim=0) if self.use_sigmoid_cls: scores = scores.sigmoid() else: scores = scores.softmax(dim=1) return accuracy(scores, labels, thresh=self.score_threshold) def collect_loss_level_single(self, cls_loss: Tensor, reg_loss: Tensor, assigned_gt_inds: Tensor, labels_seq: Tensor) -> Tensor: """Get the average loss in each FPN level w.r.t. each gt label. Args: cls_loss (Tensor): Classification loss of each feature map pixel, shape (num_anchor, num_class) reg_loss (Tensor): Regression loss of each feature map pixel, shape (num_anchor, 4) assigned_gt_inds (Tensor): It indicates which gt the prior is assigned to (0-based, -1: no assignment). shape (num_anchor), labels_seq: The rank of labels. shape (num_gt) Returns: Tensor: shape (num_gt), average loss of each gt in this level """ if len(reg_loss.shape) == 2: # iou loss has shape (num_prior, 4) reg_loss = reg_loss.sum(dim=-1) # sum loss in tblr dims if len(cls_loss.shape) == 2: cls_loss = cls_loss.sum(dim=-1) # sum loss in class dims loss = cls_loss + reg_loss assert loss.size(0) == assigned_gt_inds.size(0) # Default loss value is 1e6 for a layer where no anchor is positive # to ensure it will not be chosen to back-propagate gradient losses_ = loss.new_full(labels_seq.shape, 1e6) for i, l in enumerate(labels_seq): match = assigned_gt_inds == l if match.any(): losses_[i] = loss[match].mean() return losses_, def reweight_loss_single(self, cls_loss: Tensor, reg_loss: Tensor, assigned_gt_inds: Tensor, labels: Tensor, level: int, min_levels: Tensor) -> tuple: """Reweight loss values at each level. Reassign loss values at each level by masking those where the pre-calculated loss is too large. Then return the reduced losses. Args: cls_loss (Tensor): Element-wise classification loss. Shape: (num_anchors, num_classes) reg_loss (Tensor): Element-wise regression loss. Shape: (num_anchors, 4) assigned_gt_inds (Tensor): The gt indices that each anchor bbox is assigned to. -1 denotes a negative anchor, otherwise it is the gt index (0-based). Shape: (num_anchors, ), labels (Tensor): Label assigned to anchors. Shape: (num_anchors, ). level (int): The current level index in the pyramid (0-4 for RetinaNet) min_levels (Tensor): The best-matching level for each gt. Shape: (num_gts, ), Returns: tuple: - cls_loss: Reduced corrected classification loss. Scalar. - reg_loss: Reduced corrected regression loss. Scalar. - pos_flags (Tensor): Corrected bool tensor indicating the \ final positive anchors. Shape: (num_anchors, ). """ loc_weight = torch.ones_like(reg_loss) cls_weight = torch.ones_like(cls_loss) pos_flags = assigned_gt_inds >= 0 # positive pixel flag pos_indices = torch.nonzero(pos_flags, as_tuple=False).flatten() if pos_flags.any(): # pos pixels exist pos_assigned_gt_inds = assigned_gt_inds[pos_flags] zeroing_indices = (min_levels[pos_assigned_gt_inds] != level) neg_indices = pos_indices[zeroing_indices] if neg_indices.numel(): pos_flags[neg_indices] = 0 loc_weight[neg_indices] = 0 # Only the weight corresponding to the label is # zeroed out if not selected zeroing_labels = labels[neg_indices] assert (zeroing_labels >= 0).all() cls_weight[neg_indices, zeroing_labels] = 0 # Weighted loss for both cls and reg loss cls_loss = weight_reduce_loss(cls_loss, cls_weight, reduction='sum') reg_loss = weight_reduce_loss(reg_loss, loc_weight, reduction='sum') return cls_loss, reg_loss, pos_flags ================================================ FILE: mmdet/models/dense_heads/ga_retina_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple import torch.nn as nn from mmcv.cnn import ConvModule from mmcv.ops import MaskedConv2d from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import OptConfigType, OptMultiConfig from .guided_anchor_head import FeatureAdaption, GuidedAnchorHead @MODELS.register_module() class GARetinaHead(GuidedAnchorHead): """Guided-Anchor-based RetinaNet head.""" def __init__(self, num_classes: int, in_channels: int, stacked_convs: int = 4, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, init_cfg: OptMultiConfig = None, **kwargs) -> None: if init_cfg is None: init_cfg = dict( type='Normal', layer='Conv2d', std=0.01, override=[ dict( type='Normal', name='conv_loc', std=0.01, bias_prob=0.01), dict( type='Normal', name='retina_cls', std=0.01, bias_prob=0.01) ]) self.stacked_convs = stacked_convs self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg super().__init__( num_classes=num_classes, in_channels=in_channels, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" self.relu = nn.ReLU(inplace=True) self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.reg_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.conv_loc = nn.Conv2d(self.feat_channels, 1, 1) num_anchors = self.square_anchor_generator.num_base_priors[0] self.conv_shape = nn.Conv2d(self.feat_channels, num_anchors * 2, 1) self.feature_adaption_cls = FeatureAdaption( self.feat_channels, self.feat_channels, kernel_size=3, deform_groups=self.deform_groups) self.feature_adaption_reg = FeatureAdaption( self.feat_channels, self.feat_channels, kernel_size=3, deform_groups=self.deform_groups) self.retina_cls = MaskedConv2d( self.feat_channels, self.num_base_priors * self.cls_out_channels, 3, padding=1) self.retina_reg = MaskedConv2d( self.feat_channels, self.num_base_priors * 4, 3, padding=1) def forward_single(self, x: Tensor) -> Tuple[Tensor]: """Forward feature map of a single scale level.""" cls_feat = x reg_feat = x for cls_conv in self.cls_convs: cls_feat = cls_conv(cls_feat) for reg_conv in self.reg_convs: reg_feat = reg_conv(reg_feat) loc_pred = self.conv_loc(cls_feat) shape_pred = self.conv_shape(reg_feat) cls_feat = self.feature_adaption_cls(cls_feat, shape_pred) reg_feat = self.feature_adaption_reg(reg_feat, shape_pred) if not self.training: mask = loc_pred.sigmoid()[0] >= self.loc_filter_thr else: mask = None cls_score = self.retina_cls(cls_feat, mask) bbox_pred = self.retina_reg(reg_feat, mask) return cls_score, bbox_pred, shape_pred, loc_pred ================================================ FILE: mmdet/models/dense_heads/ga_rpn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import List, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.ops import nms from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import ConfigType, InstanceList, MultiConfig, OptInstanceList from .guided_anchor_head import GuidedAnchorHead @MODELS.register_module() class GARPNHead(GuidedAnchorHead): """Guided-Anchor-based RPN head.""" def __init__(self, in_channels: int, num_classes: int = 1, init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='conv_loc', std=0.01, bias_prob=0.01)), **kwargs) -> None: super().__init__( num_classes=num_classes, in_channels=in_channels, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" self.rpn_conv = nn.Conv2d( self.in_channels, self.feat_channels, 3, padding=1) super(GARPNHead, self)._init_layers() def forward_single(self, x: Tensor) -> Tuple[Tensor]: """Forward feature of a single scale level.""" x = self.rpn_conv(x) x = F.relu(x, inplace=True) (cls_score, bbox_pred, shape_pred, loc_pred) = super().forward_single(x) return cls_score, bbox_pred, shape_pred, loc_pred def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], shape_preds: List[Tensor], loc_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level has shape (N, num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). shape_preds (list[Tensor]): shape predictions for each scale level with shape (N, 1, H, W). loc_preds (list[Tensor]): location predictions for each scale level with shape (N, num_anchors * 2, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: A dictionary of loss components. """ losses = super().loss_by_feat( cls_scores, bbox_preds, shape_preds, loc_preds, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) return dict( loss_rpn_cls=losses['loss_cls'], loss_rpn_bbox=losses['loss_bbox'], loss_anchor_shape=losses['loss_shape'], loss_anchor_loc=losses['loss_loc']) def _predict_by_feat_single(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], mlvl_anchors: List[Tensor], mlvl_masks: List[Tensor], img_meta: dict, cfg: ConfigType, rescale: bool = False) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: cls_scores (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). mlvl_anchors (list[Tensor]): Each element in the list is the anchors of a single level in feature pyramid. it has shape (num_priors, 4). mlvl_masks (list[Tensor]): Each element in the list is location masks of a single level. img_meta (dict): Image meta info. cfg (:obj:`ConfigDict` or dict): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ cfg = self.test_cfg if cfg is None else cfg cfg = copy.deepcopy(cfg) assert cfg.nms.get('type', 'nms') == 'nms', 'GARPNHead only support ' \ 'naive nms.' mlvl_proposals = [] for idx in range(len(cls_scores)): rpn_cls_score = cls_scores[idx] rpn_bbox_pred = bbox_preds[idx] anchors = mlvl_anchors[idx] mask = mlvl_masks[idx] assert rpn_cls_score.size()[-2:] == rpn_bbox_pred.size()[-2:] # if no location is kept, end. if mask.sum() == 0: continue rpn_cls_score = rpn_cls_score.permute(1, 2, 0) if self.use_sigmoid_cls: rpn_cls_score = rpn_cls_score.reshape(-1) scores = rpn_cls_score.sigmoid() else: rpn_cls_score = rpn_cls_score.reshape(-1, 2) # remind that we set FG labels to [0, num_class-1] # since mmdet v2.0 # BG cat_id: num_class scores = rpn_cls_score.softmax(dim=1)[:, :-1] # filter scores, bbox_pred w.r.t. mask. # anchors are filtered in get_anchors() beforehand. scores = scores[mask] rpn_bbox_pred = rpn_bbox_pred.permute(1, 2, 0).reshape(-1, 4)[mask, :] if scores.dim() == 0: rpn_bbox_pred = rpn_bbox_pred.unsqueeze(0) anchors = anchors.unsqueeze(0) scores = scores.unsqueeze(0) # filter anchors, bbox_pred, scores w.r.t. scores if cfg.nms_pre > 0 and scores.shape[0] > cfg.nms_pre: _, topk_inds = scores.topk(cfg.nms_pre) rpn_bbox_pred = rpn_bbox_pred[topk_inds, :] anchors = anchors[topk_inds, :] scores = scores[topk_inds] # get proposals w.r.t. anchors and rpn_bbox_pred proposals = self.bbox_coder.decode( anchors, rpn_bbox_pred, max_shape=img_meta['img_shape']) # filter out too small bboxes if cfg.min_bbox_size >= 0: w = proposals[:, 2] - proposals[:, 0] h = proposals[:, 3] - proposals[:, 1] valid_mask = (w > cfg.min_bbox_size) & (h > cfg.min_bbox_size) if not valid_mask.all(): proposals = proposals[valid_mask] scores = scores[valid_mask] # NMS in current level proposals, _ = nms(proposals, scores, cfg.nms.iou_threshold) proposals = proposals[:cfg.nms_post, :] mlvl_proposals.append(proposals) proposals = torch.cat(mlvl_proposals, 0) if cfg.get('nms_across_levels', False): # NMS across multi levels proposals, _ = nms(proposals[:, :4], proposals[:, -1], cfg.nms.iou_threshold) proposals = proposals[:cfg.max_per_img, :] else: scores = proposals[:, 4] num = min(cfg.max_per_img, proposals.shape[0]) _, topk_inds = scores.topk(num) proposals = proposals[topk_inds, :] bboxes = proposals[:, :-1] scores = proposals[:, -1] if rescale: assert img_meta.get('scale_factor') is not None bboxes /= bboxes.new_tensor(img_meta['scale_factor']).repeat( (1, 2)) results = InstanceData() results.bboxes = bboxes results.scores = scores results.labels = scores.new_zeros(scores.size(0), dtype=torch.long) return results ================================================ FILE: mmdet/models/dense_heads/gfl_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Sequence, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule, Scale from mmengine.config import ConfigDict from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures.bbox import bbox_overlaps from mmdet.utils import (ConfigType, InstanceList, MultiConfig, OptConfigType, OptInstanceList, reduce_mean) from ..task_modules.prior_generators import anchor_inside_flags from ..task_modules.samplers import PseudoSampler from ..utils import (filter_scores_and_topk, images_to_levels, multi_apply, unmap) from .anchor_head import AnchorHead class Integral(nn.Module): """A fixed layer for calculating integral result from distribution. This layer calculates the target location by :math: ``sum{P(y_i) * y_i}``, P(y_i) denotes the softmax vector that represents the discrete distribution y_i denotes the discrete set, usually {0, 1, 2, ..., reg_max} Args: reg_max (int): The maximal value of the discrete set. Defaults to 16. You may want to reset it according to your new dataset or related settings. """ def __init__(self, reg_max: int = 16) -> None: super().__init__() self.reg_max = reg_max self.register_buffer('project', torch.linspace(0, self.reg_max, self.reg_max + 1)) def forward(self, x: Tensor) -> Tensor: """Forward feature from the regression head to get integral result of bounding box location. Args: x (Tensor): Features of the regression head, shape (N, 4*(n+1)), n is self.reg_max. Returns: x (Tensor): Integral result of box locations, i.e., distance offsets from the box center in four directions, shape (N, 4). """ x = F.softmax(x.reshape(-1, self.reg_max + 1), dim=1) x = F.linear(x, self.project.type_as(x)).reshape(-1, 4) return x @MODELS.register_module() class GFLHead(AnchorHead): """Generalized Focal Loss: Learning Qualified and Distributed Bounding Boxes for Dense Object Detection. GFL head structure is similar with ATSS, however GFL uses 1) joint representation for classification and localization quality, and 2) flexible General distribution for bounding box locations, which are supervised by Quality Focal Loss (QFL) and Distribution Focal Loss (DFL), respectively https://arxiv.org/abs/2006.04388 Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. stacked_convs (int): Number of conv layers in cls and reg tower. Defaults to 4. conv_cfg (:obj:`ConfigDict` or dict, optional): dictionary to construct and config conv layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict): dictionary to construct and config norm layer. Default: dict(type='GN', num_groups=32, requires_grad=True). loss_qfl (:obj:`ConfigDict` or dict): Config of Quality Focal Loss (QFL). bbox_coder (:obj:`ConfigDict` or dict): Config of bbox coder. Defaults to 'DistancePointBBoxCoder'. reg_max (int): Max value of integral set :math: ``{0, ..., reg_max}`` in QFL setting. Defaults to 16. init_cfg (:obj:`ConfigDict` or dict or list[dict] or list[:obj:`ConfigDict`]): Initialization config dict. Example: >>> self = GFLHead(11, 7) >>> feats = [torch.rand(1, 7, s, s) for s in [4, 8, 16, 32, 64]] >>> cls_quality_score, bbox_pred = self.forward(feats) >>> assert len(cls_quality_score) == len(self.scales) """ def __init__(self, num_classes: int, in_channels: int, stacked_convs: int = 4, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict( type='GN', num_groups=32, requires_grad=True), loss_dfl: ConfigType = dict( type='DistributionFocalLoss', loss_weight=0.25), bbox_coder: ConfigType = dict(type='DistancePointBBoxCoder'), reg_max: int = 16, init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='gfl_cls', std=0.01, bias_prob=0.01)), **kwargs) -> None: self.stacked_convs = stacked_convs self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.reg_max = reg_max super().__init__( num_classes=num_classes, in_channels=in_channels, bbox_coder=bbox_coder, init_cfg=init_cfg, **kwargs) if self.train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) if self.train_cfg.get('sampler', None) is not None: self.sampler = TASK_UTILS.build( self.train_cfg['sampler'], default_args=dict(context=self)) else: self.sampler = PseudoSampler(context=self) self.integral = Integral(self.reg_max) self.loss_dfl = MODELS.build(loss_dfl) def _init_layers(self) -> None: """Initialize layers of the head.""" self.relu = nn.ReLU() self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.reg_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) assert self.num_anchors == 1, 'anchor free version' self.gfl_cls = nn.Conv2d( self.feat_channels, self.cls_out_channels, 3, padding=1) self.gfl_reg = nn.Conv2d( self.feat_channels, 4 * (self.reg_max + 1), 3, padding=1) self.scales = nn.ModuleList( [Scale(1.0) for _ in self.prior_generator.strides]) def forward(self, x: Tuple[Tensor]) -> Tuple[List[Tensor]]: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Usually a tuple of classification scores and bbox prediction - cls_scores (list[Tensor]): Classification and quality (IoU) joint scores for all scale levels, each is a 4D-tensor, the channel number is num_classes. - bbox_preds (list[Tensor]): Box distribution logits for all scale levels, each is a 4D-tensor, the channel number is 4*(n+1), n is max value of integral set. """ return multi_apply(self.forward_single, x, self.scales) def forward_single(self, x: Tensor, scale: Scale) -> Sequence[Tensor]: """Forward feature of a single scale level. Args: x (Tensor): Features of a single scale level. scale (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize the bbox prediction. Returns: tuple: - cls_score (Tensor): Cls and quality joint scores for a single scale level the channel number is num_classes. - bbox_pred (Tensor): Box distribution logits for a single scale level, the channel number is 4*(n+1), n is max value of integral set. """ cls_feat = x reg_feat = x for cls_conv in self.cls_convs: cls_feat = cls_conv(cls_feat) for reg_conv in self.reg_convs: reg_feat = reg_conv(reg_feat) cls_score = self.gfl_cls(cls_feat) bbox_pred = scale(self.gfl_reg(reg_feat)).float() return cls_score, bbox_pred def anchor_center(self, anchors: Tensor) -> Tensor: """Get anchor centers from anchors. Args: anchors (Tensor): Anchor list with shape (N, 4), ``xyxy`` format. Returns: Tensor: Anchor centers with shape (N, 2), ``xy`` format. """ anchors_cx = (anchors[..., 2] + anchors[..., 0]) / 2 anchors_cy = (anchors[..., 3] + anchors[..., 1]) / 2 return torch.stack([anchors_cx, anchors_cy], dim=-1) def loss_by_feat_single(self, anchors: Tensor, cls_score: Tensor, bbox_pred: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, stride: Tuple[int], avg_factor: int) -> dict: """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: anchors (Tensor): Box reference for each scale level with shape (N, num_total_anchors, 4). cls_score (Tensor): Cls and quality joint scores for each scale level has shape (N, num_classes, H, W). bbox_pred (Tensor): Box distribution logits for each scale level with shape (N, 4*(n+1), H, W), n is max value of integral set. labels (Tensor): Labels of each anchors with shape (N, num_total_anchors). label_weights (Tensor): Label weights of each anchor with shape (N, num_total_anchors) bbox_targets (Tensor): BBox regression targets of each anchor weight shape (N, num_total_anchors, 4). stride (Tuple[int]): Stride in this scale level. avg_factor (int): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using `PseudoSampler`, `avg_factor` is usually equal to the number of positive priors. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert stride[0] == stride[1], 'h stride is not equal to w stride!' anchors = anchors.reshape(-1, 4) cls_score = cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4 * (self.reg_max + 1)) bbox_targets = bbox_targets.reshape(-1, 4) labels = labels.reshape(-1) label_weights = label_weights.reshape(-1) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = self.num_classes pos_inds = ((labels >= 0) & (labels < bg_class_ind)).nonzero().squeeze(1) score = label_weights.new_zeros(labels.shape) if len(pos_inds) > 0: pos_bbox_targets = bbox_targets[pos_inds] pos_bbox_pred = bbox_pred[pos_inds] pos_anchors = anchors[pos_inds] pos_anchor_centers = self.anchor_center(pos_anchors) / stride[0] weight_targets = cls_score.detach().sigmoid() weight_targets = weight_targets.max(dim=1)[0][pos_inds] pos_bbox_pred_corners = self.integral(pos_bbox_pred) pos_decode_bbox_pred = self.bbox_coder.decode( pos_anchor_centers, pos_bbox_pred_corners) pos_decode_bbox_targets = pos_bbox_targets / stride[0] score[pos_inds] = bbox_overlaps( pos_decode_bbox_pred.detach(), pos_decode_bbox_targets, is_aligned=True) pred_corners = pos_bbox_pred.reshape(-1, self.reg_max + 1) target_corners = self.bbox_coder.encode(pos_anchor_centers, pos_decode_bbox_targets, self.reg_max).reshape(-1) # regression loss loss_bbox = self.loss_bbox( pos_decode_bbox_pred, pos_decode_bbox_targets, weight=weight_targets, avg_factor=1.0) # dfl loss loss_dfl = self.loss_dfl( pred_corners, target_corners, weight=weight_targets[:, None].expand(-1, 4).reshape(-1), avg_factor=4.0) else: loss_bbox = bbox_pred.sum() * 0 loss_dfl = bbox_pred.sum() * 0 weight_targets = bbox_pred.new_tensor(0) # cls (qfl) loss loss_cls = self.loss_cls( cls_score, (labels, score), weight=label_weights, avg_factor=avg_factor) return loss_cls, loss_bbox, loss_dfl, weight_targets.sum() def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Cls and quality scores for each scale level has shape (N, num_classes, H, W). bbox_preds (list[Tensor]): Box distribution logits for each scale level with shape (N, 4*(n+1), H, W), n is max value of integral set. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets avg_factor = reduce_mean( torch.tensor(avg_factor, dtype=torch.float, device=device)).item() losses_cls, losses_bbox, losses_dfl,\ avg_factor = multi_apply( self.loss_by_feat_single, anchor_list, cls_scores, bbox_preds, labels_list, label_weights_list, bbox_targets_list, self.prior_generator.strides, avg_factor=avg_factor) avg_factor = sum(avg_factor) avg_factor = reduce_mean(avg_factor).clamp_(min=1).item() losses_bbox = list(map(lambda x: x / avg_factor, losses_bbox)) losses_dfl = list(map(lambda x: x / avg_factor, losses_dfl)) return dict( loss_cls=losses_cls, loss_bbox=losses_bbox, loss_dfl=losses_dfl) def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], score_factor_list: List[Tensor], mlvl_priors: List[Tensor], img_meta: dict, cfg: ConfigDict, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: cls_score_list (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). score_factor_list (list[Tensor]): Score factor from all scale levels of a single image. GFL head does not need this value. mlvl_priors (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid, has shape (num_priors, 4). img_meta (dict): Image meta info. cfg (:obj: `ConfigDict`): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: tuple[Tensor]: Results of detected bboxes and labels. If with_nms is False and mlvl_score_factor is None, return mlvl_bboxes and mlvl_scores, else return mlvl_bboxes, mlvl_scores and mlvl_score_factor. Usually with_nms is False is used for aug test. If with_nms is True, then return the following format - det_bboxes (Tensor): Predicted bboxes with shape [num_bboxes, 5], where the first 4 columns are bounding box positions (tl_x, tl_y, br_x, br_y) and the 5-th column are scores between 0 and 1. - det_labels (Tensor): Predicted labels of the corresponding box with shape [num_bboxes]. """ cfg = self.test_cfg if cfg is None else cfg img_shape = img_meta['img_shape'] nms_pre = cfg.get('nms_pre', -1) mlvl_bboxes = [] mlvl_scores = [] mlvl_labels = [] for level_idx, (cls_score, bbox_pred, stride, priors) in enumerate( zip(cls_score_list, bbox_pred_list, self.prior_generator.strides, mlvl_priors)): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] assert stride[0] == stride[1] bbox_pred = bbox_pred.permute(1, 2, 0) bbox_pred = self.integral(bbox_pred) * stride[0] scores = cls_score.permute(1, 2, 0).reshape( -1, self.cls_out_channels).sigmoid() # After https://github.com/open-mmlab/mmdetection/pull/6268/, # this operation keeps fewer bboxes under the same `nms_pre`. # There is no difference in performance for most models. If you # find a slight drop in performance, you can set a larger # `nms_pre` than before. results = filter_scores_and_topk( scores, cfg.score_thr, nms_pre, dict(bbox_pred=bbox_pred, priors=priors)) scores, labels, _, filtered_results = results bbox_pred = filtered_results['bbox_pred'] priors = filtered_results['priors'] bboxes = self.bbox_coder.decode( self.anchor_center(priors), bbox_pred, max_shape=img_shape) mlvl_bboxes.append(bboxes) mlvl_scores.append(scores) mlvl_labels.append(labels) results = InstanceData() results.bboxes = torch.cat(mlvl_bboxes) results.scores = torch.cat(mlvl_scores) results.labels = torch.cat(mlvl_labels) return self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta) def get_targets(self, anchor_list: List[Tensor], valid_flag_list: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs=True) -> tuple: """Get targets for GFL head. This method is almost the same as `AnchorHead.get_targets()`. Besides returning the targets as the parent method does, it also returns the anchors as the first element of the returned tuple. """ num_imgs = len(batch_img_metas) assert len(anchor_list) == len(valid_flag_list) == num_imgs # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] num_level_anchors_list = [num_level_anchors] * num_imgs # concat all level anchors and flags to a single tensor for i in range(num_imgs): assert len(anchor_list[i]) == len(valid_flag_list[i]) anchor_list[i] = torch.cat(anchor_list[i]) valid_flag_list[i] = torch.cat(valid_flag_list[i]) # compute targets for each image if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None] * num_imgs (all_anchors, all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, pos_inds_list, neg_inds_list, sampling_results_list) = multi_apply( self._get_targets_single, anchor_list, valid_flag_list, num_level_anchors_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs) # Get `avg_factor` of all images, which calculate in `SamplingResult`. # When using sampling method, avg_factor is usually the sum of # positive and negative priors. When using `PseudoSampler`, # `avg_factor` is usually equal to the number of positive priors. avg_factor = sum( [results.avg_factor for results in sampling_results_list]) # split targets to a list w.r.t. multiple levels anchors_list = images_to_levels(all_anchors, num_level_anchors) labels_list = images_to_levels(all_labels, num_level_anchors) label_weights_list = images_to_levels(all_label_weights, num_level_anchors) bbox_targets_list = images_to_levels(all_bbox_targets, num_level_anchors) bbox_weights_list = images_to_levels(all_bbox_weights, num_level_anchors) return (anchors_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) def _get_targets_single(self, flat_anchors: Tensor, valid_flags: Tensor, num_level_anchors: List[int], gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None, unmap_outputs: bool = True) -> tuple: """Compute regression, classification targets for anchors in a single image. Args: flat_anchors (Tensor): Multi-level anchors of the image, which are concatenated into a single tensor of shape (num_anchors, 4) valid_flags (Tensor): Multi level valid flags of the image, which are concatenated into a single tensor of shape (num_anchors,). num_level_anchors (list[int]): Number of anchors of each scale level. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. Returns: tuple: N is the number of total anchors in the image. - anchors (Tensor): All anchors in the image with shape (N, 4). - labels (Tensor): Labels of all anchors in the image with shape (N,). - label_weights (Tensor): Label weights of all anchor in the image with shape (N,). - bbox_targets (Tensor): BBox targets of all anchors in the image with shape (N, 4). - bbox_weights (Tensor): BBox weights of all anchors in the image with shape (N, 4). - pos_inds (Tensor): Indices of positive anchor with shape (num_pos,). - neg_inds (Tensor): Indices of negative anchor with shape (num_neg,). - sampling_result (:obj:`SamplingResult`): Sampling results. """ inside_flags = anchor_inside_flags(flat_anchors, valid_flags, img_meta['img_shape'][:2], self.train_cfg['allowed_border']) if not inside_flags.any(): raise ValueError( 'There is no valid anchor inside the image boundary. Please ' 'check the image size and anchor sizes, or set ' '``allowed_border`` to -1 to skip the condition.') # assign gt and sample anchors anchors = flat_anchors[inside_flags, :] num_level_anchors_inside = self.get_num_level_anchors_inside( num_level_anchors, inside_flags) pred_instances = InstanceData(priors=anchors) assign_result = self.assigner.assign( pred_instances=pred_instances, num_level_priors=num_level_anchors_inside, gt_instances=gt_instances, gt_instances_ignore=gt_instances_ignore) sampling_result = self.sampler.sample( assign_result=assign_result, pred_instances=pred_instances, gt_instances=gt_instances) num_valid_anchors = anchors.shape[0] bbox_targets = torch.zeros_like(anchors) bbox_weights = torch.zeros_like(anchors) labels = anchors.new_full((num_valid_anchors, ), self.num_classes, dtype=torch.long) label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: pos_bbox_targets = sampling_result.pos_gt_bboxes bbox_targets[pos_inds, :] = pos_bbox_targets bbox_weights[pos_inds, :] = 1.0 labels[pos_inds] = sampling_result.pos_gt_labels if self.train_cfg['pos_weight'] <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = self.train_cfg['pos_weight'] if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 # map up to original set of anchors if unmap_outputs: num_total_anchors = flat_anchors.size(0) anchors = unmap(anchors, num_total_anchors, inside_flags) labels = unmap( labels, num_total_anchors, inside_flags, fill=self.num_classes) label_weights = unmap(label_weights, num_total_anchors, inside_flags) bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) return (anchors, labels, label_weights, bbox_targets, bbox_weights, pos_inds, neg_inds, sampling_result) def get_num_level_anchors_inside(self, num_level_anchors: List[int], inside_flags: Tensor) -> List[int]: """Get the number of valid anchors in every level.""" split_inside_flags = torch.split(inside_flags, num_level_anchors) num_level_anchors_inside = [ int(flags.sum()) for flags in split_inside_flags ] return num_level_anchors_inside ================================================ FILE: mmdet/models/dense_heads/guided_anchor_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch import torch.nn as nn from mmcv.ops import DeformConv2d, MaskedConv2d from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.utils import (ConfigType, InstanceList, MultiConfig, OptConfigType, OptInstanceList) from ..layers import multiclass_nms from ..task_modules.prior_generators import anchor_inside_flags, calc_region from ..task_modules.samplers import PseudoSampler from ..utils import images_to_levels, multi_apply, unmap from .anchor_head import AnchorHead class FeatureAdaption(BaseModule): """Feature Adaption Module. Feature Adaption Module is implemented based on DCN v1. It uses anchor shape prediction rather than feature map to predict offsets of deform conv layer. Args: in_channels (int): Number of channels in the input feature map. out_channels (int): Number of channels in the output feature map. kernel_size (int): Deformable conv kernel size. Defaults to 3. deform_groups (int): Deformable conv group size. Defaults to 4. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or \ list[dict], optional): Initialization config dict. """ def __init__( self, in_channels: int, out_channels: int, kernel_size: int = 3, deform_groups: int = 4, init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.1, override=dict(type='Normal', name='conv_adaption', std=0.01)) ) -> None: super().__init__(init_cfg=init_cfg) offset_channels = kernel_size * kernel_size * 2 self.conv_offset = nn.Conv2d( 2, deform_groups * offset_channels, 1, bias=False) self.conv_adaption = DeformConv2d( in_channels, out_channels, kernel_size=kernel_size, padding=(kernel_size - 1) // 2, deform_groups=deform_groups) self.relu = nn.ReLU(inplace=True) def forward(self, x: Tensor, shape: Tensor) -> Tensor: offset = self.conv_offset(shape.detach()) x = self.relu(self.conv_adaption(x, offset)) return x @MODELS.register_module() class GuidedAnchorHead(AnchorHead): """Guided-Anchor-based head (GA-RPN, GA-RetinaNet, etc.). This GuidedAnchorHead will predict high-quality feature guided anchors and locations where anchors will be kept in inference. There are mainly 3 categories of bounding-boxes. - Sampled 9 pairs for target assignment. (approxes) - The square boxes where the predicted anchors are based on. (squares) - Guided anchors. Please refer to https://arxiv.org/abs/1901.03278 for more details. Args: num_classes (int): Number of classes. in_channels (int): Number of channels in the input feature map. feat_channels (int): Number of hidden channels. Defaults to 256. approx_anchor_generator (:obj:`ConfigDict` or dict): Config dict for approx generator square_anchor_generator (:obj:`ConfigDict` or dict): Config dict for square generator anchor_coder (:obj:`ConfigDict` or dict): Config dict for anchor coder bbox_coder (:obj:`ConfigDict` or dict): Config dict for bbox coder reg_decoded_bbox (bool): If true, the regression loss would be applied directly on decoded bounding boxes, converting both the predicted boxes and regression targets to absolute coordinates format. Defaults to False. It should be `True` when using `IoULoss`, `GIoULoss`, or `DIoULoss` in the bbox head. deform_groups: (int): Group number of DCN in FeatureAdaption module. Defaults to 4. loc_filter_thr (float): Threshold to filter out unconcerned regions. Defaults to 0.01. loss_loc (:obj:`ConfigDict` or dict): Config of location loss. loss_shape (:obj:`ConfigDict` or dict): Config of anchor shape loss. loss_cls (:obj:`ConfigDict` or dict): Config of classification loss. loss_bbox (:obj:`ConfigDict` or dict): Config of bbox regression loss. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or \ list[dict], optional): Initialization config dict. """ def __init__( self, num_classes: int, in_channels: int, feat_channels: int = 256, approx_anchor_generator: ConfigType = dict( type='AnchorGenerator', octave_base_scale=8, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), square_anchor_generator: ConfigType = dict( type='AnchorGenerator', ratios=[1.0], scales=[8], strides=[4, 8, 16, 32, 64]), anchor_coder: ConfigType = dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), bbox_coder: ConfigType = dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), reg_decoded_bbox: bool = False, deform_groups: int = 4, loc_filter_thr: float = 0.01, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, loss_loc: ConfigType = dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_shape: ConfigType = dict( type='BoundedIoULoss', beta=0.2, loss_weight=1.0), loss_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox: ConfigType = dict( type='SmoothL1Loss', beta=1.0, loss_weight=1.0), init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='conv_loc', std=0.01, lbias_prob=0.01)) ) -> None: super(AnchorHead, self).__init__(init_cfg=init_cfg) self.in_channels = in_channels self.num_classes = num_classes self.feat_channels = feat_channels self.deform_groups = deform_groups self.loc_filter_thr = loc_filter_thr # build approx_anchor_generator and square_anchor_generator assert (approx_anchor_generator['octave_base_scale'] == square_anchor_generator['scales'][0]) assert (approx_anchor_generator['strides'] == square_anchor_generator['strides']) self.approx_anchor_generator = TASK_UTILS.build( approx_anchor_generator) self.square_anchor_generator = TASK_UTILS.build( square_anchor_generator) self.approxs_per_octave = self.approx_anchor_generator \ .num_base_priors[0] self.reg_decoded_bbox = reg_decoded_bbox # one anchor per location self.num_base_priors = self.square_anchor_generator.num_base_priors[0] self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) self.loc_focal_loss = loss_loc['type'] in ['FocalLoss'] if self.use_sigmoid_cls: self.cls_out_channels = self.num_classes else: self.cls_out_channels = self.num_classes + 1 # build bbox_coder self.anchor_coder = TASK_UTILS.build(anchor_coder) self.bbox_coder = TASK_UTILS.build(bbox_coder) # build losses self.loss_loc = MODELS.build(loss_loc) self.loss_shape = MODELS.build(loss_shape) self.loss_cls = MODELS.build(loss_cls) self.loss_bbox = MODELS.build(loss_bbox) self.train_cfg = train_cfg self.test_cfg = test_cfg if self.train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) # use PseudoSampler when no sampler in train_cfg if train_cfg.get('sampler', None) is not None: self.sampler = TASK_UTILS.build( self.train_cfg['sampler'], default_args=dict(context=self)) else: self.sampler = PseudoSampler() self.ga_assigner = TASK_UTILS.build(self.train_cfg['ga_assigner']) if train_cfg.get('ga_sampler', None) is not None: self.ga_sampler = TASK_UTILS.build( self.train_cfg['ga_sampler'], default_args=dict(context=self)) else: self.ga_sampler = PseudoSampler() self._init_layers() def _init_layers(self) -> None: """Initialize layers of the head.""" self.relu = nn.ReLU(inplace=True) self.conv_loc = nn.Conv2d(self.in_channels, 1, 1) self.conv_shape = nn.Conv2d(self.in_channels, self.num_base_priors * 2, 1) self.feature_adaption = FeatureAdaption( self.in_channels, self.feat_channels, kernel_size=3, deform_groups=self.deform_groups) self.conv_cls = MaskedConv2d( self.feat_channels, self.num_base_priors * self.cls_out_channels, 1) self.conv_reg = MaskedConv2d(self.feat_channels, self.num_base_priors * 4, 1) def forward_single(self, x: Tensor) -> Tuple[Tensor]: """Forward feature of a single scale level.""" loc_pred = self.conv_loc(x) shape_pred = self.conv_shape(x) x = self.feature_adaption(x, shape_pred) # masked conv is only used during inference for speed-up if not self.training: mask = loc_pred.sigmoid()[0] >= self.loc_filter_thr else: mask = None cls_score = self.conv_cls(x, mask) bbox_pred = self.conv_reg(x, mask) return cls_score, bbox_pred, shape_pred, loc_pred def forward(self, x: List[Tensor]) -> Tuple[List[Tensor]]: """Forward features from the upstream network.""" return multi_apply(self.forward_single, x) def get_sampled_approxs(self, featmap_sizes: List[Tuple[int, int]], batch_img_metas: List[dict], device: str = 'cuda') -> tuple: """Get sampled approxs and inside flags according to feature map sizes. Args: featmap_sizes (list[tuple]): Multi-level feature map sizes. batch_img_metas (list[dict]): Image meta info. device (str): device for returned tensors Returns: tuple: approxes of each image, inside flags of each image """ num_imgs = len(batch_img_metas) # since feature map sizes of all images are the same, we only compute # approxes for one time multi_level_approxs = self.approx_anchor_generator.grid_priors( featmap_sizes, device=device) approxs_list = [multi_level_approxs for _ in range(num_imgs)] # for each image, we compute inside flags of multi level approxes inside_flag_list = [] for img_id, img_meta in enumerate(batch_img_metas): multi_level_flags = [] multi_level_approxs = approxs_list[img_id] # obtain valid flags for each approx first multi_level_approx_flags = self.approx_anchor_generator \ .valid_flags(featmap_sizes, img_meta['pad_shape'], device=device) for i, flags in enumerate(multi_level_approx_flags): approxs = multi_level_approxs[i] inside_flags_list = [] for j in range(self.approxs_per_octave): split_valid_flags = flags[j::self.approxs_per_octave] split_approxs = approxs[j::self.approxs_per_octave, :] inside_flags = anchor_inside_flags( split_approxs, split_valid_flags, img_meta['img_shape'][:2], self.train_cfg['allowed_border']) inside_flags_list.append(inside_flags) # inside_flag for a position is true if any anchor in this # position is true inside_flags = ( torch.stack(inside_flags_list, 0).sum(dim=0) > 0) multi_level_flags.append(inside_flags) inside_flag_list.append(multi_level_flags) return approxs_list, inside_flag_list def get_anchors(self, featmap_sizes: List[Tuple[int, int]], shape_preds: List[Tensor], loc_preds: List[Tensor], batch_img_metas: List[dict], use_loc_filter: bool = False, device: str = 'cuda') -> tuple: """Get squares according to feature map sizes and guided anchors. Args: featmap_sizes (list[tuple]): Multi-level feature map sizes. shape_preds (list[tensor]): Multi-level shape predictions. loc_preds (list[tensor]): Multi-level location predictions. batch_img_metas (list[dict]): Image meta info. use_loc_filter (bool): Use loc filter or not. Defaults to False device (str): device for returned tensors. Defaults to `cuda`. Returns: tuple: square approxs of each image, guided anchors of each image, loc masks of each image. """ num_imgs = len(batch_img_metas) num_levels = len(featmap_sizes) # since feature map sizes of all images are the same, we only compute # squares for one time multi_level_squares = self.square_anchor_generator.grid_priors( featmap_sizes, device=device) squares_list = [multi_level_squares for _ in range(num_imgs)] # for each image, we compute multi level guided anchors guided_anchors_list = [] loc_mask_list = [] for img_id, img_meta in enumerate(batch_img_metas): multi_level_guided_anchors = [] multi_level_loc_mask = [] for i in range(num_levels): squares = squares_list[img_id][i] shape_pred = shape_preds[i][img_id] loc_pred = loc_preds[i][img_id] guided_anchors, loc_mask = self._get_guided_anchors_single( squares, shape_pred, loc_pred, use_loc_filter=use_loc_filter) multi_level_guided_anchors.append(guided_anchors) multi_level_loc_mask.append(loc_mask) guided_anchors_list.append(multi_level_guided_anchors) loc_mask_list.append(multi_level_loc_mask) return squares_list, guided_anchors_list, loc_mask_list def _get_guided_anchors_single( self, squares: Tensor, shape_pred: Tensor, loc_pred: Tensor, use_loc_filter: bool = False) -> Tuple[Tensor]: """Get guided anchors and loc masks for a single level. Args: squares (tensor): Squares of a single level. shape_pred (tensor): Shape predictions of a single level. loc_pred (tensor): Loc predictions of a single level. use_loc_filter (list[tensor]): Use loc filter or not. Defaults to False. Returns: tuple: guided anchors, location masks """ # calculate location filtering mask loc_pred = loc_pred.sigmoid().detach() if use_loc_filter: loc_mask = loc_pred >= self.loc_filter_thr else: loc_mask = loc_pred >= 0.0 mask = loc_mask.permute(1, 2, 0).expand(-1, -1, self.num_base_priors) mask = mask.contiguous().view(-1) # calculate guided anchors squares = squares[mask] anchor_deltas = shape_pred.permute(1, 2, 0).contiguous().view( -1, 2).detach()[mask] bbox_deltas = anchor_deltas.new_full(squares.size(), 0) bbox_deltas[:, 2:] = anchor_deltas guided_anchors = self.anchor_coder.decode( squares, bbox_deltas, wh_ratio_clip=1e-6) return guided_anchors, mask def ga_loc_targets(self, batch_gt_instances: InstanceList, featmap_sizes: List[Tuple[int, int]]) -> tuple: """Compute location targets for guided anchoring. Each feature map is divided into positive, negative and ignore regions. - positive regions: target 1, weight 1 - ignore regions: target 0, weight 0 - negative regions: target 0, weight 0.1 Args: batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. featmap_sizes (list[tuple]): Multi level sizes of each feature maps. Returns: tuple: Returns a tuple containing location targets. """ anchor_scale = self.approx_anchor_generator.octave_base_scale anchor_strides = self.approx_anchor_generator.strides # Currently only supports same stride in x and y direction. for stride in anchor_strides: assert (stride[0] == stride[1]) anchor_strides = [stride[0] for stride in anchor_strides] center_ratio = self.train_cfg['center_ratio'] ignore_ratio = self.train_cfg['ignore_ratio'] img_per_gpu = len(batch_gt_instances) num_lvls = len(featmap_sizes) r1 = (1 - center_ratio) / 2 r2 = (1 - ignore_ratio) / 2 all_loc_targets = [] all_loc_weights = [] all_ignore_map = [] for lvl_id in range(num_lvls): h, w = featmap_sizes[lvl_id] loc_targets = torch.zeros( img_per_gpu, 1, h, w, device=batch_gt_instances[0].bboxes.device, dtype=torch.float32) loc_weights = torch.full_like(loc_targets, -1) ignore_map = torch.zeros_like(loc_targets) all_loc_targets.append(loc_targets) all_loc_weights.append(loc_weights) all_ignore_map.append(ignore_map) for img_id in range(img_per_gpu): gt_bboxes = batch_gt_instances[img_id].bboxes scale = torch.sqrt((gt_bboxes[:, 2] - gt_bboxes[:, 0]) * (gt_bboxes[:, 3] - gt_bboxes[:, 1])) min_anchor_size = scale.new_full( (1, ), float(anchor_scale * anchor_strides[0])) # assign gt bboxes to different feature levels w.r.t. their scales target_lvls = torch.floor( torch.log2(scale) - torch.log2(min_anchor_size) + 0.5) target_lvls = target_lvls.clamp(min=0, max=num_lvls - 1).long() for gt_id in range(gt_bboxes.size(0)): lvl = target_lvls[gt_id].item() # rescaled to corresponding feature map gt_ = gt_bboxes[gt_id, :4] / anchor_strides[lvl] # calculate ignore regions ignore_x1, ignore_y1, ignore_x2, ignore_y2 = calc_region( gt_, r2, featmap_sizes[lvl]) # calculate positive (center) regions ctr_x1, ctr_y1, ctr_x2, ctr_y2 = calc_region( gt_, r1, featmap_sizes[lvl]) all_loc_targets[lvl][img_id, 0, ctr_y1:ctr_y2 + 1, ctr_x1:ctr_x2 + 1] = 1 all_loc_weights[lvl][img_id, 0, ignore_y1:ignore_y2 + 1, ignore_x1:ignore_x2 + 1] = 0 all_loc_weights[lvl][img_id, 0, ctr_y1:ctr_y2 + 1, ctr_x1:ctr_x2 + 1] = 1 # calculate ignore map on nearby low level feature if lvl > 0: d_lvl = lvl - 1 # rescaled to corresponding feature map gt_ = gt_bboxes[gt_id, :4] / anchor_strides[d_lvl] ignore_x1, ignore_y1, ignore_x2, ignore_y2 = calc_region( gt_, r2, featmap_sizes[d_lvl]) all_ignore_map[d_lvl][img_id, 0, ignore_y1:ignore_y2 + 1, ignore_x1:ignore_x2 + 1] = 1 # calculate ignore map on nearby high level feature if lvl < num_lvls - 1: u_lvl = lvl + 1 # rescaled to corresponding feature map gt_ = gt_bboxes[gt_id, :4] / anchor_strides[u_lvl] ignore_x1, ignore_y1, ignore_x2, ignore_y2 = calc_region( gt_, r2, featmap_sizes[u_lvl]) all_ignore_map[u_lvl][img_id, 0, ignore_y1:ignore_y2 + 1, ignore_x1:ignore_x2 + 1] = 1 for lvl_id in range(num_lvls): # ignore negative regions w.r.t. ignore map all_loc_weights[lvl_id][(all_loc_weights[lvl_id] < 0) & (all_ignore_map[lvl_id] > 0)] = 0 # set negative regions with weight 0.1 all_loc_weights[lvl_id][all_loc_weights[lvl_id] < 0] = 0.1 # loc average factor to balance loss loc_avg_factor = sum( [t.size(0) * t.size(-1) * t.size(-2) for t in all_loc_targets]) / 200 return all_loc_targets, all_loc_weights, loc_avg_factor def _ga_shape_target_single(self, flat_approxs: Tensor, inside_flags: Tensor, flat_squares: Tensor, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData], img_meta: dict, unmap_outputs: bool = True) -> tuple: """Compute guided anchoring targets. This function returns sampled anchors and gt bboxes directly rather than calculates regression targets. Args: flat_approxs (Tensor): flat approxs of a single image, shape (n, 4) inside_flags (Tensor): inside flags of a single image, shape (n, ). flat_squares (Tensor): flat squares of a single image, shape (approxs_per_octave * n, 4) gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. img_meta (dict): Meta info of a single image. unmap_outputs (bool): unmap outputs or not. Returns: tuple: Returns a tuple containing shape targets of each image. """ if not inside_flags.any(): raise ValueError( 'There is no valid anchor inside the image boundary. Please ' 'check the image size and anchor sizes, or set ' '``allowed_border`` to -1 to skip the condition.') # assign gt and sample anchors num_square = flat_squares.size(0) approxs = flat_approxs.view(num_square, self.approxs_per_octave, 4) approxs = approxs[inside_flags, ...] squares = flat_squares[inside_flags, :] pred_instances = InstanceData() pred_instances.priors = squares pred_instances.approxs = approxs assign_result = self.ga_assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances, gt_instances_ignore=gt_instances_ignore) sampling_result = self.ga_sampler.sample( assign_result=assign_result, pred_instances=pred_instances, gt_instances=gt_instances) bbox_anchors = torch.zeros_like(squares) bbox_gts = torch.zeros_like(squares) bbox_weights = torch.zeros_like(squares) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: bbox_anchors[pos_inds, :] = sampling_result.pos_bboxes bbox_gts[pos_inds, :] = sampling_result.pos_gt_bboxes bbox_weights[pos_inds, :] = 1.0 # map up to original set of anchors if unmap_outputs: num_total_anchors = flat_squares.size(0) bbox_anchors = unmap(bbox_anchors, num_total_anchors, inside_flags) bbox_gts = unmap(bbox_gts, num_total_anchors, inside_flags) bbox_weights = unmap(bbox_weights, num_total_anchors, inside_flags) return (bbox_anchors, bbox_gts, bbox_weights, pos_inds, neg_inds, sampling_result) def ga_shape_targets(self, approx_list: List[List[Tensor]], inside_flag_list: List[List[Tensor]], square_list: List[List[Tensor]], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs: bool = True) -> tuple: """Compute guided anchoring targets. Args: approx_list (list[list[Tensor]]): Multi level approxs of each image. inside_flag_list (list[list[Tensor]]): Multi level inside flags of each image. square_list (list[list[Tensor]]): Multi level squares of each image. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): unmap outputs or not. Defaults to None. Returns: tuple: Returns a tuple containing shape targets. """ num_imgs = len(batch_img_metas) assert len(approx_list) == len(inside_flag_list) == len( square_list) == num_imgs # anchor number of multi levels num_level_squares = [squares.size(0) for squares in square_list[0]] # concat all level anchors and flags to a single tensor inside_flag_flat_list = [] approx_flat_list = [] square_flat_list = [] for i in range(num_imgs): assert len(square_list[i]) == len(inside_flag_list[i]) inside_flag_flat_list.append(torch.cat(inside_flag_list[i])) approx_flat_list.append(torch.cat(approx_list[i])) square_flat_list.append(torch.cat(square_list[i])) # compute targets for each image if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None for _ in range(num_imgs)] (all_bbox_anchors, all_bbox_gts, all_bbox_weights, pos_inds_list, neg_inds_list, sampling_results_list) = multi_apply( self._ga_shape_target_single, approx_flat_list, inside_flag_flat_list, square_flat_list, batch_gt_instances, batch_gt_instances_ignore, batch_img_metas, unmap_outputs=unmap_outputs) # sampled anchors of all images avg_factor = sum( [results.avg_factor for results in sampling_results_list]) # split targets to a list w.r.t. multiple levels bbox_anchors_list = images_to_levels(all_bbox_anchors, num_level_squares) bbox_gts_list = images_to_levels(all_bbox_gts, num_level_squares) bbox_weights_list = images_to_levels(all_bbox_weights, num_level_squares) return (bbox_anchors_list, bbox_gts_list, bbox_weights_list, avg_factor) def loss_shape_single(self, shape_pred: Tensor, bbox_anchors: Tensor, bbox_gts: Tensor, anchor_weights: Tensor, avg_factor: int) -> Tensor: """Compute shape loss in single level.""" shape_pred = shape_pred.permute(0, 2, 3, 1).contiguous().view(-1, 2) bbox_anchors = bbox_anchors.contiguous().view(-1, 4) bbox_gts = bbox_gts.contiguous().view(-1, 4) anchor_weights = anchor_weights.contiguous().view(-1, 4) bbox_deltas = bbox_anchors.new_full(bbox_anchors.size(), 0) bbox_deltas[:, 2:] += shape_pred # filter out negative samples to speed-up weighted_bounded_iou_loss inds = torch.nonzero( anchor_weights[:, 0] > 0, as_tuple=False).squeeze(1) bbox_deltas_ = bbox_deltas[inds] bbox_anchors_ = bbox_anchors[inds] bbox_gts_ = bbox_gts[inds] anchor_weights_ = anchor_weights[inds] pred_anchors_ = self.anchor_coder.decode( bbox_anchors_, bbox_deltas_, wh_ratio_clip=1e-6) loss_shape = self.loss_shape( pred_anchors_, bbox_gts_, anchor_weights_, avg_factor=avg_factor) return loss_shape def loss_loc_single(self, loc_pred: Tensor, loc_target: Tensor, loc_weight: Tensor, avg_factor: float) -> Tensor: """Compute location loss in single level.""" loss_loc = self.loss_loc( loc_pred.reshape(-1, 1), loc_target.reshape(-1).long(), loc_weight.reshape(-1), avg_factor=avg_factor) return loss_loc def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], shape_preds: List[Tensor], loc_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level has shape (N, num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). shape_preds (list[Tensor]): shape predictions for each scale level with shape (N, 1, H, W). loc_preds (list[Tensor]): location predictions for each scale level with shape (N, num_anchors * 2, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.approx_anchor_generator.num_levels device = cls_scores[0].device # get loc targets loc_targets, loc_weights, loc_avg_factor = self.ga_loc_targets( batch_gt_instances, featmap_sizes) # get sampled approxes approxs_list, inside_flag_list = self.get_sampled_approxs( featmap_sizes, batch_img_metas, device=device) # get squares and guided anchors squares_list, guided_anchors_list, _ = self.get_anchors( featmap_sizes, shape_preds, loc_preds, batch_img_metas, device=device) # get shape targets shape_targets = self.ga_shape_targets(approxs_list, inside_flag_list, squares_list, batch_gt_instances, batch_img_metas) (bbox_anchors_list, bbox_gts_list, anchor_weights_list, ga_avg_factor) = shape_targets # get anchor targets cls_reg_targets = self.get_targets( guided_anchors_list, inside_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets # anchor number of multi levels num_level_anchors = [ anchors.size(0) for anchors in guided_anchors_list[0] ] # concat all level anchors to a single tensor concat_anchor_list = [] for i in range(len(guided_anchors_list)): concat_anchor_list.append(torch.cat(guided_anchors_list[i])) all_anchor_list = images_to_levels(concat_anchor_list, num_level_anchors) # get classification and bbox regression losses losses_cls, losses_bbox = multi_apply( self.loss_by_feat_single, cls_scores, bbox_preds, all_anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor=avg_factor) # get anchor location loss losses_loc = [] for i in range(len(loc_preds)): loss_loc = self.loss_loc_single( loc_preds[i], loc_targets[i], loc_weights[i], avg_factor=loc_avg_factor) losses_loc.append(loss_loc) # get anchor shape loss losses_shape = [] for i in range(len(shape_preds)): loss_shape = self.loss_shape_single( shape_preds[i], bbox_anchors_list[i], bbox_gts_list[i], anchor_weights_list[i], avg_factor=ga_avg_factor) losses_shape.append(loss_shape) return dict( loss_cls=losses_cls, loss_bbox=losses_bbox, loss_shape=losses_shape, loss_loc=losses_loc) def predict_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], shape_preds: List[Tensor], loc_preds: List[Tensor], batch_img_metas: List[dict], cfg: OptConfigType = None, rescale: bool = False) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. Args: cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * 4, H, W). shape_preds (list[Tensor]): shape predictions for each scale level with shape (N, 1, H, W). loc_preds (list[Tensor]): location predictions for each scale level with shape (N, num_anchors * 2, H, W). batch_img_metas (list[dict], Optional): Batch image meta info. Defaults to None. cfg (ConfigDict, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(cls_scores) == len(bbox_preds) == len(shape_preds) == len( loc_preds) num_levels = len(cls_scores) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] device = cls_scores[0].device # get guided anchors _, guided_anchors, loc_masks = self.get_anchors( featmap_sizes, shape_preds, loc_preds, batch_img_metas, use_loc_filter=not self.training, device=device) result_list = [] for img_id in range(len(batch_img_metas)): cls_score_list = [ cls_scores[i][img_id].detach() for i in range(num_levels) ] bbox_pred_list = [ bbox_preds[i][img_id].detach() for i in range(num_levels) ] guided_anchor_list = [ guided_anchors[img_id][i].detach() for i in range(num_levels) ] loc_mask_list = [ loc_masks[img_id][i].detach() for i in range(num_levels) ] proposals = self._predict_by_feat_single( cls_scores=cls_score_list, bbox_preds=bbox_pred_list, mlvl_anchors=guided_anchor_list, mlvl_masks=loc_mask_list, img_meta=batch_img_metas[img_id], cfg=cfg, rescale=rescale) result_list.append(proposals) return result_list def _predict_by_feat_single(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], mlvl_anchors: List[Tensor], mlvl_masks: List[Tensor], img_meta: dict, cfg: ConfigType, rescale: bool = False) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: cls_scores (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). mlvl_anchors (list[Tensor]): Each element in the list is the anchors of a single level in feature pyramid. it has shape (num_priors, 4). mlvl_masks (list[Tensor]): Each element in the list is location masks of a single level. img_meta (dict): Image meta info. cfg (:obj:`ConfigDict` or dict): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ cfg = self.test_cfg if cfg is None else cfg assert len(cls_scores) == len(bbox_preds) == len(mlvl_anchors) mlvl_bbox_preds = [] mlvl_valid_anchors = [] mlvl_scores = [] for cls_score, bbox_pred, anchors, mask in zip(cls_scores, bbox_preds, mlvl_anchors, mlvl_masks): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] # if no location is kept, end. if mask.sum() == 0: continue # reshape scores and bbox_pred cls_score = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) if self.use_sigmoid_cls: scores = cls_score.sigmoid() else: scores = cls_score.softmax(-1) bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) # filter scores, bbox_pred w.r.t. mask. # anchors are filtered in get_anchors() beforehand. scores = scores[mask, :] bbox_pred = bbox_pred[mask, :] if scores.dim() == 0: anchors = anchors.unsqueeze(0) scores = scores.unsqueeze(0) bbox_pred = bbox_pred.unsqueeze(0) # filter anchors, bbox_pred, scores w.r.t. scores nms_pre = cfg.get('nms_pre', -1) if nms_pre > 0 and scores.shape[0] > nms_pre: if self.use_sigmoid_cls: max_scores, _ = scores.max(dim=1) else: # remind that we set FG labels to [0, num_class-1] # since mmdet v2.0 # BG cat_id: num_class max_scores, _ = scores[:, :-1].max(dim=1) _, topk_inds = max_scores.topk(nms_pre) anchors = anchors[topk_inds, :] bbox_pred = bbox_pred[topk_inds, :] scores = scores[topk_inds, :] mlvl_bbox_preds.append(bbox_pred) mlvl_valid_anchors.append(anchors) mlvl_scores.append(scores) mlvl_bbox_preds = torch.cat(mlvl_bbox_preds) mlvl_anchors = torch.cat(mlvl_valid_anchors) mlvl_scores = torch.cat(mlvl_scores) mlvl_bboxes = self.bbox_coder.decode( mlvl_anchors, mlvl_bbox_preds, max_shape=img_meta['img_shape']) if rescale: assert img_meta.get('scale_factor') is not None mlvl_bboxes /= mlvl_bboxes.new_tensor( img_meta['scale_factor']).repeat((1, 2)) if self.use_sigmoid_cls: # Add a dummy background class to the backend when using sigmoid # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 # BG cat_id: num_class padding = mlvl_scores.new_zeros(mlvl_scores.shape[0], 1) mlvl_scores = torch.cat([mlvl_scores, padding], dim=1) # multi class NMS det_bboxes, det_labels = multiclass_nms(mlvl_bboxes, mlvl_scores, cfg.score_thr, cfg.nms, cfg.max_per_img) results = InstanceData() results.bboxes = det_bboxes[:, :-1] results.scores = det_bboxes[:, -1] results.labels = det_labels return results ================================================ FILE: mmdet/models/dense_heads/lad_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional import torch from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox_overlaps from mmdet.utils import InstanceList, OptInstanceList from ..utils import levels_to_images, multi_apply, unpack_gt_instances from .paa_head import PAAHead @MODELS.register_module() class LADHead(PAAHead): """Label Assignment Head from the paper: `Improving Object Detection by Label Assignment Distillation `_""" def get_label_assignment( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], iou_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> tuple: """Get label assignment (from teacher). Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W) iou_preds (list[Tensor]): iou_preds for each scale level with shape (N, num_anchors * 1, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: tuple: Returns a tuple containing label assignment variables. - labels (Tensor): Labels of all anchors, each with shape (num_anchors,). - labels_weight (Tensor): Label weights of all anchor. each with shape (num_anchors,). - bboxes_target (Tensor): BBox targets of all anchors. each with shape (num_anchors, 4). - bboxes_weight (Tensor): BBox weights of all anchors. each with shape (num_anchors, 4). - pos_inds_flatten (Tensor): Contains all index of positive sample in all anchor. - pos_anchors (Tensor): Positive anchors. - num_pos (int): Number of positive anchors. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, ) (labels, labels_weight, bboxes_target, bboxes_weight, pos_inds, pos_gt_index) = cls_reg_targets cls_scores = levels_to_images(cls_scores) cls_scores = [ item.reshape(-1, self.cls_out_channels) for item in cls_scores ] bbox_preds = levels_to_images(bbox_preds) bbox_preds = [item.reshape(-1, 4) for item in bbox_preds] pos_losses_list, = multi_apply(self.get_pos_loss, anchor_list, cls_scores, bbox_preds, labels, labels_weight, bboxes_target, bboxes_weight, pos_inds) with torch.no_grad(): reassign_labels, reassign_label_weight, \ reassign_bbox_weights, num_pos = multi_apply( self.paa_reassign, pos_losses_list, labels, labels_weight, bboxes_weight, pos_inds, pos_gt_index, anchor_list) num_pos = sum(num_pos) # convert all tensor list to a flatten tensor labels = torch.cat(reassign_labels, 0).view(-1) flatten_anchors = torch.cat( [torch.cat(item, 0) for item in anchor_list]) labels_weight = torch.cat(reassign_label_weight, 0).view(-1) bboxes_target = torch.cat(bboxes_target, 0).view(-1, bboxes_target[0].size(-1)) pos_inds_flatten = ((labels >= 0) & (labels < self.num_classes)).nonzero().reshape(-1) if num_pos: pos_anchors = flatten_anchors[pos_inds_flatten] else: pos_anchors = None label_assignment_results = (labels, labels_weight, bboxes_target, bboxes_weight, pos_inds_flatten, pos_anchors, num_pos) return label_assignment_results def loss(self, x: List[Tensor], label_assignment_results: tuple, batch_data_samples: SampleList) -> dict: """Forward train with the available label assignment (student receives from teacher). Args: x (list[Tensor]): Features from FPN. label_assignment_results (tuple): As the outputs defined in the function `self.get_label_assignment`. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: losses: (dict[str, Tensor]): A dictionary of loss components. """ outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, batch_img_metas \ = outputs outs = self(x) loss_inputs = outs + (batch_gt_instances, batch_img_metas) losses = self.loss_by_feat( *loss_inputs, batch_gt_instances_ignore=batch_gt_instances_ignore, label_assignment_results=label_assignment_results) return losses def loss_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], iou_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, label_assignment_results: Optional[tuple] = None) -> dict: """Compute losses of the head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W) iou_preds (list[Tensor]): iou_preds for each scale level with shape (N, num_anchors * 1, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. label_assignment_results (tuple, optional): As the outputs defined in the function `self.get_ label_assignment`. Returns: dict[str, Tensor]: A dictionary of loss gmm_assignment. """ (labels, labels_weight, bboxes_target, bboxes_weight, pos_inds_flatten, pos_anchors, num_pos) = label_assignment_results cls_scores = levels_to_images(cls_scores) cls_scores = [ item.reshape(-1, self.cls_out_channels) for item in cls_scores ] bbox_preds = levels_to_images(bbox_preds) bbox_preds = [item.reshape(-1, 4) for item in bbox_preds] iou_preds = levels_to_images(iou_preds) iou_preds = [item.reshape(-1, 1) for item in iou_preds] # convert all tensor list to a flatten tensor cls_scores = torch.cat(cls_scores, 0).view(-1, cls_scores[0].size(-1)) bbox_preds = torch.cat(bbox_preds, 0).view(-1, bbox_preds[0].size(-1)) iou_preds = torch.cat(iou_preds, 0).view(-1, iou_preds[0].size(-1)) losses_cls = self.loss_cls( cls_scores, labels, labels_weight, avg_factor=max(num_pos, len(batch_img_metas))) # avoid num_pos=0 if num_pos: pos_bbox_pred = self.bbox_coder.decode( pos_anchors, bbox_preds[pos_inds_flatten]) pos_bbox_target = bboxes_target[pos_inds_flatten] iou_target = bbox_overlaps( pos_bbox_pred.detach(), pos_bbox_target, is_aligned=True) losses_iou = self.loss_centerness( iou_preds[pos_inds_flatten], iou_target.unsqueeze(-1), avg_factor=num_pos) losses_bbox = self.loss_bbox( pos_bbox_pred, pos_bbox_target, avg_factor=num_pos) else: losses_iou = iou_preds.sum() * 0 losses_bbox = bbox_preds.sum() * 0 return dict( loss_cls=losses_cls, loss_bbox=losses_bbox, loss_iou=losses_iou) ================================================ FILE: mmdet/models/dense_heads/ld_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import torch from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox_overlaps from mmdet.utils import ConfigType, InstanceList, OptInstanceList, reduce_mean from ..utils import multi_apply, unpack_gt_instances from .gfl_head import GFLHead @MODELS.register_module() class LDHead(GFLHead): """Localization distillation Head. (Short description) It utilizes the learned bbox distributions to transfer the localization dark knowledge from teacher to student. Original paper: `Localization Distillation for Object Detection. `_ Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. loss_ld (:obj:`ConfigDict` or dict): Config of Localization Distillation Loss (LD), T is the temperature for distillation. """ def __init__(self, num_classes: int, in_channels: int, loss_ld: ConfigType = dict( type='LocalizationDistillationLoss', loss_weight=0.25, T=10), **kwargs) -> dict: super().__init__( num_classes=num_classes, in_channels=in_channels, **kwargs) self.loss_ld = MODELS.build(loss_ld) def loss_by_feat_single(self, anchors: Tensor, cls_score: Tensor, bbox_pred: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, stride: Tuple[int], soft_targets: Tensor, avg_factor: int): """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: anchors (Tensor): Box reference for each scale level with shape (N, num_total_anchors, 4). cls_score (Tensor): Cls and quality joint scores for each scale level has shape (N, num_classes, H, W). bbox_pred (Tensor): Box distribution logits for each scale level with shape (N, 4*(n+1), H, W), n is max value of integral set. labels (Tensor): Labels of each anchors with shape (N, num_total_anchors). label_weights (Tensor): Label weights of each anchor with shape (N, num_total_anchors) bbox_targets (Tensor): BBox regression targets of each anchor weight shape (N, num_total_anchors, 4). stride (tuple): Stride in this scale level. soft_targets (Tensor): Soft BBox regression targets. avg_factor (int): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using `PseudoSampler`, `avg_factor` is usually equal to the number of positive priors. Returns: dict[tuple, Tensor]: Loss components and weight targets. """ assert stride[0] == stride[1], 'h stride is not equal to w stride!' anchors = anchors.reshape(-1, 4) cls_score = cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4 * (self.reg_max + 1)) soft_targets = soft_targets.permute(0, 2, 3, 1).reshape(-1, 4 * (self.reg_max + 1)) bbox_targets = bbox_targets.reshape(-1, 4) labels = labels.reshape(-1) label_weights = label_weights.reshape(-1) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = self.num_classes pos_inds = ((labels >= 0) & (labels < bg_class_ind)).nonzero().squeeze(1) score = label_weights.new_zeros(labels.shape) if len(pos_inds) > 0: pos_bbox_targets = bbox_targets[pos_inds] pos_bbox_pred = bbox_pred[pos_inds] pos_anchors = anchors[pos_inds] pos_anchor_centers = self.anchor_center(pos_anchors) / stride[0] weight_targets = cls_score.detach().sigmoid() weight_targets = weight_targets.max(dim=1)[0][pos_inds] pos_bbox_pred_corners = self.integral(pos_bbox_pred) pos_decode_bbox_pred = self.bbox_coder.decode( pos_anchor_centers, pos_bbox_pred_corners) pos_decode_bbox_targets = pos_bbox_targets / stride[0] score[pos_inds] = bbox_overlaps( pos_decode_bbox_pred.detach(), pos_decode_bbox_targets, is_aligned=True) pred_corners = pos_bbox_pred.reshape(-1, self.reg_max + 1) pos_soft_targets = soft_targets[pos_inds] soft_corners = pos_soft_targets.reshape(-1, self.reg_max + 1) target_corners = self.bbox_coder.encode(pos_anchor_centers, pos_decode_bbox_targets, self.reg_max).reshape(-1) # regression loss loss_bbox = self.loss_bbox( pos_decode_bbox_pred, pos_decode_bbox_targets, weight=weight_targets, avg_factor=1.0) # dfl loss loss_dfl = self.loss_dfl( pred_corners, target_corners, weight=weight_targets[:, None].expand(-1, 4).reshape(-1), avg_factor=4.0) # ld loss loss_ld = self.loss_ld( pred_corners, soft_corners, weight=weight_targets[:, None].expand(-1, 4).reshape(-1), avg_factor=4.0) else: loss_ld = bbox_pred.sum() * 0 loss_bbox = bbox_pred.sum() * 0 loss_dfl = bbox_pred.sum() * 0 weight_targets = bbox_pred.new_tensor(0) # cls (qfl) loss loss_cls = self.loss_cls( cls_score, (labels, score), weight=label_weights, avg_factor=avg_factor) return loss_cls, loss_bbox, loss_dfl, loss_ld, weight_targets.sum() def loss(self, x: List[Tensor], out_teacher: Tuple[Tensor], batch_data_samples: SampleList) -> dict: """ Args: x (list[Tensor]): Features from FPN. out_teacher (tuple[Tensor]): The output of teacher. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: tuple[dict, list]: The loss components and proposals of each image. - losses (dict[str, Tensor]): A dictionary of loss components. - proposal_list (list[Tensor]): Proposals of each image. """ outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, batch_img_metas \ = outputs outs = self(x) soft_targets = out_teacher[1] loss_inputs = outs + (batch_gt_instances, batch_img_metas, soft_targets) losses = self.loss_by_feat( *loss_inputs, batch_gt_instances_ignore=batch_gt_instances_ignore) return losses def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], soft_targets: List[Tensor], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Compute losses of the head. Args: cls_scores (list[Tensor]): Cls and quality scores for each scale level has shape (N, num_classes, H, W). bbox_preds (list[Tensor]): Box distribution logits for each scale level with shape (N, 4*(n+1), H, W), n is max value of integral set. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. soft_targets (list[Tensor]): Soft BBox regression targets. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets avg_factor = reduce_mean( torch.tensor(avg_factor, dtype=torch.float, device=device)).item() losses_cls, losses_bbox, losses_dfl, losses_ld, \ avg_factor = multi_apply( self.loss_by_feat_single, anchor_list, cls_scores, bbox_preds, labels_list, label_weights_list, bbox_targets_list, self.prior_generator.strides, soft_targets, avg_factor=avg_factor) avg_factor = sum(avg_factor) + 1e-6 avg_factor = reduce_mean(avg_factor).item() losses_bbox = [x / avg_factor for x in losses_bbox] losses_dfl = [x / avg_factor for x in losses_dfl] return dict( loss_cls=losses_cls, loss_bbox=losses_bbox, loss_dfl=losses_dfl, loss_ld=losses_ld) ================================================ FILE: mmdet/models/dense_heads/mask2former_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import List, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import Conv2d from mmcv.ops import point_sample from mmengine.model import ModuleList, caffe2_xavier_init from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures import SampleList from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig, reduce_mean from ..layers import Mask2FormerTransformerDecoder, SinePositionalEncoding from ..utils import get_uncertain_point_coords_with_randomness from .anchor_free_head import AnchorFreeHead from .maskformer_head import MaskFormerHead @MODELS.register_module() class Mask2FormerHead(MaskFormerHead): """Implements the Mask2Former head. See `Masked-attention Mask Transformer for Universal Image Segmentation `_ for details. Args: in_channels (list[int]): Number of channels in the input feature map. feat_channels (int): Number of channels for features. out_channels (int): Number of channels for output. num_things_classes (int): Number of things. num_stuff_classes (int): Number of stuff. num_queries (int): Number of query in Transformer decoder. pixel_decoder (:obj:`ConfigDict` or dict): Config for pixel decoder. Defaults to None. enforce_decoder_input_project (bool, optional): Whether to add a layer to change the embed_dim of tranformer encoder in pixel decoder to the embed_dim of transformer decoder. Defaults to False. transformer_decoder (:obj:`ConfigDict` or dict): Config for transformer decoder. Defaults to None. positional_encoding (:obj:`ConfigDict` or dict): Config for transformer decoder position encoding. Defaults to dict(num_feats=128, normalize=True). loss_cls (:obj:`ConfigDict` or dict): Config of the classification loss. Defaults to None. loss_mask (:obj:`ConfigDict` or dict): Config of the mask loss. Defaults to None. loss_dice (:obj:`ConfigDict` or dict): Config of the dice loss. Defaults to None. train_cfg (:obj:`ConfigDict` or dict, optional): Training config of Mask2Former head. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of Mask2Former head. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, in_channels: List[int], feat_channels: int, out_channels: int, num_things_classes: int = 80, num_stuff_classes: int = 53, num_queries: int = 100, num_transformer_feat_level: int = 3, pixel_decoder: ConfigType = ..., enforce_decoder_input_project: bool = False, transformer_decoder: ConfigType = ..., positional_encoding: ConfigType = dict( num_feats=128, normalize=True), loss_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=2.0, reduction='mean', class_weight=[1.0] * 133 + [0.1]), loss_mask: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, reduction='mean', loss_weight=5.0), loss_dice: ConfigType = dict( type='DiceLoss', use_sigmoid=True, activate=True, reduction='mean', naive_dice=True, eps=1.0, loss_weight=5.0), train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: OptMultiConfig = None, **kwargs) -> None: super(AnchorFreeHead, self).__init__(init_cfg=init_cfg) self.num_things_classes = num_things_classes self.num_stuff_classes = num_stuff_classes self.num_classes = self.num_things_classes + self.num_stuff_classes self.num_queries = num_queries self.num_transformer_feat_level = num_transformer_feat_level self.num_heads = transformer_decoder.layer_cfg.cross_attn_cfg.num_heads self.num_transformer_decoder_layers = transformer_decoder.num_layers assert pixel_decoder.encoder.layer_cfg. \ self_attn_cfg.num_levels == num_transformer_feat_level pixel_decoder_ = copy.deepcopy(pixel_decoder) pixel_decoder_.update( in_channels=in_channels, feat_channels=feat_channels, out_channels=out_channels) self.pixel_decoder = MODELS.build(pixel_decoder_) self.transformer_decoder = Mask2FormerTransformerDecoder( **transformer_decoder) self.decoder_embed_dims = self.transformer_decoder.embed_dims self.decoder_input_projs = ModuleList() # from low resolution to high resolution for _ in range(num_transformer_feat_level): if (self.decoder_embed_dims != feat_channels or enforce_decoder_input_project): self.decoder_input_projs.append( Conv2d( feat_channels, self.decoder_embed_dims, kernel_size=1)) else: self.decoder_input_projs.append(nn.Identity()) self.decoder_positional_encoding = SinePositionalEncoding( **positional_encoding) self.query_embed = nn.Embedding(self.num_queries, feat_channels) self.query_feat = nn.Embedding(self.num_queries, feat_channels) # from low resolution to high resolution self.level_embed = nn.Embedding(self.num_transformer_feat_level, feat_channels) self.cls_embed = nn.Linear(feat_channels, self.num_classes + 1) self.mask_embed = nn.Sequential( nn.Linear(feat_channels, feat_channels), nn.ReLU(inplace=True), nn.Linear(feat_channels, feat_channels), nn.ReLU(inplace=True), nn.Linear(feat_channels, out_channels)) self.test_cfg = test_cfg self.train_cfg = train_cfg if train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) self.sampler = TASK_UTILS.build( self.train_cfg['sampler'], default_args=dict(context=self)) self.num_points = self.train_cfg.get('num_points', 12544) self.oversample_ratio = self.train_cfg.get('oversample_ratio', 3.0) self.importance_sample_ratio = self.train_cfg.get( 'importance_sample_ratio', 0.75) self.class_weight = loss_cls.class_weight self.loss_cls = MODELS.build(loss_cls) self.loss_mask = MODELS.build(loss_mask) self.loss_dice = MODELS.build(loss_dice) def init_weights(self) -> None: for m in self.decoder_input_projs: if isinstance(m, Conv2d): caffe2_xavier_init(m, bias=0) self.pixel_decoder.init_weights() for p in self.transformer_decoder.parameters(): if p.dim() > 1: nn.init.xavier_normal_(p) def _get_targets_single(self, cls_score: Tensor, mask_pred: Tensor, gt_instances: InstanceData, img_meta: dict) -> Tuple[Tensor]: """Compute classification and mask targets for one image. Args: cls_score (Tensor): Mask score logits from a single decoder layer for one image. Shape (num_queries, cls_out_channels). mask_pred (Tensor): Mask logits for a single decoder layer for one image. Shape (num_queries, h, w). gt_instances (:obj:`InstanceData`): It contains ``labels`` and ``masks``. img_meta (dict): Image informtation. Returns: tuple[Tensor]: A tuple containing the following for one image. - labels (Tensor): Labels of each image. \ shape (num_queries, ). - label_weights (Tensor): Label weights of each image. \ shape (num_queries, ). - mask_targets (Tensor): Mask targets of each image. \ shape (num_queries, h, w). - mask_weights (Tensor): Mask weights of each image. \ shape (num_queries, ). - pos_inds (Tensor): Sampled positive indices for each \ image. - neg_inds (Tensor): Sampled negative indices for each \ image. - sampling_result (:obj:`SamplingResult`): Sampling results. """ gt_labels = gt_instances.labels gt_masks = gt_instances.masks # sample points num_queries = cls_score.shape[0] num_gts = gt_labels.shape[0] point_coords = torch.rand((1, self.num_points, 2), device=cls_score.device) # shape (num_queries, num_points) mask_points_pred = point_sample( mask_pred.unsqueeze(1), point_coords.repeat(num_queries, 1, 1)).squeeze(1) # shape (num_gts, num_points) gt_points_masks = point_sample( gt_masks.unsqueeze(1).float(), point_coords.repeat(num_gts, 1, 1)).squeeze(1) sampled_gt_instances = InstanceData( labels=gt_labels, masks=gt_points_masks) sampled_pred_instances = InstanceData( scores=cls_score, masks=mask_points_pred) # assign and sample assign_result = self.assigner.assign( pred_instances=sampled_pred_instances, gt_instances=sampled_gt_instances, img_meta=img_meta) pred_instances = InstanceData(scores=cls_score, masks=mask_pred) sampling_result = self.sampler.sample( assign_result=assign_result, pred_instances=pred_instances, gt_instances=gt_instances) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds # label target labels = gt_labels.new_full((self.num_queries, ), self.num_classes, dtype=torch.long) labels[pos_inds] = gt_labels[sampling_result.pos_assigned_gt_inds] label_weights = gt_labels.new_ones((self.num_queries, )) # mask target mask_targets = gt_masks[sampling_result.pos_assigned_gt_inds] mask_weights = mask_pred.new_zeros((self.num_queries, )) mask_weights[pos_inds] = 1.0 return (labels, label_weights, mask_targets, mask_weights, pos_inds, neg_inds, sampling_result) def _loss_by_feat_single(self, cls_scores: Tensor, mask_preds: Tensor, batch_gt_instances: List[InstanceData], batch_img_metas: List[dict]) -> Tuple[Tensor]: """Loss function for outputs from a single decoder layer. Args: cls_scores (Tensor): Mask score logits from a single decoder layer for all images. Shape (batch_size, num_queries, cls_out_channels). Note `cls_out_channels` should includes background. mask_preds (Tensor): Mask logits for a pixel decoder for all images. Shape (batch_size, num_queries, h, w). batch_gt_instances (list[obj:`InstanceData`]): each contains ``labels`` and ``masks``. batch_img_metas (list[dict]): List of image meta information. Returns: tuple[Tensor]: Loss components for outputs from a single \ decoder layer. """ num_imgs = cls_scores.size(0) cls_scores_list = [cls_scores[i] for i in range(num_imgs)] mask_preds_list = [mask_preds[i] for i in range(num_imgs)] (labels_list, label_weights_list, mask_targets_list, mask_weights_list, avg_factor) = self.get_targets(cls_scores_list, mask_preds_list, batch_gt_instances, batch_img_metas) # shape (batch_size, num_queries) labels = torch.stack(labels_list, dim=0) # shape (batch_size, num_queries) label_weights = torch.stack(label_weights_list, dim=0) # shape (num_total_gts, h, w) mask_targets = torch.cat(mask_targets_list, dim=0) # shape (batch_size, num_queries) mask_weights = torch.stack(mask_weights_list, dim=0) # classfication loss # shape (batch_size * num_queries, ) cls_scores = cls_scores.flatten(0, 1) labels = labels.flatten(0, 1) label_weights = label_weights.flatten(0, 1) class_weight = cls_scores.new_tensor(self.class_weight) loss_cls = self.loss_cls( cls_scores, labels, label_weights, avg_factor=class_weight[labels].sum()) num_total_masks = reduce_mean(cls_scores.new_tensor([avg_factor])) num_total_masks = max(num_total_masks, 1) # extract positive ones # shape (batch_size, num_queries, h, w) -> (num_total_gts, h, w) mask_preds = mask_preds[mask_weights > 0] if mask_targets.shape[0] == 0: # zero match loss_dice = mask_preds.sum() loss_mask = mask_preds.sum() return loss_cls, loss_mask, loss_dice with torch.no_grad(): points_coords = get_uncertain_point_coords_with_randomness( mask_preds.unsqueeze(1), None, self.num_points, self.oversample_ratio, self.importance_sample_ratio) # shape (num_total_gts, h, w) -> (num_total_gts, num_points) mask_point_targets = point_sample( mask_targets.unsqueeze(1).float(), points_coords).squeeze(1) # shape (num_queries, h, w) -> (num_queries, num_points) mask_point_preds = point_sample( mask_preds.unsqueeze(1), points_coords).squeeze(1) # dice loss loss_dice = self.loss_dice( mask_point_preds, mask_point_targets, avg_factor=num_total_masks) # mask loss # shape (num_queries, num_points) -> (num_queries * num_points, ) mask_point_preds = mask_point_preds.reshape(-1) # shape (num_total_gts, num_points) -> (num_total_gts * num_points, ) mask_point_targets = mask_point_targets.reshape(-1) loss_mask = self.loss_mask( mask_point_preds, mask_point_targets, avg_factor=num_total_masks * self.num_points) return loss_cls, loss_mask, loss_dice def _forward_head(self, decoder_out: Tensor, mask_feature: Tensor, attn_mask_target_size: Tuple[int, int]) -> Tuple[Tensor]: """Forward for head part which is called after every decoder layer. Args: decoder_out (Tensor): in shape (batch_size, num_queries, c). mask_feature (Tensor): in shape (batch_size, c, h, w). attn_mask_target_size (tuple[int, int]): target attention mask size. Returns: tuple: A tuple contain three elements. - cls_pred (Tensor): Classification scores in shape \ (batch_size, num_queries, cls_out_channels). \ Note `cls_out_channels` should includes background. - mask_pred (Tensor): Mask scores in shape \ (batch_size, num_queries,h, w). - attn_mask (Tensor): Attention mask in shape \ (batch_size * num_heads, num_queries, h, w). """ decoder_out = self.transformer_decoder.post_norm(decoder_out) # shape (num_queries, batch_size, c) cls_pred = self.cls_embed(decoder_out) # shape (num_queries, batch_size, c) mask_embed = self.mask_embed(decoder_out) # shape (num_queries, batch_size, h, w) mask_pred = torch.einsum('bqc,bchw->bqhw', mask_embed, mask_feature) attn_mask = F.interpolate( mask_pred, attn_mask_target_size, mode='bilinear', align_corners=False) # shape (num_queries, batch_size, h, w) -> # (batch_size * num_head, num_queries, h, w) attn_mask = attn_mask.flatten(2).unsqueeze(1).repeat( (1, self.num_heads, 1, 1)).flatten(0, 1) attn_mask = attn_mask.sigmoid() < 0.5 attn_mask = attn_mask.detach() return cls_pred, mask_pred, attn_mask def forward(self, x: List[Tensor], batch_data_samples: SampleList) -> Tuple[List[Tensor]]: """Forward function. Args: x (list[Tensor]): Multi scale Features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: tuple[list[Tensor]]: A tuple contains two elements. - cls_pred_list (list[Tensor)]: Classification logits \ for each decoder layer. Each is a 3D-tensor with shape \ (batch_size, num_queries, cls_out_channels). \ Note `cls_out_channels` should includes background. - mask_pred_list (list[Tensor]): Mask logits for each \ decoder layer. Each with shape (batch_size, num_queries, \ h, w). """ batch_img_metas = [ data_sample.metainfo for data_sample in batch_data_samples ] batch_size = len(batch_img_metas) mask_features, multi_scale_memorys = self.pixel_decoder(x) # multi_scale_memorys (from low resolution to high resolution) decoder_inputs = [] decoder_positional_encodings = [] for i in range(self.num_transformer_feat_level): decoder_input = self.decoder_input_projs[i](multi_scale_memorys[i]) # shape (batch_size, c, h, w) -> (batch_size, h*w, c) decoder_input = decoder_input.flatten(2).permute(0, 2, 1) level_embed = self.level_embed.weight[i].view(1, 1, -1) decoder_input = decoder_input + level_embed # shape (batch_size, c, h, w) -> (batch_size, h*w, c) mask = decoder_input.new_zeros( (batch_size, ) + multi_scale_memorys[i].shape[-2:], dtype=torch.bool) decoder_positional_encoding = self.decoder_positional_encoding( mask) decoder_positional_encoding = decoder_positional_encoding.flatten( 2).permute(0, 2, 1) decoder_inputs.append(decoder_input) decoder_positional_encodings.append(decoder_positional_encoding) # shape (num_queries, c) -> (batch_size, num_queries, c) query_feat = self.query_feat.weight.unsqueeze(0).repeat( (batch_size, 1, 1)) query_embed = self.query_embed.weight.unsqueeze(0).repeat( (batch_size, 1, 1)) cls_pred_list = [] mask_pred_list = [] cls_pred, mask_pred, attn_mask = self._forward_head( query_feat, mask_features, multi_scale_memorys[0].shape[-2:]) cls_pred_list.append(cls_pred) mask_pred_list.append(mask_pred) for i in range(self.num_transformer_decoder_layers): level_idx = i % self.num_transformer_feat_level # if a mask is all True(all background), then set it all False. attn_mask[torch.where( attn_mask.sum(-1) == attn_mask.shape[-1])] = False # cross_attn + self_attn layer = self.transformer_decoder.layers[i] query_feat = layer( query=query_feat, key=decoder_inputs[level_idx], value=decoder_inputs[level_idx], query_pos=query_embed, key_pos=decoder_positional_encodings[level_idx], cross_attn_mask=attn_mask, query_key_padding_mask=None, # here we do not apply masking on padded region key_padding_mask=None) cls_pred, mask_pred, attn_mask = self._forward_head( query_feat, mask_features, multi_scale_memorys[ (i + 1) % self.num_transformer_feat_level].shape[-2:]) cls_pred_list.append(cls_pred) mask_pred_list.append(mask_pred) return cls_pred_list, mask_pred_list ================================================ FILE: mmdet/models/dense_heads/maskformer_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Optional, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import Conv2d from mmengine.model import caffe2_xavier_init from mmengine.structures import InstanceData, PixelData from torch import Tensor from mmdet.models.layers.pixel_decoder import PixelDecoder from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures import SampleList from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptMultiConfig, reduce_mean) from ..layers import DetrTransformerDecoder, SinePositionalEncoding from ..utils import multi_apply, preprocess_panoptic_gt from .anchor_free_head import AnchorFreeHead @MODELS.register_module() class MaskFormerHead(AnchorFreeHead): """Implements the MaskFormer head. See `Per-Pixel Classification is Not All You Need for Semantic Segmentation `_ for details. Args: in_channels (list[int]): Number of channels in the input feature map. feat_channels (int): Number of channels for feature. out_channels (int): Number of channels for output. num_things_classes (int): Number of things. num_stuff_classes (int): Number of stuff. num_queries (int): Number of query in Transformer. pixel_decoder (:obj:`ConfigDict` or dict): Config for pixel decoder. enforce_decoder_input_project (bool): Whether to add a layer to change the embed_dim of transformer encoder in pixel decoder to the embed_dim of transformer decoder. Defaults to False. transformer_decoder (:obj:`ConfigDict` or dict): Config for transformer decoder. positional_encoding (:obj:`ConfigDict` or dict): Config for transformer decoder position encoding. loss_cls (:obj:`ConfigDict` or dict): Config of the classification loss. Defaults to `CrossEntropyLoss`. loss_mask (:obj:`ConfigDict` or dict): Config of the mask loss. Defaults to `FocalLoss`. loss_dice (:obj:`ConfigDict` or dict): Config of the dice loss. Defaults to `DiceLoss`. train_cfg (:obj:`ConfigDict` or dict, optional): Training config of MaskFormer head. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of MaskFormer head. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, in_channels: List[int], feat_channels: int, out_channels: int, num_things_classes: int = 80, num_stuff_classes: int = 53, num_queries: int = 100, pixel_decoder: ConfigType = ..., enforce_decoder_input_project: bool = False, transformer_decoder: ConfigType = ..., positional_encoding: ConfigType = dict( num_feats=128, normalize=True), loss_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0, class_weight=[1.0] * 133 + [0.1]), loss_mask: ConfigType = dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=20.0), loss_dice: ConfigType = dict( type='DiceLoss', use_sigmoid=True, activate=True, naive_dice=True, loss_weight=1.0), train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: OptMultiConfig = None, **kwargs) -> None: super(AnchorFreeHead, self).__init__(init_cfg=init_cfg) self.num_things_classes = num_things_classes self.num_stuff_classes = num_stuff_classes self.num_classes = self.num_things_classes + self.num_stuff_classes self.num_queries = num_queries pixel_decoder.update( in_channels=in_channels, feat_channels=feat_channels, out_channels=out_channels) self.pixel_decoder = MODELS.build(pixel_decoder) self.transformer_decoder = DetrTransformerDecoder( **transformer_decoder) self.decoder_embed_dims = self.transformer_decoder.embed_dims if type(self.pixel_decoder) == PixelDecoder and ( self.decoder_embed_dims != in_channels[-1] or enforce_decoder_input_project): self.decoder_input_proj = Conv2d( in_channels[-1], self.decoder_embed_dims, kernel_size=1) else: self.decoder_input_proj = nn.Identity() self.decoder_pe = SinePositionalEncoding(**positional_encoding) self.query_embed = nn.Embedding(self.num_queries, out_channels) self.cls_embed = nn.Linear(feat_channels, self.num_classes + 1) self.mask_embed = nn.Sequential( nn.Linear(feat_channels, feat_channels), nn.ReLU(inplace=True), nn.Linear(feat_channels, feat_channels), nn.ReLU(inplace=True), nn.Linear(feat_channels, out_channels)) self.test_cfg = test_cfg self.train_cfg = train_cfg if train_cfg: self.assigner = TASK_UTILS.build(train_cfg['assigner']) self.sampler = TASK_UTILS.build( train_cfg['sampler'], default_args=dict(context=self)) self.class_weight = loss_cls.class_weight self.loss_cls = MODELS.build(loss_cls) self.loss_mask = MODELS.build(loss_mask) self.loss_dice = MODELS.build(loss_dice) def init_weights(self) -> None: if isinstance(self.decoder_input_proj, Conv2d): caffe2_xavier_init(self.decoder_input_proj, bias=0) self.pixel_decoder.init_weights() for p in self.transformer_decoder.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) def preprocess_gt( self, batch_gt_instances: InstanceList, batch_gt_semantic_segs: List[Optional[PixelData]]) -> InstanceList: """Preprocess the ground truth for all images. Args: batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``labels``, each is ground truth labels of each bbox, with shape (num_gts, ) and ``masks``, each is ground truth masks of each instances of a image, shape (num_gts, h, w). gt_semantic_seg (list[Optional[PixelData]]): Ground truth of semantic segmentation, each with the shape (1, h, w). [0, num_thing_class - 1] means things, [num_thing_class, num_class-1] means stuff, 255 means VOID. It's None when training instance segmentation. Returns: list[obj:`InstanceData`]: each contains the following keys - labels (Tensor): Ground truth class indices\ for a image, with shape (n, ), n is the sum of\ number of stuff type and number of instance in a image. - masks (Tensor): Ground truth mask for a\ image, with shape (n, h, w). """ num_things_list = [self.num_things_classes] * len(batch_gt_instances) num_stuff_list = [self.num_stuff_classes] * len(batch_gt_instances) gt_labels_list = [ gt_instances['labels'] for gt_instances in batch_gt_instances ] gt_masks_list = [ gt_instances['masks'] for gt_instances in batch_gt_instances ] gt_semantic_segs = [ None if gt_semantic_seg is None else gt_semantic_seg.sem_seg for gt_semantic_seg in batch_gt_semantic_segs ] targets = multi_apply(preprocess_panoptic_gt, gt_labels_list, gt_masks_list, gt_semantic_segs, num_things_list, num_stuff_list) labels, masks = targets batch_gt_instances = [ InstanceData(labels=label, masks=mask) for label, mask in zip(labels, masks) ] return batch_gt_instances def get_targets( self, cls_scores_list: List[Tensor], mask_preds_list: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], return_sampling_results: bool = False ) -> Tuple[List[Union[Tensor, int]]]: """Compute classification and mask targets for all images for a decoder layer. Args: cls_scores_list (list[Tensor]): Mask score logits from a single decoder layer for all images. Each with shape (num_queries, cls_out_channels). mask_preds_list (list[Tensor]): Mask logits from a single decoder layer for all images. Each with shape (num_queries, h, w). batch_gt_instances (list[obj:`InstanceData`]): each contains ``labels`` and ``masks``. batch_img_metas (list[dict]): List of image meta information. return_sampling_results (bool): Whether to return the sampling results. Defaults to False. Returns: tuple: a tuple containing the following targets. - labels_list (list[Tensor]): Labels of all images.\ Each with shape (num_queries, ). - label_weights_list (list[Tensor]): Label weights\ of all images. Each with shape (num_queries, ). - mask_targets_list (list[Tensor]): Mask targets of\ all images. Each with shape (num_queries, h, w). - mask_weights_list (list[Tensor]): Mask weights of\ all images. Each with shape (num_queries, ). - avg_factor (int): Average factor that is used to average\ the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using `MaskPseudoSampler`, `avg_factor` is usually equal to the number of positive priors. additional_returns: This function enables user-defined returns from `self._get_targets_single`. These returns are currently refined to properties at each feature map (i.e. having HxW dimension). The results will be concatenated after the end. """ results = multi_apply(self._get_targets_single, cls_scores_list, mask_preds_list, batch_gt_instances, batch_img_metas) (labels_list, label_weights_list, mask_targets_list, mask_weights_list, pos_inds_list, neg_inds_list, sampling_results_list) = results[:7] rest_results = list(results[7:]) avg_factor = sum( [results.avg_factor for results in sampling_results_list]) res = (labels_list, label_weights_list, mask_targets_list, mask_weights_list, avg_factor) if return_sampling_results: res = res + (sampling_results_list) return res + tuple(rest_results) def _get_targets_single(self, cls_score: Tensor, mask_pred: Tensor, gt_instances: InstanceData, img_meta: dict) -> Tuple[Tensor]: """Compute classification and mask targets for one image. Args: cls_score (Tensor): Mask score logits from a single decoder layer for one image. Shape (num_queries, cls_out_channels). mask_pred (Tensor): Mask logits for a single decoder layer for one image. Shape (num_queries, h, w). gt_instances (:obj:`InstanceData`): It contains ``labels`` and ``masks``. img_meta (dict): Image informtation. Returns: tuple: a tuple containing the following for one image. - labels (Tensor): Labels of each image. shape (num_queries, ). - label_weights (Tensor): Label weights of each image. shape (num_queries, ). - mask_targets (Tensor): Mask targets of each image. shape (num_queries, h, w). - mask_weights (Tensor): Mask weights of each image. shape (num_queries, ). - pos_inds (Tensor): Sampled positive indices for each image. - neg_inds (Tensor): Sampled negative indices for each image. - sampling_result (:obj:`SamplingResult`): Sampling results. """ gt_masks = gt_instances.masks gt_labels = gt_instances.labels target_shape = mask_pred.shape[-2:] if gt_masks.shape[0] > 0: gt_masks_downsampled = F.interpolate( gt_masks.unsqueeze(1).float(), target_shape, mode='nearest').squeeze(1).long() else: gt_masks_downsampled = gt_masks pred_instances = InstanceData(scores=cls_score, masks=mask_pred) downsampled_gt_instances = InstanceData( labels=gt_labels, masks=gt_masks_downsampled) # assign and sample assign_result = self.assigner.assign( pred_instances=pred_instances, gt_instances=downsampled_gt_instances, img_meta=img_meta) sampling_result = self.sampler.sample( assign_result=assign_result, pred_instances=pred_instances, gt_instances=gt_instances) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds # label target labels = gt_labels.new_full((self.num_queries, ), self.num_classes, dtype=torch.long) labels[pos_inds] = gt_labels[sampling_result.pos_assigned_gt_inds] label_weights = gt_labels.new_ones(self.num_queries) # mask target mask_targets = gt_masks[sampling_result.pos_assigned_gt_inds] mask_weights = mask_pred.new_zeros((self.num_queries, )) mask_weights[pos_inds] = 1.0 return (labels, label_weights, mask_targets, mask_weights, pos_inds, neg_inds, sampling_result) def loss_by_feat(self, all_cls_scores: Tensor, all_mask_preds: Tensor, batch_gt_instances: List[InstanceData], batch_img_metas: List[dict]) -> Dict[str, Tensor]: """Loss function. Args: all_cls_scores (Tensor): Classification scores for all decoder layers with shape (num_decoder, batch_size, num_queries, cls_out_channels). Note `cls_out_channels` should includes background. all_mask_preds (Tensor): Mask scores for all decoder layers with shape (num_decoder, batch_size, num_queries, h, w). batch_gt_instances (list[obj:`InstanceData`]): each contains ``labels`` and ``masks``. batch_img_metas (list[dict]): List of image meta information. Returns: dict[str, Tensor]: A dictionary of loss components. """ num_dec_layers = len(all_cls_scores) batch_gt_instances_list = [ batch_gt_instances for _ in range(num_dec_layers) ] img_metas_list = [batch_img_metas for _ in range(num_dec_layers)] losses_cls, losses_mask, losses_dice = multi_apply( self._loss_by_feat_single, all_cls_scores, all_mask_preds, batch_gt_instances_list, img_metas_list) loss_dict = dict() # loss from the last decoder layer loss_dict['loss_cls'] = losses_cls[-1] loss_dict['loss_mask'] = losses_mask[-1] loss_dict['loss_dice'] = losses_dice[-1] # loss from other decoder layers num_dec_layer = 0 for loss_cls_i, loss_mask_i, loss_dice_i in zip( losses_cls[:-1], losses_mask[:-1], losses_dice[:-1]): loss_dict[f'd{num_dec_layer}.loss_cls'] = loss_cls_i loss_dict[f'd{num_dec_layer}.loss_mask'] = loss_mask_i loss_dict[f'd{num_dec_layer}.loss_dice'] = loss_dice_i num_dec_layer += 1 return loss_dict def _loss_by_feat_single(self, cls_scores: Tensor, mask_preds: Tensor, batch_gt_instances: List[InstanceData], batch_img_metas: List[dict]) -> Tuple[Tensor]: """Loss function for outputs from a single decoder layer. Args: cls_scores (Tensor): Mask score logits from a single decoder layer for all images. Shape (batch_size, num_queries, cls_out_channels). Note `cls_out_channels` should includes background. mask_preds (Tensor): Mask logits for a pixel decoder for all images. Shape (batch_size, num_queries, h, w). batch_gt_instances (list[obj:`InstanceData`]): each contains ``labels`` and ``masks``. batch_img_metas (list[dict]): List of image meta information. Returns: tuple[Tensor]: Loss components for outputs from a single decoder\ layer. """ num_imgs = cls_scores.size(0) cls_scores_list = [cls_scores[i] for i in range(num_imgs)] mask_preds_list = [mask_preds[i] for i in range(num_imgs)] (labels_list, label_weights_list, mask_targets_list, mask_weights_list, avg_factor) = self.get_targets(cls_scores_list, mask_preds_list, batch_gt_instances, batch_img_metas) # shape (batch_size, num_queries) labels = torch.stack(labels_list, dim=0) # shape (batch_size, num_queries) label_weights = torch.stack(label_weights_list, dim=0) # shape (num_total_gts, h, w) mask_targets = torch.cat(mask_targets_list, dim=0) # shape (batch_size, num_queries) mask_weights = torch.stack(mask_weights_list, dim=0) # classfication loss # shape (batch_size * num_queries, ) cls_scores = cls_scores.flatten(0, 1) labels = labels.flatten(0, 1) label_weights = label_weights.flatten(0, 1) class_weight = cls_scores.new_tensor(self.class_weight) loss_cls = self.loss_cls( cls_scores, labels, label_weights, avg_factor=class_weight[labels].sum()) num_total_masks = reduce_mean(cls_scores.new_tensor([avg_factor])) num_total_masks = max(num_total_masks, 1) # extract positive ones # shape (batch_size, num_queries, h, w) -> (num_total_gts, h, w) mask_preds = mask_preds[mask_weights > 0] target_shape = mask_targets.shape[-2:] if mask_targets.shape[0] == 0: # zero match loss_dice = mask_preds.sum() loss_mask = mask_preds.sum() return loss_cls, loss_mask, loss_dice # upsample to shape of target # shape (num_total_gts, h, w) mask_preds = F.interpolate( mask_preds.unsqueeze(1), target_shape, mode='bilinear', align_corners=False).squeeze(1) # dice loss loss_dice = self.loss_dice( mask_preds, mask_targets, avg_factor=num_total_masks) # mask loss # FocalLoss support input of shape (n, num_class) h, w = mask_preds.shape[-2:] # shape (num_total_gts, h, w) -> (num_total_gts * h * w, 1) mask_preds = mask_preds.reshape(-1, 1) # shape (num_total_gts, h, w) -> (num_total_gts * h * w) mask_targets = mask_targets.reshape(-1) # target is (1 - mask_targets) !!! loss_mask = self.loss_mask( mask_preds, 1 - mask_targets, avg_factor=num_total_masks * h * w) return loss_cls, loss_mask, loss_dice def forward(self, x: Tuple[Tensor], batch_data_samples: SampleList) -> Tuple[Tensor]: """Forward function. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: tuple[Tensor]: a tuple contains two elements. - all_cls_scores (Tensor): Classification scores for each\ scale level. Each is a 4D-tensor with shape\ (num_decoder, batch_size, num_queries, cls_out_channels).\ Note `cls_out_channels` should includes background. - all_mask_preds (Tensor): Mask scores for each decoder\ layer. Each with shape (num_decoder, batch_size,\ num_queries, h, w). """ batch_img_metas = [ data_sample.metainfo for data_sample in batch_data_samples ] batch_size = len(batch_img_metas) input_img_h, input_img_w = batch_img_metas[0]['batch_input_shape'] padding_mask = x[-1].new_ones((batch_size, input_img_h, input_img_w), dtype=torch.float32) for i in range(batch_size): img_h, img_w = batch_img_metas[i]['img_shape'] padding_mask[i, :img_h, :img_w] = 0 padding_mask = F.interpolate( padding_mask.unsqueeze(1), size=x[-1].shape[-2:], mode='nearest').to(torch.bool).squeeze(1) # when backbone is swin, memory is output of last stage of swin. # when backbone is r50, memory is output of tranformer encoder. mask_features, memory = self.pixel_decoder(x, batch_img_metas) pos_embed = self.decoder_pe(padding_mask) memory = self.decoder_input_proj(memory) # shape (batch_size, c, h, w) -> (batch_size, h*w, c) memory = memory.flatten(2).permute(0, 2, 1) pos_embed = pos_embed.flatten(2).permute(0, 2, 1) # shape (batch_size, h * w) padding_mask = padding_mask.flatten(1) # shape = (num_queries, embed_dims) query_embed = self.query_embed.weight # shape = (batch_size, num_queries, embed_dims) query_embed = query_embed.unsqueeze(0).repeat(batch_size, 1, 1) target = torch.zeros_like(query_embed) # shape (num_decoder, num_queries, batch_size, embed_dims) out_dec = self.transformer_decoder( query=target, key=memory, value=memory, query_pos=query_embed, key_pos=pos_embed, key_padding_mask=padding_mask) # cls_scores all_cls_scores = self.cls_embed(out_dec) # mask_preds mask_embed = self.mask_embed(out_dec) all_mask_preds = torch.einsum('lbqc,bchw->lbqhw', mask_embed, mask_features) return all_cls_scores, all_mask_preds def loss( self, x: Tuple[Tensor], batch_data_samples: SampleList, ) -> Dict[str, Tensor]: """Perform forward propagation and loss calculation of the panoptic head on the features of the upstream network. Args: x (tuple[Tensor]): Multi-level features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: dict[str, Tensor]: a dictionary of loss components """ batch_img_metas = [] batch_gt_instances = [] batch_gt_semantic_segs = [] for data_sample in batch_data_samples: batch_img_metas.append(data_sample.metainfo) batch_gt_instances.append(data_sample.gt_instances) if 'gt_sem_seg' in data_sample: batch_gt_semantic_segs.append(data_sample.gt_sem_seg) else: batch_gt_semantic_segs.append(None) # forward all_cls_scores, all_mask_preds = self(x, batch_data_samples) # preprocess ground truth batch_gt_instances = self.preprocess_gt(batch_gt_instances, batch_gt_semantic_segs) # loss losses = self.loss_by_feat(all_cls_scores, all_mask_preds, batch_gt_instances, batch_img_metas) return losses def predict(self, x: Tuple[Tensor], batch_data_samples: SampleList) -> Tuple[Tensor]: """Test without augmentaton. Args: x (tuple[Tensor]): Multi-level features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: tuple[Tensor]: A tuple contains two tensors. - mask_cls_results (Tensor): Mask classification logits,\ shape (batch_size, num_queries, cls_out_channels). Note `cls_out_channels` should includes background. - mask_pred_results (Tensor): Mask logits, shape \ (batch_size, num_queries, h, w). """ batch_img_metas = [ data_sample.metainfo for data_sample in batch_data_samples ] all_cls_scores, all_mask_preds = self(x, batch_data_samples) mask_cls_results = all_cls_scores[-1] mask_pred_results = all_mask_preds[-1] # upsample masks img_shape = batch_img_metas[0]['batch_input_shape'] mask_pred_results = F.interpolate( mask_pred_results, size=(img_shape[0], img_shape[1]), mode='bilinear', align_corners=False) return mask_cls_results, mask_pred_results ================================================ FILE: mmdet/models/dense_heads/nasfcos_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import torch.nn as nn from mmcv.cnn import ConvModule, Scale from mmdet.models.dense_heads.fcos_head import FCOSHead from mmdet.registry import MODELS from mmdet.utils import OptMultiConfig @MODELS.register_module() class NASFCOSHead(FCOSHead): """Anchor-free head used in `NASFCOS `_. It is quite similar with FCOS head, except for the searched structure of classification branch and bbox regression branch, where a structure of "dconv3x3, conv3x3, dconv3x3, conv1x1" is utilized instead. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. strides (Sequence[int] or Sequence[Tuple[int, int]]): Strides of points in multiple feature levels. Defaults to (4, 8, 16, 32, 64). regress_ranges (Sequence[Tuple[int, int]]): Regress range of multiple level points. center_sampling (bool): If true, use center sampling. Defaults to False. center_sample_radius (float): Radius of center sampling. Defaults to 1.5. norm_on_bbox (bool): If true, normalize the regression targets with FPN strides. Defaults to False. centerness_on_reg (bool): If true, position centerness on the regress branch. Please refer to https://github.com/tianzhi0549/FCOS/issues/89#issuecomment-516877042. Defaults to False. conv_bias (bool or str): If specified as `auto`, it will be decided by the norm_cfg. Bias of conv will be set as True if `norm_cfg` is None, otherwise False. Defaults to "auto". loss_cls (:obj:`ConfigDict` or dict): Config of classification loss. loss_bbox (:obj:`ConfigDict` or dict): Config of localization loss. loss_centerness (:obj:`ConfigDict`, or dict): Config of centerness loss. norm_cfg (:obj:`ConfigDict` or dict): dictionary to construct and config norm layer. Defaults to ``norm_cfg=dict(type='GN', num_groups=32, requires_grad=True)``. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], opitonal): Initialization config dict. """ # noqa: E501 def __init__(self, *args, init_cfg: OptMultiConfig = None, **kwargs) -> None: if init_cfg is None: init_cfg = [ dict(type='Caffe2Xavier', layer=['ConvModule', 'Conv2d']), dict( type='Normal', std=0.01, override=[ dict(name='conv_reg'), dict(name='conv_centerness'), dict( name='conv_cls', type='Normal', std=0.01, bias_prob=0.01) ]), ] super().__init__(*args, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" dconv3x3_config = dict( type='DCNv2', kernel_size=3, use_bias=True, deform_groups=2, padding=1) conv3x3_config = dict(type='Conv', kernel_size=3, padding=1) conv1x1_config = dict(type='Conv', kernel_size=1) self.arch_config = [ dconv3x3_config, conv3x3_config, dconv3x3_config, conv1x1_config ] self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() for i, op_ in enumerate(self.arch_config): op = copy.deepcopy(op_) chn = self.in_channels if i == 0 else self.feat_channels assert isinstance(op, dict) use_bias = op.pop('use_bias', False) padding = op.pop('padding', 0) kernel_size = op.pop('kernel_size') module = ConvModule( chn, self.feat_channels, kernel_size, stride=1, padding=padding, norm_cfg=self.norm_cfg, bias=use_bias, conv_cfg=op) self.cls_convs.append(copy.deepcopy(module)) self.reg_convs.append(copy.deepcopy(module)) self.conv_cls = nn.Conv2d( self.feat_channels, self.cls_out_channels, 3, padding=1) self.conv_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1) self.conv_centerness = nn.Conv2d(self.feat_channels, 1, 3, padding=1) self.scales = nn.ModuleList([Scale(1.0) for _ in self.strides]) ================================================ FILE: mmdet/models/dense_heads/paa_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import numpy as np import torch from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures.bbox import bbox_overlaps from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptInstanceList) from ..layers import multiclass_nms from ..utils import levels_to_images, multi_apply from . import ATSSHead EPS = 1e-12 try: import sklearn.mixture as skm except ImportError: skm = None @MODELS.register_module() class PAAHead(ATSSHead): """Head of PAAAssignment: Probabilistic Anchor Assignment with IoU Prediction for Object Detection. Code is modified from the `official github repo `_. More details can be found in the `paper `_ . Args: topk (int): Select topk samples with smallest loss in each level. score_voting (bool): Whether to use score voting in post-process. covariance_type : String describing the type of covariance parameters to be used in :class:`sklearn.mixture.GaussianMixture`. It must be one of: - 'full': each component has its own general covariance matrix - 'tied': all components share the same general covariance matrix - 'diag': each component has its own diagonal covariance matrix - 'spherical': each component has its own single variance Default: 'diag'. From 'full' to 'spherical', the gmm fitting process is faster yet the performance could be influenced. For most cases, 'diag' should be a good choice. """ def __init__(self, *args, topk: int = 9, score_voting: bool = True, covariance_type: str = 'diag', **kwargs): # topk used in paa reassign process self.topk = topk self.with_score_voting = score_voting self.covariance_type = covariance_type super().__init__(*args, **kwargs) def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], iou_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W) iou_preds (list[Tensor]): iou_preds for each scale level with shape (N, num_anchors * 1, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss gmm_assignment. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, ) (labels, labels_weight, bboxes_target, bboxes_weight, pos_inds, pos_gt_index) = cls_reg_targets cls_scores = levels_to_images(cls_scores) cls_scores = [ item.reshape(-1, self.cls_out_channels) for item in cls_scores ] bbox_preds = levels_to_images(bbox_preds) bbox_preds = [item.reshape(-1, 4) for item in bbox_preds] iou_preds = levels_to_images(iou_preds) iou_preds = [item.reshape(-1, 1) for item in iou_preds] pos_losses_list, = multi_apply(self.get_pos_loss, anchor_list, cls_scores, bbox_preds, labels, labels_weight, bboxes_target, bboxes_weight, pos_inds) with torch.no_grad(): reassign_labels, reassign_label_weight, \ reassign_bbox_weights, num_pos = multi_apply( self.paa_reassign, pos_losses_list, labels, labels_weight, bboxes_weight, pos_inds, pos_gt_index, anchor_list) num_pos = sum(num_pos) # convert all tensor list to a flatten tensor cls_scores = torch.cat(cls_scores, 0).view(-1, cls_scores[0].size(-1)) bbox_preds = torch.cat(bbox_preds, 0).view(-1, bbox_preds[0].size(-1)) iou_preds = torch.cat(iou_preds, 0).view(-1, iou_preds[0].size(-1)) labels = torch.cat(reassign_labels, 0).view(-1) flatten_anchors = torch.cat( [torch.cat(item, 0) for item in anchor_list]) labels_weight = torch.cat(reassign_label_weight, 0).view(-1) bboxes_target = torch.cat(bboxes_target, 0).view(-1, bboxes_target[0].size(-1)) pos_inds_flatten = ((labels >= 0) & (labels < self.num_classes)).nonzero().reshape(-1) losses_cls = self.loss_cls( cls_scores, labels, labels_weight, avg_factor=max(num_pos, len(batch_img_metas))) # avoid num_pos=0 if num_pos: pos_bbox_pred = self.bbox_coder.decode( flatten_anchors[pos_inds_flatten], bbox_preds[pos_inds_flatten]) pos_bbox_target = bboxes_target[pos_inds_flatten] iou_target = bbox_overlaps( pos_bbox_pred.detach(), pos_bbox_target, is_aligned=True) losses_iou = self.loss_centerness( iou_preds[pos_inds_flatten], iou_target.unsqueeze(-1), avg_factor=num_pos) losses_bbox = self.loss_bbox( pos_bbox_pred, pos_bbox_target, iou_target.clamp(min=EPS), avg_factor=iou_target.sum()) else: losses_iou = iou_preds.sum() * 0 losses_bbox = bbox_preds.sum() * 0 return dict( loss_cls=losses_cls, loss_bbox=losses_bbox, loss_iou=losses_iou) def get_pos_loss(self, anchors: List[Tensor], cls_score: Tensor, bbox_pred: Tensor, label: Tensor, label_weight: Tensor, bbox_target: dict, bbox_weight: Tensor, pos_inds: Tensor) -> Tensor: """Calculate loss of all potential positive samples obtained from first match process. Args: anchors (list[Tensor]): Anchors of each scale. cls_score (Tensor): Box scores of single image with shape (num_anchors, num_classes) bbox_pred (Tensor): Box energies / deltas of single image with shape (num_anchors, 4) label (Tensor): classification target of each anchor with shape (num_anchors,) label_weight (Tensor): Classification loss weight of each anchor with shape (num_anchors). bbox_target (dict): Regression target of each anchor with shape (num_anchors, 4). bbox_weight (Tensor): Bbox weight of each anchor with shape (num_anchors, 4). pos_inds (Tensor): Index of all positive samples got from first assign process. Returns: Tensor: Losses of all positive samples in single image. """ if not len(pos_inds): return cls_score.new([]), anchors_all_level = torch.cat(anchors, 0) pos_scores = cls_score[pos_inds] pos_bbox_pred = bbox_pred[pos_inds] pos_label = label[pos_inds] pos_label_weight = label_weight[pos_inds] pos_bbox_target = bbox_target[pos_inds] pos_bbox_weight = bbox_weight[pos_inds] pos_anchors = anchors_all_level[pos_inds] pos_bbox_pred = self.bbox_coder.decode(pos_anchors, pos_bbox_pred) # to keep loss dimension loss_cls = self.loss_cls( pos_scores, pos_label, pos_label_weight, avg_factor=1.0, reduction_override='none') loss_bbox = self.loss_bbox( pos_bbox_pred, pos_bbox_target, pos_bbox_weight, avg_factor=1.0, # keep same loss weight before reassign reduction_override='none') loss_cls = loss_cls.sum(-1) pos_loss = loss_bbox + loss_cls return pos_loss, def paa_reassign(self, pos_losses: Tensor, label: Tensor, label_weight: Tensor, bbox_weight: Tensor, pos_inds: Tensor, pos_gt_inds: Tensor, anchors: List[Tensor]) -> tuple: """Fit loss to GMM distribution and separate positive, ignore, negative samples again with GMM model. Args: pos_losses (Tensor): Losses of all positive samples in single image. label (Tensor): classification target of each anchor with shape (num_anchors,) label_weight (Tensor): Classification loss weight of each anchor with shape (num_anchors). bbox_weight (Tensor): Bbox weight of each anchor with shape (num_anchors, 4). pos_inds (Tensor): Index of all positive samples got from first assign process. pos_gt_inds (Tensor): Gt_index of all positive samples got from first assign process. anchors (list[Tensor]): Anchors of each scale. Returns: tuple: Usually returns a tuple containing learning targets. - label (Tensor): classification target of each anchor after paa assign, with shape (num_anchors,) - label_weight (Tensor): Classification loss weight of each anchor after paa assign, with shape (num_anchors). - bbox_weight (Tensor): Bbox weight of each anchor with shape (num_anchors, 4). - num_pos (int): The number of positive samples after paa assign. """ if not len(pos_inds): return label, label_weight, bbox_weight, 0 label = label.clone() label_weight = label_weight.clone() bbox_weight = bbox_weight.clone() num_gt = pos_gt_inds.max() + 1 num_level = len(anchors) num_anchors_each_level = [item.size(0) for item in anchors] num_anchors_each_level.insert(0, 0) inds_level_interval = np.cumsum(num_anchors_each_level) pos_level_mask = [] for i in range(num_level): mask = (pos_inds >= inds_level_interval[i]) & ( pos_inds < inds_level_interval[i + 1]) pos_level_mask.append(mask) pos_inds_after_paa = [label.new_tensor([])] ignore_inds_after_paa = [label.new_tensor([])] for gt_ind in range(num_gt): pos_inds_gmm = [] pos_loss_gmm = [] gt_mask = pos_gt_inds == gt_ind for level in range(num_level): level_mask = pos_level_mask[level] level_gt_mask = level_mask & gt_mask value, topk_inds = pos_losses[level_gt_mask].topk( min(level_gt_mask.sum(), self.topk), largest=False) pos_inds_gmm.append(pos_inds[level_gt_mask][topk_inds]) pos_loss_gmm.append(value) pos_inds_gmm = torch.cat(pos_inds_gmm) pos_loss_gmm = torch.cat(pos_loss_gmm) # fix gmm need at least two sample if len(pos_inds_gmm) < 2: continue device = pos_inds_gmm.device pos_loss_gmm, sort_inds = pos_loss_gmm.sort() pos_inds_gmm = pos_inds_gmm[sort_inds] pos_loss_gmm = pos_loss_gmm.view(-1, 1).cpu().numpy() min_loss, max_loss = pos_loss_gmm.min(), pos_loss_gmm.max() means_init = np.array([min_loss, max_loss]).reshape(2, 1) weights_init = np.array([0.5, 0.5]) precisions_init = np.array([1.0, 1.0]).reshape(2, 1, 1) # full if self.covariance_type == 'spherical': precisions_init = precisions_init.reshape(2) elif self.covariance_type == 'diag': precisions_init = precisions_init.reshape(2, 1) elif self.covariance_type == 'tied': precisions_init = np.array([[1.0]]) if skm is None: raise ImportError('Please run "pip install sklearn" ' 'to install sklearn first.') gmm = skm.GaussianMixture( 2, weights_init=weights_init, means_init=means_init, precisions_init=precisions_init, covariance_type=self.covariance_type) gmm.fit(pos_loss_gmm) gmm_assignment = gmm.predict(pos_loss_gmm) scores = gmm.score_samples(pos_loss_gmm) gmm_assignment = torch.from_numpy(gmm_assignment).to(device) scores = torch.from_numpy(scores).to(device) pos_inds_temp, ignore_inds_temp = self.gmm_separation_scheme( gmm_assignment, scores, pos_inds_gmm) pos_inds_after_paa.append(pos_inds_temp) ignore_inds_after_paa.append(ignore_inds_temp) pos_inds_after_paa = torch.cat(pos_inds_after_paa) ignore_inds_after_paa = torch.cat(ignore_inds_after_paa) reassign_mask = (pos_inds.unsqueeze(1) != pos_inds_after_paa).all(1) reassign_ids = pos_inds[reassign_mask] label[reassign_ids] = self.num_classes label_weight[ignore_inds_after_paa] = 0 bbox_weight[reassign_ids] = 0 num_pos = len(pos_inds_after_paa) return label, label_weight, bbox_weight, num_pos def gmm_separation_scheme(self, gmm_assignment: Tensor, scores: Tensor, pos_inds_gmm: Tensor) -> Tuple[Tensor, Tensor]: """A general separation scheme for gmm model. It separates a GMM distribution of candidate samples into three parts, 0 1 and uncertain areas, and you can implement other separation schemes by rewriting this function. Args: gmm_assignment (Tensor): The prediction of GMM which is of shape (num_samples,). The 0/1 value indicates the distribution that each sample comes from. scores (Tensor): The probability of sample coming from the fit GMM distribution. The tensor is of shape (num_samples,). pos_inds_gmm (Tensor): All the indexes of samples which are used to fit GMM model. The tensor is of shape (num_samples,) Returns: tuple[Tensor, Tensor]: The indices of positive and ignored samples. - pos_inds_temp (Tensor): Indices of positive samples. - ignore_inds_temp (Tensor): Indices of ignore samples. """ # The implementation is (c) in Fig.3 in origin paper instead of (b). # You can refer to issues such as # https://github.com/kkhoot/PAA/issues/8 and # https://github.com/kkhoot/PAA/issues/9. fgs = gmm_assignment == 0 pos_inds_temp = fgs.new_tensor([], dtype=torch.long) ignore_inds_temp = fgs.new_tensor([], dtype=torch.long) if fgs.nonzero().numel(): _, pos_thr_ind = scores[fgs].topk(1) pos_inds_temp = pos_inds_gmm[fgs][:pos_thr_ind + 1] ignore_inds_temp = pos_inds_gmm.new_tensor([]) return pos_inds_temp, ignore_inds_temp def get_targets(self, anchor_list: List[List[Tensor]], valid_flag_list: List[List[Tensor]], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs: bool = True) -> tuple: """Get targets for PAA head. This method is almost the same as `AnchorHead.get_targets()`. We direct return the results from _get_targets_single instead map it to levels by images_to_levels function. Args: anchor_list (list[list[Tensor]]): Multi level anchors of each image. The outer list indicates images, and the inner list corresponds to feature levels of the image. Each element of the inner list is a tensor of shape (num_anchors, 4). valid_flag_list (list[list[Tensor]]): Multi level valid flags of each image. The outer list indicates images, and the inner list corresponds to feature levels of the image. Each element of the inner list is a tensor of shape (num_anchors, ) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. Returns: tuple: Usually returns a tuple containing learning targets. - labels (list[Tensor]): Labels of all anchors, each with shape (num_anchors,). - label_weights (list[Tensor]): Label weights of all anchor. each with shape (num_anchors,). - bbox_targets (list[Tensor]): BBox targets of all anchors. each with shape (num_anchors, 4). - bbox_weights (list[Tensor]): BBox weights of all anchors. each with shape (num_anchors, 4). - pos_inds (list[Tensor]): Contains all index of positive sample in all anchor. - gt_inds (list[Tensor]): Contains all gt_index of positive sample in all anchor. """ num_imgs = len(batch_img_metas) assert len(anchor_list) == len(valid_flag_list) == num_imgs concat_anchor_list = [] concat_valid_flag_list = [] for i in range(num_imgs): assert len(anchor_list[i]) == len(valid_flag_list[i]) concat_anchor_list.append(torch.cat(anchor_list[i])) concat_valid_flag_list.append(torch.cat(valid_flag_list[i])) # compute targets for each image if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None] * num_imgs results = multi_apply( self._get_targets_single, concat_anchor_list, concat_valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs) (labels, label_weights, bbox_targets, bbox_weights, valid_pos_inds, valid_neg_inds, sampling_result) = results # Due to valid flag of anchors, we have to calculate the real pos_inds # in origin anchor set. pos_inds = [] for i, single_labels in enumerate(labels): pos_mask = (0 <= single_labels) & ( single_labels < self.num_classes) pos_inds.append(pos_mask.nonzero().view(-1)) gt_inds = [item.pos_assigned_gt_inds for item in sampling_result] return (labels, label_weights, bbox_targets, bbox_weights, pos_inds, gt_inds) def _get_targets_single(self, flat_anchors: Tensor, valid_flags: Tensor, gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None, unmap_outputs: bool = True) -> tuple: """Compute regression and classification targets for anchors in a single image. This method is same as `AnchorHead._get_targets_single()`. """ assert unmap_outputs, 'We must map outputs back to the original' \ 'set of anchors in PAAhead' return super(ATSSHead, self)._get_targets_single( flat_anchors, valid_flags, gt_instances, img_meta, gt_instances_ignore, unmap_outputs=True) def predict_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], score_factors: Optional[List[Tensor]] = None, batch_img_metas: Optional[List[dict]] = None, cfg: OptConfigType = None, rescale: bool = False, with_nms: bool = True) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. This method is same as `BaseDenseHead.get_results()`. """ assert with_nms, 'PAA only supports "with_nms=True" now and it ' \ 'means PAAHead does not support ' \ 'test-time augmentation' return super().predict_by_feat( cls_scores=cls_scores, bbox_preds=bbox_preds, score_factors=score_factors, batch_img_metas=batch_img_metas, cfg=cfg, rescale=rescale, with_nms=with_nms) def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], score_factor_list: List[Tensor], mlvl_priors: List[Tensor], img_meta: dict, cfg: OptConfigType = None, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: cls_score_list (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). score_factor_list (list[Tensor]): Score factors from all scale levels of a single image, each item has shape (num_priors * 1, H, W). mlvl_priors (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid, has shape (num_priors, 4). img_meta (dict): Image meta info. cfg (:obj:`ConfigDict` or dict, optional): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Default: False. with_nms (bool): If True, do nms before return boxes. Default: True. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ cfg = self.test_cfg if cfg is None else cfg img_shape = img_meta['img_shape'] nms_pre = cfg.get('nms_pre', -1) mlvl_bboxes = [] mlvl_scores = [] mlvl_score_factors = [] for level_idx, (cls_score, bbox_pred, score_factor, priors) in \ enumerate(zip(cls_score_list, bbox_pred_list, score_factor_list, mlvl_priors)): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] scores = cls_score.permute(1, 2, 0).reshape( -1, self.cls_out_channels).sigmoid() bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) score_factor = score_factor.permute(1, 2, 0).reshape(-1).sigmoid() if 0 < nms_pre < scores.shape[0]: max_scores, _ = (scores * score_factor[:, None]).sqrt().max(dim=1) _, topk_inds = max_scores.topk(nms_pre) priors = priors[topk_inds, :] bbox_pred = bbox_pred[topk_inds, :] scores = scores[topk_inds, :] score_factor = score_factor[topk_inds] bboxes = self.bbox_coder.decode( priors, bbox_pred, max_shape=img_shape) mlvl_bboxes.append(bboxes) mlvl_scores.append(scores) mlvl_score_factors.append(score_factor) results = InstanceData() results.bboxes = torch.cat(mlvl_bboxes) results.scores = torch.cat(mlvl_scores) results.score_factors = torch.cat(mlvl_score_factors) return self._bbox_post_process(results, cfg, rescale, with_nms, img_meta) def _bbox_post_process(self, results: InstanceData, cfg: ConfigType, rescale: bool = False, with_nms: bool = True, img_meta: Optional[dict] = None): """bbox post-processing method. The boxes would be rescaled to the original image scale and do the nms operation. Usually with_nms is False is used for aug test. Args: results (:obj:`InstaceData`): Detection instance results, each item has shape (num_bboxes, ). cfg (:obj:`ConfigDict` or dict): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Default: False. with_nms (bool): If True, do nms before return boxes. Default: True. img_meta (dict, optional): Image meta info. Defaults to None. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ if rescale: results.bboxes /= results.bboxes.new_tensor( img_meta['scale_factor']).repeat((1, 2)) # Add a dummy background class to the backend when using sigmoid # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 # BG cat_id: num_class padding = results.scores.new_zeros(results.scores.shape[0], 1) mlvl_scores = torch.cat([results.scores, padding], dim=1) mlvl_nms_scores = (mlvl_scores * results.score_factors[:, None]).sqrt() det_bboxes, det_labels = multiclass_nms( results.bboxes, mlvl_nms_scores, cfg.score_thr, cfg.nms, cfg.max_per_img, score_factors=None) if self.with_score_voting and len(det_bboxes) > 0: det_bboxes, det_labels = self.score_voting(det_bboxes, det_labels, results.bboxes, mlvl_nms_scores, cfg.score_thr) nms_results = InstanceData() nms_results.bboxes = det_bboxes[:, :-1] nms_results.scores = det_bboxes[:, -1] nms_results.labels = det_labels return nms_results def score_voting(self, det_bboxes: Tensor, det_labels: Tensor, mlvl_bboxes: Tensor, mlvl_nms_scores: Tensor, score_thr: float) -> Tuple[Tensor, Tensor]: """Implementation of score voting method works on each remaining boxes after NMS procedure. Args: det_bboxes (Tensor): Remaining boxes after NMS procedure, with shape (k, 5), each dimension means (x1, y1, x2, y2, score). det_labels (Tensor): The label of remaining boxes, with shape (k, 1),Labels are 0-based. mlvl_bboxes (Tensor): All boxes before the NMS procedure, with shape (num_anchors,4). mlvl_nms_scores (Tensor): The scores of all boxes which is used in the NMS procedure, with shape (num_anchors, num_class) score_thr (float): The score threshold of bboxes. Returns: tuple: Usually returns a tuple containing voting results. - det_bboxes_voted (Tensor): Remaining boxes after score voting procedure, with shape (k, 5), each dimension means (x1, y1, x2, y2, score). - det_labels_voted (Tensor): Label of remaining bboxes after voting, with shape (num_anchors,). """ candidate_mask = mlvl_nms_scores > score_thr candidate_mask_nonzeros = candidate_mask.nonzero(as_tuple=False) candidate_inds = candidate_mask_nonzeros[:, 0] candidate_labels = candidate_mask_nonzeros[:, 1] candidate_bboxes = mlvl_bboxes[candidate_inds] candidate_scores = mlvl_nms_scores[candidate_mask] det_bboxes_voted = [] det_labels_voted = [] for cls in range(self.cls_out_channels): candidate_cls_mask = candidate_labels == cls if not candidate_cls_mask.any(): continue candidate_cls_scores = candidate_scores[candidate_cls_mask] candidate_cls_bboxes = candidate_bboxes[candidate_cls_mask] det_cls_mask = det_labels == cls det_cls_bboxes = det_bboxes[det_cls_mask].view( -1, det_bboxes.size(-1)) det_candidate_ious = bbox_overlaps(det_cls_bboxes[:, :4], candidate_cls_bboxes) for det_ind in range(len(det_cls_bboxes)): single_det_ious = det_candidate_ious[det_ind] pos_ious_mask = single_det_ious > 0.01 pos_ious = single_det_ious[pos_ious_mask] pos_bboxes = candidate_cls_bboxes[pos_ious_mask] pos_scores = candidate_cls_scores[pos_ious_mask] pis = (torch.exp(-(1 - pos_ious)**2 / 0.025) * pos_scores)[:, None] voted_box = torch.sum( pis * pos_bboxes, dim=0) / torch.sum( pis, dim=0) voted_score = det_cls_bboxes[det_ind][-1:][None, :] det_bboxes_voted.append( torch.cat((voted_box[None, :], voted_score), dim=1)) det_labels_voted.append(cls) det_bboxes_voted = torch.cat(det_bboxes_voted, dim=0) det_labels_voted = det_labels.new_tensor(det_labels_voted) return det_bboxes_voted, det_labels_voted ================================================ FILE: mmdet/models/dense_heads/pisa_retinanet_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List import torch from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import InstanceList, OptInstanceList from ..losses import carl_loss, isr_p from ..utils import images_to_levels from .retina_head import RetinaHead @MODELS.register_module() class PISARetinaHead(RetinaHead): """PISA Retinanet Head. The head owns the same structure with Retinanet Head, but differs in two aspects: 1. Importance-based Sample Reweighting Positive (ISR-P) is applied to change the positive loss weights. 2. Classification-aware regression loss is adopted as a third loss. """ def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Compute losses of the head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: Loss dict, comprise classification loss, regression loss and carl loss. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) label_channels = self.cls_out_channels if self.use_sigmoid_cls else 1 cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, return_sampling_results=True) if cls_reg_targets is None: return None (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor, sampling_results_list) = cls_reg_targets # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] # concat all level anchors and flags to a single tensor concat_anchor_list = [] for i in range(len(anchor_list)): concat_anchor_list.append(torch.cat(anchor_list[i])) all_anchor_list = images_to_levels(concat_anchor_list, num_level_anchors) num_imgs = len(batch_img_metas) flatten_cls_scores = [ cls_score.permute(0, 2, 3, 1).reshape(num_imgs, -1, label_channels) for cls_score in cls_scores ] flatten_cls_scores = torch.cat( flatten_cls_scores, dim=1).reshape(-1, flatten_cls_scores[0].size(-1)) flatten_bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, 4) for bbox_pred in bbox_preds ] flatten_bbox_preds = torch.cat( flatten_bbox_preds, dim=1).view(-1, flatten_bbox_preds[0].size(-1)) flatten_labels = torch.cat(labels_list, dim=1).reshape(-1) flatten_label_weights = torch.cat( label_weights_list, dim=1).reshape(-1) flatten_anchors = torch.cat(all_anchor_list, dim=1).reshape(-1, 4) flatten_bbox_targets = torch.cat( bbox_targets_list, dim=1).reshape(-1, 4) flatten_bbox_weights = torch.cat( bbox_weights_list, dim=1).reshape(-1, 4) # Apply ISR-P isr_cfg = self.train_cfg.get('isr', None) if isr_cfg is not None: all_targets = (flatten_labels, flatten_label_weights, flatten_bbox_targets, flatten_bbox_weights) with torch.no_grad(): all_targets = isr_p( flatten_cls_scores, flatten_bbox_preds, all_targets, flatten_anchors, sampling_results_list, bbox_coder=self.bbox_coder, loss_cls=self.loss_cls, num_class=self.num_classes, **self.train_cfg['isr']) (flatten_labels, flatten_label_weights, flatten_bbox_targets, flatten_bbox_weights) = all_targets # For convenience we compute loss once instead separating by fpn level, # so that we don't need to separate the weights by level again. # The result should be the same losses_cls = self.loss_cls( flatten_cls_scores, flatten_labels, flatten_label_weights, avg_factor=avg_factor) losses_bbox = self.loss_bbox( flatten_bbox_preds, flatten_bbox_targets, flatten_bbox_weights, avg_factor=avg_factor) loss_dict = dict(loss_cls=losses_cls, loss_bbox=losses_bbox) # CARL Loss carl_cfg = self.train_cfg.get('carl', None) if carl_cfg is not None: loss_carl = carl_loss( flatten_cls_scores, flatten_labels, flatten_bbox_preds, flatten_bbox_targets, self.loss_bbox, **self.train_cfg['carl'], avg_factor=avg_factor, sigmoid=True, num_class=self.num_classes) loss_dict.update(loss_carl) return loss_dict ================================================ FILE: mmdet/models/dense_heads/pisa_ssd_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Union import torch from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import InstanceList, OptInstanceList from ..losses import CrossEntropyLoss, SmoothL1Loss, carl_loss, isr_p from ..utils import multi_apply from .ssd_head import SSDHead # TODO: add loss evaluator for SSD @MODELS.register_module() class PISASSDHead(SSDHead): """Implementation of `PISA SSD head `_ Args: num_classes (int): Number of categories excluding the background category. in_channels (Sequence[int]): Number of channels in the input feature map. stacked_convs (int): Number of conv layers in cls and reg tower. Defaults to 0. feat_channels (int): Number of hidden channels when stacked_convs > 0. Defaults to 256. use_depthwise (bool): Whether to use DepthwiseSeparableConv. Defaults to False. conv_cfg (:obj:`ConfigDict` or dict, Optional): Dictionary to construct and config conv layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict, Optional): Dictionary to construct and config norm layer. Defaults to None. act_cfg (:obj:`ConfigDict` or dict, Optional): Dictionary to construct and config activation layer. Defaults to None. anchor_generator (:obj:`ConfigDict` or dict): Config dict for anchor generator. bbox_coder (:obj:`ConfigDict` or dict): Config of bounding box coder. reg_decoded_bbox (bool): If true, the regression loss would be applied directly on decoded bounding boxes, converting both the predicted boxes and regression targets to absolute coordinates format. Defaults to False. It should be `True` when using `IoULoss`, `GIoULoss`, or `DIoULoss` in the bbox head. train_cfg (:obj:`ConfigDict` or dict, Optional): Training config of anchor head. test_cfg (:obj:`ConfigDict` or dict, Optional): Testing config of anchor head. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], Optional): Initialization config dict. """ # noqa: W605 def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Union[List[Tensor], Tensor]]: """Compute losses of the head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Union[List[Tensor], Tensor]]: A dictionary of loss components. the dict has components below: - loss_cls (list[Tensor]): A list containing each feature map \ classification loss. - loss_bbox (list[Tensor]): A list containing each feature map \ regression loss. - loss_carl (Tensor): The loss of CARL. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, unmap_outputs=False, return_sampling_results=True) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor, sampling_results_list) = cls_reg_targets num_images = len(batch_img_metas) all_cls_scores = torch.cat([ s.permute(0, 2, 3, 1).reshape( num_images, -1, self.cls_out_channels) for s in cls_scores ], 1) all_labels = torch.cat(labels_list, -1).view(num_images, -1) all_label_weights = torch.cat(label_weights_list, -1).view(num_images, -1) all_bbox_preds = torch.cat([ b.permute(0, 2, 3, 1).reshape(num_images, -1, 4) for b in bbox_preds ], -2) all_bbox_targets = torch.cat(bbox_targets_list, -2).view(num_images, -1, 4) all_bbox_weights = torch.cat(bbox_weights_list, -2).view(num_images, -1, 4) # concat all level anchors to a single tensor all_anchors = [] for i in range(num_images): all_anchors.append(torch.cat(anchor_list[i])) isr_cfg = self.train_cfg.get('isr', None) all_targets = (all_labels.view(-1), all_label_weights.view(-1), all_bbox_targets.view(-1, 4), all_bbox_weights.view(-1, 4)) # apply ISR-P if isr_cfg is not None: all_targets = isr_p( all_cls_scores.view(-1, all_cls_scores.size(-1)), all_bbox_preds.view(-1, 4), all_targets, torch.cat(all_anchors), sampling_results_list, loss_cls=CrossEntropyLoss(), bbox_coder=self.bbox_coder, **self.train_cfg['isr'], num_class=self.num_classes) (new_labels, new_label_weights, new_bbox_targets, new_bbox_weights) = all_targets all_labels = new_labels.view(all_labels.shape) all_label_weights = new_label_weights.view(all_label_weights.shape) all_bbox_targets = new_bbox_targets.view(all_bbox_targets.shape) all_bbox_weights = new_bbox_weights.view(all_bbox_weights.shape) # add CARL loss carl_loss_cfg = self.train_cfg.get('carl', None) if carl_loss_cfg is not None: loss_carl = carl_loss( all_cls_scores.view(-1, all_cls_scores.size(-1)), all_targets[0], all_bbox_preds.view(-1, 4), all_targets[2], SmoothL1Loss(beta=1.), **self.train_cfg['carl'], avg_factor=avg_factor, num_class=self.num_classes) # check NaN and Inf assert torch.isfinite(all_cls_scores).all().item(), \ 'classification scores become infinite or NaN!' assert torch.isfinite(all_bbox_preds).all().item(), \ 'bbox predications become infinite or NaN!' losses_cls, losses_bbox = multi_apply( self.loss_by_feat_single, all_cls_scores, all_bbox_preds, all_anchors, all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, avg_factor=avg_factor) loss_dict = dict(loss_cls=losses_cls, loss_bbox=losses_bbox) if carl_loss_cfg is not None: loss_dict.update(loss_carl) return loss_dict ================================================ FILE: mmdet/models/dense_heads/reppoints_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Sequence, Tuple import numpy as np import torch import torch.nn as nn from mmcv.cnn import ConvModule from mmcv.ops import DeformConv2d from mmengine.config import ConfigDict from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.utils import ConfigType, InstanceList, MultiConfig, OptInstanceList from ..task_modules.prior_generators import MlvlPointGenerator from ..task_modules.samplers import PseudoSampler from ..utils import (filter_scores_and_topk, images_to_levels, multi_apply, unmap) from .anchor_free_head import AnchorFreeHead @MODELS.register_module() class RepPointsHead(AnchorFreeHead): """RepPoint head. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. point_feat_channels (int): Number of channels of points features. num_points (int): Number of points. gradient_mul (float): The multiplier to gradients from points refinement and recognition. point_strides (Sequence[int]): points strides. point_base_scale (int): bbox scale for assigning labels. loss_cls (:obj:`ConfigDict` or dict): Config of classification loss. loss_bbox_init (:obj:`ConfigDict` or dict): Config of initial points loss. loss_bbox_refine (:obj:`ConfigDict` or dict): Config of points loss in refinement. use_grid_points (bool): If we use bounding box representation, the reppoints is represented as grid points on the bounding box. center_init (bool): Whether to use center point assignment. transform_method (str): The methods to transform RepPoints to bbox. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict]): Initialization config dict. """ # noqa: W605 def __init__(self, num_classes: int, in_channels: int, point_feat_channels: int = 256, num_points: int = 9, gradient_mul: float = 0.1, point_strides: Sequence[int] = [8, 16, 32, 64, 128], point_base_scale: int = 4, loss_cls: ConfigType = dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox_init: ConfigType = dict( type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=0.5), loss_bbox_refine: ConfigType = dict( type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0), use_grid_points: bool = False, center_init: bool = True, transform_method: str = 'moment', moment_mul: float = 0.01, init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='reppoints_cls_out', std=0.01, bias_prob=0.01)), **kwargs) -> None: self.num_points = num_points self.point_feat_channels = point_feat_channels self.use_grid_points = use_grid_points self.center_init = center_init # we use deform conv to extract points features self.dcn_kernel = int(np.sqrt(num_points)) self.dcn_pad = int((self.dcn_kernel - 1) / 2) assert self.dcn_kernel * self.dcn_kernel == num_points, \ 'The points number should be a square number.' assert self.dcn_kernel % 2 == 1, \ 'The points number should be an odd square number.' dcn_base = np.arange(-self.dcn_pad, self.dcn_pad + 1).astype(np.float64) dcn_base_y = np.repeat(dcn_base, self.dcn_kernel) dcn_base_x = np.tile(dcn_base, self.dcn_kernel) dcn_base_offset = np.stack([dcn_base_y, dcn_base_x], axis=1).reshape( (-1)) self.dcn_base_offset = torch.tensor(dcn_base_offset).view(1, -1, 1, 1) super().__init__( num_classes=num_classes, in_channels=in_channels, loss_cls=loss_cls, init_cfg=init_cfg, **kwargs) self.gradient_mul = gradient_mul self.point_base_scale = point_base_scale self.point_strides = point_strides self.prior_generator = MlvlPointGenerator( self.point_strides, offset=0.) if self.train_cfg: self.init_assigner = TASK_UTILS.build( self.train_cfg['init']['assigner']) self.refine_assigner = TASK_UTILS.build( self.train_cfg['refine']['assigner']) if self.train_cfg.get('sampler', None) is not None: self.sampler = TASK_UTILS.build( self.train_cfg['sampler'], default_args=dict(context=self)) else: self.sampler = PseudoSampler(context=self) self.transform_method = transform_method if self.transform_method == 'moment': self.moment_transfer = nn.Parameter( data=torch.zeros(2), requires_grad=True) self.moment_mul = moment_mul self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) if self.use_sigmoid_cls: self.cls_out_channels = self.num_classes else: self.cls_out_channels = self.num_classes + 1 self.loss_bbox_init = MODELS.build(loss_bbox_init) self.loss_bbox_refine = MODELS.build(loss_bbox_refine) def _init_layers(self) -> None: """Initialize layers of the head.""" self.relu = nn.ReLU(inplace=True) self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.reg_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) pts_out_dim = 4 if self.use_grid_points else 2 * self.num_points self.reppoints_cls_conv = DeformConv2d(self.feat_channels, self.point_feat_channels, self.dcn_kernel, 1, self.dcn_pad) self.reppoints_cls_out = nn.Conv2d(self.point_feat_channels, self.cls_out_channels, 1, 1, 0) self.reppoints_pts_init_conv = nn.Conv2d(self.feat_channels, self.point_feat_channels, 3, 1, 1) self.reppoints_pts_init_out = nn.Conv2d(self.point_feat_channels, pts_out_dim, 1, 1, 0) self.reppoints_pts_refine_conv = DeformConv2d(self.feat_channels, self.point_feat_channels, self.dcn_kernel, 1, self.dcn_pad) self.reppoints_pts_refine_out = nn.Conv2d(self.point_feat_channels, pts_out_dim, 1, 1, 0) def points2bbox(self, pts: Tensor, y_first: bool = True) -> Tensor: """Converting the points set into bounding box. Args: pts (Tensor): the input points sets (fields), each points set (fields) is represented as 2n scalar. y_first (bool): if y_first=True, the point set is represented as [y1, x1, y2, x2 ... yn, xn], otherwise the point set is represented as [x1, y1, x2, y2 ... xn, yn]. Defaults to True. Returns: Tensor: each points set is converting to a bbox [x1, y1, x2, y2]. """ pts_reshape = pts.view(pts.shape[0], -1, 2, *pts.shape[2:]) pts_y = pts_reshape[:, :, 0, ...] if y_first else pts_reshape[:, :, 1, ...] pts_x = pts_reshape[:, :, 1, ...] if y_first else pts_reshape[:, :, 0, ...] if self.transform_method == 'minmax': bbox_left = pts_x.min(dim=1, keepdim=True)[0] bbox_right = pts_x.max(dim=1, keepdim=True)[0] bbox_up = pts_y.min(dim=1, keepdim=True)[0] bbox_bottom = pts_y.max(dim=1, keepdim=True)[0] bbox = torch.cat([bbox_left, bbox_up, bbox_right, bbox_bottom], dim=1) elif self.transform_method == 'partial_minmax': pts_y = pts_y[:, :4, ...] pts_x = pts_x[:, :4, ...] bbox_left = pts_x.min(dim=1, keepdim=True)[0] bbox_right = pts_x.max(dim=1, keepdim=True)[0] bbox_up = pts_y.min(dim=1, keepdim=True)[0] bbox_bottom = pts_y.max(dim=1, keepdim=True)[0] bbox = torch.cat([bbox_left, bbox_up, bbox_right, bbox_bottom], dim=1) elif self.transform_method == 'moment': pts_y_mean = pts_y.mean(dim=1, keepdim=True) pts_x_mean = pts_x.mean(dim=1, keepdim=True) pts_y_std = torch.std(pts_y - pts_y_mean, dim=1, keepdim=True) pts_x_std = torch.std(pts_x - pts_x_mean, dim=1, keepdim=True) moment_transfer = (self.moment_transfer * self.moment_mul) + ( self.moment_transfer.detach() * (1 - self.moment_mul)) moment_width_transfer = moment_transfer[0] moment_height_transfer = moment_transfer[1] half_width = pts_x_std * torch.exp(moment_width_transfer) half_height = pts_y_std * torch.exp(moment_height_transfer) bbox = torch.cat([ pts_x_mean - half_width, pts_y_mean - half_height, pts_x_mean + half_width, pts_y_mean + half_height ], dim=1) else: raise NotImplementedError return bbox def gen_grid_from_reg(self, reg: Tensor, previous_boxes: Tensor) -> Tuple[Tensor]: """Base on the previous bboxes and regression values, we compute the regressed bboxes and generate the grids on the bboxes. Args: reg (Tensor): the regression value to previous bboxes. previous_boxes (Tensor): previous bboxes. Returns: Tuple[Tensor]: generate grids on the regressed bboxes. """ b, _, h, w = reg.shape bxy = (previous_boxes[:, :2, ...] + previous_boxes[:, 2:, ...]) / 2. bwh = (previous_boxes[:, 2:, ...] - previous_boxes[:, :2, ...]).clamp(min=1e-6) grid_topleft = bxy + bwh * reg[:, :2, ...] - 0.5 * bwh * torch.exp( reg[:, 2:, ...]) grid_wh = bwh * torch.exp(reg[:, 2:, ...]) grid_left = grid_topleft[:, [0], ...] grid_top = grid_topleft[:, [1], ...] grid_width = grid_wh[:, [0], ...] grid_height = grid_wh[:, [1], ...] intervel = torch.linspace(0., 1., self.dcn_kernel).view( 1, self.dcn_kernel, 1, 1).type_as(reg) grid_x = grid_left + grid_width * intervel grid_x = grid_x.unsqueeze(1).repeat(1, self.dcn_kernel, 1, 1, 1) grid_x = grid_x.view(b, -1, h, w) grid_y = grid_top + grid_height * intervel grid_y = grid_y.unsqueeze(2).repeat(1, 1, self.dcn_kernel, 1, 1) grid_y = grid_y.view(b, -1, h, w) grid_yx = torch.stack([grid_y, grid_x], dim=2) grid_yx = grid_yx.view(b, -1, h, w) regressed_bbox = torch.cat([ grid_left, grid_top, grid_left + grid_width, grid_top + grid_height ], 1) return grid_yx, regressed_bbox def forward(self, feats: Tuple[Tensor]) -> Tuple[Tensor]: return multi_apply(self.forward_single, feats) def forward_single(self, x: Tensor) -> Tuple[Tensor]: """Forward feature map of a single FPN level.""" dcn_base_offset = self.dcn_base_offset.type_as(x) # If we use center_init, the initial reppoints is from center points. # If we use bounding bbox representation, the initial reppoints is # from regular grid placed on a pre-defined bbox. if self.use_grid_points or not self.center_init: scale = self.point_base_scale / 2 points_init = dcn_base_offset / dcn_base_offset.max() * scale bbox_init = x.new_tensor([-scale, -scale, scale, scale]).view(1, 4, 1, 1) else: points_init = 0 cls_feat = x pts_feat = x for cls_conv in self.cls_convs: cls_feat = cls_conv(cls_feat) for reg_conv in self.reg_convs: pts_feat = reg_conv(pts_feat) # initialize reppoints pts_out_init = self.reppoints_pts_init_out( self.relu(self.reppoints_pts_init_conv(pts_feat))) if self.use_grid_points: pts_out_init, bbox_out_init = self.gen_grid_from_reg( pts_out_init, bbox_init.detach()) else: pts_out_init = pts_out_init + points_init # refine and classify reppoints pts_out_init_grad_mul = (1 - self.gradient_mul) * pts_out_init.detach( ) + self.gradient_mul * pts_out_init dcn_offset = pts_out_init_grad_mul - dcn_base_offset cls_out = self.reppoints_cls_out( self.relu(self.reppoints_cls_conv(cls_feat, dcn_offset))) pts_out_refine = self.reppoints_pts_refine_out( self.relu(self.reppoints_pts_refine_conv(pts_feat, dcn_offset))) if self.use_grid_points: pts_out_refine, bbox_out_refine = self.gen_grid_from_reg( pts_out_refine, bbox_out_init.detach()) else: pts_out_refine = pts_out_refine + pts_out_init.detach() if self.training: return cls_out, pts_out_init, pts_out_refine else: return cls_out, self.points2bbox(pts_out_refine) def get_points(self, featmap_sizes: List[Tuple[int]], batch_img_metas: List[dict], device: str) -> tuple: """Get points according to feature map sizes. Args: featmap_sizes (list[tuple]): Multi-level feature map sizes. batch_img_metas (list[dict]): Image meta info. Returns: tuple: points of each image, valid flags of each image """ num_imgs = len(batch_img_metas) # since feature map sizes of all images are the same, we only compute # points center for one time multi_level_points = self.prior_generator.grid_priors( featmap_sizes, device=device, with_stride=True) points_list = [[point.clone() for point in multi_level_points] for _ in range(num_imgs)] # for each image, we compute valid flags of multi level grids valid_flag_list = [] for img_id, img_meta in enumerate(batch_img_metas): multi_level_flags = self.prior_generator.valid_flags( featmap_sizes, img_meta['pad_shape'], device=device) valid_flag_list.append(multi_level_flags) return points_list, valid_flag_list def centers_to_bboxes(self, point_list: List[Tensor]) -> List[Tensor]: """Get bboxes according to center points. Only used in :class:`MaxIoUAssigner`. """ bbox_list = [] for i_img, point in enumerate(point_list): bbox = [] for i_lvl in range(len(self.point_strides)): scale = self.point_base_scale * self.point_strides[i_lvl] * 0.5 bbox_shift = torch.Tensor([-scale, -scale, scale, scale]).view(1, 4).type_as(point[0]) bbox_center = torch.cat( [point[i_lvl][:, :2], point[i_lvl][:, :2]], dim=1) bbox.append(bbox_center + bbox_shift) bbox_list.append(bbox) return bbox_list def offset_to_pts(self, center_list: List[Tensor], pred_list: List[Tensor]) -> List[Tensor]: """Change from point offset to point coordinate.""" pts_list = [] for i_lvl in range(len(self.point_strides)): pts_lvl = [] for i_img in range(len(center_list)): pts_center = center_list[i_img][i_lvl][:, :2].repeat( 1, self.num_points) pts_shift = pred_list[i_lvl][i_img] yx_pts_shift = pts_shift.permute(1, 2, 0).view( -1, 2 * self.num_points) y_pts_shift = yx_pts_shift[..., 0::2] x_pts_shift = yx_pts_shift[..., 1::2] xy_pts_shift = torch.stack([x_pts_shift, y_pts_shift], -1) xy_pts_shift = xy_pts_shift.view(*yx_pts_shift.shape[:-1], -1) pts = xy_pts_shift * self.point_strides[i_lvl] + pts_center pts_lvl.append(pts) pts_lvl = torch.stack(pts_lvl, 0) pts_list.append(pts_lvl) return pts_list def _get_targets_single(self, flat_proposals: Tensor, valid_flags: Tensor, gt_instances: InstanceData, gt_instances_ignore: InstanceData, stage: str = 'init', unmap_outputs: bool = True) -> tuple: """Compute corresponding GT box and classification targets for proposals. Args: flat_proposals (Tensor): Multi level points of a image. valid_flags (Tensor): Multi level valid flags of a image. gt_instances (InstanceData): It usually includes ``bboxes`` and ``labels`` attributes. gt_instances_ignore (InstanceData): It includes ``bboxes`` attribute data that is ignored during training and testing. stage (str): 'init' or 'refine'. Generate target for init stage or refine stage. Defaults to 'init'. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. Returns: tuple: - labels (Tensor): Labels of each level. - label_weights (Tensor): Label weights of each level. - bbox_targets (Tensor): BBox targets of each level. - bbox_weights (Tensor): BBox weights of each level. - pos_inds (Tensor): positive samples indexes. - neg_inds (Tensor): negative samples indexes. - sampling_result (:obj:`SamplingResult`): Sampling results. """ inside_flags = valid_flags if not inside_flags.any(): raise ValueError( 'There is no valid proposal inside the image boundary. Please ' 'check the image size.') # assign gt and sample proposals proposals = flat_proposals[inside_flags, :] pred_instances = InstanceData(priors=proposals) if stage == 'init': assigner = self.init_assigner pos_weight = self.train_cfg['init']['pos_weight'] else: assigner = self.refine_assigner pos_weight = self.train_cfg['refine']['pos_weight'] assign_result = assigner.assign(pred_instances, gt_instances, gt_instances_ignore) sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) num_valid_proposals = proposals.shape[0] bbox_gt = proposals.new_zeros([num_valid_proposals, 4]) pos_proposals = torch.zeros_like(proposals) proposals_weights = proposals.new_zeros([num_valid_proposals, 4]) labels = proposals.new_full((num_valid_proposals, ), self.num_classes, dtype=torch.long) label_weights = proposals.new_zeros( num_valid_proposals, dtype=torch.float) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: bbox_gt[pos_inds, :] = sampling_result.pos_gt_bboxes pos_proposals[pos_inds, :] = proposals[pos_inds, :] proposals_weights[pos_inds, :] = 1.0 labels[pos_inds] = sampling_result.pos_gt_labels if pos_weight <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = pos_weight if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 # map up to original set of proposals if unmap_outputs: num_total_proposals = flat_proposals.size(0) labels = unmap( labels, num_total_proposals, inside_flags, fill=self.num_classes) # fill bg label label_weights = unmap(label_weights, num_total_proposals, inside_flags) bbox_gt = unmap(bbox_gt, num_total_proposals, inside_flags) pos_proposals = unmap(pos_proposals, num_total_proposals, inside_flags) proposals_weights = unmap(proposals_weights, num_total_proposals, inside_flags) return (labels, label_weights, bbox_gt, pos_proposals, proposals_weights, pos_inds, neg_inds, sampling_result) def get_targets(self, proposals_list: List[Tensor], valid_flag_list: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, stage: str = 'init', unmap_outputs: bool = True, return_sampling_results: bool = False) -> tuple: """Compute corresponding GT box and classification targets for proposals. Args: proposals_list (list[Tensor]): Multi level points/bboxes of each image. valid_flag_list (list[Tensor]): Multi level valid flags of each image. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. stage (str): 'init' or 'refine'. Generate target for init stage or refine stage. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. return_sampling_results (bool): Whether to return the sampling results. Defaults to False. Returns: tuple: - labels_list (list[Tensor]): Labels of each level. - label_weights_list (list[Tensor]): Label weights of each level. - bbox_gt_list (list[Tensor]): Ground truth bbox of each level. - proposals_list (list[Tensor]): Proposals(points/bboxes) of each level. - proposal_weights_list (list[Tensor]): Proposal weights of each level. - avg_factor (int): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using `PseudoSampler`, `avg_factor` is usually equal to the number of positive priors. """ assert stage in ['init', 'refine'] num_imgs = len(batch_img_metas) assert len(proposals_list) == len(valid_flag_list) == num_imgs # points number of multi levels num_level_proposals = [points.size(0) for points in proposals_list[0]] # concat all level points and flags to a single tensor for i in range(num_imgs): assert len(proposals_list[i]) == len(valid_flag_list[i]) proposals_list[i] = torch.cat(proposals_list[i]) valid_flag_list[i] = torch.cat(valid_flag_list[i]) if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None] * num_imgs (all_labels, all_label_weights, all_bbox_gt, all_proposals, all_proposal_weights, pos_inds_list, neg_inds_list, sampling_results_list) = multi_apply( self._get_targets_single, proposals_list, valid_flag_list, batch_gt_instances, batch_gt_instances_ignore, stage=stage, unmap_outputs=unmap_outputs) # sampled points of all images avg_refactor = sum( [results.avg_factor for results in sampling_results_list]) labels_list = images_to_levels(all_labels, num_level_proposals) label_weights_list = images_to_levels(all_label_weights, num_level_proposals) bbox_gt_list = images_to_levels(all_bbox_gt, num_level_proposals) proposals_list = images_to_levels(all_proposals, num_level_proposals) proposal_weights_list = images_to_levels(all_proposal_weights, num_level_proposals) res = (labels_list, label_weights_list, bbox_gt_list, proposals_list, proposal_weights_list, avg_refactor) if return_sampling_results: res = res + (sampling_results_list, ) return res def loss_by_feat_single(self, cls_score: Tensor, pts_pred_init: Tensor, pts_pred_refine: Tensor, labels: Tensor, label_weights, bbox_gt_init: Tensor, bbox_weights_init: Tensor, bbox_gt_refine: Tensor, bbox_weights_refine: Tensor, stride: int, avg_factor_init: int, avg_factor_refine: int) -> Tuple[Tensor]: """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: cls_score (Tensor): Box scores for each scale level Has shape (N, num_classes, h_i, w_i). pts_pred_init (Tensor): Points of shape (batch_size, h_i * w_i, num_points * 2). pts_pred_refine (Tensor): Points refined of shape (batch_size, h_i * w_i, num_points * 2). labels (Tensor): Ground truth class indices with shape (batch_size, h_i * w_i). label_weights (Tensor): Label weights of shape (batch_size, h_i * w_i). bbox_gt_init (Tensor): BBox regression targets in the init stage of shape (batch_size, h_i * w_i, 4). bbox_weights_init (Tensor): BBox regression loss weights in the init stage of shape (batch_size, h_i * w_i, 4). bbox_gt_refine (Tensor): BBox regression targets in the refine stage of shape (batch_size, h_i * w_i, 4). bbox_weights_refine (Tensor): BBox regression loss weights in the refine stage of shape (batch_size, h_i * w_i, 4). stride (int): Point stride. avg_factor_init (int): Average factor that is used to average the loss in the init stage. avg_factor_refine (int): Average factor that is used to average the loss in the refine stage. Returns: Tuple[Tensor]: loss components. """ # classification loss labels = labels.reshape(-1) label_weights = label_weights.reshape(-1) cls_score = cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) cls_score = cls_score.contiguous() loss_cls = self.loss_cls( cls_score, labels, label_weights, avg_factor=avg_factor_refine) # points loss bbox_gt_init = bbox_gt_init.reshape(-1, 4) bbox_weights_init = bbox_weights_init.reshape(-1, 4) bbox_pred_init = self.points2bbox( pts_pred_init.reshape(-1, 2 * self.num_points), y_first=False) bbox_gt_refine = bbox_gt_refine.reshape(-1, 4) bbox_weights_refine = bbox_weights_refine.reshape(-1, 4) bbox_pred_refine = self.points2bbox( pts_pred_refine.reshape(-1, 2 * self.num_points), y_first=False) normalize_term = self.point_base_scale * stride loss_pts_init = self.loss_bbox_init( bbox_pred_init / normalize_term, bbox_gt_init / normalize_term, bbox_weights_init, avg_factor=avg_factor_init) loss_pts_refine = self.loss_bbox_refine( bbox_pred_refine / normalize_term, bbox_gt_refine / normalize_term, bbox_weights_refine, avg_factor=avg_factor_refine) return loss_cls, loss_pts_init, loss_pts_refine def loss_by_feat( self, cls_scores: List[Tensor], pts_preds_init: List[Tensor], pts_preds_refine: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, Tensor]: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level, each is a 4D-tensor, of shape (batch_size, num_classes, h, w). pts_preds_init (list[Tensor]): Points for each scale level, each is a 3D-tensor, of shape (batch_size, h_i * w_i, num_points * 2). pts_preds_refine (list[Tensor]): Points refined for each scale level, each is a 3D-tensor, of shape (batch_size, h_i * w_i, num_points * 2). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] device = cls_scores[0].device # target for initial stage center_list, valid_flag_list = self.get_points(featmap_sizes, batch_img_metas, device) pts_coordinate_preds_init = self.offset_to_pts(center_list, pts_preds_init) if self.train_cfg['init']['assigner']['type'] == 'PointAssigner': # Assign target for center list candidate_list = center_list else: # transform center list to bbox list and # assign target for bbox list bbox_list = self.centers_to_bboxes(center_list) candidate_list = bbox_list cls_reg_targets_init = self.get_targets( proposals_list=candidate_list, valid_flag_list=valid_flag_list, batch_gt_instances=batch_gt_instances, batch_img_metas=batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, stage='init', return_sampling_results=False) (*_, bbox_gt_list_init, candidate_list_init, bbox_weights_list_init, avg_factor_init) = cls_reg_targets_init # target for refinement stage center_list, valid_flag_list = self.get_points(featmap_sizes, batch_img_metas, device) pts_coordinate_preds_refine = self.offset_to_pts( center_list, pts_preds_refine) bbox_list = [] for i_img, center in enumerate(center_list): bbox = [] for i_lvl in range(len(pts_preds_refine)): bbox_preds_init = self.points2bbox( pts_preds_init[i_lvl].detach()) bbox_shift = bbox_preds_init * self.point_strides[i_lvl] bbox_center = torch.cat( [center[i_lvl][:, :2], center[i_lvl][:, :2]], dim=1) bbox.append(bbox_center + bbox_shift[i_img].permute(1, 2, 0).reshape(-1, 4)) bbox_list.append(bbox) cls_reg_targets_refine = self.get_targets( proposals_list=bbox_list, valid_flag_list=valid_flag_list, batch_gt_instances=batch_gt_instances, batch_img_metas=batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, stage='refine', return_sampling_results=False) (labels_list, label_weights_list, bbox_gt_list_refine, candidate_list_refine, bbox_weights_list_refine, avg_factor_refine) = cls_reg_targets_refine # compute loss losses_cls, losses_pts_init, losses_pts_refine = multi_apply( self.loss_by_feat_single, cls_scores, pts_coordinate_preds_init, pts_coordinate_preds_refine, labels_list, label_weights_list, bbox_gt_list_init, bbox_weights_list_init, bbox_gt_list_refine, bbox_weights_list_refine, self.point_strides, avg_factor_init=avg_factor_init, avg_factor_refine=avg_factor_refine) loss_dict_all = { 'loss_cls': losses_cls, 'loss_pts_init': losses_pts_init, 'loss_pts_refine': losses_pts_refine } return loss_dict_all # Same as base_dense_head/_get_bboxes_single except self._bbox_decode def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], score_factor_list: List[Tensor], mlvl_priors: List[Tensor], img_meta: dict, cfg: ConfigDict, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform outputs of a single image into bbox predictions. Args: cls_score_list (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). score_factor_list (list[Tensor]): Score factor from all scale levels of a single image. RepPoints head does not need this value. mlvl_priors (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid, has shape (num_priors, 2). img_meta (dict): Image meta info. cfg (:obj:`ConfigDict`): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ cfg = self.test_cfg if cfg is None else cfg assert len(cls_score_list) == len(bbox_pred_list) img_shape = img_meta['img_shape'] nms_pre = cfg.get('nms_pre', -1) mlvl_bboxes = [] mlvl_scores = [] mlvl_labels = [] for level_idx, (cls_score, bbox_pred, priors) in enumerate( zip(cls_score_list, bbox_pred_list, mlvl_priors)): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) cls_score = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) if self.use_sigmoid_cls: scores = cls_score.sigmoid() else: scores = cls_score.softmax(-1)[:, :-1] # After https://github.com/open-mmlab/mmdetection/pull/6268/, # this operation keeps fewer bboxes under the same `nms_pre`. # There is no difference in performance for most models. If you # find a slight drop in performance, you can set a larger # `nms_pre` than before. results = filter_scores_and_topk( scores, cfg.score_thr, nms_pre, dict(bbox_pred=bbox_pred, priors=priors)) scores, labels, _, filtered_results = results bbox_pred = filtered_results['bbox_pred'] priors = filtered_results['priors'] bboxes = self._bbox_decode(priors, bbox_pred, self.point_strides[level_idx], img_shape) mlvl_bboxes.append(bboxes) mlvl_scores.append(scores) mlvl_labels.append(labels) results = InstanceData() results.bboxes = torch.cat(mlvl_bboxes) results.scores = torch.cat(mlvl_scores) results.labels = torch.cat(mlvl_labels) return self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta) def _bbox_decode(self, points: Tensor, bbox_pred: Tensor, stride: int, max_shape: Tuple[int, int]) -> Tensor: """Decode the prediction to bounding box. Args: points (Tensor): shape (h_i * w_i, 2). bbox_pred (Tensor): shape (h_i * w_i, 4). stride (int): Stride for bbox_pred in different level. max_shape (Tuple[int, int]): image shape. Returns: Tensor: Bounding boxes decoded. """ bbox_pos_center = torch.cat([points[:, :2], points[:, :2]], dim=1) bboxes = bbox_pred * stride + bbox_pos_center x1 = bboxes[:, 0].clamp(min=0, max=max_shape[1]) y1 = bboxes[:, 1].clamp(min=0, max=max_shape[0]) x2 = bboxes[:, 2].clamp(min=0, max=max_shape[1]) y2 = bboxes[:, 3].clamp(min=0, max=max_shape[0]) decoded_bboxes = torch.stack([x1, y1, x2, y2], dim=-1) return decoded_bboxes ================================================ FILE: mmdet/models/dense_heads/retina_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn as nn from mmcv.cnn import ConvModule from mmdet.registry import MODELS from .anchor_head import AnchorHead @MODELS.register_module() class RetinaHead(AnchorHead): r"""An anchor-based head used in `RetinaNet `_. The head contains two subnetworks. The first classifies anchor boxes and the second regresses deltas for the anchors. Example: >>> import torch >>> self = RetinaHead(11, 7) >>> x = torch.rand(1, 7, 32, 32) >>> cls_score, bbox_pred = self.forward_single(x) >>> # Each anchor predicts a score for each class except background >>> cls_per_anchor = cls_score.shape[1] / self.num_anchors >>> box_per_anchor = bbox_pred.shape[1] / self.num_anchors >>> assert cls_per_anchor == (self.num_classes) >>> assert box_per_anchor == 4 """ def __init__(self, num_classes, in_channels, stacked_convs=4, conv_cfg=None, norm_cfg=None, anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), init_cfg=dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='retina_cls', std=0.01, bias_prob=0.01)), **kwargs): assert stacked_convs >= 0, \ '`stacked_convs` must be non-negative integers, ' \ f'but got {stacked_convs} instead.' self.stacked_convs = stacked_convs self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg super(RetinaHead, self).__init__( num_classes, in_channels, anchor_generator=anchor_generator, init_cfg=init_cfg, **kwargs) def _init_layers(self): """Initialize layers of the head.""" self.relu = nn.ReLU(inplace=True) self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() in_channels = self.in_channels for i in range(self.stacked_convs): self.cls_convs.append( ConvModule( in_channels, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.reg_convs.append( ConvModule( in_channels, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) in_channels = self.feat_channels self.retina_cls = nn.Conv2d( in_channels, self.num_base_priors * self.cls_out_channels, 3, padding=1) reg_dim = self.bbox_coder.encode_size self.retina_reg = nn.Conv2d( in_channels, self.num_base_priors * reg_dim, 3, padding=1) def forward_single(self, x): """Forward feature of a single scale level. Args: x (Tensor): Features of a single scale level. Returns: tuple: cls_score (Tensor): Cls scores for a single scale level the channels number is num_anchors * num_classes. bbox_pred (Tensor): Box energies / deltas for a single scale level, the channels number is num_anchors * 4. """ cls_feat = x reg_feat = x for cls_conv in self.cls_convs: cls_feat = cls_conv(cls_feat) for reg_conv in self.reg_convs: reg_feat = reg_conv(reg_feat) cls_score = self.retina_cls(cls_feat) bbox_pred = self.retina_reg(reg_feat) return cls_score, bbox_pred ================================================ FILE: mmdet/models/dense_heads/retina_sepbn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple import torch.nn as nn from mmcv.cnn import ConvModule from mmengine.model import bias_init_with_prob, normal_init from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import OptConfigType, OptMultiConfig from .anchor_head import AnchorHead @MODELS.register_module() class RetinaSepBNHead(AnchorHead): """"RetinaHead with separate BN. In RetinaHead, conv/norm layers are shared across different FPN levels, while in RetinaSepBNHead, conv layers are shared across different FPN levels, but BN layers are separated. """ def __init__(self, num_classes: int, num_ins: int, in_channels: int, stacked_convs: int = 4, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, init_cfg: OptMultiConfig = None, **kwargs) -> None: assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' self.stacked_convs = stacked_convs self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.num_ins = num_ins super().__init__( num_classes=num_classes, in_channels=in_channels, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" self.relu = nn.ReLU(inplace=True) self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() for i in range(self.num_ins): cls_convs = nn.ModuleList() reg_convs = nn.ModuleList() for j in range(self.stacked_convs): chn = self.in_channels if j == 0 else self.feat_channels cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) reg_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.cls_convs.append(cls_convs) self.reg_convs.append(reg_convs) for i in range(self.stacked_convs): for j in range(1, self.num_ins): self.cls_convs[j][i].conv = self.cls_convs[0][i].conv self.reg_convs[j][i].conv = self.reg_convs[0][i].conv self.retina_cls = nn.Conv2d( self.feat_channels, self.num_base_priors * self.cls_out_channels, 3, padding=1) self.retina_reg = nn.Conv2d( self.feat_channels, self.num_base_priors * 4, 3, padding=1) def init_weights(self) -> None: """Initialize weights of the head.""" super().init_weights() for m in self.cls_convs[0]: normal_init(m.conv, std=0.01) for m in self.reg_convs[0]: normal_init(m.conv, std=0.01) bias_cls = bias_init_with_prob(0.01) normal_init(self.retina_cls, std=0.01, bias=bias_cls) normal_init(self.retina_reg, std=0.01) def forward(self, feats: Tuple[Tensor]) -> tuple: """Forward features from the upstream network. Args: feats (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Usually a tuple of classification scores and bbox prediction - cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_anchors * num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_anchors * 4. """ cls_scores = [] bbox_preds = [] for i, x in enumerate(feats): cls_feat = feats[i] reg_feat = feats[i] for cls_conv in self.cls_convs[i]: cls_feat = cls_conv(cls_feat) for reg_conv in self.reg_convs[i]: reg_feat = reg_conv(reg_feat) cls_score = self.retina_cls(cls_feat) bbox_pred = self.retina_reg(reg_feat) cls_scores.append(cls_score) bbox_preds.append(bbox_pred) return cls_scores, bbox_preds ================================================ FILE: mmdet/models/dense_heads/rpn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import List, Optional, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmcv.ops import batched_nms from mmengine.config import ConfigDict from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures.bbox import (cat_boxes, empty_box_as, get_box_tensor, get_box_wh, scale_boxes) from mmdet.utils import InstanceList, MultiConfig, OptInstanceList from .anchor_head import AnchorHead @MODELS.register_module() class RPNHead(AnchorHead): """Implementation of RPN head. Args: in_channels (int): Number of channels in the input feature map. num_classes (int): Number of categories excluding the background category. Defaults to 1. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or \ list[dict]): Initialization config dict. num_convs (int): Number of convolution layers in the head. Defaults to 1. """ # noqa: W605 def __init__(self, in_channels: int, num_classes: int = 1, init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.01), num_convs: int = 1, **kwargs) -> None: self.num_convs = num_convs assert num_classes == 1 super().__init__( num_classes=num_classes, in_channels=in_channels, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" if self.num_convs > 1: rpn_convs = [] for i in range(self.num_convs): if i == 0: in_channels = self.in_channels else: in_channels = self.feat_channels # use ``inplace=False`` to avoid error: one of the variables # needed for gradient computation has been modified by an # inplace operation. rpn_convs.append( ConvModule( in_channels, self.feat_channels, 3, padding=1, inplace=False)) self.rpn_conv = nn.Sequential(*rpn_convs) else: self.rpn_conv = nn.Conv2d( self.in_channels, self.feat_channels, 3, padding=1) self.rpn_cls = nn.Conv2d(self.feat_channels, self.num_base_priors * self.cls_out_channels, 1) reg_dim = self.bbox_coder.encode_size self.rpn_reg = nn.Conv2d(self.feat_channels, self.num_base_priors * reg_dim, 1) def forward_single(self, x: Tensor) -> Tuple[Tensor, Tensor]: """Forward feature of a single scale level. Args: x (Tensor): Features of a single scale level. Returns: tuple: cls_score (Tensor): Cls scores for a single scale level \ the channels number is num_base_priors * num_classes. bbox_pred (Tensor): Box energies / deltas for a single scale \ level, the channels number is num_base_priors * 4. """ x = self.rpn_conv(x) x = F.relu(x) rpn_cls_score = self.rpn_cls(x) rpn_bbox_pred = self.rpn_reg(x) return rpn_cls_score, rpn_bbox_pred def loss_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) \ -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level, has shape (N, num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). batch_gt_instances (list[obj:InstanceData]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[obj:InstanceData], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Returns: dict[str, Tensor]: A dictionary of loss components. """ losses = super().loss_by_feat( cls_scores, bbox_preds, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) return dict( loss_rpn_cls=losses['loss_cls'], loss_rpn_bbox=losses['loss_bbox']) def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], score_factor_list: List[Tensor], mlvl_priors: List[Tensor], img_meta: dict, cfg: ConfigDict, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: cls_score_list (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). score_factor_list (list[Tensor]): Be compatible with BaseDenseHead. Not used in RPNHead. mlvl_priors (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid. In all anchor-based methods, it has shape (num_priors, 4). In all anchor-free methods, it has shape (num_priors, 2) when `with_stride=True`, otherwise it still has shape (num_priors, 4). img_meta (dict): Image meta info. cfg (ConfigDict, optional): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ cfg = self.test_cfg if cfg is None else cfg cfg = copy.deepcopy(cfg) img_shape = img_meta['img_shape'] nms_pre = cfg.get('nms_pre', -1) mlvl_bbox_preds = [] mlvl_valid_priors = [] mlvl_scores = [] level_ids = [] for level_idx, (cls_score, bbox_pred, priors) in \ enumerate(zip(cls_score_list, bbox_pred_list, mlvl_priors)): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] reg_dim = self.bbox_coder.encode_size bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, reg_dim) cls_score = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) if self.use_sigmoid_cls: scores = cls_score.sigmoid() else: # remind that we set FG labels to [0] since mmdet v2.0 # BG cat_id: 1 scores = cls_score.softmax(-1)[:, :-1] scores = torch.squeeze(scores) if 0 < nms_pre < scores.shape[0]: # sort is faster than topk # _, topk_inds = scores.topk(cfg.nms_pre) ranked_scores, rank_inds = scores.sort(descending=True) topk_inds = rank_inds[:nms_pre] scores = ranked_scores[:nms_pre] bbox_pred = bbox_pred[topk_inds, :] priors = priors[topk_inds] mlvl_bbox_preds.append(bbox_pred) mlvl_valid_priors.append(priors) mlvl_scores.append(scores) # use level id to implement the separate level nms level_ids.append( scores.new_full((scores.size(0), ), level_idx, dtype=torch.long)) bbox_pred = torch.cat(mlvl_bbox_preds) priors = cat_boxes(mlvl_valid_priors) bboxes = self.bbox_coder.decode(priors, bbox_pred, max_shape=img_shape) results = InstanceData() results.bboxes = bboxes results.scores = torch.cat(mlvl_scores) results.level_ids = torch.cat(level_ids) return self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, img_meta=img_meta) def _bbox_post_process(self, results: InstanceData, cfg: ConfigDict, rescale: bool = False, with_nms: bool = True, img_meta: Optional[dict] = None) -> InstanceData: """bbox post-processing method. The boxes would be rescaled to the original image scale and do the nms operation. Args: results (:obj:`InstaceData`): Detection instance results, each item has shape (num_bboxes, ). cfg (ConfigDict): Test / postprocessing configuration. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Default to True. img_meta (dict, optional): Image meta info. Defaults to None. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert with_nms, '`with_nms` must be True in RPNHead' if rescale: assert img_meta.get('scale_factor') is not None scale_factor = [1 / s for s in img_meta['scale_factor']] results.bboxes = scale_boxes(results.bboxes, scale_factor) # filter small size bboxes if cfg.get('min_bbox_size', -1) >= 0: w, h = get_box_wh(results.bboxes) valid_mask = (w > cfg.min_bbox_size) & (h > cfg.min_bbox_size) if not valid_mask.all(): results = results[valid_mask] if results.bboxes.numel() > 0: bboxes = get_box_tensor(results.bboxes) det_bboxes, keep_idxs = batched_nms(bboxes, results.scores, results.level_ids, cfg.nms) results = results[keep_idxs] # some nms would reweight the score, such as softnms results.scores = det_bboxes[:, -1] results = results[:cfg.max_per_img] # TODO: This would unreasonably show the 0th class label # in visualization results.labels = results.scores.new_zeros( len(results), dtype=torch.long) del results.level_ids else: # To avoid some potential error results_ = InstanceData() results_.bboxes = empty_box_as(results.bboxes) results_.scores = results.scores.new_zeros(0) results_.labels = results.scores.new_zeros(0) results = results_ return results ================================================ FILE: mmdet/models/dense_heads/rtmdet_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple, Union import torch import torch.nn as nn from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule, Scale, is_norm from mmengine.model import bias_init_with_prob, constant_init, normal_init from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures.bbox import distance2bbox from mmdet.utils import ConfigType, InstanceList, OptInstanceList, reduce_mean from ..layers.transformer import inverse_sigmoid from ..task_modules import anchor_inside_flags from ..utils import (images_to_levels, multi_apply, sigmoid_geometric_mean, unmap) from .atss_head import ATSSHead @MODELS.register_module() class RTMDetHead(ATSSHead): """Detection Head of RTMDet. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. with_objectness (bool): Whether to add an objectness branch. Defaults to True. act_cfg (:obj:`ConfigDict` or dict): Config dict for activation layer. Default: dict(type='ReLU') """ def __init__(self, num_classes: int, in_channels: int, with_objectness: bool = True, act_cfg: ConfigType = dict(type='ReLU'), **kwargs) -> None: self.act_cfg = act_cfg self.with_objectness = with_objectness super().__init__(num_classes, in_channels, **kwargs) if self.train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) def _init_layers(self): """Initialize layers of the head.""" self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) self.reg_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) pred_pad_size = self.pred_kernel_size // 2 self.rtm_cls = nn.Conv2d( self.feat_channels, self.num_base_priors * self.cls_out_channels, self.pred_kernel_size, padding=pred_pad_size) self.rtm_reg = nn.Conv2d( self.feat_channels, self.num_base_priors * 4, self.pred_kernel_size, padding=pred_pad_size) if self.with_objectness: self.rtm_obj = nn.Conv2d( self.feat_channels, 1, self.pred_kernel_size, padding=pred_pad_size) self.scales = nn.ModuleList( [Scale(1.0) for _ in self.prior_generator.strides]) def init_weights(self) -> None: """Initialize weights of the head.""" for m in self.modules(): if isinstance(m, nn.Conv2d): normal_init(m, mean=0, std=0.01) if is_norm(m): constant_init(m, 1) bias_cls = bias_init_with_prob(0.01) normal_init(self.rtm_cls, std=0.01, bias=bias_cls) normal_init(self.rtm_reg, std=0.01) if self.with_objectness: normal_init(self.rtm_obj, std=0.01, bias=bias_cls) def forward(self, feats: Tuple[Tensor, ...]) -> tuple: """Forward features from the upstream network. Args: feats (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Usually a tuple of classification scores and bbox prediction - cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * 4. """ cls_scores = [] bbox_preds = [] for idx, (x, scale, stride) in enumerate( zip(feats, self.scales, self.prior_generator.strides)): cls_feat = x reg_feat = x for cls_layer in self.cls_convs: cls_feat = cls_layer(cls_feat) cls_score = self.rtm_cls(cls_feat) for reg_layer in self.reg_convs: reg_feat = reg_layer(reg_feat) if self.with_objectness: objectness = self.rtm_obj(reg_feat) cls_score = inverse_sigmoid( sigmoid_geometric_mean(cls_score, objectness)) reg_dist = scale(self.rtm_reg(reg_feat).exp()).float() * stride[0] cls_scores.append(cls_score) bbox_preds.append(reg_dist) return tuple(cls_scores), tuple(bbox_preds) def loss_by_feat_single(self, cls_score: Tensor, bbox_pred: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, assign_metrics: Tensor, stride: List[int]): """Compute loss of a single scale level. Args: cls_score (Tensor): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W). bbox_pred (Tensor): Decoded bboxes for each scale level with shape (N, num_anchors * 4, H, W). labels (Tensor): Labels of each anchors with shape (N, num_total_anchors). label_weights (Tensor): Label weights of each anchor with shape (N, num_total_anchors). bbox_targets (Tensor): BBox regression targets of each anchor with shape (N, num_total_anchors, 4). assign_metrics (Tensor): Assign metrics with shape (N, num_total_anchors). stride (List[int]): Downsample stride of the feature map. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert stride[0] == stride[1], 'h stride is not equal to w stride!' cls_score = cls_score.permute(0, 2, 3, 1).reshape( -1, self.cls_out_channels).contiguous() bbox_pred = bbox_pred.reshape(-1, 4) bbox_targets = bbox_targets.reshape(-1, 4) labels = labels.reshape(-1) assign_metrics = assign_metrics.reshape(-1) label_weights = label_weights.reshape(-1) targets = (labels, assign_metrics) loss_cls = self.loss_cls( cls_score, targets, label_weights, avg_factor=1.0) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = self.num_classes pos_inds = ((labels >= 0) & (labels < bg_class_ind)).nonzero().squeeze(1) if len(pos_inds) > 0: pos_bbox_targets = bbox_targets[pos_inds] pos_bbox_pred = bbox_pred[pos_inds] pos_decode_bbox_pred = pos_bbox_pred pos_decode_bbox_targets = pos_bbox_targets # regression loss pos_bbox_weight = assign_metrics[pos_inds] loss_bbox = self.loss_bbox( pos_decode_bbox_pred, pos_decode_bbox_targets, weight=pos_bbox_weight, avg_factor=1.0) else: loss_bbox = bbox_pred.sum() * 0 pos_bbox_weight = bbox_targets.new_tensor(0.) return loss_cls, loss_bbox, assign_metrics.sum(), pos_bbox_weight.sum() def loss_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None): """Compute losses of the head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Decoded box for each scale level with shape (N, num_anchors * 4, H, W) in [tl_x, tl_y, br_x, br_y] format. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ num_imgs = len(batch_img_metas) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) flatten_cls_scores = torch.cat([ cls_score.permute(0, 2, 3, 1).reshape(num_imgs, -1, self.cls_out_channels) for cls_score in cls_scores ], 1) decoded_bboxes = [] for anchor, bbox_pred in zip(anchor_list[0], bbox_preds): anchor = anchor.reshape(-1, 4) bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, 4) bbox_pred = distance2bbox(anchor, bbox_pred) decoded_bboxes.append(bbox_pred) flatten_bboxes = torch.cat(decoded_bboxes, 1) cls_reg_targets = self.get_targets( flatten_cls_scores, flatten_bboxes, anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (anchor_list, labels_list, label_weights_list, bbox_targets_list, assign_metrics_list, sampling_results_list) = cls_reg_targets losses_cls, losses_bbox,\ cls_avg_factors, bbox_avg_factors = multi_apply( self.loss_by_feat_single, cls_scores, decoded_bboxes, labels_list, label_weights_list, bbox_targets_list, assign_metrics_list, self.prior_generator.strides) cls_avg_factor = reduce_mean(sum(cls_avg_factors)).clamp_(min=1).item() losses_cls = list(map(lambda x: x / cls_avg_factor, losses_cls)) bbox_avg_factor = reduce_mean( sum(bbox_avg_factors)).clamp_(min=1).item() losses_bbox = list(map(lambda x: x / bbox_avg_factor, losses_bbox)) return dict(loss_cls=losses_cls, loss_bbox=losses_bbox) def get_targets(self, cls_scores: Tensor, bbox_preds: Tensor, anchor_list: List[List[Tensor]], valid_flag_list: List[List[Tensor]], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs=True): """Compute regression and classification targets for anchors in multiple images. Args: cls_scores (Tensor): Classification predictions of images, a 3D-Tensor with shape [num_imgs, num_priors, num_classes]. bbox_preds (Tensor): Decoded bboxes predictions of one image, a 3D-Tensor with shape [num_imgs, num_priors, 4] in [tl_x, tl_y, br_x, br_y] format. anchor_list (list[list[Tensor]]): Multi level anchors of each image. The outer list indicates images, and the inner list corresponds to feature levels of the image. Each element of the inner list is a tensor of shape (num_anchors, 4). valid_flag_list (list[list[Tensor]]): Multi level valid flags of each image. The outer list indicates images, and the inner list corresponds to feature levels of the image. Each element of the inner list is a tensor of shape (num_anchors, ) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. Returns: tuple: a tuple containing learning targets. - anchors_list (list[list[Tensor]]): Anchors of each level. - labels_list (list[Tensor]): Labels of each level. - label_weights_list (list[Tensor]): Label weights of each level. - bbox_targets_list (list[Tensor]): BBox targets of each level. - assign_metrics_list (list[Tensor]): alignment metrics of each level. """ num_imgs = len(batch_img_metas) assert len(anchor_list) == len(valid_flag_list) == num_imgs # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] # concat all level anchors and flags to a single tensor for i in range(num_imgs): assert len(anchor_list[i]) == len(valid_flag_list[i]) anchor_list[i] = torch.cat(anchor_list[i]) valid_flag_list[i] = torch.cat(valid_flag_list[i]) # compute targets for each image if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None] * num_imgs # anchor_list: list(b * [-1, 4]) (all_anchors, all_labels, all_label_weights, all_bbox_targets, all_assign_metrics, sampling_results_list) = multi_apply( self._get_targets_single, cls_scores.detach(), bbox_preds.detach(), anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs) # no valid anchors if any([labels is None for labels in all_labels]): return None # split targets to a list w.r.t. multiple levels anchors_list = images_to_levels(all_anchors, num_level_anchors) labels_list = images_to_levels(all_labels, num_level_anchors) label_weights_list = images_to_levels(all_label_weights, num_level_anchors) bbox_targets_list = images_to_levels(all_bbox_targets, num_level_anchors) assign_metrics_list = images_to_levels(all_assign_metrics, num_level_anchors) return (anchors_list, labels_list, label_weights_list, bbox_targets_list, assign_metrics_list, sampling_results_list) def _get_targets_single(self, cls_scores: Tensor, bbox_preds: Tensor, flat_anchors: Tensor, valid_flags: Tensor, gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None, unmap_outputs=True): """Compute regression, classification targets for anchors in a single image. Args: cls_scores (list(Tensor)): Box scores for each image. bbox_preds (list(Tensor)): Box energies / deltas for each image. flat_anchors (Tensor): Multi-level anchors of the image, which are concatenated into a single tensor of shape (num_anchors ,4) valid_flags (Tensor): Multi level valid flags of the image, which are concatenated into a single tensor of shape (num_anchors,). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. Returns: tuple: N is the number of total anchors in the image. - anchors (Tensor): All anchors in the image with shape (N, 4). - labels (Tensor): Labels of all anchors in the image with shape (N,). - label_weights (Tensor): Label weights of all anchor in the image with shape (N,). - bbox_targets (Tensor): BBox targets of all anchors in the image with shape (N, 4). - norm_alignment_metrics (Tensor): Normalized alignment metrics of all priors in the image with shape (N,). """ inside_flags = anchor_inside_flags(flat_anchors, valid_flags, img_meta['img_shape'][:2], self.train_cfg['allowed_border']) if not inside_flags.any(): return (None, ) * 7 # assign gt and sample anchors anchors = flat_anchors[inside_flags, :] pred_instances = InstanceData( scores=cls_scores[inside_flags, :], bboxes=bbox_preds[inside_flags, :], priors=anchors) assign_result = self.assigner.assign(pred_instances, gt_instances, gt_instances_ignore) sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) num_valid_anchors = anchors.shape[0] bbox_targets = torch.zeros_like(anchors) labels = anchors.new_full((num_valid_anchors, ), self.num_classes, dtype=torch.long) label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) assign_metrics = anchors.new_zeros( num_valid_anchors, dtype=torch.float) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: # point-based pos_bbox_targets = sampling_result.pos_gt_bboxes bbox_targets[pos_inds, :] = pos_bbox_targets labels[pos_inds] = sampling_result.pos_gt_labels if self.train_cfg['pos_weight'] <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = self.train_cfg['pos_weight'] if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 class_assigned_gt_inds = torch.unique( sampling_result.pos_assigned_gt_inds) for gt_inds in class_assigned_gt_inds: gt_class_inds = pos_inds[sampling_result.pos_assigned_gt_inds == gt_inds] assign_metrics[gt_class_inds] = assign_result.max_overlaps[ gt_class_inds] # map up to original set of anchors if unmap_outputs: num_total_anchors = flat_anchors.size(0) anchors = unmap(anchors, num_total_anchors, inside_flags) labels = unmap( labels, num_total_anchors, inside_flags, fill=self.num_classes) label_weights = unmap(label_weights, num_total_anchors, inside_flags) bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) assign_metrics = unmap(assign_metrics, num_total_anchors, inside_flags) return (anchors, labels, label_weights, bbox_targets, assign_metrics, sampling_result) def get_anchors(self, featmap_sizes: List[tuple], batch_img_metas: List[dict], device: Union[torch.device, str] = 'cuda') \ -> Tuple[List[List[Tensor]], List[List[Tensor]]]: """Get anchors according to feature map sizes. Args: featmap_sizes (list[tuple]): Multi-level feature map sizes. batch_img_metas (list[dict]): Image meta info. device (torch.device or str): Device for returned tensors. Defaults to cuda. Returns: tuple: - anchor_list (list[list[Tensor]]): Anchors of each image. - valid_flag_list (list[list[Tensor]]): Valid flags of each image. """ num_imgs = len(batch_img_metas) # since feature map sizes of all images are the same, we only compute # anchors for one time multi_level_anchors = self.prior_generator.grid_priors( featmap_sizes, device=device, with_stride=True) anchor_list = [multi_level_anchors for _ in range(num_imgs)] # for each image, we compute valid flags of multi level anchors valid_flag_list = [] for img_id, img_meta in enumerate(batch_img_metas): multi_level_flags = self.prior_generator.valid_flags( featmap_sizes, img_meta['pad_shape'], device) valid_flag_list.append(multi_level_flags) return anchor_list, valid_flag_list @MODELS.register_module() class RTMDetSepBNHead(RTMDetHead): """RTMDetHead with separated BN layers and shared conv layers. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. share_conv (bool): Whether to share conv layers between stages. Defaults to True. use_depthwise (bool): Whether to use depthwise separable convolution in head. Defaults to False. norm_cfg (:obj:`ConfigDict` or dict)): Config dict for normalization layer. Defaults to dict(type='BN', momentum=0.03, eps=0.001). act_cfg (:obj:`ConfigDict` or dict)): Config dict for activation layer. Defaults to dict(type='SiLU'). pred_kernel_size (int): Kernel size of prediction layer. Defaults to 1. """ def __init__(self, num_classes: int, in_channels: int, share_conv: bool = True, use_depthwise: bool = False, norm_cfg: ConfigType = dict( type='BN', momentum=0.03, eps=0.001), act_cfg: ConfigType = dict(type='SiLU'), pred_kernel_size: int = 1, exp_on_reg=False, **kwargs) -> None: self.share_conv = share_conv self.exp_on_reg = exp_on_reg self.use_depthwise = use_depthwise super().__init__( num_classes, in_channels, norm_cfg=norm_cfg, act_cfg=act_cfg, pred_kernel_size=pred_kernel_size, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" conv = DepthwiseSeparableConvModule \ if self.use_depthwise else ConvModule self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() self.rtm_cls = nn.ModuleList() self.rtm_reg = nn.ModuleList() if self.with_objectness: self.rtm_obj = nn.ModuleList() for n in range(len(self.prior_generator.strides)): cls_convs = nn.ModuleList() reg_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels cls_convs.append( conv( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) reg_convs.append( conv( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) self.cls_convs.append(cls_convs) self.reg_convs.append(reg_convs) self.rtm_cls.append( nn.Conv2d( self.feat_channels, self.num_base_priors * self.cls_out_channels, self.pred_kernel_size, padding=self.pred_kernel_size // 2)) self.rtm_reg.append( nn.Conv2d( self.feat_channels, self.num_base_priors * 4, self.pred_kernel_size, padding=self.pred_kernel_size // 2)) if self.with_objectness: self.rtm_obj.append( nn.Conv2d( self.feat_channels, 1, self.pred_kernel_size, padding=self.pred_kernel_size // 2)) if self.share_conv: for n in range(len(self.prior_generator.strides)): for i in range(self.stacked_convs): self.cls_convs[n][i].conv = self.cls_convs[0][i].conv self.reg_convs[n][i].conv = self.reg_convs[0][i].conv def init_weights(self) -> None: """Initialize weights of the head.""" for m in self.modules(): if isinstance(m, nn.Conv2d): normal_init(m, mean=0, std=0.01) if is_norm(m): constant_init(m, 1) bias_cls = bias_init_with_prob(0.01) for rtm_cls, rtm_reg in zip(self.rtm_cls, self.rtm_reg): normal_init(rtm_cls, std=0.01, bias=bias_cls) normal_init(rtm_reg, std=0.01) if self.with_objectness: for rtm_obj in self.rtm_obj: normal_init(rtm_obj, std=0.01, bias=bias_cls) def forward(self, feats: Tuple[Tensor, ...]) -> tuple: """Forward features from the upstream network. Args: feats (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Usually a tuple of classification scores and bbox prediction - cls_scores (tuple[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_anchors * num_classes. - bbox_preds (tuple[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_anchors * 4. """ cls_scores = [] bbox_preds = [] for idx, (x, stride) in enumerate( zip(feats, self.prior_generator.strides)): cls_feat = x reg_feat = x for cls_layer in self.cls_convs[idx]: cls_feat = cls_layer(cls_feat) cls_score = self.rtm_cls[idx](cls_feat) for reg_layer in self.reg_convs[idx]: reg_feat = reg_layer(reg_feat) if self.with_objectness: objectness = self.rtm_obj[idx](reg_feat) cls_score = inverse_sigmoid( sigmoid_geometric_mean(cls_score, objectness)) if self.exp_on_reg: reg_dist = self.rtm_reg[idx](reg_feat).exp() * stride[0] else: reg_dist = self.rtm_reg[idx](reg_feat) * stride[0] cls_scores.append(cls_score) bbox_preds.append(reg_dist) return tuple(cls_scores), tuple(bbox_preds) ================================================ FILE: mmdet/models/dense_heads/rtmdet_ins_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import math from typing import List, Optional, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule, is_norm from mmcv.ops import batched_nms from mmengine.model import (BaseModule, bias_init_with_prob, constant_init, normal_init) from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.layers.transformer import inverse_sigmoid from mmdet.models.utils import (filter_scores_and_topk, multi_apply, select_single_mlvl, sigmoid_geometric_mean) from mmdet.registry import MODELS from mmdet.structures.bbox import (cat_boxes, distance2bbox, get_box_tensor, get_box_wh, scale_boxes) from mmdet.utils import ConfigType, InstanceList, OptInstanceList, reduce_mean from .rtmdet_head import RTMDetHead @MODELS.register_module() class RTMDetInsHead(RTMDetHead): """Detection Head of RTMDet-Ins. Args: num_prototypes (int): Number of mask prototype features extracted from the mask head. Defaults to 8. dyconv_channels (int): Channel of the dynamic conv layers. Defaults to 8. num_dyconvs (int): Number of the dynamic convolution layers. Defaults to 3. mask_loss_stride (int): Down sample stride of the masks for loss computation. Defaults to 4. loss_mask (:obj:`ConfigDict` or dict): Config dict for mask loss. """ def __init__(self, *args, num_prototypes: int = 8, dyconv_channels: int = 8, num_dyconvs: int = 3, mask_loss_stride: int = 4, loss_mask=dict( type='DiceLoss', loss_weight=2.0, eps=5e-6, reduction='mean'), **kwargs) -> None: self.num_prototypes = num_prototypes self.num_dyconvs = num_dyconvs self.dyconv_channels = dyconv_channels self.mask_loss_stride = mask_loss_stride super().__init__(*args, **kwargs) self.loss_mask = MODELS.build(loss_mask) def _init_layers(self) -> None: """Initialize layers of the head.""" super()._init_layers() # a branch to predict kernels of dynamic convs self.kernel_convs = nn.ModuleList() # calculate num dynamic parameters weight_nums, bias_nums = [], [] for i in range(self.num_dyconvs): if i == 0: weight_nums.append( # mask prototype and coordinate features (self.num_prototypes + 2) * self.dyconv_channels) bias_nums.append(self.dyconv_channels * 1) elif i == self.num_dyconvs - 1: weight_nums.append(self.dyconv_channels * 1) bias_nums.append(1) else: weight_nums.append(self.dyconv_channels * self.dyconv_channels) bias_nums.append(self.dyconv_channels * 1) self.weight_nums = weight_nums self.bias_nums = bias_nums self.num_gen_params = sum(weight_nums) + sum(bias_nums) for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels self.kernel_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) pred_pad_size = self.pred_kernel_size // 2 self.rtm_kernel = nn.Conv2d( self.feat_channels, self.num_gen_params, self.pred_kernel_size, padding=pred_pad_size) self.mask_head = MaskFeatModule( in_channels=self.in_channels, feat_channels=self.feat_channels, stacked_convs=4, num_levels=len(self.prior_generator.strides), num_prototypes=self.num_prototypes, act_cfg=self.act_cfg, norm_cfg=self.norm_cfg) def forward(self, feats: Tuple[Tensor, ...]) -> tuple: """Forward features from the upstream network. Args: feats (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Usually a tuple of classification scores and bbox prediction - cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * 4. - kernel_preds (list[Tensor]): Dynamic conv kernels for all scale levels, each is a 4D-tensor, the channels number is num_gen_params. - mask_feat (Tensor): Output feature of the mask head. Each is a 4D-tensor, the channels number is num_prototypes. """ mask_feat = self.mask_head(feats) cls_scores = [] bbox_preds = [] kernel_preds = [] for idx, (x, scale, stride) in enumerate( zip(feats, self.scales, self.prior_generator.strides)): cls_feat = x reg_feat = x kernel_feat = x for cls_layer in self.cls_convs: cls_feat = cls_layer(cls_feat) cls_score = self.rtm_cls(cls_feat) for kernel_layer in self.kernel_convs: kernel_feat = kernel_layer(kernel_feat) kernel_pred = self.rtm_kernel(kernel_feat) for reg_layer in self.reg_convs: reg_feat = reg_layer(reg_feat) if self.with_objectness: objectness = self.rtm_obj(reg_feat) cls_score = inverse_sigmoid( sigmoid_geometric_mean(cls_score, objectness)) reg_dist = scale(self.rtm_reg(reg_feat)) * stride[0] cls_scores.append(cls_score) bbox_preds.append(reg_dist) kernel_preds.append(kernel_pred) return tuple(cls_scores), tuple(bbox_preds), tuple( kernel_preds), mask_feat def predict_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], kernel_preds: List[Tensor], mask_feat: Tensor, score_factors: Optional[List[Tensor]] = None, batch_img_metas: Optional[List[dict]] = None, cfg: Optional[ConfigType] = None, rescale: bool = False, with_nms: bool = True) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. Note: When score_factors is not None, the cls_scores are usually multiplied by it then obtain the real score used in NMS, such as CenterNess in FCOS, IoU branch in ATSS. Args: cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * 4, H, W). kernel_preds (list[Tensor]): Kernel predictions of dynamic convs for all scale levels, each is a 4D-tensor, has shape (batch_size, num_params, H, W). mask_feat (Tensor): Mask prototype features extracted from the mask head, has shape (batch_size, num_prototypes, H, W). score_factors (list[Tensor], optional): Score factor for all scale level, each is a 4D-tensor, has shape (batch_size, num_priors * 1, H, W). Defaults to None. batch_img_metas (list[dict], Optional): Batch image meta info. Defaults to None. cfg (ConfigDict, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, h, w). """ assert len(cls_scores) == len(bbox_preds) if score_factors is None: # e.g. Retina, FreeAnchor, Foveabox, etc. with_score_factors = False else: # e.g. FCOS, PAA, ATSS, AutoAssign, etc. with_score_factors = True assert len(cls_scores) == len(score_factors) num_levels = len(cls_scores) featmap_sizes = [cls_scores[i].shape[-2:] for i in range(num_levels)] mlvl_priors = self.prior_generator.grid_priors( featmap_sizes, dtype=cls_scores[0].dtype, device=cls_scores[0].device, with_stride=True) result_list = [] for img_id in range(len(batch_img_metas)): img_meta = batch_img_metas[img_id] cls_score_list = select_single_mlvl( cls_scores, img_id, detach=True) bbox_pred_list = select_single_mlvl( bbox_preds, img_id, detach=True) kernel_pred_list = select_single_mlvl( kernel_preds, img_id, detach=True) if with_score_factors: score_factor_list = select_single_mlvl( score_factors, img_id, detach=True) else: score_factor_list = [None for _ in range(num_levels)] results = self._predict_by_feat_single( cls_score_list=cls_score_list, bbox_pred_list=bbox_pred_list, kernel_pred_list=kernel_pred_list, mask_feat=mask_feat[img_id], score_factor_list=score_factor_list, mlvl_priors=mlvl_priors, img_meta=img_meta, cfg=cfg, rescale=rescale, with_nms=with_nms) result_list.append(results) return result_list def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], kernel_pred_list: List[Tensor], mask_feat: Tensor, score_factor_list: List[Tensor], mlvl_priors: List[Tensor], img_meta: dict, cfg: ConfigType, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox and mask results. Args: cls_score_list (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). kernel_preds (list[Tensor]): Kernel predictions of dynamic convs for all scale levels of a single image, each is a 4D-tensor, has shape (num_params, H, W). mask_feat (Tensor): Mask prototype features of a single image extracted from the mask head, has shape (num_prototypes, H, W). score_factor_list (list[Tensor]): Score factor from all scale levels of a single image, each item has shape (num_priors * 1, H, W). mlvl_priors (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid. In all anchor-based methods, it has shape (num_priors, 4). In all anchor-free methods, it has shape (num_priors, 2) when `with_stride=True`, otherwise it still has shape (num_priors, 4). img_meta (dict): Image meta info. cfg (mmengine.Config): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, h, w). """ if score_factor_list[0] is None: # e.g. Retina, FreeAnchor, etc. with_score_factors = False else: # e.g. FCOS, PAA, ATSS, etc. with_score_factors = True cfg = self.test_cfg if cfg is None else cfg cfg = copy.deepcopy(cfg) img_shape = img_meta['img_shape'] nms_pre = cfg.get('nms_pre', -1) mlvl_bbox_preds = [] mlvl_kernels = [] mlvl_valid_priors = [] mlvl_scores = [] mlvl_labels = [] if with_score_factors: mlvl_score_factors = [] else: mlvl_score_factors = None for level_idx, (cls_score, bbox_pred, kernel_pred, score_factor, priors) in \ enumerate(zip(cls_score_list, bbox_pred_list, kernel_pred_list, score_factor_list, mlvl_priors)): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] dim = self.bbox_coder.encode_size bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, dim) if with_score_factors: score_factor = score_factor.permute(1, 2, 0).reshape(-1).sigmoid() cls_score = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) kernel_pred = kernel_pred.permute(1, 2, 0).reshape( -1, self.num_gen_params) if self.use_sigmoid_cls: scores = cls_score.sigmoid() else: # remind that we set FG labels to [0, num_class-1] # since mmdet v2.0 # BG cat_id: num_class scores = cls_score.softmax(-1)[:, :-1] # After https://github.com/open-mmlab/mmdetection/pull/6268/, # this operation keeps fewer bboxes under the same `nms_pre`. # There is no difference in performance for most models. If you # find a slight drop in performance, you can set a larger # `nms_pre` than before. score_thr = cfg.get('score_thr', 0) results = filter_scores_and_topk( scores, score_thr, nms_pre, dict( bbox_pred=bbox_pred, priors=priors, kernel_pred=kernel_pred)) scores, labels, keep_idxs, filtered_results = results bbox_pred = filtered_results['bbox_pred'] priors = filtered_results['priors'] kernel_pred = filtered_results['kernel_pred'] if with_score_factors: score_factor = score_factor[keep_idxs] mlvl_bbox_preds.append(bbox_pred) mlvl_valid_priors.append(priors) mlvl_scores.append(scores) mlvl_labels.append(labels) mlvl_kernels.append(kernel_pred) if with_score_factors: mlvl_score_factors.append(score_factor) bbox_pred = torch.cat(mlvl_bbox_preds) priors = cat_boxes(mlvl_valid_priors) bboxes = self.bbox_coder.decode( priors[..., :2], bbox_pred, max_shape=img_shape) results = InstanceData() results.bboxes = bboxes results.priors = priors results.scores = torch.cat(mlvl_scores) results.labels = torch.cat(mlvl_labels) results.kernels = torch.cat(mlvl_kernels) if with_score_factors: results.score_factors = torch.cat(mlvl_score_factors) return self._bbox_mask_post_process( results=results, mask_feat=mask_feat, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta) def _bbox_mask_post_process( self, results: InstanceData, mask_feat, cfg: ConfigType, rescale: bool = False, with_nms: bool = True, img_meta: Optional[dict] = None) -> InstanceData: """bbox and mask post-processing method. The boxes would be rescaled to the original image scale and do the nms operation. Usually `with_nms` is False is used for aug test. Args: results (:obj:`InstaceData`): Detection instance results, each item has shape (num_bboxes, ). cfg (ConfigDict): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Default to False. with_nms (bool): If True, do nms before return boxes. Default to True. img_meta (dict, optional): Image meta info. Defaults to None. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, h, w). """ stride = self.prior_generator.strides[0][0] if rescale: assert img_meta.get('scale_factor') is not None scale_factor = [1 / s for s in img_meta['scale_factor']] results.bboxes = scale_boxes(results.bboxes, scale_factor) if hasattr(results, 'score_factors'): # TODO： Add sqrt operation in order to be consistent with # the paper. score_factors = results.pop('score_factors') results.scores = results.scores * score_factors # filter small size bboxes if cfg.get('min_bbox_size', -1) >= 0: w, h = get_box_wh(results.bboxes) valid_mask = (w > cfg.min_bbox_size) & (h > cfg.min_bbox_size) if not valid_mask.all(): results = results[valid_mask] # TODO: deal with `with_nms` and `nms_cfg=None` in test_cfg assert with_nms, 'with_nms must be True for RTMDet-Ins' if results.bboxes.numel() > 0: bboxes = get_box_tensor(results.bboxes) det_bboxes, keep_idxs = batched_nms(bboxes, results.scores, results.labels, cfg.nms) results = results[keep_idxs] # some nms would reweight the score, such as softnms results.scores = det_bboxes[:, -1] results = results[:cfg.max_per_img] # process masks mask_logits = self._mask_predict_by_feat_single( mask_feat, results.kernels, results.priors) mask_logits = F.interpolate( mask_logits.unsqueeze(0), scale_factor=stride, mode='bilinear') if rescale: ori_h, ori_w = img_meta['ori_shape'][:2] mask_logits = F.interpolate( mask_logits, size=[ math.ceil(mask_logits.shape[-2] * scale_factor[0]), math.ceil(mask_logits.shape[-1] * scale_factor[1]) ], mode='bilinear', align_corners=False)[..., :ori_h, :ori_w] masks = mask_logits.sigmoid().squeeze(0) masks = masks > cfg.mask_thr_binary results.masks = masks else: h, w = img_meta['ori_shape'][:2] if rescale else img_meta[ 'img_shape'][:2] results.masks = torch.zeros( size=(results.bboxes.shape[0], h, w), dtype=torch.bool, device=results.bboxes.device) return results def parse_dynamic_params(self, flatten_kernels: Tensor) -> tuple: """split kernel head prediction to conv weight and bias.""" n_inst = flatten_kernels.size(0) n_layers = len(self.weight_nums) params_splits = list( torch.split_with_sizes( flatten_kernels, self.weight_nums + self.bias_nums, dim=1)) weight_splits = params_splits[:n_layers] bias_splits = params_splits[n_layers:] for i in range(n_layers): if i < n_layers - 1: weight_splits[i] = weight_splits[i].reshape( n_inst * self.dyconv_channels, -1, 1, 1) bias_splits[i] = bias_splits[i].reshape(n_inst * self.dyconv_channels) else: weight_splits[i] = weight_splits[i].reshape(n_inst, -1, 1, 1) bias_splits[i] = bias_splits[i].reshape(n_inst) return weight_splits, bias_splits def _mask_predict_by_feat_single(self, mask_feat: Tensor, kernels: Tensor, priors: Tensor) -> Tensor: """Generate mask logits from mask features with dynamic convs. Args: mask_feat (Tensor): Mask prototype features. Has shape (num_prototypes, H, W). kernels (Tensor): Kernel parameters for each instance. Has shape (num_instance, num_params) priors (Tensor): Center priors for each instance. Has shape (num_instance, 4). Returns: Tensor: Instance segmentation masks for each instance. Has shape (num_instance, H, W). """ num_inst = priors.shape[0] h, w = mask_feat.size()[-2:] if num_inst < 1: return torch.empty( size=(num_inst, h, w), dtype=mask_feat.dtype, device=mask_feat.device) if len(mask_feat.shape) < 4: mask_feat.unsqueeze(0) coord = self.prior_generator.single_level_grid_priors( (h, w), level_idx=0).reshape(1, -1, 2) num_inst = priors.shape[0] points = priors[:, :2].reshape(-1, 1, 2) strides = priors[:, 2:].reshape(-1, 1, 2) relative_coord = (points - coord).permute(0, 2, 1) / ( strides[..., 0].reshape(-1, 1, 1) * 8) relative_coord = relative_coord.reshape(num_inst, 2, h, w) mask_feat = torch.cat( [relative_coord, mask_feat.repeat(num_inst, 1, 1, 1)], dim=1) weights, biases = self.parse_dynamic_params(kernels) n_layers = len(weights) x = mask_feat.reshape(1, -1, h, w) for i, (weight, bias) in enumerate(zip(weights, biases)): x = F.conv2d( x, weight, bias=bias, stride=1, padding=0, groups=num_inst) if i < n_layers - 1: x = F.relu(x) x = x.reshape(num_inst, h, w) return x def loss_mask_by_feat(self, mask_feats: Tensor, flatten_kernels: Tensor, sampling_results_list: list, batch_gt_instances: InstanceList) -> Tensor: """Compute instance segmentation loss. Args: mask_feats (list[Tensor]): Mask prototype features extracted from the mask head. Has shape (N, num_prototypes, H, W) flatten_kernels (list[Tensor]): Kernels of the dynamic conv layers. Has shape (N, num_instances, num_params) sampling_results_list (list[:obj:`SamplingResults`]) Batch of assignment results. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. Returns: Tensor: The mask loss tensor. """ batch_pos_mask_logits = [] pos_gt_masks = [] for idx, (mask_feat, kernels, sampling_results, gt_instances) in enumerate( zip(mask_feats, flatten_kernels, sampling_results_list, batch_gt_instances)): pos_priors = sampling_results.pos_priors pos_inds = sampling_results.pos_inds pos_kernels = kernels[pos_inds] # n_pos, num_gen_params pos_mask_logits = self._mask_predict_by_feat_single( mask_feat, pos_kernels, pos_priors) if gt_instances.masks.numel() == 0: gt_masks = torch.empty_like(gt_instances.masks) else: gt_masks = gt_instances.masks[ sampling_results.pos_assigned_gt_inds, :] batch_pos_mask_logits.append(pos_mask_logits) pos_gt_masks.append(gt_masks) pos_gt_masks = torch.cat(pos_gt_masks, 0) batch_pos_mask_logits = torch.cat(batch_pos_mask_logits, 0) # avg_factor num_pos = batch_pos_mask_logits.shape[0] num_pos = reduce_mean(mask_feats.new_tensor([num_pos ])).clamp_(min=1).item() if batch_pos_mask_logits.shape[0] == 0: return mask_feats.sum() * 0 scale = self.prior_generator.strides[0][0] // self.mask_loss_stride # upsample pred masks batch_pos_mask_logits = F.interpolate( batch_pos_mask_logits.unsqueeze(0), scale_factor=scale, mode='bilinear', align_corners=False).squeeze(0) # downsample gt masks pos_gt_masks = pos_gt_masks[:, self.mask_loss_stride // 2::self.mask_loss_stride, self.mask_loss_stride // 2::self.mask_loss_stride] loss_mask = self.loss_mask( batch_pos_mask_logits, pos_gt_masks, weight=None, avg_factor=num_pos) return loss_mask def loss_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], kernel_preds: List[Tensor], mask_feat: Tensor, batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None): """Compute losses of the head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Decoded box for each scale level with shape (N, num_anchors * 4, H, W) in [tl_x, tl_y, br_x, br_y] format. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ num_imgs = len(batch_img_metas) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) flatten_cls_scores = torch.cat([ cls_score.permute(0, 2, 3, 1).reshape(num_imgs, -1, self.cls_out_channels) for cls_score in cls_scores ], 1) flatten_kernels = torch.cat([ kernel_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, self.num_gen_params) for kernel_pred in kernel_preds ], 1) decoded_bboxes = [] for anchor, bbox_pred in zip(anchor_list[0], bbox_preds): anchor = anchor.reshape(-1, 4) bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, 4) bbox_pred = distance2bbox(anchor, bbox_pred) decoded_bboxes.append(bbox_pred) flatten_bboxes = torch.cat(decoded_bboxes, 1) for gt_instances in batch_gt_instances: gt_instances.masks = gt_instances.masks.to_tensor( dtype=torch.bool, device=device) cls_reg_targets = self.get_targets( flatten_cls_scores, flatten_bboxes, anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (anchor_list, labels_list, label_weights_list, bbox_targets_list, assign_metrics_list, sampling_results_list) = cls_reg_targets losses_cls, losses_bbox,\ cls_avg_factors, bbox_avg_factors = multi_apply( self.loss_by_feat_single, cls_scores, decoded_bboxes, labels_list, label_weights_list, bbox_targets_list, assign_metrics_list, self.prior_generator.strides) cls_avg_factor = reduce_mean(sum(cls_avg_factors)).clamp_(min=1).item() losses_cls = list(map(lambda x: x / cls_avg_factor, losses_cls)) bbox_avg_factor = reduce_mean( sum(bbox_avg_factors)).clamp_(min=1).item() losses_bbox = list(map(lambda x: x / bbox_avg_factor, losses_bbox)) loss_mask = self.loss_mask_by_feat(mask_feat, flatten_kernels, sampling_results_list, batch_gt_instances) loss = dict( loss_cls=losses_cls, loss_bbox=losses_bbox, loss_mask=loss_mask) return loss class MaskFeatModule(BaseModule): """Mask feature head used in RTMDet-Ins. Args: in_channels (int): Number of channels in the input feature map. feat_channels (int): Number of hidden channels of the mask feature map branch. num_levels (int): The starting feature map level from RPN that will be used to predict the mask feature map. num_prototypes (int): Number of output channel of the mask feature map branch. This is the channel count of the mask feature map that to be dynamically convolved with the predicted kernel. stacked_convs (int): Number of convs in mask feature branch. act_cfg (:obj:`ConfigDict` or dict): Config dict for activation layer. Default: dict(type='ReLU', inplace=True) norm_cfg (dict): Config dict for normalization layer. Default: None. """ def __init__( self, in_channels: int, feat_channels: int = 256, stacked_convs: int = 4, num_levels: int = 3, num_prototypes: int = 8, act_cfg: ConfigType = dict(type='ReLU', inplace=True), norm_cfg: ConfigType = dict(type='BN') ) -> None: super().__init__(init_cfg=None) self.num_levels = num_levels self.fusion_conv = nn.Conv2d(num_levels * in_channels, in_channels, 1) convs = [] for i in range(stacked_convs): in_c = in_channels if i == 0 else feat_channels convs.append( ConvModule( in_c, feat_channels, 3, padding=1, act_cfg=act_cfg, norm_cfg=norm_cfg)) self.stacked_convs = nn.Sequential(*convs) self.projection = nn.Conv2d( feat_channels, num_prototypes, kernel_size=1) def forward(self, features: Tuple[Tensor, ...]) -> Tensor: # multi-level feature fusion fusion_feats = [features[0]] size = features[0].shape[-2:] for i in range(1, self.num_levels): f = F.interpolate(features[i], size=size, mode='bilinear') fusion_feats.append(f) fusion_feats = torch.cat(fusion_feats, dim=1) fusion_feats = self.fusion_conv(fusion_feats) # pred mask feats mask_features = self.stacked_convs(fusion_feats) mask_features = self.projection(mask_features) return mask_features @MODELS.register_module() class RTMDetInsSepBNHead(RTMDetInsHead): """Detection Head of RTMDet-Ins with sep-bn layers. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. share_conv (bool): Whether to share conv layers between stages. Defaults to True. norm_cfg (:obj:`ConfigDict` or dict)): Config dict for normalization layer. Defaults to dict(type='BN'). act_cfg (:obj:`ConfigDict` or dict)): Config dict for activation layer. Defaults to dict(type='SiLU', inplace=True). pred_kernel_size (int): Kernel size of prediction layer. Defaults to 1. """ def __init__(self, num_classes: int, in_channels: int, share_conv: bool = True, with_objectness: bool = False, norm_cfg: ConfigType = dict(type='BN', requires_grad=True), act_cfg: ConfigType = dict(type='SiLU', inplace=True), pred_kernel_size: int = 1, **kwargs) -> None: self.share_conv = share_conv super().__init__( num_classes, in_channels, norm_cfg=norm_cfg, act_cfg=act_cfg, pred_kernel_size=pred_kernel_size, with_objectness=with_objectness, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() self.kernel_convs = nn.ModuleList() self.rtm_cls = nn.ModuleList() self.rtm_reg = nn.ModuleList() self.rtm_kernel = nn.ModuleList() self.rtm_obj = nn.ModuleList() # calculate num dynamic parameters weight_nums, bias_nums = [], [] for i in range(self.num_dyconvs): if i == 0: weight_nums.append( (self.num_prototypes + 2) * self.dyconv_channels) bias_nums.append(self.dyconv_channels) elif i == self.num_dyconvs - 1: weight_nums.append(self.dyconv_channels) bias_nums.append(1) else: weight_nums.append(self.dyconv_channels * self.dyconv_channels) bias_nums.append(self.dyconv_channels) self.weight_nums = weight_nums self.bias_nums = bias_nums self.num_gen_params = sum(weight_nums) + sum(bias_nums) pred_pad_size = self.pred_kernel_size // 2 for n in range(len(self.prior_generator.strides)): cls_convs = nn.ModuleList() reg_convs = nn.ModuleList() kernel_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) reg_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) kernel_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) self.cls_convs.append(cls_convs) self.reg_convs.append(cls_convs) self.kernel_convs.append(kernel_convs) self.rtm_cls.append( nn.Conv2d( self.feat_channels, self.num_base_priors * self.cls_out_channels, self.pred_kernel_size, padding=pred_pad_size)) self.rtm_reg.append( nn.Conv2d( self.feat_channels, self.num_base_priors * 4, self.pred_kernel_size, padding=pred_pad_size)) self.rtm_kernel.append( nn.Conv2d( self.feat_channels, self.num_gen_params, self.pred_kernel_size, padding=pred_pad_size)) if self.with_objectness: self.rtm_obj.append( nn.Conv2d( self.feat_channels, 1, self.pred_kernel_size, padding=pred_pad_size)) if self.share_conv: for n in range(len(self.prior_generator.strides)): for i in range(self.stacked_convs): self.cls_convs[n][i].conv = self.cls_convs[0][i].conv self.reg_convs[n][i].conv = self.reg_convs[0][i].conv self.mask_head = MaskFeatModule( in_channels=self.in_channels, feat_channels=self.feat_channels, stacked_convs=4, num_levels=len(self.prior_generator.strides), num_prototypes=self.num_prototypes, act_cfg=self.act_cfg, norm_cfg=self.norm_cfg) def init_weights(self) -> None: """Initialize weights of the head.""" for m in self.modules(): if isinstance(m, nn.Conv2d): normal_init(m, mean=0, std=0.01) if is_norm(m): constant_init(m, 1) bias_cls = bias_init_with_prob(0.01) for rtm_cls, rtm_reg, rtm_kernel in zip(self.rtm_cls, self.rtm_reg, self.rtm_kernel): normal_init(rtm_cls, std=0.01, bias=bias_cls) normal_init(rtm_reg, std=0.01, bias=1) if self.with_objectness: for rtm_obj in self.rtm_obj: normal_init(rtm_obj, std=0.01, bias=bias_cls) def forward(self, feats: Tuple[Tensor, ...]) -> tuple: """Forward features from the upstream network. Args: feats (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Usually a tuple of classification scores and bbox prediction - cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * 4. - kernel_preds (list[Tensor]): Dynamic conv kernels for all scale levels, each is a 4D-tensor, the channels number is num_gen_params. - mask_feat (Tensor): Output feature of the mask head. Each is a 4D-tensor, the channels number is num_prototypes. """ mask_feat = self.mask_head(feats) cls_scores = [] bbox_preds = [] kernel_preds = [] for idx, (x, stride) in enumerate( zip(feats, self.prior_generator.strides)): cls_feat = x reg_feat = x kernel_feat = x for cls_layer in self.cls_convs[idx]: cls_feat = cls_layer(cls_feat) cls_score = self.rtm_cls[idx](cls_feat) for kernel_layer in self.kernel_convs[idx]: kernel_feat = kernel_layer(kernel_feat) kernel_pred = self.rtm_kernel[idx](kernel_feat) for reg_layer in self.reg_convs[idx]: reg_feat = reg_layer(reg_feat) if self.with_objectness: objectness = self.rtm_obj[idx](reg_feat) cls_score = inverse_sigmoid( sigmoid_geometric_mean(cls_score, objectness)) reg_dist = F.relu(self.rtm_reg[idx](reg_feat)) * stride[0] cls_scores.append(cls_score) bbox_preds.append(reg_dist) kernel_preds.append(kernel_pred) return tuple(cls_scores), tuple(bbox_preds), tuple( kernel_preds), mask_feat ================================================ FILE: mmdet/models/dense_heads/sabl_retina_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple, Union import numpy as np import torch import torch.nn as nn from mmcv.cnn import ConvModule from mmengine.config import ConfigDict from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.utils import (ConfigType, InstanceList, MultiConfig, OptConfigType, OptInstanceList) from ..task_modules.samplers import PseudoSampler from ..utils import (filter_scores_and_topk, images_to_levels, multi_apply, unmap) from .base_dense_head import BaseDenseHead from .guided_anchor_head import GuidedAnchorHead @MODELS.register_module() class SABLRetinaHead(BaseDenseHead): """Side-Aware Boundary Localization (SABL) for RetinaNet. The anchor generation, assigning and sampling in SABLRetinaHead are the same as GuidedAnchorHead for guided anchoring. Please refer to https://arxiv.org/abs/1912.04260 for more details. Args: num_classes (int): Number of classes. in_channels (int): Number of channels in the input feature map. stacked_convs (int): Number of Convs for classification and regression branches. Defaults to 4. feat_channels (int): Number of hidden channels. Defaults to 256. approx_anchor_generator (:obj:`ConfigType` or dict): Config dict for approx generator. square_anchor_generator (:obj:`ConfigDict` or dict): Config dict for square generator. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for ConvModule. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for Norm Layer. Defaults to None. bbox_coder (:obj:`ConfigDict` or dict): Config dict for bbox coder. reg_decoded_bbox (bool): If true, the regression loss would be applied directly on decoded bounding boxes, converting both the predicted boxes and regression targets to absolute coordinates format. Default False. It should be ``True`` when using ``IoULoss``, ``GIoULoss``, or ``DIoULoss`` in the bbox head. train_cfg (:obj:`ConfigDict` or dict, optional): Training config of SABLRetinaHead. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of SABLRetinaHead. loss_cls (:obj:`ConfigDict` or dict): Config of classification loss. loss_bbox_cls (:obj:`ConfigDict` or dict): Config of classification loss for bbox branch. loss_bbox_reg (:obj:`ConfigDict` or dict): Config of regression loss for bbox branch. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. """ def __init__( self, num_classes: int, in_channels: int, stacked_convs: int = 4, feat_channels: int = 256, approx_anchor_generator: ConfigType = dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), square_anchor_generator: ConfigType = dict( type='AnchorGenerator', ratios=[1.0], scales=[4], strides=[8, 16, 32, 64, 128]), conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, bbox_coder: ConfigType = dict( type='BucketingBBoxCoder', num_buckets=14, scale_factor=3.0), reg_decoded_bbox: bool = False, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, loss_cls: ConfigType = dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.5), loss_bbox_reg: ConfigType = dict( type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.5), init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='retina_cls', std=0.01, bias_prob=0.01)) ) -> None: super().__init__(init_cfg=init_cfg) self.in_channels = in_channels self.num_classes = num_classes self.feat_channels = feat_channels self.num_buckets = bbox_coder['num_buckets'] self.side_num = int(np.ceil(self.num_buckets / 2)) assert (approx_anchor_generator['octave_base_scale'] == square_anchor_generator['scales'][0]) assert (approx_anchor_generator['strides'] == square_anchor_generator['strides']) self.approx_anchor_generator = TASK_UTILS.build( approx_anchor_generator) self.square_anchor_generator = TASK_UTILS.build( square_anchor_generator) self.approxs_per_octave = ( self.approx_anchor_generator.num_base_priors[0]) # one anchor per location self.num_base_priors = self.square_anchor_generator.num_base_priors[0] self.stacked_convs = stacked_convs self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.reg_decoded_bbox = reg_decoded_bbox self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) if self.use_sigmoid_cls: self.cls_out_channels = num_classes else: self.cls_out_channels = num_classes + 1 self.bbox_coder = TASK_UTILS.build(bbox_coder) self.loss_cls = MODELS.build(loss_cls) self.loss_bbox_cls = MODELS.build(loss_bbox_cls) self.loss_bbox_reg = MODELS.build(loss_bbox_reg) self.train_cfg = train_cfg self.test_cfg = test_cfg if self.train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) # use PseudoSampler when sampling is False if 'sampler' in self.train_cfg: self.sampler = TASK_UTILS.build( self.train_cfg['sampler'], default_args=dict(context=self)) else: self.sampler = PseudoSampler(context=self) self._init_layers() def _init_layers(self) -> None: self.relu = nn.ReLU(inplace=True) self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.reg_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.retina_cls = nn.Conv2d( self.feat_channels, self.cls_out_channels, 3, padding=1) self.retina_bbox_reg = nn.Conv2d( self.feat_channels, self.side_num * 4, 3, padding=1) self.retina_bbox_cls = nn.Conv2d( self.feat_channels, self.side_num * 4, 3, padding=1) def forward_single(self, x: Tensor) -> Tuple[Tensor, Tensor]: cls_feat = x reg_feat = x for cls_conv in self.cls_convs: cls_feat = cls_conv(cls_feat) for reg_conv in self.reg_convs: reg_feat = reg_conv(reg_feat) cls_score = self.retina_cls(cls_feat) bbox_cls_pred = self.retina_bbox_cls(reg_feat) bbox_reg_pred = self.retina_bbox_reg(reg_feat) bbox_pred = (bbox_cls_pred, bbox_reg_pred) return cls_score, bbox_pred def forward(self, feats: List[Tensor]) -> Tuple[List[Tensor]]: return multi_apply(self.forward_single, feats) def get_anchors( self, featmap_sizes: List[tuple], img_metas: List[dict], device: Union[torch.device, str] = 'cuda' ) -> Tuple[List[List[Tensor]], List[List[Tensor]]]: """Get squares according to feature map sizes and guided anchors. Args: featmap_sizes (list[tuple]): Multi-level feature map sizes. img_metas (list[dict]): Image meta info. device (torch.device | str): device for returned tensors Returns: tuple: square approxs of each image """ num_imgs = len(img_metas) # since feature map sizes of all images are the same, we only compute # squares for one time multi_level_squares = self.square_anchor_generator.grid_priors( featmap_sizes, device=device) squares_list = [multi_level_squares for _ in range(num_imgs)] return squares_list def get_targets(self, approx_list: List[List[Tensor]], inside_flag_list: List[List[Tensor]], square_list: List[List[Tensor]], batch_gt_instances: InstanceList, batch_img_metas, batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs=True) -> tuple: """Compute bucketing targets. Args: approx_list (list[list[Tensor]]): Multi level approxs of each image. inside_flag_list (list[list[Tensor]]): Multi level inside flags of each image. square_list (list[list[Tensor]]): Multi level squares of each image. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. Returns: tuple: Returns a tuple containing learning targets. - labels_list (list[Tensor]): Labels of each level. - label_weights_list (list[Tensor]): Label weights of each level. - bbox_cls_targets_list (list[Tensor]): BBox cls targets of \ each level. - bbox_cls_weights_list (list[Tensor]): BBox cls weights of \ each level. - bbox_reg_targets_list (list[Tensor]): BBox reg targets of \ each level. - bbox_reg_weights_list (list[Tensor]): BBox reg weights of \ each level. - num_total_pos (int): Number of positive samples in all images. - num_total_neg (int): Number of negative samples in all images. """ num_imgs = len(batch_img_metas) assert len(approx_list) == len(inside_flag_list) == len( square_list) == num_imgs # anchor number of multi levels num_level_squares = [squares.size(0) for squares in square_list[0]] # concat all level anchors and flags to a single tensor inside_flag_flat_list = [] approx_flat_list = [] square_flat_list = [] for i in range(num_imgs): assert len(square_list[i]) == len(inside_flag_list[i]) inside_flag_flat_list.append(torch.cat(inside_flag_list[i])) approx_flat_list.append(torch.cat(approx_list[i])) square_flat_list.append(torch.cat(square_list[i])) # compute targets for each image if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None for _ in range(num_imgs)] (all_labels, all_label_weights, all_bbox_cls_targets, all_bbox_cls_weights, all_bbox_reg_targets, all_bbox_reg_weights, pos_inds_list, neg_inds_list, sampling_results_list) = multi_apply( self._get_targets_single, approx_flat_list, inside_flag_flat_list, square_flat_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs) # sampled anchors of all images avg_factor = sum( [results.avg_factor for results in sampling_results_list]) # split targets to a list w.r.t. multiple levels labels_list = images_to_levels(all_labels, num_level_squares) label_weights_list = images_to_levels(all_label_weights, num_level_squares) bbox_cls_targets_list = images_to_levels(all_bbox_cls_targets, num_level_squares) bbox_cls_weights_list = images_to_levels(all_bbox_cls_weights, num_level_squares) bbox_reg_targets_list = images_to_levels(all_bbox_reg_targets, num_level_squares) bbox_reg_weights_list = images_to_levels(all_bbox_reg_weights, num_level_squares) return (labels_list, label_weights_list, bbox_cls_targets_list, bbox_cls_weights_list, bbox_reg_targets_list, bbox_reg_weights_list, avg_factor) def _get_targets_single(self, flat_approxs: Tensor, inside_flags: Tensor, flat_squares: Tensor, gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None, unmap_outputs: bool = True) -> tuple: """Compute regression and classification targets for anchors in a single image. Args: flat_approxs (Tensor): flat approxs of a single image, shape (n, 4) inside_flags (Tensor): inside flags of a single image, shape (n, ). flat_squares (Tensor): flat squares of a single image, shape (approxs_per_octave * n, 4) gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Defaults to True. Returns: tuple: - labels_list (Tensor): Labels in a single image. - label_weights (Tensor): Label weights in a single image. - bbox_cls_targets (Tensor): BBox cls targets in a single image. - bbox_cls_weights (Tensor): BBox cls weights in a single image. - bbox_reg_targets (Tensor): BBox reg targets in a single image. - bbox_reg_weights (Tensor): BBox reg weights in a single image. - num_total_pos (int): Number of positive samples in a single \ image. - num_total_neg (int): Number of negative samples in a single \ image. - sampling_result (:obj:`SamplingResult`): Sampling result object. """ if not inside_flags.any(): raise ValueError( 'There is no valid anchor inside the image boundary. Please ' 'check the image size and anchor sizes, or set ' '``allowed_border`` to -1 to skip the condition.') # assign gt and sample anchors num_square = flat_squares.size(0) approxs = flat_approxs.view(num_square, self.approxs_per_octave, 4) approxs = approxs[inside_flags, ...] squares = flat_squares[inside_flags, :] pred_instances = InstanceData() pred_instances.priors = squares pred_instances.approxs = approxs assign_result = self.assigner.assign(pred_instances, gt_instances, gt_instances_ignore) sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) num_valid_squares = squares.shape[0] bbox_cls_targets = squares.new_zeros( (num_valid_squares, self.side_num * 4)) bbox_cls_weights = squares.new_zeros( (num_valid_squares, self.side_num * 4)) bbox_reg_targets = squares.new_zeros( (num_valid_squares, self.side_num * 4)) bbox_reg_weights = squares.new_zeros( (num_valid_squares, self.side_num * 4)) labels = squares.new_full((num_valid_squares, ), self.num_classes, dtype=torch.long) label_weights = squares.new_zeros(num_valid_squares, dtype=torch.float) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: (pos_bbox_reg_targets, pos_bbox_reg_weights, pos_bbox_cls_targets, pos_bbox_cls_weights) = self.bbox_coder.encode( sampling_result.pos_bboxes, sampling_result.pos_gt_bboxes) bbox_cls_targets[pos_inds, :] = pos_bbox_cls_targets bbox_reg_targets[pos_inds, :] = pos_bbox_reg_targets bbox_cls_weights[pos_inds, :] = pos_bbox_cls_weights bbox_reg_weights[pos_inds, :] = pos_bbox_reg_weights labels[pos_inds] = sampling_result.pos_gt_labels if self.train_cfg['pos_weight'] <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = self.train_cfg['pos_weight'] if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 # map up to original set of anchors if unmap_outputs: num_total_anchors = flat_squares.size(0) labels = unmap( labels, num_total_anchors, inside_flags, fill=self.num_classes) label_weights = unmap(label_weights, num_total_anchors, inside_flags) bbox_cls_targets = unmap(bbox_cls_targets, num_total_anchors, inside_flags) bbox_cls_weights = unmap(bbox_cls_weights, num_total_anchors, inside_flags) bbox_reg_targets = unmap(bbox_reg_targets, num_total_anchors, inside_flags) bbox_reg_weights = unmap(bbox_reg_weights, num_total_anchors, inside_flags) return (labels, label_weights, bbox_cls_targets, bbox_cls_weights, bbox_reg_targets, bbox_reg_weights, pos_inds, neg_inds, sampling_result) def loss_by_feat_single(self, cls_score: Tensor, bbox_pred: Tensor, labels: Tensor, label_weights: Tensor, bbox_cls_targets: Tensor, bbox_cls_weights: Tensor, bbox_reg_targets: Tensor, bbox_reg_weights: Tensor, avg_factor: float) -> Tuple[Tensor]: """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: cls_score (Tensor): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W). bbox_pred (Tensor): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). labels (Tensor): Labels in a single image. label_weights (Tensor): Label weights in a single level. bbox_cls_targets (Tensor): BBox cls targets in a single level. bbox_cls_weights (Tensor): BBox cls weights in a single level. bbox_reg_targets (Tensor): BBox reg targets in a single level. bbox_reg_weights (Tensor): BBox reg weights in a single level. avg_factor (int): Average factor that is used to average the loss. Returns: tuple: loss components. """ # classification loss labels = labels.reshape(-1) label_weights = label_weights.reshape(-1) cls_score = cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) loss_cls = self.loss_cls( cls_score, labels, label_weights, avg_factor=avg_factor) # regression loss bbox_cls_targets = bbox_cls_targets.reshape(-1, self.side_num * 4) bbox_cls_weights = bbox_cls_weights.reshape(-1, self.side_num * 4) bbox_reg_targets = bbox_reg_targets.reshape(-1, self.side_num * 4) bbox_reg_weights = bbox_reg_weights.reshape(-1, self.side_num * 4) (bbox_cls_pred, bbox_reg_pred) = bbox_pred bbox_cls_pred = bbox_cls_pred.permute(0, 2, 3, 1).reshape( -1, self.side_num * 4) bbox_reg_pred = bbox_reg_pred.permute(0, 2, 3, 1).reshape( -1, self.side_num * 4) loss_bbox_cls = self.loss_bbox_cls( bbox_cls_pred, bbox_cls_targets.long(), bbox_cls_weights, avg_factor=avg_factor * 4 * self.side_num) loss_bbox_reg = self.loss_bbox_reg( bbox_reg_pred, bbox_reg_targets, bbox_reg_weights, avg_factor=avg_factor * 4 * self.bbox_coder.offset_topk) return loss_cls, loss_bbox_cls, loss_bbox_reg def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level has shape (N, num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.approx_anchor_generator.num_levels device = cls_scores[0].device # get sampled approxes approxs_list, inside_flag_list = GuidedAnchorHead.get_sampled_approxs( self, featmap_sizes, batch_img_metas, device=device) square_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( approxs_list, inside_flag_list, square_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (labels_list, label_weights_list, bbox_cls_targets_list, bbox_cls_weights_list, bbox_reg_targets_list, bbox_reg_weights_list, avg_factor) = cls_reg_targets losses_cls, losses_bbox_cls, losses_bbox_reg = multi_apply( self.loss_by_feat_single, cls_scores, bbox_preds, labels_list, label_weights_list, bbox_cls_targets_list, bbox_cls_weights_list, bbox_reg_targets_list, bbox_reg_weights_list, avg_factor=avg_factor) return dict( loss_cls=losses_cls, loss_bbox_cls=losses_bbox_cls, loss_bbox_reg=losses_bbox_reg) def predict_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_img_metas: List[dict], cfg: Optional[ConfigDict] = None, rescale: bool = False, with_nms: bool = True) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. Note: When score_factors is not None, the cls_scores are usually multiplied by it then obtain the real score used in NMS, such as CenterNess in FCOS, IoU branch in ATSS. Args: cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * 4, H, W). batch_img_metas (list[dict], Optional): Batch image meta info. cfg (:obj:`ConfigDict`, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(cls_scores) == len(bbox_preds) num_levels = len(cls_scores) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] device = cls_scores[0].device mlvl_anchors = self.get_anchors( featmap_sizes, batch_img_metas, device=device) result_list = [] for img_id in range(len(batch_img_metas)): cls_score_list = [ cls_scores[i][img_id].detach() for i in range(num_levels) ] bbox_cls_pred_list = [ bbox_preds[i][0][img_id].detach() for i in range(num_levels) ] bbox_reg_pred_list = [ bbox_preds[i][1][img_id].detach() for i in range(num_levels) ] proposals = self._predict_by_feat_single( cls_scores=cls_score_list, bbox_cls_preds=bbox_cls_pred_list, bbox_reg_preds=bbox_reg_pred_list, mlvl_anchors=mlvl_anchors[img_id], img_meta=batch_img_metas[img_id], cfg=cfg, rescale=rescale, with_nms=with_nms) result_list.append(proposals) return result_list def _predict_by_feat_single(self, cls_scores: List[Tensor], bbox_cls_preds: List[Tensor], bbox_reg_preds: List[Tensor], mlvl_anchors: List[Tensor], img_meta: dict, cfg: ConfigDict, rescale: bool = False, with_nms: bool = True) -> InstanceData: cfg = self.test_cfg if cfg is None else cfg nms_pre = cfg.get('nms_pre', -1) mlvl_bboxes = [] mlvl_scores = [] mlvl_confids = [] mlvl_labels = [] assert len(cls_scores) == len(bbox_cls_preds) == len( bbox_reg_preds) == len(mlvl_anchors) for cls_score, bbox_cls_pred, bbox_reg_pred, anchors in zip( cls_scores, bbox_cls_preds, bbox_reg_preds, mlvl_anchors): assert cls_score.size()[-2:] == bbox_cls_pred.size( )[-2:] == bbox_reg_pred.size()[-2::] cls_score = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) if self.use_sigmoid_cls: scores = cls_score.sigmoid() else: scores = cls_score.softmax(-1)[:, :-1] bbox_cls_pred = bbox_cls_pred.permute(1, 2, 0).reshape( -1, self.side_num * 4) bbox_reg_pred = bbox_reg_pred.permute(1, 2, 0).reshape( -1, self.side_num * 4) # After https://github.com/open-mmlab/mmdetection/pull/6268/, # this operation keeps fewer bboxes under the same `nms_pre`. # There is no difference in performance for most models. If you # find a slight drop in performance, you can set a larger # `nms_pre` than before. results = filter_scores_and_topk( scores, cfg.score_thr, nms_pre, dict( anchors=anchors, bbox_cls_pred=bbox_cls_pred, bbox_reg_pred=bbox_reg_pred)) scores, labels, _, filtered_results = results anchors = filtered_results['anchors'] bbox_cls_pred = filtered_results['bbox_cls_pred'] bbox_reg_pred = filtered_results['bbox_reg_pred'] bbox_preds = [ bbox_cls_pred.contiguous(), bbox_reg_pred.contiguous() ] bboxes, confids = self.bbox_coder.decode( anchors.contiguous(), bbox_preds, max_shape=img_meta['img_shape']) mlvl_bboxes.append(bboxes) mlvl_scores.append(scores) mlvl_confids.append(confids) mlvl_labels.append(labels) results = InstanceData() results.bboxes = torch.cat(mlvl_bboxes) results.scores = torch.cat(mlvl_scores) results.score_factors = torch.cat(mlvl_confids) results.labels = torch.cat(mlvl_labels) return self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta) ================================================ FILE: mmdet/models/dense_heads/solo_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import mmcv import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.utils.misc import floordiv from mmdet.registry import MODELS from mmdet.utils import ConfigType, InstanceList, MultiConfig, OptConfigType from ..layers import mask_matrix_nms from ..utils import center_of_mass, generate_coordinate, multi_apply from .base_mask_head import BaseMaskHead @MODELS.register_module() class SOLOHead(BaseMaskHead): """SOLO mask head used in `SOLO: Segmenting Objects by Locations. `_ Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. feat_channels (int): Number of hidden channels. Used in child classes. Defaults to 256. stacked_convs (int): Number of stacking convs of the head. Defaults to 4. strides (tuple): Downsample factor of each feature map. scale_ranges (tuple[tuple[int, int]]): Area range of multiple level masks, in the format [(min1, max1), (min2, max2), ...]. A range of (16, 64) means the area range between (16, 64). pos_scale (float): Constant scale factor to control the center region. num_grids (list[int]): Divided image into a uniform grids, each feature map has a different grid value. The number of output channels is grid ** 2. Defaults to [40, 36, 24, 16, 12]. cls_down_index (int): The index of downsample operation in classification branch. Defaults to 0. loss_mask (dict): Config of mask loss. loss_cls (dict): Config of classification loss. norm_cfg (dict): Dictionary to construct and config norm layer. Defaults to norm_cfg=dict(type='GN', num_groups=32, requires_grad=True). train_cfg (dict): Training config of head. test_cfg (dict): Testing config of head. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__( self, num_classes: int, in_channels: int, feat_channels: int = 256, stacked_convs: int = 4, strides: tuple = (4, 8, 16, 32, 64), scale_ranges: tuple = ((8, 32), (16, 64), (32, 128), (64, 256), (128, 512)), pos_scale: float = 0.2, num_grids: list = [40, 36, 24, 16, 12], cls_down_index: int = 0, loss_mask: ConfigType = dict( type='DiceLoss', use_sigmoid=True, loss_weight=3.0), loss_cls: ConfigType = dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), norm_cfg: ConfigType = dict( type='GN', num_groups=32, requires_grad=True), train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: MultiConfig = [ dict(type='Normal', layer='Conv2d', std=0.01), dict( type='Normal', std=0.01, bias_prob=0.01, override=dict(name='conv_mask_list')), dict( type='Normal', std=0.01, bias_prob=0.01, override=dict(name='conv_cls')) ] ) -> None: super().__init__(init_cfg=init_cfg) self.num_classes = num_classes self.cls_out_channels = self.num_classes self.in_channels = in_channels self.feat_channels = feat_channels self.stacked_convs = stacked_convs self.strides = strides self.num_grids = num_grids # number of FPN feats self.num_levels = len(strides) assert self.num_levels == len(scale_ranges) == len(num_grids) self.scale_ranges = scale_ranges self.pos_scale = pos_scale self.cls_down_index = cls_down_index self.loss_cls = MODELS.build(loss_cls) self.loss_mask = MODELS.build(loss_mask) self.norm_cfg = norm_cfg self.init_cfg = init_cfg self.train_cfg = train_cfg self.test_cfg = test_cfg self._init_layers() def _init_layers(self) -> None: """Initialize layers of the head.""" self.mask_convs = nn.ModuleList() self.cls_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels + 2 if i == 0 else self.feat_channels self.mask_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, norm_cfg=self.norm_cfg)) chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, norm_cfg=self.norm_cfg)) self.conv_mask_list = nn.ModuleList() for num_grid in self.num_grids: self.conv_mask_list.append( nn.Conv2d(self.feat_channels, num_grid**2, 1)) self.conv_cls = nn.Conv2d( self.feat_channels, self.cls_out_channels, 3, padding=1) def resize_feats(self, x: Tuple[Tensor]) -> List[Tensor]: """Downsample the first feat and upsample last feat in feats. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: list[Tensor]: Features after resizing, each is a 4D-tensor. """ out = [] for i in range(len(x)): if i == 0: out.append( F.interpolate(x[0], scale_factor=0.5, mode='bilinear')) elif i == len(x) - 1: out.append( F.interpolate( x[i], size=x[i - 1].shape[-2:], mode='bilinear')) else: out.append(x[i]) return out def forward(self, x: Tuple[Tensor]) -> tuple: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of classification scores and mask prediction. - mlvl_mask_preds (list[Tensor]): Multi-level mask prediction. Each element in the list has shape (batch_size, num_grids**2 ,h ,w). - mlvl_cls_preds (list[Tensor]): Multi-level scores. Each element in the list has shape (batch_size, num_classes, num_grids ,num_grids). """ assert len(x) == self.num_levels feats = self.resize_feats(x) mlvl_mask_preds = [] mlvl_cls_preds = [] for i in range(self.num_levels): x = feats[i] mask_feat = x cls_feat = x # generate and concat the coordinate coord_feat = generate_coordinate(mask_feat.size(), mask_feat.device) mask_feat = torch.cat([mask_feat, coord_feat], 1) for mask_layer in (self.mask_convs): mask_feat = mask_layer(mask_feat) mask_feat = F.interpolate( mask_feat, scale_factor=2, mode='bilinear') mask_preds = self.conv_mask_list[i](mask_feat) # cls branch for j, cls_layer in enumerate(self.cls_convs): if j == self.cls_down_index: num_grid = self.num_grids[i] cls_feat = F.interpolate( cls_feat, size=num_grid, mode='bilinear') cls_feat = cls_layer(cls_feat) cls_pred = self.conv_cls(cls_feat) if not self.training: feat_wh = feats[0].size()[-2:] upsampled_size = (feat_wh[0] * 2, feat_wh[1] * 2) mask_preds = F.interpolate( mask_preds.sigmoid(), size=upsampled_size, mode='bilinear') cls_pred = cls_pred.sigmoid() # get local maximum local_max = F.max_pool2d(cls_pred, 2, stride=1, padding=1) keep_mask = local_max[:, :, :-1, :-1] == cls_pred cls_pred = cls_pred * keep_mask mlvl_mask_preds.append(mask_preds) mlvl_cls_preds.append(cls_pred) return mlvl_mask_preds, mlvl_cls_preds def loss_by_feat(self, mlvl_mask_preds: List[Tensor], mlvl_cls_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], **kwargs) -> dict: """Calculate the loss based on the features extracted by the mask head. Args: mlvl_mask_preds (list[Tensor]): Multi-level mask prediction. Each element in the list has shape (batch_size, num_grids**2 ,h ,w). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``masks``, and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of multiple images. Returns: dict[str, Tensor]: A dictionary of loss components. """ num_levels = self.num_levels num_imgs = len(batch_img_metas) featmap_sizes = [featmap.size()[-2:] for featmap in mlvl_mask_preds] # `BoolTensor` in `pos_masks` represent # whether the corresponding point is # positive pos_mask_targets, labels, pos_masks = multi_apply( self._get_targets_single, batch_gt_instances, featmap_sizes=featmap_sizes) # change from the outside list meaning multi images # to the outside list meaning multi levels mlvl_pos_mask_targets = [[] for _ in range(num_levels)] mlvl_pos_mask_preds = [[] for _ in range(num_levels)] mlvl_pos_masks = [[] for _ in range(num_levels)] mlvl_labels = [[] for _ in range(num_levels)] for img_id in range(num_imgs): assert num_levels == len(pos_mask_targets[img_id]) for lvl in range(num_levels): mlvl_pos_mask_targets[lvl].append( pos_mask_targets[img_id][lvl]) mlvl_pos_mask_preds[lvl].append( mlvl_mask_preds[lvl][img_id, pos_masks[img_id][lvl], ...]) mlvl_pos_masks[lvl].append(pos_masks[img_id][lvl].flatten()) mlvl_labels[lvl].append(labels[img_id][lvl].flatten()) # cat multiple image temp_mlvl_cls_preds = [] for lvl in range(num_levels): mlvl_pos_mask_targets[lvl] = torch.cat( mlvl_pos_mask_targets[lvl], dim=0) mlvl_pos_mask_preds[lvl] = torch.cat( mlvl_pos_mask_preds[lvl], dim=0) mlvl_pos_masks[lvl] = torch.cat(mlvl_pos_masks[lvl], dim=0) mlvl_labels[lvl] = torch.cat(mlvl_labels[lvl], dim=0) temp_mlvl_cls_preds.append(mlvl_cls_preds[lvl].permute( 0, 2, 3, 1).reshape(-1, self.cls_out_channels)) num_pos = sum(item.sum() for item in mlvl_pos_masks) # dice loss loss_mask = [] for pred, target in zip(mlvl_pos_mask_preds, mlvl_pos_mask_targets): if pred.size()[0] == 0: loss_mask.append(pred.sum().unsqueeze(0)) continue loss_mask.append( self.loss_mask(pred, target, reduction_override='none')) if num_pos > 0: loss_mask = torch.cat(loss_mask).sum() / num_pos else: loss_mask = torch.cat(loss_mask).mean() flatten_labels = torch.cat(mlvl_labels) flatten_cls_preds = torch.cat(temp_mlvl_cls_preds) loss_cls = self.loss_cls( flatten_cls_preds, flatten_labels, avg_factor=num_pos + 1) return dict(loss_mask=loss_mask, loss_cls=loss_cls) def _get_targets_single(self, gt_instances: InstanceData, featmap_sizes: Optional[list] = None) -> tuple: """Compute targets for predictions of single image. Args: gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes``, ``labels``, and ``masks`` attributes. featmap_sizes (list[:obj:`torch.size`]): Size of each feature map from feature pyramid, each element means (feat_h, feat_w). Defaults to None. Returns: Tuple: Usually returns a tuple containing targets for predictions. - mlvl_pos_mask_targets (list[Tensor]): Each element represent the binary mask targets for positive points in this level, has shape (num_pos, out_h, out_w). - mlvl_labels (list[Tensor]): Each element is classification labels for all points in this level, has shape (num_grid, num_grid). - mlvl_pos_masks (list[Tensor]): Each element is a `BoolTensor` to represent whether the corresponding point in single level is positive, has shape (num_grid **2). """ gt_labels = gt_instances.labels device = gt_labels.device gt_bboxes = gt_instances.bboxes gt_areas = torch.sqrt((gt_bboxes[:, 2] - gt_bboxes[:, 0]) * (gt_bboxes[:, 3] - gt_bboxes[:, 1])) gt_masks = gt_instances.masks.to_tensor( dtype=torch.bool, device=device) mlvl_pos_mask_targets = [] mlvl_labels = [] mlvl_pos_masks = [] for (lower_bound, upper_bound), stride, featmap_size, num_grid \ in zip(self.scale_ranges, self.strides, featmap_sizes, self.num_grids): mask_target = torch.zeros( [num_grid**2, featmap_size[0], featmap_size[1]], dtype=torch.uint8, device=device) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes labels = torch.zeros([num_grid, num_grid], dtype=torch.int64, device=device) + self.num_classes pos_mask = torch.zeros([num_grid**2], dtype=torch.bool, device=device) gt_inds = ((gt_areas >= lower_bound) & (gt_areas <= upper_bound)).nonzero().flatten() if len(gt_inds) == 0: mlvl_pos_mask_targets.append( mask_target.new_zeros(0, featmap_size[0], featmap_size[1])) mlvl_labels.append(labels) mlvl_pos_masks.append(pos_mask) continue hit_gt_bboxes = gt_bboxes[gt_inds] hit_gt_labels = gt_labels[gt_inds] hit_gt_masks = gt_masks[gt_inds, ...] pos_w_ranges = 0.5 * (hit_gt_bboxes[:, 2] - hit_gt_bboxes[:, 0]) * self.pos_scale pos_h_ranges = 0.5 * (hit_gt_bboxes[:, 3] - hit_gt_bboxes[:, 1]) * self.pos_scale # Make sure hit_gt_masks has a value valid_mask_flags = hit_gt_masks.sum(dim=-1).sum(dim=-1) > 0 output_stride = stride / 2 for gt_mask, gt_label, pos_h_range, pos_w_range, \ valid_mask_flag in \ zip(hit_gt_masks, hit_gt_labels, pos_h_ranges, pos_w_ranges, valid_mask_flags): if not valid_mask_flag: continue upsampled_size = (featmap_sizes[0][0] * 4, featmap_sizes[0][1] * 4) center_h, center_w = center_of_mass(gt_mask) coord_w = int( floordiv((center_w / upsampled_size[1]), (1. / num_grid), rounding_mode='trunc')) coord_h = int( floordiv((center_h / upsampled_size[0]), (1. / num_grid), rounding_mode='trunc')) # left, top, right, down top_box = max( 0, int( floordiv( (center_h - pos_h_range) / upsampled_size[0], (1. / num_grid), rounding_mode='trunc'))) down_box = min( num_grid - 1, int( floordiv( (center_h + pos_h_range) / upsampled_size[0], (1. / num_grid), rounding_mode='trunc'))) left_box = max( 0, int( floordiv( (center_w - pos_w_range) / upsampled_size[1], (1. / num_grid), rounding_mode='trunc'))) right_box = min( num_grid - 1, int( floordiv( (center_w + pos_w_range) / upsampled_size[1], (1. / num_grid), rounding_mode='trunc'))) top = max(top_box, coord_h - 1) down = min(down_box, coord_h + 1) left = max(coord_w - 1, left_box) right = min(right_box, coord_w + 1) labels[top:(down + 1), left:(right + 1)] = gt_label # ins gt_mask = np.uint8(gt_mask.cpu().numpy()) # Follow the original implementation, F.interpolate is # different from cv2 and opencv gt_mask = mmcv.imrescale(gt_mask, scale=1. / output_stride) gt_mask = torch.from_numpy(gt_mask).to(device=device) for i in range(top, down + 1): for j in range(left, right + 1): index = int(i * num_grid + j) mask_target[index, :gt_mask.shape[0], :gt_mask. shape[1]] = gt_mask pos_mask[index] = True mlvl_pos_mask_targets.append(mask_target[pos_mask]) mlvl_labels.append(labels) mlvl_pos_masks.append(pos_mask) return mlvl_pos_mask_targets, mlvl_labels, mlvl_pos_masks def predict_by_feat(self, mlvl_mask_preds: List[Tensor], mlvl_cls_scores: List[Tensor], batch_img_metas: List[dict], **kwargs) -> InstanceList: """Transform a batch of output features extracted from the head into mask results. Args: mlvl_mask_preds (list[Tensor]): Multi-level mask prediction. Each element in the list has shape (batch_size, num_grids**2 ,h ,w). mlvl_cls_scores (list[Tensor]): Multi-level scores. Each element in the list has shape (batch_size, num_classes, num_grids ,num_grids). batch_img_metas (list[dict]): Meta information of all images. Returns: list[:obj:`InstanceData`]: Processed results of multiple images.Each :obj:`InstanceData` usually contains following keys. - scores (Tensor): Classification scores, has shape (num_instance,). - labels (Tensor): Has shape (num_instances,). - masks (Tensor): Processed mask results, has shape (num_instances, h, w). """ mlvl_cls_scores = [ item.permute(0, 2, 3, 1) for item in mlvl_cls_scores ] assert len(mlvl_mask_preds) == len(mlvl_cls_scores) num_levels = len(mlvl_cls_scores) results_list = [] for img_id in range(len(batch_img_metas)): cls_pred_list = [ mlvl_cls_scores[lvl][img_id].view(-1, self.cls_out_channels) for lvl in range(num_levels) ] mask_pred_list = [ mlvl_mask_preds[lvl][img_id] for lvl in range(num_levels) ] cls_pred_list = torch.cat(cls_pred_list, dim=0) mask_pred_list = torch.cat(mask_pred_list, dim=0) img_meta = batch_img_metas[img_id] results = self._predict_by_feat_single( cls_pred_list, mask_pred_list, img_meta=img_meta) results_list.append(results) return results_list def _predict_by_feat_single(self, cls_scores: Tensor, mask_preds: Tensor, img_meta: dict, cfg: OptConfigType = None) -> InstanceData: """Transform a single image's features extracted from the head into mask results. Args: cls_scores (Tensor): Classification score of all points in single image, has shape (num_points, num_classes). mask_preds (Tensor): Mask prediction of all points in single image, has shape (num_points, feat_h, feat_w). img_meta (dict): Meta information of corresponding image. cfg (dict, optional): Config used in test phase. Defaults to None. Returns: :obj:`InstanceData`: Processed results of single image. it usually contains following keys. - scores (Tensor): Classification scores, has shape (num_instance,). - labels (Tensor): Has shape (num_instances,). - masks (Tensor): Processed mask results, has shape (num_instances, h, w). """ def empty_results(cls_scores, ori_shape): """Generate a empty results.""" results = InstanceData() results.scores = cls_scores.new_ones(0) results.masks = cls_scores.new_zeros(0, *ori_shape) results.labels = cls_scores.new_ones(0) results.bboxes = cls_scores.new_zeros(0, 4) return results cfg = self.test_cfg if cfg is None else cfg assert len(cls_scores) == len(mask_preds) featmap_size = mask_preds.size()[-2:] h, w = img_meta['img_shape'][:2] upsampled_size = (featmap_size[0] * 4, featmap_size[1] * 4) score_mask = (cls_scores > cfg.score_thr) cls_scores = cls_scores[score_mask] if len(cls_scores) == 0: return empty_results(cls_scores, img_meta['ori_shape'][:2]) inds = score_mask.nonzero() cls_labels = inds[:, 1] # Filter the mask mask with an area is smaller than # stride of corresponding feature level lvl_interval = cls_labels.new_tensor(self.num_grids).pow(2).cumsum(0) strides = cls_scores.new_ones(lvl_interval[-1]) strides[:lvl_interval[0]] *= self.strides[0] for lvl in range(1, self.num_levels): strides[lvl_interval[lvl - 1]:lvl_interval[lvl]] *= self.strides[lvl] strides = strides[inds[:, 0]] mask_preds = mask_preds[inds[:, 0]] masks = mask_preds > cfg.mask_thr sum_masks = masks.sum((1, 2)).float() keep = sum_masks > strides if keep.sum() == 0: return empty_results(cls_scores, img_meta['ori_shape'][:2]) masks = masks[keep] mask_preds = mask_preds[keep] sum_masks = sum_masks[keep] cls_scores = cls_scores[keep] cls_labels = cls_labels[keep] # maskness. mask_scores = (mask_preds * masks).sum((1, 2)) / sum_masks cls_scores *= mask_scores scores, labels, _, keep_inds = mask_matrix_nms( masks, cls_labels, cls_scores, mask_area=sum_masks, nms_pre=cfg.nms_pre, max_num=cfg.max_per_img, kernel=cfg.kernel, sigma=cfg.sigma, filter_thr=cfg.filter_thr) # mask_matrix_nms may return an empty Tensor if len(keep_inds) == 0: return empty_results(cls_scores, img_meta['ori_shape'][:2]) mask_preds = mask_preds[keep_inds] mask_preds = F.interpolate( mask_preds.unsqueeze(0), size=upsampled_size, mode='bilinear')[:, :, :h, :w] mask_preds = F.interpolate( mask_preds, size=img_meta['ori_shape'][:2], mode='bilinear').squeeze(0) masks = mask_preds > cfg.mask_thr results = InstanceData() results.masks = masks results.labels = labels results.scores = scores # create an empty bbox in InstanceData to avoid bugs when # calculating metrics. results.bboxes = results.scores.new_zeros(len(scores), 4) return results @MODELS.register_module() class DecoupledSOLOHead(SOLOHead): """Decoupled SOLO mask head used in `SOLO: Segmenting Objects by Locations. `_ Args: init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, *args, init_cfg: MultiConfig = [ dict(type='Normal', layer='Conv2d', std=0.01), dict( type='Normal', std=0.01, bias_prob=0.01, override=dict(name='conv_mask_list_x')), dict( type='Normal', std=0.01, bias_prob=0.01, override=dict(name='conv_mask_list_y')), dict( type='Normal', std=0.01, bias_prob=0.01, override=dict(name='conv_cls')) ], **kwargs) -> None: super().__init__(*args, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: self.mask_convs_x = nn.ModuleList() self.mask_convs_y = nn.ModuleList() self.cls_convs = nn.ModuleList() for i in range(self.stacked_convs): chn = self.in_channels + 1 if i == 0 else self.feat_channels self.mask_convs_x.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, norm_cfg=self.norm_cfg)) self.mask_convs_y.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, norm_cfg=self.norm_cfg)) chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, norm_cfg=self.norm_cfg)) self.conv_mask_list_x = nn.ModuleList() self.conv_mask_list_y = nn.ModuleList() for num_grid in self.num_grids: self.conv_mask_list_x.append( nn.Conv2d(self.feat_channels, num_grid, 3, padding=1)) self.conv_mask_list_y.append( nn.Conv2d(self.feat_channels, num_grid, 3, padding=1)) self.conv_cls = nn.Conv2d( self.feat_channels, self.cls_out_channels, 3, padding=1) def forward(self, x: Tuple[Tensor]) -> Tuple: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of classification scores and mask prediction. - mlvl_mask_preds_x (list[Tensor]): Multi-level mask prediction from x branch. Each element in the list has shape (batch_size, num_grids ,h ,w). - mlvl_mask_preds_y (list[Tensor]): Multi-level mask prediction from y branch. Each element in the list has shape (batch_size, num_grids ,h ,w). - mlvl_cls_preds (list[Tensor]): Multi-level scores. Each element in the list has shape (batch_size, num_classes, num_grids ,num_grids). """ assert len(x) == self.num_levels feats = self.resize_feats(x) mask_preds_x = [] mask_preds_y = [] cls_preds = [] for i in range(self.num_levels): x = feats[i] mask_feat = x cls_feat = x # generate and concat the coordinate coord_feat = generate_coordinate(mask_feat.size(), mask_feat.device) mask_feat_x = torch.cat([mask_feat, coord_feat[:, 0:1, ...]], 1) mask_feat_y = torch.cat([mask_feat, coord_feat[:, 1:2, ...]], 1) for mask_layer_x, mask_layer_y in \ zip(self.mask_convs_x, self.mask_convs_y): mask_feat_x = mask_layer_x(mask_feat_x) mask_feat_y = mask_layer_y(mask_feat_y) mask_feat_x = F.interpolate( mask_feat_x, scale_factor=2, mode='bilinear') mask_feat_y = F.interpolate( mask_feat_y, scale_factor=2, mode='bilinear') mask_pred_x = self.conv_mask_list_x[i](mask_feat_x) mask_pred_y = self.conv_mask_list_y[i](mask_feat_y) # cls branch for j, cls_layer in enumerate(self.cls_convs): if j == self.cls_down_index: num_grid = self.num_grids[i] cls_feat = F.interpolate( cls_feat, size=num_grid, mode='bilinear') cls_feat = cls_layer(cls_feat) cls_pred = self.conv_cls(cls_feat) if not self.training: feat_wh = feats[0].size()[-2:] upsampled_size = (feat_wh[0] * 2, feat_wh[1] * 2) mask_pred_x = F.interpolate( mask_pred_x.sigmoid(), size=upsampled_size, mode='bilinear') mask_pred_y = F.interpolate( mask_pred_y.sigmoid(), size=upsampled_size, mode='bilinear') cls_pred = cls_pred.sigmoid() # get local maximum local_max = F.max_pool2d(cls_pred, 2, stride=1, padding=1) keep_mask = local_max[:, :, :-1, :-1] == cls_pred cls_pred = cls_pred * keep_mask mask_preds_x.append(mask_pred_x) mask_preds_y.append(mask_pred_y) cls_preds.append(cls_pred) return mask_preds_x, mask_preds_y, cls_preds def loss_by_feat(self, mlvl_mask_preds_x: List[Tensor], mlvl_mask_preds_y: List[Tensor], mlvl_cls_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], **kwargs) -> dict: """Calculate the loss based on the features extracted by the mask head. Args: mlvl_mask_preds_x (list[Tensor]): Multi-level mask prediction from x branch. Each element in the list has shape (batch_size, num_grids ,h ,w). mlvl_mask_preds_y (list[Tensor]): Multi-level mask prediction from y branch. Each element in the list has shape (batch_size, num_grids ,h ,w). mlvl_cls_preds (list[Tensor]): Multi-level scores. Each element in the list has shape (batch_size, num_classes, num_grids ,num_grids). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``masks``, and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of multiple images. Returns: dict[str, Tensor]: A dictionary of loss components. """ num_levels = self.num_levels num_imgs = len(batch_img_metas) featmap_sizes = [featmap.size()[-2:] for featmap in mlvl_mask_preds_x] pos_mask_targets, labels, xy_pos_indexes = multi_apply( self._get_targets_single, batch_gt_instances, featmap_sizes=featmap_sizes) # change from the outside list meaning multi images # to the outside list meaning multi levels mlvl_pos_mask_targets = [[] for _ in range(num_levels)] mlvl_pos_mask_preds_x = [[] for _ in range(num_levels)] mlvl_pos_mask_preds_y = [[] for _ in range(num_levels)] mlvl_labels = [[] for _ in range(num_levels)] for img_id in range(num_imgs): for lvl in range(num_levels): mlvl_pos_mask_targets[lvl].append( pos_mask_targets[img_id][lvl]) mlvl_pos_mask_preds_x[lvl].append( mlvl_mask_preds_x[lvl][img_id, xy_pos_indexes[img_id][lvl][:, 1]]) mlvl_pos_mask_preds_y[lvl].append( mlvl_mask_preds_y[lvl][img_id, xy_pos_indexes[img_id][lvl][:, 0]]) mlvl_labels[lvl].append(labels[img_id][lvl].flatten()) # cat multiple image temp_mlvl_cls_preds = [] for lvl in range(num_levels): mlvl_pos_mask_targets[lvl] = torch.cat( mlvl_pos_mask_targets[lvl], dim=0) mlvl_pos_mask_preds_x[lvl] = torch.cat( mlvl_pos_mask_preds_x[lvl], dim=0) mlvl_pos_mask_preds_y[lvl] = torch.cat( mlvl_pos_mask_preds_y[lvl], dim=0) mlvl_labels[lvl] = torch.cat(mlvl_labels[lvl], dim=0) temp_mlvl_cls_preds.append(mlvl_cls_preds[lvl].permute( 0, 2, 3, 1).reshape(-1, self.cls_out_channels)) num_pos = 0. # dice loss loss_mask = [] for pred_x, pred_y, target in \ zip(mlvl_pos_mask_preds_x, mlvl_pos_mask_preds_y, mlvl_pos_mask_targets): num_masks = pred_x.size(0) if num_masks == 0: # make sure can get grad loss_mask.append((pred_x.sum() + pred_y.sum()).unsqueeze(0)) continue num_pos += num_masks pred_mask = pred_y.sigmoid() * pred_x.sigmoid() loss_mask.append( self.loss_mask(pred_mask, target, reduction_override='none')) if num_pos > 0: loss_mask = torch.cat(loss_mask).sum() / num_pos else: loss_mask = torch.cat(loss_mask).mean() # cate flatten_labels = torch.cat(mlvl_labels) flatten_cls_preds = torch.cat(temp_mlvl_cls_preds) loss_cls = self.loss_cls( flatten_cls_preds, flatten_labels, avg_factor=num_pos + 1) return dict(loss_mask=loss_mask, loss_cls=loss_cls) def _get_targets_single(self, gt_instances: InstanceData, featmap_sizes: Optional[list] = None) -> tuple: """Compute targets for predictions of single image. Args: gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes``, ``labels``, and ``masks`` attributes. featmap_sizes (list[:obj:`torch.size`]): Size of each feature map from feature pyramid, each element means (feat_h, feat_w). Defaults to None. Returns: Tuple: Usually returns a tuple containing targets for predictions. - mlvl_pos_mask_targets (list[Tensor]): Each element represent the binary mask targets for positive points in this level, has shape (num_pos, out_h, out_w). - mlvl_labels (list[Tensor]): Each element is classification labels for all points in this level, has shape (num_grid, num_grid). - mlvl_xy_pos_indexes (list[Tensor]): Each element in the list contains the index of positive samples in corresponding level, has shape (num_pos, 2), last dimension 2 present (index_x, index_y). """ mlvl_pos_mask_targets, mlvl_labels, mlvl_pos_masks = \ super()._get_targets_single(gt_instances, featmap_sizes=featmap_sizes) mlvl_xy_pos_indexes = [(item - self.num_classes).nonzero() for item in mlvl_labels] return mlvl_pos_mask_targets, mlvl_labels, mlvl_xy_pos_indexes def predict_by_feat(self, mlvl_mask_preds_x: List[Tensor], mlvl_mask_preds_y: List[Tensor], mlvl_cls_scores: List[Tensor], batch_img_metas: List[dict], **kwargs) -> InstanceList: """Transform a batch of output features extracted from the head into mask results. Args: mlvl_mask_preds_x (list[Tensor]): Multi-level mask prediction from x branch. Each element in the list has shape (batch_size, num_grids ,h ,w). mlvl_mask_preds_y (list[Tensor]): Multi-level mask prediction from y branch. Each element in the list has shape (batch_size, num_grids ,h ,w). mlvl_cls_scores (list[Tensor]): Multi-level scores. Each element in the list has shape (batch_size, num_classes ,num_grids ,num_grids). batch_img_metas (list[dict]): Meta information of all images. Returns: list[:obj:`InstanceData`]: Processed results of multiple images.Each :obj:`InstanceData` usually contains following keys. - scores (Tensor): Classification scores, has shape (num_instance,). - labels (Tensor): Has shape (num_instances,). - masks (Tensor): Processed mask results, has shape (num_instances, h, w). """ mlvl_cls_scores = [ item.permute(0, 2, 3, 1) for item in mlvl_cls_scores ] assert len(mlvl_mask_preds_x) == len(mlvl_cls_scores) num_levels = len(mlvl_cls_scores) results_list = [] for img_id in range(len(batch_img_metas)): cls_pred_list = [ mlvl_cls_scores[i][img_id].view( -1, self.cls_out_channels).detach() for i in range(num_levels) ] mask_pred_list_x = [ mlvl_mask_preds_x[i][img_id] for i in range(num_levels) ] mask_pred_list_y = [ mlvl_mask_preds_y[i][img_id] for i in range(num_levels) ] cls_pred_list = torch.cat(cls_pred_list, dim=0) mask_pred_list_x = torch.cat(mask_pred_list_x, dim=0) mask_pred_list_y = torch.cat(mask_pred_list_y, dim=0) img_meta = batch_img_metas[img_id] results = self._predict_by_feat_single( cls_pred_list, mask_pred_list_x, mask_pred_list_y, img_meta=img_meta) results_list.append(results) return results_list def _predict_by_feat_single(self, cls_scores: Tensor, mask_preds_x: Tensor, mask_preds_y: Tensor, img_meta: dict, cfg: OptConfigType = None) -> InstanceData: """Transform a single image's features extracted from the head into mask results. Args: cls_scores (Tensor): Classification score of all points in single image, has shape (num_points, num_classes). mask_preds_x (Tensor): Mask prediction of x branch of all points in single image, has shape (sum_num_grids, feat_h, feat_w). mask_preds_y (Tensor): Mask prediction of y branch of all points in single image, has shape (sum_num_grids, feat_h, feat_w). img_meta (dict): Meta information of corresponding image. cfg (dict): Config used in test phase. Returns: :obj:`InstanceData`: Processed results of single image. it usually contains following keys. - scores (Tensor): Classification scores, has shape (num_instance,). - labels (Tensor): Has shape (num_instances,). - masks (Tensor): Processed mask results, has shape (num_instances, h, w). """ def empty_results(cls_scores, ori_shape): """Generate a empty results.""" results = InstanceData() results.scores = cls_scores.new_ones(0) results.masks = cls_scores.new_zeros(0, *ori_shape) results.labels = cls_scores.new_ones(0) results.bboxes = cls_scores.new_zeros(0, 4) return results cfg = self.test_cfg if cfg is None else cfg featmap_size = mask_preds_x.size()[-2:] h, w = img_meta['img_shape'][:2] upsampled_size = (featmap_size[0] * 4, featmap_size[1] * 4) score_mask = (cls_scores > cfg.score_thr) cls_scores = cls_scores[score_mask] inds = score_mask.nonzero() lvl_interval = inds.new_tensor(self.num_grids).pow(2).cumsum(0) num_all_points = lvl_interval[-1] lvl_start_index = inds.new_ones(num_all_points) num_grids = inds.new_ones(num_all_points) seg_size = inds.new_tensor(self.num_grids).cumsum(0) mask_lvl_start_index = inds.new_ones(num_all_points) strides = inds.new_ones(num_all_points) lvl_start_index[:lvl_interval[0]] *= 0 mask_lvl_start_index[:lvl_interval[0]] *= 0 num_grids[:lvl_interval[0]] *= self.num_grids[0] strides[:lvl_interval[0]] *= self.strides[0] for lvl in range(1, self.num_levels): lvl_start_index[lvl_interval[lvl - 1]:lvl_interval[lvl]] *= \ lvl_interval[lvl - 1] mask_lvl_start_index[lvl_interval[lvl - 1]:lvl_interval[lvl]] *= \ seg_size[lvl - 1] num_grids[lvl_interval[lvl - 1]:lvl_interval[lvl]] *= \ self.num_grids[lvl] strides[lvl_interval[lvl - 1]:lvl_interval[lvl]] *= \ self.strides[lvl] lvl_start_index = lvl_start_index[inds[:, 0]] mask_lvl_start_index = mask_lvl_start_index[inds[:, 0]] num_grids = num_grids[inds[:, 0]] strides = strides[inds[:, 0]] y_lvl_offset = (inds[:, 0] - lvl_start_index) // num_grids x_lvl_offset = (inds[:, 0] - lvl_start_index) % num_grids y_inds = mask_lvl_start_index + y_lvl_offset x_inds = mask_lvl_start_index + x_lvl_offset cls_labels = inds[:, 1] mask_preds = mask_preds_x[x_inds, ...] * mask_preds_y[y_inds, ...] masks = mask_preds > cfg.mask_thr sum_masks = masks.sum((1, 2)).float() keep = sum_masks > strides if keep.sum() == 0: return empty_results(cls_scores, img_meta['ori_shape'][:2]) masks = masks[keep] mask_preds = mask_preds[keep] sum_masks = sum_masks[keep] cls_scores = cls_scores[keep] cls_labels = cls_labels[keep] # maskness. mask_scores = (mask_preds * masks).sum((1, 2)) / sum_masks cls_scores *= mask_scores scores, labels, _, keep_inds = mask_matrix_nms( masks, cls_labels, cls_scores, mask_area=sum_masks, nms_pre=cfg.nms_pre, max_num=cfg.max_per_img, kernel=cfg.kernel, sigma=cfg.sigma, filter_thr=cfg.filter_thr) # mask_matrix_nms may return an empty Tensor if len(keep_inds) == 0: return empty_results(cls_scores, img_meta['ori_shape'][:2]) mask_preds = mask_preds[keep_inds] mask_preds = F.interpolate( mask_preds.unsqueeze(0), size=upsampled_size, mode='bilinear')[:, :, :h, :w] mask_preds = F.interpolate( mask_preds, size=img_meta['ori_shape'][:2], mode='bilinear').squeeze(0) masks = mask_preds > cfg.mask_thr results = InstanceData() results.masks = masks results.labels = labels results.scores = scores # create an empty bbox in InstanceData to avoid bugs when # calculating metrics. results.bboxes = results.scores.new_zeros(len(scores), 4) return results @MODELS.register_module() class DecoupledSOLOLightHead(DecoupledSOLOHead): """Decoupled Light SOLO mask head used in `SOLO: Segmenting Objects by Locations `_ Args: with_dcn (bool): Whether use dcn in mask_convs and cls_convs, Defaults to False. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, *args, dcn_cfg: OptConfigType = None, init_cfg: MultiConfig = [ dict(type='Normal', layer='Conv2d', std=0.01), dict( type='Normal', std=0.01, bias_prob=0.01, override=dict(name='conv_mask_list_x')), dict( type='Normal', std=0.01, bias_prob=0.01, override=dict(name='conv_mask_list_y')), dict( type='Normal', std=0.01, bias_prob=0.01, override=dict(name='conv_cls')) ], **kwargs) -> None: assert dcn_cfg is None or isinstance(dcn_cfg, dict) self.dcn_cfg = dcn_cfg super().__init__(*args, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: self.mask_convs = nn.ModuleList() self.cls_convs = nn.ModuleList() for i in range(self.stacked_convs): if self.dcn_cfg is not None \ and i == self.stacked_convs - 1: conv_cfg = self.dcn_cfg else: conv_cfg = None chn = self.in_channels + 2 if i == 0 else self.feat_channels self.mask_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=self.norm_cfg)) chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=self.norm_cfg)) self.conv_mask_list_x = nn.ModuleList() self.conv_mask_list_y = nn.ModuleList() for num_grid in self.num_grids: self.conv_mask_list_x.append( nn.Conv2d(self.feat_channels, num_grid, 3, padding=1)) self.conv_mask_list_y.append( nn.Conv2d(self.feat_channels, num_grid, 3, padding=1)) self.conv_cls = nn.Conv2d( self.feat_channels, self.cls_out_channels, 3, padding=1) def forward(self, x: Tuple[Tensor]) -> Tuple: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of classification scores and mask prediction. - mlvl_mask_preds_x (list[Tensor]): Multi-level mask prediction from x branch. Each element in the list has shape (batch_size, num_grids ,h ,w). - mlvl_mask_preds_y (list[Tensor]): Multi-level mask prediction from y branch. Each element in the list has shape (batch_size, num_grids ,h ,w). - mlvl_cls_preds (list[Tensor]): Multi-level scores. Each element in the list has shape (batch_size, num_classes, num_grids ,num_grids). """ assert len(x) == self.num_levels feats = self.resize_feats(x) mask_preds_x = [] mask_preds_y = [] cls_preds = [] for i in range(self.num_levels): x = feats[i] mask_feat = x cls_feat = x # generate and concat the coordinate coord_feat = generate_coordinate(mask_feat.size(), mask_feat.device) mask_feat = torch.cat([mask_feat, coord_feat], 1) for mask_layer in self.mask_convs: mask_feat = mask_layer(mask_feat) mask_feat = F.interpolate( mask_feat, scale_factor=2, mode='bilinear') mask_pred_x = self.conv_mask_list_x[i](mask_feat) mask_pred_y = self.conv_mask_list_y[i](mask_feat) # cls branch for j, cls_layer in enumerate(self.cls_convs): if j == self.cls_down_index: num_grid = self.num_grids[i] cls_feat = F.interpolate( cls_feat, size=num_grid, mode='bilinear') cls_feat = cls_layer(cls_feat) cls_pred = self.conv_cls(cls_feat) if not self.training: feat_wh = feats[0].size()[-2:] upsampled_size = (feat_wh[0] * 2, feat_wh[1] * 2) mask_pred_x = F.interpolate( mask_pred_x.sigmoid(), size=upsampled_size, mode='bilinear') mask_pred_y = F.interpolate( mask_pred_y.sigmoid(), size=upsampled_size, mode='bilinear') cls_pred = cls_pred.sigmoid() # get local maximum local_max = F.max_pool2d(cls_pred, 2, stride=1, padding=1) keep_mask = local_max[:, :, :-1, :-1] == cls_pred cls_pred = cls_pred * keep_mask mask_preds_x.append(mask_pred_x) mask_preds_y.append(mask_pred_y) cls_preds.append(cls_pred) return mask_preds_x, mask_preds_y, cls_preds ================================================ FILE: mmdet/models/dense_heads/solov2_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings from typing import List, Optional, Tuple import mmcv import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.utils.misc import floordiv from mmdet.registry import MODELS from mmdet.utils import ConfigType, InstanceList, MultiConfig, OptConfigType from ..layers import mask_matrix_nms from ..utils import center_of_mass, generate_coordinate, multi_apply from .solo_head import SOLOHead class MaskFeatModule(BaseModule): """SOLOv2 mask feature map branch used in `SOLOv2: Dynamic and Fast Instance Segmentation. `_ Args: in_channels (int): Number of channels in the input feature map. feat_channels (int): Number of hidden channels of the mask feature map branch. start_level (int): The starting feature map level from RPN that will be used to predict the mask feature map. end_level (int): The ending feature map level from rpn that will be used to predict the mask feature map. out_channels (int): Number of output channels of the mask feature map branch. This is the channel count of the mask feature map that to be dynamically convolved with the predicted kernel. mask_stride (int): Downsample factor of the mask feature map output. Defaults to 4. conv_cfg (dict): Config dict for convolution layer. Default: None. norm_cfg (dict): Config dict for normalization layer. Default: None. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__( self, in_channels: int, feat_channels: int, start_level: int, end_level: int, out_channels: int, mask_stride: int = 4, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, init_cfg: MultiConfig = [ dict(type='Normal', layer='Conv2d', std=0.01) ] ) -> None: super().__init__(init_cfg=init_cfg) self.in_channels = in_channels self.feat_channels = feat_channels self.start_level = start_level self.end_level = end_level self.mask_stride = mask_stride assert start_level >= 0 and end_level >= start_level self.out_channels = out_channels self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self._init_layers() self.fp16_enabled = False def _init_layers(self) -> None: """Initialize layers of the head.""" self.convs_all_levels = nn.ModuleList() for i in range(self.start_level, self.end_level + 1): convs_per_level = nn.Sequential() if i == 0: convs_per_level.add_module( f'conv{i}', ConvModule( self.in_channels, self.feat_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, inplace=False)) self.convs_all_levels.append(convs_per_level) continue for j in range(i): if j == 0: if i == self.end_level: chn = self.in_channels + 2 else: chn = self.in_channels convs_per_level.add_module( f'conv{j}', ConvModule( chn, self.feat_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, inplace=False)) convs_per_level.add_module( f'upsample{j}', nn.Upsample( scale_factor=2, mode='bilinear', align_corners=False)) continue convs_per_level.add_module( f'conv{j}', ConvModule( self.feat_channels, self.feat_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, inplace=False)) convs_per_level.add_module( f'upsample{j}', nn.Upsample( scale_factor=2, mode='bilinear', align_corners=False)) self.convs_all_levels.append(convs_per_level) self.conv_pred = ConvModule( self.feat_channels, self.out_channels, 1, padding=0, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg) def forward(self, x: Tuple[Tensor]) -> Tensor: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: Tensor: The predicted mask feature map. """ inputs = x[self.start_level:self.end_level + 1] assert len(inputs) == (self.end_level - self.start_level + 1) feature_add_all_level = self.convs_all_levels[0](inputs[0]) for i in range(1, len(inputs)): input_p = inputs[i] if i == len(inputs) - 1: coord_feat = generate_coordinate(input_p.size(), input_p.device) input_p = torch.cat([input_p, coord_feat], 1) feature_add_all_level = feature_add_all_level + \ self.convs_all_levels[i](input_p) feature_pred = self.conv_pred(feature_add_all_level) return feature_pred @MODELS.register_module() class SOLOV2Head(SOLOHead): """SOLOv2 mask head used in `SOLOv2: Dynamic and Fast Instance Segmentation. `_ Args: mask_feature_head (dict): Config of SOLOv2MaskFeatHead. dynamic_conv_size (int): Dynamic Conv kernel size. Defaults to 1. dcn_cfg (dict): Dcn conv configurations in kernel_convs and cls_conv. Defaults to None. dcn_apply_to_all_conv (bool): Whether to use dcn in every layer of kernel_convs and cls_convs, or only the last layer. It shall be set `True` for the normal version of SOLOv2 and `False` for the light-weight version. Defaults to True. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, *args, mask_feature_head: ConfigType, dynamic_conv_size: int = 1, dcn_cfg: OptConfigType = None, dcn_apply_to_all_conv: bool = True, init_cfg: MultiConfig = [ dict(type='Normal', layer='Conv2d', std=0.01), dict( type='Normal', std=0.01, bias_prob=0.01, override=dict(name='conv_cls')) ], **kwargs) -> None: assert dcn_cfg is None or isinstance(dcn_cfg, dict) self.dcn_cfg = dcn_cfg self.with_dcn = dcn_cfg is not None self.dcn_apply_to_all_conv = dcn_apply_to_all_conv self.dynamic_conv_size = dynamic_conv_size mask_out_channels = mask_feature_head.get('out_channels') self.kernel_out_channels = \ mask_out_channels * self.dynamic_conv_size * self.dynamic_conv_size super().__init__(*args, init_cfg=init_cfg, **kwargs) # update the in_channels of mask_feature_head if mask_feature_head.get('in_channels', None) is not None: if mask_feature_head.in_channels != self.in_channels: warnings.warn('The `in_channels` of SOLOv2MaskFeatHead and ' 'SOLOv2Head should be same, changing ' 'mask_feature_head.in_channels to ' f'{self.in_channels}') mask_feature_head.update(in_channels=self.in_channels) else: mask_feature_head.update(in_channels=self.in_channels) self.mask_feature_head = MaskFeatModule(**mask_feature_head) self.mask_stride = self.mask_feature_head.mask_stride self.fp16_enabled = False def _init_layers(self) -> None: """Initialize layers of the head.""" self.cls_convs = nn.ModuleList() self.kernel_convs = nn.ModuleList() conv_cfg = None for i in range(self.stacked_convs): if self.with_dcn: if self.dcn_apply_to_all_conv: conv_cfg = self.dcn_cfg elif i == self.stacked_convs - 1: # light head conv_cfg = self.dcn_cfg chn = self.in_channels + 2 if i == 0 else self.feat_channels self.kernel_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=self.norm_cfg, bias=self.norm_cfg is None)) chn = self.in_channels if i == 0 else self.feat_channels self.cls_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=self.norm_cfg, bias=self.norm_cfg is None)) self.conv_cls = nn.Conv2d( self.feat_channels, self.cls_out_channels, 3, padding=1) self.conv_kernel = nn.Conv2d( self.feat_channels, self.kernel_out_channels, 3, padding=1) def forward(self, x): """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of classification scores, mask prediction, and mask features. - mlvl_kernel_preds (list[Tensor]): Multi-level dynamic kernel prediction. The kernel is used to generate instance segmentation masks by dynamic convolution. Each element in the list has shape (batch_size, kernel_out_channels, num_grids, num_grids). - mlvl_cls_preds (list[Tensor]): Multi-level scores. Each element in the list has shape (batch_size, num_classes, num_grids, num_grids). - mask_feats (Tensor): Unified mask feature map used to generate instance segmentation masks by dynamic convolution. Has shape (batch_size, mask_out_channels, h, w). """ assert len(x) == self.num_levels mask_feats = self.mask_feature_head(x) ins_kernel_feats = self.resize_feats(x) mlvl_kernel_preds = [] mlvl_cls_preds = [] for i in range(self.num_levels): ins_kernel_feat = ins_kernel_feats[i] # ins branch # concat coord coord_feat = generate_coordinate(ins_kernel_feat.size(), ins_kernel_feat.device) ins_kernel_feat = torch.cat([ins_kernel_feat, coord_feat], 1) # kernel branch kernel_feat = ins_kernel_feat kernel_feat = F.interpolate( kernel_feat, size=self.num_grids[i], mode='bilinear', align_corners=False) cate_feat = kernel_feat[:, :-2, :, :] kernel_feat = kernel_feat.contiguous() for i, kernel_conv in enumerate(self.kernel_convs): kernel_feat = kernel_conv(kernel_feat) kernel_pred = self.conv_kernel(kernel_feat) # cate branch cate_feat = cate_feat.contiguous() for i, cls_conv in enumerate(self.cls_convs): cate_feat = cls_conv(cate_feat) cate_pred = self.conv_cls(cate_feat) mlvl_kernel_preds.append(kernel_pred) mlvl_cls_preds.append(cate_pred) return mlvl_kernel_preds, mlvl_cls_preds, mask_feats def _get_targets_single(self, gt_instances: InstanceData, featmap_sizes: Optional[list] = None) -> tuple: """Compute targets for predictions of single image. Args: gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes``, ``labels``, and ``masks`` attributes. featmap_sizes (list[:obj:`torch.size`]): Size of each feature map from feature pyramid, each element means (feat_h, feat_w). Defaults to None. Returns: Tuple: Usually returns a tuple containing targets for predictions. - mlvl_pos_mask_targets (list[Tensor]): Each element represent the binary mask targets for positive points in this level, has shape (num_pos, out_h, out_w). - mlvl_labels (list[Tensor]): Each element is classification labels for all points in this level, has shape (num_grid, num_grid). - mlvl_pos_masks (list[Tensor]): Each element is a `BoolTensor` to represent whether the corresponding point in single level is positive, has shape (num_grid **2). - mlvl_pos_indexes (list[list]): Each element in the list contains the positive index in corresponding level, has shape (num_pos). """ gt_labels = gt_instances.labels device = gt_labels.device gt_bboxes = gt_instances.bboxes gt_areas = torch.sqrt((gt_bboxes[:, 2] - gt_bboxes[:, 0]) * (gt_bboxes[:, 3] - gt_bboxes[:, 1])) gt_masks = gt_instances.masks.to_tensor( dtype=torch.bool, device=device) mlvl_pos_mask_targets = [] mlvl_pos_indexes = [] mlvl_labels = [] mlvl_pos_masks = [] for (lower_bound, upper_bound), num_grid \ in zip(self.scale_ranges, self.num_grids): mask_target = [] # FG cat_id: [0, num_classes -1], BG cat_id: num_classes pos_index = [] labels = torch.zeros([num_grid, num_grid], dtype=torch.int64, device=device) + self.num_classes pos_mask = torch.zeros([num_grid**2], dtype=torch.bool, device=device) gt_inds = ((gt_areas >= lower_bound) & (gt_areas <= upper_bound)).nonzero().flatten() if len(gt_inds) == 0: mlvl_pos_mask_targets.append( torch.zeros([0, featmap_sizes[0], featmap_sizes[1]], dtype=torch.uint8, device=device)) mlvl_labels.append(labels) mlvl_pos_masks.append(pos_mask) mlvl_pos_indexes.append([]) continue hit_gt_bboxes = gt_bboxes[gt_inds] hit_gt_labels = gt_labels[gt_inds] hit_gt_masks = gt_masks[gt_inds, ...] pos_w_ranges = 0.5 * (hit_gt_bboxes[:, 2] - hit_gt_bboxes[:, 0]) * self.pos_scale pos_h_ranges = 0.5 * (hit_gt_bboxes[:, 3] - hit_gt_bboxes[:, 1]) * self.pos_scale # Make sure hit_gt_masks has a value valid_mask_flags = hit_gt_masks.sum(dim=-1).sum(dim=-1) > 0 for gt_mask, gt_label, pos_h_range, pos_w_range, \ valid_mask_flag in \ zip(hit_gt_masks, hit_gt_labels, pos_h_ranges, pos_w_ranges, valid_mask_flags): if not valid_mask_flag: continue upsampled_size = (featmap_sizes[0] * self.mask_stride, featmap_sizes[1] * self.mask_stride) center_h, center_w = center_of_mass(gt_mask) coord_w = int( floordiv((center_w / upsampled_size[1]), (1. / num_grid), rounding_mode='trunc')) coord_h = int( floordiv((center_h / upsampled_size[0]), (1. / num_grid), rounding_mode='trunc')) # left, top, right, down top_box = max( 0, int( floordiv( (center_h - pos_h_range) / upsampled_size[0], (1. / num_grid), rounding_mode='trunc'))) down_box = min( num_grid - 1, int( floordiv( (center_h + pos_h_range) / upsampled_size[0], (1. / num_grid), rounding_mode='trunc'))) left_box = max( 0, int( floordiv( (center_w - pos_w_range) / upsampled_size[1], (1. / num_grid), rounding_mode='trunc'))) right_box = min( num_grid - 1, int( floordiv( (center_w + pos_w_range) / upsampled_size[1], (1. / num_grid), rounding_mode='trunc'))) top = max(top_box, coord_h - 1) down = min(down_box, coord_h + 1) left = max(coord_w - 1, left_box) right = min(right_box, coord_w + 1) labels[top:(down + 1), left:(right + 1)] = gt_label # ins gt_mask = np.uint8(gt_mask.cpu().numpy()) # Follow the original implementation, F.interpolate is # different from cv2 and opencv gt_mask = mmcv.imrescale(gt_mask, scale=1. / self.mask_stride) gt_mask = torch.from_numpy(gt_mask).to(device=device) for i in range(top, down + 1): for j in range(left, right + 1): index = int(i * num_grid + j) this_mask_target = torch.zeros( [featmap_sizes[0], featmap_sizes[1]], dtype=torch.uint8, device=device) this_mask_target[:gt_mask.shape[0], :gt_mask. shape[1]] = gt_mask mask_target.append(this_mask_target) pos_mask[index] = True pos_index.append(index) if len(mask_target) == 0: mask_target = torch.zeros( [0, featmap_sizes[0], featmap_sizes[1]], dtype=torch.uint8, device=device) else: mask_target = torch.stack(mask_target, 0) mlvl_pos_mask_targets.append(mask_target) mlvl_labels.append(labels) mlvl_pos_masks.append(pos_mask) mlvl_pos_indexes.append(pos_index) return (mlvl_pos_mask_targets, mlvl_labels, mlvl_pos_masks, mlvl_pos_indexes) def loss_by_feat(self, mlvl_kernel_preds: List[Tensor], mlvl_cls_preds: List[Tensor], mask_feats: Tensor, batch_gt_instances: InstanceList, batch_img_metas: List[dict], **kwargs) -> dict: """Calculate the loss based on the features extracted by the mask head. Args: mlvl_kernel_preds (list[Tensor]): Multi-level dynamic kernel prediction. The kernel is used to generate instance segmentation masks by dynamic convolution. Each element in the list has shape (batch_size, kernel_out_channels, num_grids, num_grids). mlvl_cls_preds (list[Tensor]): Multi-level scores. Each element in the list has shape (batch_size, num_classes, num_grids, num_grids). mask_feats (Tensor): Unified mask feature map used to generate instance segmentation masks by dynamic convolution. Has shape (batch_size, mask_out_channels, h, w). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``masks``, and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of multiple images. Returns: dict[str, Tensor]: A dictionary of loss components. """ featmap_sizes = mask_feats.size()[-2:] pos_mask_targets, labels, pos_masks, pos_indexes = multi_apply( self._get_targets_single, batch_gt_instances, featmap_sizes=featmap_sizes) mlvl_mask_targets = [ torch.cat(lvl_mask_targets, 0) for lvl_mask_targets in zip(*pos_mask_targets) ] mlvl_pos_kernel_preds = [] for lvl_kernel_preds, lvl_pos_indexes in zip(mlvl_kernel_preds, zip(*pos_indexes)): lvl_pos_kernel_preds = [] for img_lvl_kernel_preds, img_lvl_pos_indexes in zip( lvl_kernel_preds, lvl_pos_indexes): img_lvl_pos_kernel_preds = img_lvl_kernel_preds.view( img_lvl_kernel_preds.shape[0], -1)[:, img_lvl_pos_indexes] lvl_pos_kernel_preds.append(img_lvl_pos_kernel_preds) mlvl_pos_kernel_preds.append(lvl_pos_kernel_preds) # make multilevel mlvl_mask_pred mlvl_mask_preds = [] for lvl_pos_kernel_preds in mlvl_pos_kernel_preds: lvl_mask_preds = [] for img_id, img_lvl_pos_kernel_pred in enumerate( lvl_pos_kernel_preds): if img_lvl_pos_kernel_pred.size()[-1] == 0: continue img_mask_feats = mask_feats[[img_id]] h, w = img_mask_feats.shape[-2:] num_kernel = img_lvl_pos_kernel_pred.shape[1] img_lvl_mask_pred = F.conv2d( img_mask_feats, img_lvl_pos_kernel_pred.permute(1, 0).view( num_kernel, -1, self.dynamic_conv_size, self.dynamic_conv_size), stride=1).view(-1, h, w) lvl_mask_preds.append(img_lvl_mask_pred) if len(lvl_mask_preds) == 0: lvl_mask_preds = None else: lvl_mask_preds = torch.cat(lvl_mask_preds, 0) mlvl_mask_preds.append(lvl_mask_preds) # dice loss num_pos = 0 for img_pos_masks in pos_masks: for lvl_img_pos_masks in img_pos_masks: # Fix `Tensor` object has no attribute `count_nonzero()` # in PyTorch 1.6, the type of `lvl_img_pos_masks` # should be `torch.bool`. num_pos += lvl_img_pos_masks.nonzero().numel() loss_mask = [] for lvl_mask_preds, lvl_mask_targets in zip(mlvl_mask_preds, mlvl_mask_targets): if lvl_mask_preds is None: continue loss_mask.append( self.loss_mask( lvl_mask_preds, lvl_mask_targets, reduction_override='none')) if num_pos > 0: loss_mask = torch.cat(loss_mask).sum() / num_pos else: loss_mask = mask_feats.sum() * 0 # cate flatten_labels = [ torch.cat( [img_lvl_labels.flatten() for img_lvl_labels in lvl_labels]) for lvl_labels in zip(*labels) ] flatten_labels = torch.cat(flatten_labels) flatten_cls_preds = [ lvl_cls_preds.permute(0, 2, 3, 1).reshape(-1, self.num_classes) for lvl_cls_preds in mlvl_cls_preds ] flatten_cls_preds = torch.cat(flatten_cls_preds) loss_cls = self.loss_cls( flatten_cls_preds, flatten_labels, avg_factor=num_pos + 1) return dict(loss_mask=loss_mask, loss_cls=loss_cls) def predict_by_feat(self, mlvl_kernel_preds: List[Tensor], mlvl_cls_scores: List[Tensor], mask_feats: Tensor, batch_img_metas: List[dict], **kwargs) -> InstanceList: """Transform a batch of output features extracted from the head into mask results. Args: mlvl_kernel_preds (list[Tensor]): Multi-level dynamic kernel prediction. The kernel is used to generate instance segmentation masks by dynamic convolution. Each element in the list has shape (batch_size, kernel_out_channels, num_grids, num_grids). mlvl_cls_scores (list[Tensor]): Multi-level scores. Each element in the list has shape (batch_size, num_classes, num_grids, num_grids). mask_feats (Tensor): Unified mask feature map used to generate instance segmentation masks by dynamic convolution. Has shape (batch_size, mask_out_channels, h, w). batch_img_metas (list[dict]): Meta information of all images. Returns: list[:obj:`InstanceData`]: Processed results of multiple images.Each :obj:`InstanceData` usually contains following keys. - scores (Tensor): Classification scores, has shape (num_instance,). - labels (Tensor): Has shape (num_instances,). - masks (Tensor): Processed mask results, has shape (num_instances, h, w). """ num_levels = len(mlvl_cls_scores) assert len(mlvl_kernel_preds) == len(mlvl_cls_scores) for lvl in range(num_levels): cls_scores = mlvl_cls_scores[lvl] cls_scores = cls_scores.sigmoid() local_max = F.max_pool2d(cls_scores, 2, stride=1, padding=1) keep_mask = local_max[:, :, :-1, :-1] == cls_scores cls_scores = cls_scores * keep_mask mlvl_cls_scores[lvl] = cls_scores.permute(0, 2, 3, 1) result_list = [] for img_id in range(len(batch_img_metas)): img_cls_pred = [ mlvl_cls_scores[lvl][img_id].view(-1, self.cls_out_channels) for lvl in range(num_levels) ] img_mask_feats = mask_feats[[img_id]] img_kernel_pred = [ mlvl_kernel_preds[lvl][img_id].permute(1, 2, 0).view( -1, self.kernel_out_channels) for lvl in range(num_levels) ] img_cls_pred = torch.cat(img_cls_pred, dim=0) img_kernel_pred = torch.cat(img_kernel_pred, dim=0) result = self._predict_by_feat_single( img_kernel_pred, img_cls_pred, img_mask_feats, img_meta=batch_img_metas[img_id]) result_list.append(result) return result_list def _predict_by_feat_single(self, kernel_preds: Tensor, cls_scores: Tensor, mask_feats: Tensor, img_meta: dict, cfg: OptConfigType = None) -> InstanceData: """Transform a single image's features extracted from the head into mask results. Args: kernel_preds (Tensor): Dynamic kernel prediction of all points in single image, has shape (num_points, kernel_out_channels). cls_scores (Tensor): Classification score of all points in single image, has shape (num_points, num_classes). mask_feats (Tensor): Mask prediction of all points in single image, has shape (num_points, feat_h, feat_w). img_meta (dict): Meta information of corresponding image. cfg (dict, optional): Config used in test phase. Defaults to None. Returns: :obj:`InstanceData`: Processed results of single image. it usually contains following keys. - scores (Tensor): Classification scores, has shape (num_instance,). - labels (Tensor): Has shape (num_instances,). - masks (Tensor): Processed mask results, has shape (num_instances, h, w). """ def empty_results(cls_scores, ori_shape): """Generate a empty results.""" results = InstanceData() results.scores = cls_scores.new_ones(0) results.masks = cls_scores.new_zeros(0, *ori_shape) results.labels = cls_scores.new_ones(0) results.bboxes = cls_scores.new_zeros(0, 4) return results cfg = self.test_cfg if cfg is None else cfg assert len(kernel_preds) == len(cls_scores) featmap_size = mask_feats.size()[-2:] # overall info h, w = img_meta['img_shape'][:2] upsampled_size = (featmap_size[0] * self.mask_stride, featmap_size[1] * self.mask_stride) # process. score_mask = (cls_scores > cfg.score_thr) cls_scores = cls_scores[score_mask] if len(cls_scores) == 0: return empty_results(cls_scores, img_meta['ori_shape'][:2]) # cate_labels & kernel_preds inds = score_mask.nonzero() cls_labels = inds[:, 1] kernel_preds = kernel_preds[inds[:, 0]] # trans vector. lvl_interval = cls_labels.new_tensor(self.num_grids).pow(2).cumsum(0) strides = kernel_preds.new_ones(lvl_interval[-1]) strides[:lvl_interval[0]] *= self.strides[0] for lvl in range(1, self.num_levels): strides[lvl_interval[lvl - 1]:lvl_interval[lvl]] *= self.strides[lvl] strides = strides[inds[:, 0]] # mask encoding. kernel_preds = kernel_preds.view( kernel_preds.size(0), -1, self.dynamic_conv_size, self.dynamic_conv_size) mask_preds = F.conv2d( mask_feats, kernel_preds, stride=1).squeeze(0).sigmoid() # mask. masks = mask_preds > cfg.mask_thr sum_masks = masks.sum((1, 2)).float() keep = sum_masks > strides if keep.sum() == 0: return empty_results(cls_scores, img_meta['ori_shape'][:2]) masks = masks[keep] mask_preds = mask_preds[keep] sum_masks = sum_masks[keep] cls_scores = cls_scores[keep] cls_labels = cls_labels[keep] # maskness. mask_scores = (mask_preds * masks).sum((1, 2)) / sum_masks cls_scores *= mask_scores scores, labels, _, keep_inds = mask_matrix_nms( masks, cls_labels, cls_scores, mask_area=sum_masks, nms_pre=cfg.nms_pre, max_num=cfg.max_per_img, kernel=cfg.kernel, sigma=cfg.sigma, filter_thr=cfg.filter_thr) if len(keep_inds) == 0: return empty_results(cls_scores, img_meta['ori_shape'][:2]) mask_preds = mask_preds[keep_inds] mask_preds = F.interpolate( mask_preds.unsqueeze(0), size=upsampled_size, mode='bilinear', align_corners=False)[:, :, :h, :w] mask_preds = F.interpolate( mask_preds, size=img_meta['ori_shape'][:2], mode='bilinear', align_corners=False).squeeze(0) masks = mask_preds > cfg.mask_thr results = InstanceData() results.masks = masks results.labels = labels results.scores = scores # create an empty bbox in InstanceData to avoid bugs when # calculating metrics. results.bboxes = results.scores.new_zeros(len(scores), 4) return results ================================================ FILE: mmdet/models/dense_heads/ssd_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Optional, Sequence, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.utils import ConfigType, InstanceList, MultiConfig, OptInstanceList from ..losses import smooth_l1_loss from ..task_modules.samplers import PseudoSampler from ..utils import multi_apply from .anchor_head import AnchorHead # TODO: add loss evaluator for SSD @MODELS.register_module() class SSDHead(AnchorHead): """Implementation of `SSD head `_ Args: num_classes (int): Number of categories excluding the background category. in_channels (Sequence[int]): Number of channels in the input feature map. stacked_convs (int): Number of conv layers in cls and reg tower. Defaults to 0. feat_channels (int): Number of hidden channels when stacked_convs > 0. Defaults to 256. use_depthwise (bool): Whether to use DepthwiseSeparableConv. Defaults to False. conv_cfg (:obj:`ConfigDict` or dict, Optional): Dictionary to construct and config conv layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict, Optional): Dictionary to construct and config norm layer. Defaults to None. act_cfg (:obj:`ConfigDict` or dict, Optional): Dictionary to construct and config activation layer. Defaults to None. anchor_generator (:obj:`ConfigDict` or dict): Config dict for anchor generator. bbox_coder (:obj:`ConfigDict` or dict): Config of bounding box coder. reg_decoded_bbox (bool): If true, the regression loss would be applied directly on decoded bounding boxes, converting both the predicted boxes and regression targets to absolute coordinates format. Defaults to False. It should be `True` when using `IoULoss`, `GIoULoss`, or `DIoULoss` in the bbox head. train_cfg (:obj:`ConfigDict` or dict, Optional): Training config of anchor head. test_cfg (:obj:`ConfigDict` or dict, Optional): Testing config of anchor head. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], Optional): Initialization config dict. """ # noqa: W605 def __init__( self, num_classes: int = 80, in_channels: Sequence[int] = (512, 1024, 512, 256, 256, 256), stacked_convs: int = 0, feat_channels: int = 256, use_depthwise: bool = False, conv_cfg: Optional[ConfigType] = None, norm_cfg: Optional[ConfigType] = None, act_cfg: Optional[ConfigType] = None, anchor_generator: ConfigType = dict( type='SSDAnchorGenerator', scale_major=False, input_size=300, strides=[8, 16, 32, 64, 100, 300], ratios=([2], [2, 3], [2, 3], [2, 3], [2], [2]), basesize_ratio_range=(0.1, 0.9)), bbox_coder: ConfigType = dict( type='DeltaXYWHBBoxCoder', clip_border=True, target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0], ), reg_decoded_bbox: bool = False, train_cfg: Optional[ConfigType] = None, test_cfg: Optional[ConfigType] = None, init_cfg: MultiConfig = dict( type='Xavier', layer='Conv2d', distribution='uniform', bias=0) ) -> None: super(AnchorHead, self).__init__(init_cfg=init_cfg) self.num_classes = num_classes self.in_channels = in_channels self.stacked_convs = stacked_convs self.feat_channels = feat_channels self.use_depthwise = use_depthwise self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.act_cfg = act_cfg self.cls_out_channels = num_classes + 1 # add background class self.prior_generator = TASK_UTILS.build(anchor_generator) # Usually the numbers of anchors for each level are the same # except SSD detectors. So it is an int in the most dense # heads but a list of int in SSDHead self.num_base_priors = self.prior_generator.num_base_priors self._init_layers() self.bbox_coder = TASK_UTILS.build(bbox_coder) self.reg_decoded_bbox = reg_decoded_bbox self.use_sigmoid_cls = False self.cls_focal_loss = False self.train_cfg = train_cfg self.test_cfg = test_cfg if self.train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) if self.train_cfg.get('sampler', None) is not None: self.sampler = TASK_UTILS.build( self.train_cfg['sampler'], default_args=dict(context=self)) else: self.sampler = PseudoSampler(context=self) def _init_layers(self) -> None: """Initialize layers of the head.""" self.cls_convs = nn.ModuleList() self.reg_convs = nn.ModuleList() # TODO: Use registry to choose ConvModule type conv = DepthwiseSeparableConvModule \ if self.use_depthwise else ConvModule for channel, num_base_priors in zip(self.in_channels, self.num_base_priors): cls_layers = [] reg_layers = [] in_channel = channel # build stacked conv tower, not used in default ssd for i in range(self.stacked_convs): cls_layers.append( conv( in_channel, self.feat_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) reg_layers.append( conv( in_channel, self.feat_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) in_channel = self.feat_channels # SSD-Lite head if self.use_depthwise: cls_layers.append( ConvModule( in_channel, in_channel, 3, padding=1, groups=in_channel, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) reg_layers.append( ConvModule( in_channel, in_channel, 3, padding=1, groups=in_channel, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg)) cls_layers.append( nn.Conv2d( in_channel, num_base_priors * self.cls_out_channels, kernel_size=1 if self.use_depthwise else 3, padding=0 if self.use_depthwise else 1)) reg_layers.append( nn.Conv2d( in_channel, num_base_priors * 4, kernel_size=1 if self.use_depthwise else 3, padding=0 if self.use_depthwise else 1)) self.cls_convs.append(nn.Sequential(*cls_layers)) self.reg_convs.append(nn.Sequential(*reg_layers)) def forward(self, x: Tuple[Tensor]) -> Tuple[List[Tensor], List[Tensor]]: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple[list[Tensor], list[Tensor]]: A tuple of cls_scores list and bbox_preds list. - cls_scores (list[Tensor]): Classification scores for all scale \ levels, each is a 4D-tensor, the channels number is \ num_anchors * num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for all scale \ levels, each is a 4D-tensor, the channels number is \ num_anchors * 4. """ cls_scores = [] bbox_preds = [] for feat, reg_conv, cls_conv in zip(x, self.reg_convs, self.cls_convs): cls_scores.append(cls_conv(feat)) bbox_preds.append(reg_conv(feat)) return cls_scores, bbox_preds def loss_by_feat_single(self, cls_score: Tensor, bbox_pred: Tensor, anchor: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, bbox_weights: Tensor, avg_factor: int) -> Tuple[Tensor, Tensor]: """Compute loss of a single image. Args: cls_score (Tensor): Box scores for eachimage Has shape (num_total_anchors, num_classes). bbox_pred (Tensor): Box energies / deltas for each image level with shape (num_total_anchors, 4). anchors (Tensor): Box reference for each scale level with shape (num_total_anchors, 4). labels (Tensor): Labels of each anchors with shape (num_total_anchors,). label_weights (Tensor): Label weights of each anchor with shape (num_total_anchors,) bbox_targets (Tensor): BBox regression targets of each anchor weight shape (num_total_anchors, 4). bbox_weights (Tensor): BBox regression loss weights of each anchor with shape (num_total_anchors, 4). avg_factor (int): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using `PseudoSampler`, `avg_factor` is usually equal to the number of positive priors. Returns: Tuple[Tensor, Tensor]: A tuple of cls loss and bbox loss of one feature map. """ loss_cls_all = F.cross_entropy( cls_score, labels, reduction='none') * label_weights # FG cat_id: [0, num_classes -1], BG cat_id: num_classes pos_inds = ((labels >= 0) & (labels < self.num_classes)).nonzero( as_tuple=False).reshape(-1) neg_inds = (labels == self.num_classes).nonzero( as_tuple=False).view(-1) num_pos_samples = pos_inds.size(0) num_neg_samples = self.train_cfg['neg_pos_ratio'] * num_pos_samples if num_neg_samples > neg_inds.size(0): num_neg_samples = neg_inds.size(0) topk_loss_cls_neg, _ = loss_cls_all[neg_inds].topk(num_neg_samples) loss_cls_pos = loss_cls_all[pos_inds].sum() loss_cls_neg = topk_loss_cls_neg.sum() loss_cls = (loss_cls_pos + loss_cls_neg) / avg_factor if self.reg_decoded_bbox: # When the regression loss (e.g. `IouLoss`, `GIouLoss`) # is applied directly on the decoded bounding boxes, it # decodes the already encoded coordinates to absolute format. bbox_pred = self.bbox_coder.decode(anchor, bbox_pred) loss_bbox = smooth_l1_loss( bbox_pred, bbox_targets, bbox_weights, beta=self.train_cfg['smoothl1_beta'], avg_factor=avg_factor) return loss_cls[None], loss_bbox def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None ) -> Dict[str, List[Tensor]]: """Compute losses of the head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, list[Tensor]]: A dictionary of loss components. the dict has components below: - loss_cls (list[Tensor]): A list containing each feature map \ classification loss. - loss_bbox (list[Tensor]): A list containing each feature map \ regression loss. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, unmap_outputs=True) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets num_images = len(batch_img_metas) all_cls_scores = torch.cat([ s.permute(0, 2, 3, 1).reshape( num_images, -1, self.cls_out_channels) for s in cls_scores ], 1) all_labels = torch.cat(labels_list, -1).view(num_images, -1) all_label_weights = torch.cat(label_weights_list, -1).view(num_images, -1) all_bbox_preds = torch.cat([ b.permute(0, 2, 3, 1).reshape(num_images, -1, 4) for b in bbox_preds ], -2) all_bbox_targets = torch.cat(bbox_targets_list, -2).view(num_images, -1, 4) all_bbox_weights = torch.cat(bbox_weights_list, -2).view(num_images, -1, 4) # concat all level anchors to a single tensor all_anchors = [] for i in range(num_images): all_anchors.append(torch.cat(anchor_list[i])) losses_cls, losses_bbox = multi_apply( self.loss_by_feat_single, all_cls_scores, all_bbox_preds, all_anchors, all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, avg_factor=avg_factor) return dict(loss_cls=losses_cls, loss_bbox=losses_bbox) ================================================ FILE: mmdet/models/dense_heads/tood_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule, Scale from mmcv.ops import deform_conv2d from mmengine import MessageHub from mmengine.config import ConfigDict from mmengine.model import bias_init_with_prob, normal_init from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures.bbox import distance2bbox from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptInstanceList, reduce_mean) from ..task_modules.prior_generators import anchor_inside_flags from ..utils import (filter_scores_and_topk, images_to_levels, multi_apply, sigmoid_geometric_mean, unmap) from .atss_head import ATSSHead class TaskDecomposition(nn.Module): """Task decomposition module in task-aligned predictor of TOOD. Args: feat_channels (int): Number of feature channels in TOOD head. stacked_convs (int): Number of conv layers in TOOD head. la_down_rate (int): Downsample rate of layer attention. Defaults to 8. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for normalization layer. Defaults to None. """ def __init__(self, feat_channels: int, stacked_convs: int, la_down_rate: int = 8, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None) -> None: super().__init__() self.feat_channels = feat_channels self.stacked_convs = stacked_convs self.in_channels = self.feat_channels * self.stacked_convs self.norm_cfg = norm_cfg self.layer_attention = nn.Sequential( nn.Conv2d(self.in_channels, self.in_channels // la_down_rate, 1), nn.ReLU(inplace=True), nn.Conv2d( self.in_channels // la_down_rate, self.stacked_convs, 1, padding=0), nn.Sigmoid()) self.reduction_conv = ConvModule( self.in_channels, self.feat_channels, 1, stride=1, padding=0, conv_cfg=conv_cfg, norm_cfg=norm_cfg, bias=norm_cfg is None) def init_weights(self) -> None: """Initialize the parameters.""" for m in self.layer_attention.modules(): if isinstance(m, nn.Conv2d): normal_init(m, std=0.001) normal_init(self.reduction_conv.conv, std=0.01) def forward(self, feat: Tensor, avg_feat: Optional[Tensor] = None) -> Tensor: """Forward function of task decomposition module.""" b, c, h, w = feat.shape if avg_feat is None: avg_feat = F.adaptive_avg_pool2d(feat, (1, 1)) weight = self.layer_attention(avg_feat) # here we first compute the product between layer attention weight and # conv weight, and then compute the convolution between new conv weight # and feature map, in order to save memory and FLOPs. conv_weight = weight.reshape( b, 1, self.stacked_convs, 1) * self.reduction_conv.conv.weight.reshape( 1, self.feat_channels, self.stacked_convs, self.feat_channels) conv_weight = conv_weight.reshape(b, self.feat_channels, self.in_channels) feat = feat.reshape(b, self.in_channels, h * w) feat = torch.bmm(conv_weight, feat).reshape(b, self.feat_channels, h, w) if self.norm_cfg is not None: feat = self.reduction_conv.norm(feat) feat = self.reduction_conv.activate(feat) return feat @MODELS.register_module() class TOODHead(ATSSHead): """TOODHead used in `TOOD: Task-aligned One-stage Object Detection. `_. TOOD uses Task-aligned head (T-head) and is optimized by Task Alignment Learning (TAL). Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. num_dcn (int): Number of deformable convolution in the head. Defaults to 0. anchor_type (str): If set to ``anchor_free``, the head will use centers to regress bboxes. If set to ``anchor_based``, the head will regress bboxes based on anchors. Defaults to ``anchor_free``. initial_loss_cls (:obj:`ConfigDict` or dict): Config of initial loss. Example: >>> self = TOODHead(11, 7) >>> feats = [torch.rand(1, 7, s, s) for s in [4, 8, 16, 32, 64]] >>> cls_score, bbox_pred = self.forward(feats) >>> assert len(cls_score) == len(self.scales) """ def __init__(self, num_classes: int, in_channels: int, num_dcn: int = 0, anchor_type: str = 'anchor_free', initial_loss_cls: ConfigType = dict( type='FocalLoss', use_sigmoid=True, activated=True, gamma=2.0, alpha=0.25, loss_weight=1.0), **kwargs) -> None: assert anchor_type in ['anchor_free', 'anchor_based'] self.num_dcn = num_dcn self.anchor_type = anchor_type super().__init__( num_classes=num_classes, in_channels=in_channels, **kwargs) if self.train_cfg: self.initial_epoch = self.train_cfg['initial_epoch'] self.initial_assigner = TASK_UTILS.build( self.train_cfg['initial_assigner']) self.initial_loss_cls = MODELS.build(initial_loss_cls) self.assigner = self.initial_assigner self.alignment_assigner = TASK_UTILS.build( self.train_cfg['assigner']) self.alpha = self.train_cfg['alpha'] self.beta = self.train_cfg['beta'] def _init_layers(self) -> None: """Initialize layers of the head.""" self.relu = nn.ReLU(inplace=True) self.inter_convs = nn.ModuleList() for i in range(self.stacked_convs): if i < self.num_dcn: conv_cfg = dict(type='DCNv2', deform_groups=4) else: conv_cfg = self.conv_cfg chn = self.in_channels if i == 0 else self.feat_channels self.inter_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=self.norm_cfg)) self.cls_decomp = TaskDecomposition(self.feat_channels, self.stacked_convs, self.stacked_convs * 8, self.conv_cfg, self.norm_cfg) self.reg_decomp = TaskDecomposition(self.feat_channels, self.stacked_convs, self.stacked_convs * 8, self.conv_cfg, self.norm_cfg) self.tood_cls = nn.Conv2d( self.feat_channels, self.num_base_priors * self.cls_out_channels, 3, padding=1) self.tood_reg = nn.Conv2d( self.feat_channels, self.num_base_priors * 4, 3, padding=1) self.cls_prob_module = nn.Sequential( nn.Conv2d(self.feat_channels * self.stacked_convs, self.feat_channels // 4, 1), nn.ReLU(inplace=True), nn.Conv2d(self.feat_channels // 4, 1, 3, padding=1)) self.reg_offset_module = nn.Sequential( nn.Conv2d(self.feat_channels * self.stacked_convs, self.feat_channels // 4, 1), nn.ReLU(inplace=True), nn.Conv2d(self.feat_channels // 4, 4 * 2, 3, padding=1)) self.scales = nn.ModuleList( [Scale(1.0) for _ in self.prior_generator.strides]) def init_weights(self) -> None: """Initialize weights of the head.""" bias_cls = bias_init_with_prob(0.01) for m in self.inter_convs: normal_init(m.conv, std=0.01) for m in self.cls_prob_module: if isinstance(m, nn.Conv2d): normal_init(m, std=0.01) for m in self.reg_offset_module: if isinstance(m, nn.Conv2d): normal_init(m, std=0.001) normal_init(self.cls_prob_module[-1], std=0.01, bias=bias_cls) self.cls_decomp.init_weights() self.reg_decomp.init_weights() normal_init(self.tood_cls, std=0.01, bias=bias_cls) normal_init(self.tood_reg, std=0.01) def forward(self, feats: Tuple[Tensor]) -> Tuple[List[Tensor]]: """Forward features from the upstream network. Args: feats (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Usually a tuple of classification scores and bbox prediction cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_anchors * num_classes. bbox_preds (list[Tensor]): Decoded box for all scale levels, each is a 4D-tensor, the channels number is num_anchors * 4. In [tl_x, tl_y, br_x, br_y] format. """ cls_scores = [] bbox_preds = [] for idx, (x, scale, stride) in enumerate( zip(feats, self.scales, self.prior_generator.strides)): b, c, h, w = x.shape anchor = self.prior_generator.single_level_grid_priors( (h, w), idx, device=x.device) anchor = torch.cat([anchor for _ in range(b)]) # extract task interactive features inter_feats = [] for inter_conv in self.inter_convs: x = inter_conv(x) inter_feats.append(x) feat = torch.cat(inter_feats, 1) # task decomposition avg_feat = F.adaptive_avg_pool2d(feat, (1, 1)) cls_feat = self.cls_decomp(feat, avg_feat) reg_feat = self.reg_decomp(feat, avg_feat) # cls prediction and alignment cls_logits = self.tood_cls(cls_feat) cls_prob = self.cls_prob_module(feat) cls_score = sigmoid_geometric_mean(cls_logits, cls_prob) # reg prediction and alignment if self.anchor_type == 'anchor_free': reg_dist = scale(self.tood_reg(reg_feat).exp()).float() reg_dist = reg_dist.permute(0, 2, 3, 1).reshape(-1, 4) reg_bbox = distance2bbox( self.anchor_center(anchor) / stride[0], reg_dist).reshape(b, h, w, 4).permute(0, 3, 1, 2) # (b, c, h, w) elif self.anchor_type == 'anchor_based': reg_dist = scale(self.tood_reg(reg_feat)).float() reg_dist = reg_dist.permute(0, 2, 3, 1).reshape(-1, 4) reg_bbox = self.bbox_coder.decode(anchor, reg_dist).reshape( b, h, w, 4).permute(0, 3, 1, 2) / stride[0] else: raise NotImplementedError( f'Unknown anchor type: {self.anchor_type}.' f'Please use `anchor_free` or `anchor_based`.') reg_offset = self.reg_offset_module(feat) bbox_pred = self.deform_sampling(reg_bbox.contiguous(), reg_offset.contiguous()) # After deform_sampling, some boxes will become invalid (The # left-top point is at the right or bottom of the right-bottom # point), which will make the GIoULoss negative. invalid_bbox_idx = (bbox_pred[:, [0]] > bbox_pred[:, [2]]) | \ (bbox_pred[:, [1]] > bbox_pred[:, [3]]) invalid_bbox_idx = invalid_bbox_idx.expand_as(bbox_pred) bbox_pred = torch.where(invalid_bbox_idx, reg_bbox, bbox_pred) cls_scores.append(cls_score) bbox_preds.append(bbox_pred) return tuple(cls_scores), tuple(bbox_preds) def deform_sampling(self, feat: Tensor, offset: Tensor) -> Tensor: """Sampling the feature x according to offset. Args: feat (Tensor): Feature offset (Tensor): Spatial offset for feature sampling """ # it is an equivalent implementation of bilinear interpolation b, c, h, w = feat.shape weight = feat.new_ones(c, 1, 1, 1) y = deform_conv2d(feat, offset, weight, 1, 0, 1, c, c) return y def anchor_center(self, anchors: Tensor) -> Tensor: """Get anchor centers from anchors. Args: anchors (Tensor): Anchor list with shape (N, 4), "xyxy" format. Returns: Tensor: Anchor centers with shape (N, 2), "xy" format. """ anchors_cx = (anchors[:, 2] + anchors[:, 0]) / 2 anchors_cy = (anchors[:, 3] + anchors[:, 1]) / 2 return torch.stack([anchors_cx, anchors_cy], dim=-1) def loss_by_feat_single(self, anchors: Tensor, cls_score: Tensor, bbox_pred: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, alignment_metrics: Tensor, stride: Tuple[int, int]) -> dict: """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: anchors (Tensor): Box reference for each scale level with shape (N, num_total_anchors, 4). cls_score (Tensor): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W). bbox_pred (Tensor): Decoded bboxes for each scale level with shape (N, num_anchors * 4, H, W). labels (Tensor): Labels of each anchors with shape (N, num_total_anchors). label_weights (Tensor): Label weights of each anchor with shape (N, num_total_anchors). bbox_targets (Tensor): BBox regression targets of each anchor with shape (N, num_total_anchors, 4). alignment_metrics (Tensor): Alignment metrics with shape (N, num_total_anchors). stride (Tuple[int, int]): Downsample stride of the feature map. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert stride[0] == stride[1], 'h stride is not equal to w stride!' anchors = anchors.reshape(-1, 4) cls_score = cls_score.permute(0, 2, 3, 1).reshape( -1, self.cls_out_channels).contiguous() bbox_pred = bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) bbox_targets = bbox_targets.reshape(-1, 4) labels = labels.reshape(-1) alignment_metrics = alignment_metrics.reshape(-1) label_weights = label_weights.reshape(-1) targets = labels if self.epoch < self.initial_epoch else ( labels, alignment_metrics) cls_loss_func = self.initial_loss_cls \ if self.epoch < self.initial_epoch else self.loss_cls loss_cls = cls_loss_func( cls_score, targets, label_weights, avg_factor=1.0) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = self.num_classes pos_inds = ((labels >= 0) & (labels < bg_class_ind)).nonzero().squeeze(1) if len(pos_inds) > 0: pos_bbox_targets = bbox_targets[pos_inds] pos_bbox_pred = bbox_pred[pos_inds] pos_anchors = anchors[pos_inds] pos_decode_bbox_pred = pos_bbox_pred pos_decode_bbox_targets = pos_bbox_targets / stride[0] # regression loss pos_bbox_weight = self.centerness_target( pos_anchors, pos_bbox_targets ) if self.epoch < self.initial_epoch else alignment_metrics[ pos_inds] loss_bbox = self.loss_bbox( pos_decode_bbox_pred, pos_decode_bbox_targets, weight=pos_bbox_weight, avg_factor=1.0) else: loss_bbox = bbox_pred.sum() * 0 pos_bbox_weight = bbox_targets.new_tensor(0.) return loss_cls, loss_bbox, alignment_metrics.sum( ), pos_bbox_weight.sum() def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level Has shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Decoded box for each scale level with shape (N, num_anchors * 4, H, W) in [tl_x, tl_y, br_x, br_y] format. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ num_imgs = len(batch_img_metas) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) flatten_cls_scores = torch.cat([ cls_score.permute(0, 2, 3, 1).reshape(num_imgs, -1, self.cls_out_channels) for cls_score in cls_scores ], 1) flatten_bbox_preds = torch.cat([ bbox_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, 4) * stride[0] for bbox_pred, stride in zip(bbox_preds, self.prior_generator.strides) ], 1) cls_reg_targets = self.get_targets( flatten_cls_scores, flatten_bbox_preds, anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (anchor_list, labels_list, label_weights_list, bbox_targets_list, alignment_metrics_list) = cls_reg_targets losses_cls, losses_bbox, \ cls_avg_factors, bbox_avg_factors = multi_apply( self.loss_by_feat_single, anchor_list, cls_scores, bbox_preds, labels_list, label_weights_list, bbox_targets_list, alignment_metrics_list, self.prior_generator.strides) cls_avg_factor = reduce_mean(sum(cls_avg_factors)).clamp_(min=1).item() losses_cls = list(map(lambda x: x / cls_avg_factor, losses_cls)) bbox_avg_factor = reduce_mean( sum(bbox_avg_factors)).clamp_(min=1).item() losses_bbox = list(map(lambda x: x / bbox_avg_factor, losses_bbox)) return dict(loss_cls=losses_cls, loss_bbox=losses_bbox) def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], score_factor_list: List[Tensor], mlvl_priors: List[Tensor], img_meta: dict, cfg: Optional[ConfigDict] = None, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: cls_score_list (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). score_factor_list (list[Tensor]): Score factor from all scale levels of a single image, each item has shape (num_priors * 1, H, W). mlvl_priors (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid. In all anchor-based methods, it has shape (num_priors, 4). In all anchor-free methods, it has shape (num_priors, 2) when `with_stride=True`, otherwise it still has shape (num_priors, 4). img_meta (dict): Image meta info. cfg (:obj:`ConfigDict`, optional): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: tuple[Tensor]: Results of detected bboxes and labels. If with_nms is False and mlvl_score_factor is None, return mlvl_bboxes and mlvl_scores, else return mlvl_bboxes, mlvl_scores and mlvl_score_factor. Usually with_nms is False is used for aug test. If with_nms is True, then return the following format - det_bboxes (Tensor): Predicted bboxes with shape \ [num_bboxes, 5], where the first 4 columns are bounding \ box positions (tl_x, tl_y, br_x, br_y) and the 5-th \ column are scores between 0 and 1. - det_labels (Tensor): Predicted labels of the corresponding \ box with shape [num_bboxes]. """ cfg = self.test_cfg if cfg is None else cfg nms_pre = cfg.get('nms_pre', -1) mlvl_bboxes = [] mlvl_scores = [] mlvl_labels = [] for cls_score, bbox_pred, priors, stride in zip( cls_score_list, bbox_pred_list, mlvl_priors, self.prior_generator.strides): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) * stride[0] scores = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) # After https://github.com/open-mmlab/mmdetection/pull/6268/, # this operation keeps fewer bboxes under the same `nms_pre`. # There is no difference in performance for most models. If you # find a slight drop in performance, you can set a larger # `nms_pre` than before. results = filter_scores_and_topk( scores, cfg.score_thr, nms_pre, dict(bbox_pred=bbox_pred, priors=priors)) scores, labels, keep_idxs, filtered_results = results bboxes = filtered_results['bbox_pred'] mlvl_bboxes.append(bboxes) mlvl_scores.append(scores) mlvl_labels.append(labels) results = InstanceData() results.bboxes = torch.cat(mlvl_bboxes) results.scores = torch.cat(mlvl_scores) results.labels = torch.cat(mlvl_labels) return self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta) def get_targets(self, cls_scores: List[List[Tensor]], bbox_preds: List[List[Tensor]], anchor_list: List[List[Tensor]], valid_flag_list: List[List[Tensor]], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs: bool = True) -> tuple: """Compute regression and classification targets for anchors in multiple images. Args: cls_scores (list[list[Tensor]]): Classification predictions of images, a 3D-Tensor with shape [num_imgs, num_priors, num_classes]. bbox_preds (list[list[Tensor]]): Decoded bboxes predictions of one image, a 3D-Tensor with shape [num_imgs, num_priors, 4] in [tl_x, tl_y, br_x, br_y] format. anchor_list (list[list[Tensor]]): Multi level anchors of each image. The outer list indicates images, and the inner list corresponds to feature levels of the image. Each element of the inner list is a tensor of shape (num_anchors, 4). valid_flag_list (list[list[Tensor]]): Multi level valid flags of each image. The outer list indicates images, and the inner list corresponds to feature levels of the image. Each element of the inner list is a tensor of shape (num_anchors, ) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Returns: tuple: a tuple containing learning targets. - anchors_list (list[list[Tensor]]): Anchors of each level. - labels_list (list[Tensor]): Labels of each level. - label_weights_list (list[Tensor]): Label weights of each level. - bbox_targets_list (list[Tensor]): BBox targets of each level. - norm_alignment_metrics_list (list[Tensor]): Normalized alignment metrics of each level. """ num_imgs = len(batch_img_metas) assert len(anchor_list) == len(valid_flag_list) == num_imgs # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] num_level_anchors_list = [num_level_anchors] * num_imgs # concat all level anchors and flags to a single tensor for i in range(num_imgs): assert len(anchor_list[i]) == len(valid_flag_list[i]) anchor_list[i] = torch.cat(anchor_list[i]) valid_flag_list[i] = torch.cat(valid_flag_list[i]) # compute targets for each image if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None] * num_imgs # anchor_list: list(b * [-1, 4]) # get epoch information from message hub message_hub = MessageHub.get_current_instance() self.epoch = message_hub.get_info('epoch') if self.epoch < self.initial_epoch: (all_anchors, all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, pos_inds_list, neg_inds_list, sampling_result) = multi_apply( super()._get_targets_single, anchor_list, valid_flag_list, num_level_anchors_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs) all_assign_metrics = [ weight[..., 0] for weight in all_bbox_weights ] else: (all_anchors, all_labels, all_label_weights, all_bbox_targets, all_assign_metrics) = multi_apply( self._get_targets_single, cls_scores, bbox_preds, anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs) # split targets to a list w.r.t. multiple levels anchors_list = images_to_levels(all_anchors, num_level_anchors) labels_list = images_to_levels(all_labels, num_level_anchors) label_weights_list = images_to_levels(all_label_weights, num_level_anchors) bbox_targets_list = images_to_levels(all_bbox_targets, num_level_anchors) norm_alignment_metrics_list = images_to_levels(all_assign_metrics, num_level_anchors) return (anchors_list, labels_list, label_weights_list, bbox_targets_list, norm_alignment_metrics_list) def _get_targets_single(self, cls_scores: Tensor, bbox_preds: Tensor, flat_anchors: Tensor, valid_flags: Tensor, gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None, unmap_outputs: bool = True) -> tuple: """Compute regression, classification targets for anchors in a single image. Args: cls_scores (Tensor): Box scores for each image. bbox_preds (Tensor): Box energies / deltas for each image. flat_anchors (Tensor): Multi-level anchors of the image, which are concatenated into a single tensor of shape (num_anchors ,4) valid_flags (Tensor): Multi level valid flags of the image, which are concatenated into a single tensor of shape (num_anchors,). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Returns: tuple: N is the number of total anchors in the image. anchors (Tensor): All anchors in the image with shape (N, 4). labels (Tensor): Labels of all anchors in the image with shape (N,). label_weights (Tensor): Label weights of all anchor in the image with shape (N,). bbox_targets (Tensor): BBox targets of all anchors in the image with shape (N, 4). norm_alignment_metrics (Tensor): Normalized alignment metrics of all priors in the image with shape (N,). """ inside_flags = anchor_inside_flags(flat_anchors, valid_flags, img_meta['img_shape'][:2], self.train_cfg['allowed_border']) if not inside_flags.any(): raise ValueError( 'There is no valid anchor inside the image boundary. Please ' 'check the image size and anchor sizes, or set ' '``allowed_border`` to -1 to skip the condition.') # assign gt and sample anchors anchors = flat_anchors[inside_flags, :] pred_instances = InstanceData( priors=anchors, scores=cls_scores[inside_flags, :], bboxes=bbox_preds[inside_flags, :]) assign_result = self.alignment_assigner.assign(pred_instances, gt_instances, gt_instances_ignore, self.alpha, self.beta) assign_ious = assign_result.max_overlaps assign_metrics = assign_result.assign_metrics sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) num_valid_anchors = anchors.shape[0] bbox_targets = torch.zeros_like(anchors) labels = anchors.new_full((num_valid_anchors, ), self.num_classes, dtype=torch.long) label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) norm_alignment_metrics = anchors.new_zeros( num_valid_anchors, dtype=torch.float) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: # point-based pos_bbox_targets = sampling_result.pos_gt_bboxes bbox_targets[pos_inds, :] = pos_bbox_targets labels[pos_inds] = sampling_result.pos_gt_labels if self.train_cfg['pos_weight'] <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = self.train_cfg['pos_weight'] if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 class_assigned_gt_inds = torch.unique( sampling_result.pos_assigned_gt_inds) for gt_inds in class_assigned_gt_inds: gt_class_inds = pos_inds[sampling_result.pos_assigned_gt_inds == gt_inds] pos_alignment_metrics = assign_metrics[gt_class_inds] pos_ious = assign_ious[gt_class_inds] pos_norm_alignment_metrics = pos_alignment_metrics / ( pos_alignment_metrics.max() + 10e-8) * pos_ious.max() norm_alignment_metrics[gt_class_inds] = pos_norm_alignment_metrics # map up to original set of anchors if unmap_outputs: num_total_anchors = flat_anchors.size(0) anchors = unmap(anchors, num_total_anchors, inside_flags) labels = unmap( labels, num_total_anchors, inside_flags, fill=self.num_classes) label_weights = unmap(label_weights, num_total_anchors, inside_flags) bbox_targets = unmap(bbox_targets, num_total_anchors, inside_flags) norm_alignment_metrics = unmap(norm_alignment_metrics, num_total_anchors, inside_flags) return (anchors, labels, label_weights, bbox_targets, norm_alignment_metrics) ================================================ FILE: mmdet/models/dense_heads/vfnet_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple, Union import numpy as np import torch import torch.nn as nn from mmcv.cnn import ConvModule, Scale from mmcv.ops import DeformConv2d from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures.bbox import bbox_overlaps from mmdet.utils import (ConfigType, InstanceList, MultiConfig, OptInstanceList, RangeType, reduce_mean) from ..task_modules.prior_generators import MlvlPointGenerator from ..task_modules.samplers import PseudoSampler from ..utils import multi_apply from .atss_head import ATSSHead from .fcos_head import FCOSHead INF = 1e8 @MODELS.register_module() class VFNetHead(ATSSHead, FCOSHead): """Head of `VarifocalNet (VFNet): An IoU-aware Dense Object Detector.`_. The VFNet predicts IoU-aware classification scores which mix the object presence confidence and object localization accuracy as the detection score. It is built on the FCOS architecture and uses ATSS for defining positive/negative training examples. The VFNet is trained with Varifocal Loss and empolys star-shaped deformable convolution to extract features for a bbox. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. regress_ranges (Sequence[Tuple[int, int]]): Regress range of multiple level points. center_sampling (bool): If true, use center sampling. Defaults to False. center_sample_radius (float): Radius of center sampling. Defaults to 1.5. sync_num_pos (bool): If true, synchronize the number of positive examples across GPUs. Defaults to True gradient_mul (float): The multiplier to gradients from bbox refinement and recognition. Defaults to 0.1. bbox_norm_type (str): The bbox normalization type, 'reg_denom' or 'stride'. Defaults to reg_denom loss_cls_fl (:obj:`ConfigDict` or dict): Config of focal loss. use_vfl (bool): If true, use varifocal loss for training. Defaults to True. loss_cls (:obj:`ConfigDict` or dict): Config of varifocal loss. loss_bbox (:obj:`ConfigDict` or dict): Config of localization loss, GIoU Loss. loss_bbox (:obj:`ConfigDict` or dict): Config of localization refinement loss, GIoU Loss. norm_cfg (:obj:`ConfigDict` or dict): dictionary to construct and config norm layer. Defaults to norm_cfg=dict(type='GN', num_groups=32, requires_grad=True). use_atss (bool): If true, use ATSS to define positive/negative examples. Defaults to True. anchor_generator (:obj:`ConfigDict` or dict): Config of anchor generator for ATSS. init_cfg (:obj:`ConfigDict` or dict or list[dict] or list[:obj:`ConfigDict`]): Initialization config dict. Example: >>> self = VFNetHead(11, 7) >>> feats = [torch.rand(1, 7, s, s) for s in [4, 8, 16, 32, 64]] >>> cls_score, bbox_pred, bbox_pred_refine= self.forward(feats) >>> assert len(cls_score) == len(self.scales) """ # noqa: E501 def __init__(self, num_classes: int, in_channels: int, regress_ranges: RangeType = ((-1, 64), (64, 128), (128, 256), (256, 512), (512, INF)), center_sampling: bool = False, center_sample_radius: float = 1.5, sync_num_pos: bool = True, gradient_mul: float = 0.1, bbox_norm_type: str = 'reg_denom', loss_cls_fl: ConfigType = dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), use_vfl: bool = True, loss_cls: ConfigType = dict( type='VarifocalLoss', use_sigmoid=True, alpha=0.75, gamma=2.0, iou_weighted=True, loss_weight=1.0), loss_bbox: ConfigType = dict( type='GIoULoss', loss_weight=1.5), loss_bbox_refine: ConfigType = dict( type='GIoULoss', loss_weight=2.0), norm_cfg: ConfigType = dict( type='GN', num_groups=32, requires_grad=True), use_atss: bool = True, reg_decoded_bbox: bool = True, anchor_generator: ConfigType = dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, center_offset=0.0, strides=[8, 16, 32, 64, 128]), init_cfg: MultiConfig = dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='vfnet_cls', std=0.01, bias_prob=0.01)), **kwargs) -> None: # dcn base offsets, adapted from reppoints_head.py self.num_dconv_points = 9 self.dcn_kernel = int(np.sqrt(self.num_dconv_points)) self.dcn_pad = int((self.dcn_kernel - 1) / 2) dcn_base = np.arange(-self.dcn_pad, self.dcn_pad + 1).astype(np.float64) dcn_base_y = np.repeat(dcn_base, self.dcn_kernel) dcn_base_x = np.tile(dcn_base, self.dcn_kernel) dcn_base_offset = np.stack([dcn_base_y, dcn_base_x], axis=1).reshape( (-1)) self.dcn_base_offset = torch.tensor(dcn_base_offset).view(1, -1, 1, 1) super(FCOSHead, self).__init__( num_classes=num_classes, in_channels=in_channels, norm_cfg=norm_cfg, init_cfg=init_cfg, **kwargs) self.regress_ranges = regress_ranges self.reg_denoms = [ regress_range[-1] for regress_range in regress_ranges ] self.reg_denoms[-1] = self.reg_denoms[-2] * 2 self.center_sampling = center_sampling self.center_sample_radius = center_sample_radius self.sync_num_pos = sync_num_pos self.bbox_norm_type = bbox_norm_type self.gradient_mul = gradient_mul self.use_vfl = use_vfl if self.use_vfl: self.loss_cls = MODELS.build(loss_cls) else: self.loss_cls = MODELS.build(loss_cls_fl) self.loss_bbox = MODELS.build(loss_bbox) self.loss_bbox_refine = MODELS.build(loss_bbox_refine) # for getting ATSS targets self.use_atss = use_atss self.reg_decoded_bbox = reg_decoded_bbox self.use_sigmoid_cls = loss_cls.get('use_sigmoid', False) self.anchor_center_offset = anchor_generator['center_offset'] self.num_base_priors = self.prior_generator.num_base_priors[0] if self.train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) if self.train_cfg.get('sampler', None) is not None: self.sampler = TASK_UTILS.build( self.train_cfg['sampler'], default_args=dict(context=self)) else: self.sampler = PseudoSampler() # only be used in `get_atss_targets` when `use_atss` is True self.atss_prior_generator = TASK_UTILS.build(anchor_generator) self.fcos_prior_generator = MlvlPointGenerator( anchor_generator['strides'], self.anchor_center_offset if self.use_atss else 0.5) # In order to reuse the `get_bboxes` in `BaseDenseHead. # Only be used in testing phase. self.prior_generator = self.fcos_prior_generator def _init_layers(self) -> None: """Initialize layers of the head.""" super(FCOSHead, self)._init_cls_convs() super(FCOSHead, self)._init_reg_convs() self.relu = nn.ReLU() self.vfnet_reg_conv = ConvModule( self.feat_channels, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, bias=self.conv_bias) self.vfnet_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1) self.scales = nn.ModuleList([Scale(1.0) for _ in self.strides]) self.vfnet_reg_refine_dconv = DeformConv2d( self.feat_channels, self.feat_channels, self.dcn_kernel, 1, padding=self.dcn_pad) self.vfnet_reg_refine = nn.Conv2d(self.feat_channels, 4, 3, padding=1) self.scales_refine = nn.ModuleList([Scale(1.0) for _ in self.strides]) self.vfnet_cls_dconv = DeformConv2d( self.feat_channels, self.feat_channels, self.dcn_kernel, 1, padding=self.dcn_pad) self.vfnet_cls = nn.Conv2d( self.feat_channels, self.cls_out_channels, 3, padding=1) def forward(self, x: Tuple[Tensor]) -> Tuple[List[Tensor]]: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: - cls_scores (list[Tensor]): Box iou-aware scores for each scale level, each is a 4D-tensor, the channel number is num_points * num_classes. - bbox_preds (list[Tensor]): Box offsets for each scale level, each is a 4D-tensor, the channel number is num_points * 4. - bbox_preds_refine (list[Tensor]): Refined Box offsets for each scale level, each is a 4D-tensor, the channel number is num_points * 4. """ return multi_apply(self.forward_single, x, self.scales, self.scales_refine, self.strides, self.reg_denoms) def forward_single(self, x: Tensor, scale: Scale, scale_refine: Scale, stride: int, reg_denom: int) -> tuple: """Forward features of a single scale level. Args: x (Tensor): FPN feature maps of the specified stride. scale (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize the bbox prediction. scale_refine (:obj: `mmcv.cnn.Scale`): Learnable scale module to resize the refined bbox prediction. stride (int): The corresponding stride for feature maps, used to normalize the bbox prediction when bbox_norm_type = 'stride'. reg_denom (int): The corresponding regression range for feature maps, only used to normalize the bbox prediction when bbox_norm_type = 'reg_denom'. Returns: tuple: iou-aware cls scores for each box, bbox predictions and refined bbox predictions of input feature maps. """ cls_feat = x reg_feat = x for cls_layer in self.cls_convs: cls_feat = cls_layer(cls_feat) for reg_layer in self.reg_convs: reg_feat = reg_layer(reg_feat) # predict the bbox_pred of different level reg_feat_init = self.vfnet_reg_conv(reg_feat) if self.bbox_norm_type == 'reg_denom': bbox_pred = scale( self.vfnet_reg(reg_feat_init)).float().exp() * reg_denom elif self.bbox_norm_type == 'stride': bbox_pred = scale( self.vfnet_reg(reg_feat_init)).float().exp() * stride else: raise NotImplementedError # compute star deformable convolution offsets # converting dcn_offset to reg_feat.dtype thus VFNet can be # trained with FP16 dcn_offset = self.star_dcn_offset(bbox_pred, self.gradient_mul, stride).to(reg_feat.dtype) # refine the bbox_pred reg_feat = self.relu(self.vfnet_reg_refine_dconv(reg_feat, dcn_offset)) bbox_pred_refine = scale_refine( self.vfnet_reg_refine(reg_feat)).float().exp() bbox_pred_refine = bbox_pred_refine * bbox_pred.detach() # predict the iou-aware cls score cls_feat = self.relu(self.vfnet_cls_dconv(cls_feat, dcn_offset)) cls_score = self.vfnet_cls(cls_feat) if self.training: return cls_score, bbox_pred, bbox_pred_refine else: return cls_score, bbox_pred_refine def star_dcn_offset(self, bbox_pred: Tensor, gradient_mul: float, stride: int) -> Tensor: """Compute the star deformable conv offsets. Args: bbox_pred (Tensor): Predicted bbox distance offsets (l, r, t, b). gradient_mul (float): Gradient multiplier. stride (int): The corresponding stride for feature maps, used to project the bbox onto the feature map. Returns: Tensor: The offsets for deformable convolution. """ dcn_base_offset = self.dcn_base_offset.type_as(bbox_pred) bbox_pred_grad_mul = (1 - gradient_mul) * bbox_pred.detach() + \ gradient_mul * bbox_pred # map to the feature map scale bbox_pred_grad_mul = bbox_pred_grad_mul / stride N, C, H, W = bbox_pred.size() x1 = bbox_pred_grad_mul[:, 0, :, :] y1 = bbox_pred_grad_mul[:, 1, :, :] x2 = bbox_pred_grad_mul[:, 2, :, :] y2 = bbox_pred_grad_mul[:, 3, :, :] bbox_pred_grad_mul_offset = bbox_pred.new_zeros( N, 2 * self.num_dconv_points, H, W) bbox_pred_grad_mul_offset[:, 0, :, :] = -1.0 * y1 # -y1 bbox_pred_grad_mul_offset[:, 1, :, :] = -1.0 * x1 # -x1 bbox_pred_grad_mul_offset[:, 2, :, :] = -1.0 * y1 # -y1 bbox_pred_grad_mul_offset[:, 4, :, :] = -1.0 * y1 # -y1 bbox_pred_grad_mul_offset[:, 5, :, :] = x2 # x2 bbox_pred_grad_mul_offset[:, 7, :, :] = -1.0 * x1 # -x1 bbox_pred_grad_mul_offset[:, 11, :, :] = x2 # x2 bbox_pred_grad_mul_offset[:, 12, :, :] = y2 # y2 bbox_pred_grad_mul_offset[:, 13, :, :] = -1.0 * x1 # -x1 bbox_pred_grad_mul_offset[:, 14, :, :] = y2 # y2 bbox_pred_grad_mul_offset[:, 16, :, :] = y2 # y2 bbox_pred_grad_mul_offset[:, 17, :, :] = x2 # x2 dcn_offset = bbox_pred_grad_mul_offset - dcn_base_offset return dcn_offset def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], bbox_preds_refine: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Compute loss of the head. Args: cls_scores (list[Tensor]): Box iou-aware scores for each scale level, each is a 4D-tensor, the channel number is num_points * num_classes. bbox_preds (list[Tensor]): Box offsets for each scale level, each is a 4D-tensor, the channel number is num_points * 4. bbox_preds_refine (list[Tensor]): Refined Box offsets for each scale level, each is a 4D-tensor, the channel number is num_points * 4. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert len(cls_scores) == len(bbox_preds) == len(bbox_preds_refine) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] all_level_points = self.fcos_prior_generator.grid_priors( featmap_sizes, bbox_preds[0].dtype, bbox_preds[0].device) labels, label_weights, bbox_targets, bbox_weights = self.get_targets( cls_scores, all_level_points, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) num_imgs = cls_scores[0].size(0) # flatten cls_scores, bbox_preds and bbox_preds_refine flatten_cls_scores = [ cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels).contiguous() for cls_score in cls_scores ] flatten_bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4).contiguous() for bbox_pred in bbox_preds ] flatten_bbox_preds_refine = [ bbox_pred_refine.permute(0, 2, 3, 1).reshape(-1, 4).contiguous() for bbox_pred_refine in bbox_preds_refine ] flatten_cls_scores = torch.cat(flatten_cls_scores) flatten_bbox_preds = torch.cat(flatten_bbox_preds) flatten_bbox_preds_refine = torch.cat(flatten_bbox_preds_refine) flatten_labels = torch.cat(labels) flatten_bbox_targets = torch.cat(bbox_targets) # repeat points to align with bbox_preds flatten_points = torch.cat( [points.repeat(num_imgs, 1) for points in all_level_points]) # FG cat_id: [0, num_classes - 1], BG cat_id: num_classes bg_class_ind = self.num_classes pos_inds = torch.where( ((flatten_labels >= 0) & (flatten_labels < bg_class_ind)) > 0)[0] num_pos = len(pos_inds) pos_bbox_preds = flatten_bbox_preds[pos_inds] pos_bbox_preds_refine = flatten_bbox_preds_refine[pos_inds] pos_labels = flatten_labels[pos_inds] # sync num_pos across all gpus if self.sync_num_pos: num_pos_avg_per_gpu = reduce_mean( pos_inds.new_tensor(num_pos).float()).item() num_pos_avg_per_gpu = max(num_pos_avg_per_gpu, 1.0) else: num_pos_avg_per_gpu = num_pos pos_bbox_targets = flatten_bbox_targets[pos_inds] pos_points = flatten_points[pos_inds] pos_decoded_bbox_preds = self.bbox_coder.decode( pos_points, pos_bbox_preds) pos_decoded_target_preds = self.bbox_coder.decode( pos_points, pos_bbox_targets) iou_targets_ini = bbox_overlaps( pos_decoded_bbox_preds, pos_decoded_target_preds.detach(), is_aligned=True).clamp(min=1e-6) bbox_weights_ini = iou_targets_ini.clone().detach() bbox_avg_factor_ini = reduce_mean( bbox_weights_ini.sum()).clamp_(min=1).item() pos_decoded_bbox_preds_refine = \ self.bbox_coder.decode(pos_points, pos_bbox_preds_refine) iou_targets_rf = bbox_overlaps( pos_decoded_bbox_preds_refine, pos_decoded_target_preds.detach(), is_aligned=True).clamp(min=1e-6) bbox_weights_rf = iou_targets_rf.clone().detach() bbox_avg_factor_rf = reduce_mean( bbox_weights_rf.sum()).clamp_(min=1).item() if num_pos > 0: loss_bbox = self.loss_bbox( pos_decoded_bbox_preds, pos_decoded_target_preds.detach(), weight=bbox_weights_ini, avg_factor=bbox_avg_factor_ini) loss_bbox_refine = self.loss_bbox_refine( pos_decoded_bbox_preds_refine, pos_decoded_target_preds.detach(), weight=bbox_weights_rf, avg_factor=bbox_avg_factor_rf) # build IoU-aware cls_score targets if self.use_vfl: pos_ious = iou_targets_rf.clone().detach() cls_iou_targets = torch.zeros_like(flatten_cls_scores) cls_iou_targets[pos_inds, pos_labels] = pos_ious else: loss_bbox = pos_bbox_preds.sum() * 0 loss_bbox_refine = pos_bbox_preds_refine.sum() * 0 if self.use_vfl: cls_iou_targets = torch.zeros_like(flatten_cls_scores) if self.use_vfl: loss_cls = self.loss_cls( flatten_cls_scores, cls_iou_targets, avg_factor=num_pos_avg_per_gpu) else: loss_cls = self.loss_cls( flatten_cls_scores, flatten_labels, weight=label_weights, avg_factor=num_pos_avg_per_gpu) return dict( loss_cls=loss_cls, loss_bbox=loss_bbox, loss_bbox_rf=loss_bbox_refine) def get_targets( self, cls_scores: List[Tensor], mlvl_points: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> tuple: """A wrapper for computing ATSS and FCOS targets for points in multiple images. Args: cls_scores (list[Tensor]): Box iou-aware scores for each scale level with shape (N, num_points * num_classes, H, W). mlvl_points (list[Tensor]): Points of each fpn level, each has shape (num_points, 2). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: tuple: - labels_list (list[Tensor]): Labels of each level. - label_weights (Tensor/None): Label weights of all levels. - bbox_targets_list (list[Tensor]): Regression targets of each level, (l, t, r, b). - bbox_weights (Tensor/None): Bbox weights of all levels. """ if self.use_atss: return self.get_atss_targets(cls_scores, mlvl_points, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) else: self.norm_on_bbox = False return self.get_fcos_targets(mlvl_points, batch_gt_instances) def _get_targets_single(self, *args, **kwargs): """Avoid ambiguity in multiple inheritance.""" if self.use_atss: return ATSSHead._get_targets_single(self, *args, **kwargs) else: return FCOSHead._get_targets_single(self, *args, **kwargs) def get_fcos_targets(self, points: List[Tensor], batch_gt_instances: InstanceList) -> tuple: """Compute FCOS regression and classification targets for points in multiple images. Args: points (list[Tensor]): Points of each fpn level, each has shape (num_points, 2). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. Returns: tuple: - labels (list[Tensor]): Labels of each level. - label_weights: None, to be compatible with ATSS targets. - bbox_targets (list[Tensor]): BBox targets of each level. - bbox_weights: None, to be compatible with ATSS targets. """ labels, bbox_targets = FCOSHead.get_targets(self, points, batch_gt_instances) label_weights = None bbox_weights = None return labels, label_weights, bbox_targets, bbox_weights def get_anchors(self, featmap_sizes: List[Tuple], batch_img_metas: List[dict], device: str = 'cuda') -> tuple: """Get anchors according to feature map sizes. Args: featmap_sizes (list[tuple]): Multi-level feature map sizes. batch_img_metas (list[dict]): Image meta info. device (str): Device for returned tensors Returns: tuple: - anchor_list (list[Tensor]): Anchors of each image. - valid_flag_list (list[Tensor]): Valid flags of each image. """ num_imgs = len(batch_img_metas) # since feature map sizes of all images are the same, we only compute # anchors for one time multi_level_anchors = self.atss_prior_generator.grid_priors( featmap_sizes, device=device) anchor_list = [multi_level_anchors for _ in range(num_imgs)] # for each image, we compute valid flags of multi level anchors valid_flag_list = [] for img_id, img_meta in enumerate(batch_img_metas): multi_level_flags = self.atss_prior_generator.valid_flags( featmap_sizes, img_meta['pad_shape'], device=device) valid_flag_list.append(multi_level_flags) return anchor_list, valid_flag_list def get_atss_targets( self, cls_scores: List[Tensor], mlvl_points: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> tuple: """A wrapper for computing ATSS targets for points in multiple images. Args: cls_scores (list[Tensor]): Box iou-aware scores for each scale level with shape (N, num_points * num_classes, H, W). mlvl_points (list[Tensor]): Points of each fpn level, each has shape (num_points, 2). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: tuple: - labels_list (list[Tensor]): Labels of each level. - label_weights (Tensor): Label weights of all levels. - bbox_targets_list (list[Tensor]): Regression targets of each level, (l, t, r, b). - bbox_weights (Tensor): Bbox weights of all levels. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len( featmap_sizes ) == self.atss_prior_generator.num_levels == \ self.fcos_prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = ATSSHead.get_targets( self, anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=True) (anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets bbox_targets_list = [ bbox_targets.reshape(-1, 4) for bbox_targets in bbox_targets_list ] num_imgs = len(batch_img_metas) # transform bbox_targets (x1, y1, x2, y2) into (l, t, r, b) format bbox_targets_list = self.transform_bbox_targets( bbox_targets_list, mlvl_points, num_imgs) labels_list = [labels.reshape(-1) for labels in labels_list] label_weights_list = [ label_weights.reshape(-1) for label_weights in label_weights_list ] bbox_weights_list = [ bbox_weights.reshape(-1) for bbox_weights in bbox_weights_list ] label_weights = torch.cat(label_weights_list) bbox_weights = torch.cat(bbox_weights_list) return labels_list, label_weights, bbox_targets_list, bbox_weights def transform_bbox_targets(self, decoded_bboxes: List[Tensor], mlvl_points: List[Tensor], num_imgs: int) -> List[Tensor]: """Transform bbox_targets (x1, y1, x2, y2) into (l, t, r, b) format. Args: decoded_bboxes (list[Tensor]): Regression targets of each level, in the form of (x1, y1, x2, y2). mlvl_points (list[Tensor]): Points of each fpn level, each has shape (num_points, 2). num_imgs (int): the number of images in a batch. Returns: bbox_targets (list[Tensor]): Regression targets of each level in the form of (l, t, r, b). """ # TODO: Re-implemented in Class PointCoder assert len(decoded_bboxes) == len(mlvl_points) num_levels = len(decoded_bboxes) mlvl_points = [points.repeat(num_imgs, 1) for points in mlvl_points] bbox_targets = [] for i in range(num_levels): bbox_target = self.bbox_coder.encode(mlvl_points[i], decoded_bboxes[i]) bbox_targets.append(bbox_target) return bbox_targets def _load_from_state_dict(self, state_dict: dict, prefix: str, local_metadata: dict, strict: bool, missing_keys: Union[List[str], str], unexpected_keys: Union[List[str], str], error_msgs: Union[List[str], str]) -> None: """Override the method in the parent class to avoid changing para's name.""" pass ================================================ FILE: mmdet/models/dense_heads/yolact_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import List, Optional import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.model import BaseModule, ModuleList from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptInstanceList, OptMultiConfig) from ..layers import fast_nms from ..utils import images_to_levels, multi_apply, select_single_mlvl from ..utils.misc import empty_instances from .anchor_head import AnchorHead from .base_mask_head import BaseMaskHead @MODELS.register_module() class YOLACTHead(AnchorHead): """YOLACT box head used in https://arxiv.org/abs/1904.02689. Note that YOLACT head is a light version of RetinaNet head. Four differences are described as follows: 1. YOLACT box head has three-times fewer anchors. 2. YOLACT box head shares the convs for box and cls branches. 3. YOLACT box head uses OHEM instead of Focal loss. 4. YOLACT box head predicts a set of mask coefficients for each box. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. anchor_generator (:obj:`ConfigDict` or dict): Config dict for anchor generator loss_cls (:obj:`ConfigDict` or dict): Config of classification loss. loss_bbox (:obj:`ConfigDict` or dict): Config of localization loss. num_head_convs (int): Number of the conv layers shared by box and cls branches. num_protos (int): Number of the mask coefficients. use_ohem (bool): If true, ``loss_single_OHEM`` will be used for cls loss calculation. If false, ``loss_single`` will be used. conv_cfg (:obj:`ConfigDict` or dict, optional): Dictionary to construct and config conv layer. norm_cfg (:obj:`ConfigDict` or dict, optional): Dictionary to construct and config norm layer. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. """ def __init__(self, num_classes: int, in_channels: int, anchor_generator: ConfigType = dict( type='AnchorGenerator', octave_base_scale=3, scales_per_octave=1, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), loss_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=False, reduction='none', loss_weight=1.0), loss_bbox: ConfigType = dict( type='SmoothL1Loss', beta=1.0, loss_weight=1.5), num_head_convs: int = 1, num_protos: int = 32, use_ohem: bool = True, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, init_cfg: OptMultiConfig = dict( type='Xavier', distribution='uniform', bias=0, layer='Conv2d'), **kwargs) -> None: self.num_head_convs = num_head_convs self.num_protos = num_protos self.use_ohem = use_ohem self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg super().__init__( num_classes=num_classes, in_channels=in_channels, loss_cls=loss_cls, loss_bbox=loss_bbox, anchor_generator=anchor_generator, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" self.relu = nn.ReLU(inplace=True) self.head_convs = ModuleList() for i in range(self.num_head_convs): chn = self.in_channels if i == 0 else self.feat_channels self.head_convs.append( ConvModule( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.conv_cls = nn.Conv2d( self.feat_channels, self.num_base_priors * self.cls_out_channels, 3, padding=1) self.conv_reg = nn.Conv2d( self.feat_channels, self.num_base_priors * 4, 3, padding=1) self.conv_coeff = nn.Conv2d( self.feat_channels, self.num_base_priors * self.num_protos, 3, padding=1) def forward_single(self, x: Tensor) -> tuple: """Forward feature of a single scale level. Args: x (Tensor): Features of a single scale level. Returns: tuple: - cls_score (Tensor): Cls scores for a single scale level the channels number is num_anchors * num_classes. - bbox_pred (Tensor): Box energies / deltas for a single scale level, the channels number is num_anchors * 4. - coeff_pred (Tensor): Mask coefficients for a single scale level, the channels number is num_anchors * num_protos. """ for head_conv in self.head_convs: x = head_conv(x) cls_score = self.conv_cls(x) bbox_pred = self.conv_reg(x) coeff_pred = self.conv_coeff(x).tanh() return cls_score, bbox_pred, coeff_pred def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], coeff_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the bbox head. When ``self.use_ohem == True``, it functions like ``SSDHead.loss``, otherwise, it follows ``AnchorHead.loss``. Args: cls_scores (list[Tensor]): Box scores for each scale level has shape (N, num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). coeff_preds (list[Tensor]): Mask coefficients for each scale level with shape (N, num_anchors * num_protos, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore, unmap_outputs=not self.use_ohem, return_sampling_results=True) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor, sampling_results) = cls_reg_targets if self.use_ohem: num_images = len(batch_img_metas) all_cls_scores = torch.cat([ s.permute(0, 2, 3, 1).reshape( num_images, -1, self.cls_out_channels) for s in cls_scores ], 1) all_labels = torch.cat(labels_list, -1).view(num_images, -1) all_label_weights = torch.cat(label_weights_list, -1).view(num_images, -1) all_bbox_preds = torch.cat([ b.permute(0, 2, 3, 1).reshape(num_images, -1, 4) for b in bbox_preds ], -2) all_bbox_targets = torch.cat(bbox_targets_list, -2).view(num_images, -1, 4) all_bbox_weights = torch.cat(bbox_weights_list, -2).view(num_images, -1, 4) # concat all level anchors to a single tensor all_anchors = [] for i in range(num_images): all_anchors.append(torch.cat(anchor_list[i])) # check NaN and Inf assert torch.isfinite(all_cls_scores).all().item(), \ 'classification scores become infinite or NaN!' assert torch.isfinite(all_bbox_preds).all().item(), \ 'bbox predications become infinite or NaN!' losses_cls, losses_bbox = multi_apply( self.OHEMloss_by_feat_single, all_cls_scores, all_bbox_preds, all_anchors, all_labels, all_label_weights, all_bbox_targets, all_bbox_weights, avg_factor=avg_factor) else: # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] # concat all level anchors and flags to a single tensor concat_anchor_list = [] for i in range(len(anchor_list)): concat_anchor_list.append(torch.cat(anchor_list[i])) all_anchor_list = images_to_levels(concat_anchor_list, num_level_anchors) losses_cls, losses_bbox = multi_apply( self.loss_by_feat_single, cls_scores, bbox_preds, all_anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor=avg_factor) losses = dict(loss_cls=losses_cls, loss_bbox=losses_bbox) # update `_raw_positive_infos`, which will be used when calling # `get_positive_infos`. self._raw_positive_infos.update(coeff_preds=coeff_preds) return losses def OHEMloss_by_feat_single(self, cls_score: Tensor, bbox_pred: Tensor, anchors: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, bbox_weights: Tensor, avg_factor: int) -> tuple: """Compute loss of a single image. Similar to func:``SSDHead.loss_by_feat_single`` Args: cls_score (Tensor): Box scores for eachimage Has shape (num_total_anchors, num_classes). bbox_pred (Tensor): Box energies / deltas for each image level with shape (num_total_anchors, 4). anchors (Tensor): Box reference for each scale level with shape (num_total_anchors, 4). labels (Tensor): Labels of each anchors with shape (num_total_anchors,). label_weights (Tensor): Label weights of each anchor with shape (num_total_anchors,) bbox_targets (Tensor): BBox regression targets of each anchor weight shape (num_total_anchors, 4). bbox_weights (Tensor): BBox regression loss weights of each anchor with shape (num_total_anchors, 4). avg_factor (int): Average factor that is used to average the loss. When using sampling method, avg_factor is usually the sum of positive and negative priors. When using `PseudoSampler`, `avg_factor` is usually equal to the number of positive priors. Returns: Tuple[Tensor, Tensor]: A tuple of cls loss and bbox loss of one feature map. """ loss_cls_all = self.loss_cls(cls_score, labels, label_weights) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes pos_inds = ((labels >= 0) & (labels < self.num_classes)).nonzero( as_tuple=False).reshape(-1) neg_inds = (labels == self.num_classes).nonzero( as_tuple=False).view(-1) num_pos_samples = pos_inds.size(0) if num_pos_samples == 0: num_neg_samples = neg_inds.size(0) else: num_neg_samples = self.train_cfg['neg_pos_ratio'] * \ num_pos_samples if num_neg_samples > neg_inds.size(0): num_neg_samples = neg_inds.size(0) topk_loss_cls_neg, _ = loss_cls_all[neg_inds].topk(num_neg_samples) loss_cls_pos = loss_cls_all[pos_inds].sum() loss_cls_neg = topk_loss_cls_neg.sum() loss_cls = (loss_cls_pos + loss_cls_neg) / avg_factor if self.reg_decoded_bbox: # When the regression loss (e.g. `IouLoss`, `GIouLoss`) # is applied directly on the decoded bounding boxes, it # decodes the already encoded coordinates to absolute format. bbox_pred = self.bbox_coder.decode(anchors, bbox_pred) loss_bbox = self.loss_bbox( bbox_pred, bbox_targets, bbox_weights, avg_factor=avg_factor) return loss_cls[None], loss_bbox def get_positive_infos(self) -> InstanceList: """Get positive information from sampling results. Returns: list[:obj:`InstanceData`]: Positive Information of each image, usually including positive bboxes, positive labels, positive priors, positive coeffs, etc. """ assert len(self._raw_positive_infos) > 0 sampling_results = self._raw_positive_infos['sampling_results'] num_imgs = len(sampling_results) coeff_pred_list = [] for coeff_pred_per_level in self._raw_positive_infos['coeff_preds']: coeff_pred_per_level = \ coeff_pred_per_level.permute( 0, 2, 3, 1).reshape(num_imgs, -1, self.num_protos) coeff_pred_list.append(coeff_pred_per_level) coeff_preds = torch.cat(coeff_pred_list, dim=1) pos_info_list = [] for idx, sampling_result in enumerate(sampling_results): pos_info = InstanceData() coeff_preds_single = coeff_preds[idx] pos_info.pos_assigned_gt_inds = \ sampling_result.pos_assigned_gt_inds pos_info.pos_inds = sampling_result.pos_inds pos_info.coeffs = coeff_preds_single[sampling_result.pos_inds] pos_info.bboxes = sampling_result.pos_gt_bboxes pos_info_list.append(pos_info) return pos_info_list def predict_by_feat(self, cls_scores, bbox_preds, coeff_preds, batch_img_metas, cfg=None, rescale=True, **kwargs): """Similar to func:``AnchorHead.get_bboxes``, but additionally processes coeff_preds. Args: cls_scores (list[Tensor]): Box scores for each scale level with shape (N, num_anchors * num_classes, H, W) bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W) coeff_preds (list[Tensor]): Mask coefficients for each scale level with shape (N, num_anchors * num_protos, H, W) batch_img_metas (list[dict]): Batch image meta info. cfg (:obj:`Config` | None): Test / postprocessing configuration, if None, test_cfg would be used rescale (bool): If True, return boxes in original image space. Defaults to True. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - coeffs (Tensor): the predicted mask coefficients of instance inside the corresponding box has a shape (n, num_protos). """ assert len(cls_scores) == len(bbox_preds) num_levels = len(cls_scores) device = cls_scores[0].device featmap_sizes = [cls_scores[i].shape[-2:] for i in range(num_levels)] mlvl_priors = self.prior_generator.grid_priors( featmap_sizes, device=device) result_list = [] for img_id in range(len(batch_img_metas)): img_meta = batch_img_metas[img_id] cls_score_list = select_single_mlvl(cls_scores, img_id) bbox_pred_list = select_single_mlvl(bbox_preds, img_id) coeff_pred_list = select_single_mlvl(coeff_preds, img_id) results = self._predict_by_feat_single( cls_score_list=cls_score_list, bbox_pred_list=bbox_pred_list, coeff_preds_list=coeff_pred_list, mlvl_priors=mlvl_priors, img_meta=img_meta, cfg=cfg, rescale=rescale) result_list.append(results) return result_list def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], coeff_preds_list: List[Tensor], mlvl_priors: List[Tensor], img_meta: dict, cfg: ConfigType, rescale: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Similar to func:``AnchorHead._predict_by_feat_single``, but additionally processes coeff_preds_list and uses fast NMS instead of traditional NMS. Args: cls_score_list (list[Tensor]): Box scores for a single scale level Has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas for a single scale level with shape (num_priors * 4, H, W). coeff_preds_list (list[Tensor]): Mask coefficients for a single scale level with shape (num_priors * num_protos, H, W). mlvl_priors (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid, has shape (num_priors, 4). img_meta (dict): Image meta info. cfg (mmengine.Config): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - coeffs (Tensor): the predicted mask coefficients of instance inside the corresponding box has a shape (n, num_protos). """ assert len(cls_score_list) == len(bbox_pred_list) == len(mlvl_priors) cfg = self.test_cfg if cfg is None else cfg cfg = copy.deepcopy(cfg) img_shape = img_meta['img_shape'] nms_pre = cfg.get('nms_pre', -1) mlvl_bbox_preds = [] mlvl_valid_priors = [] mlvl_scores = [] mlvl_coeffs = [] for cls_score, bbox_pred, coeff_pred, priors in \ zip(cls_score_list, bbox_pred_list, coeff_preds_list, mlvl_priors): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] cls_score = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) if self.use_sigmoid_cls: scores = cls_score.sigmoid() else: scores = cls_score.softmax(-1) bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4) coeff_pred = coeff_pred.permute(1, 2, 0).reshape(-1, self.num_protos) if 0 < nms_pre < scores.shape[0]: # Get maximum scores for foreground classes. if self.use_sigmoid_cls: max_scores, _ = scores.max(dim=1) else: # remind that we set FG labels to [0, num_class-1] # since mmdet v2.0 # BG cat_id: num_class max_scores, _ = scores[:, :-1].max(dim=1) _, topk_inds = max_scores.topk(nms_pre) priors = priors[topk_inds, :] bbox_pred = bbox_pred[topk_inds, :] scores = scores[topk_inds, :] coeff_pred = coeff_pred[topk_inds, :] mlvl_bbox_preds.append(bbox_pred) mlvl_valid_priors.append(priors) mlvl_scores.append(scores) mlvl_coeffs.append(coeff_pred) bbox_pred = torch.cat(mlvl_bbox_preds) priors = torch.cat(mlvl_valid_priors) multi_bboxes = self.bbox_coder.decode( priors, bbox_pred, max_shape=img_shape) multi_scores = torch.cat(mlvl_scores) multi_coeffs = torch.cat(mlvl_coeffs) return self._bbox_post_process( multi_bboxes=multi_bboxes, multi_scores=multi_scores, multi_coeffs=multi_coeffs, cfg=cfg, rescale=rescale, img_meta=img_meta) def _bbox_post_process(self, multi_bboxes: Tensor, multi_scores: Tensor, multi_coeffs: Tensor, cfg: ConfigType, rescale: bool = False, img_meta: Optional[dict] = None, **kwargs) -> InstanceData: """bbox post-processing method. The boxes would be rescaled to the original image scale and do the nms operation. Usually `with_nms` is False is used for aug test. Args: multi_bboxes (Tensor): Predicted bbox that concat all levels. multi_scores (Tensor): Bbox scores that concat all levels. multi_coeffs (Tensor): Mask coefficients that concat all levels. cfg (ConfigDict): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Default to False. img_meta (dict, optional): Image meta info. Defaults to None. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - coeffs (Tensor): the predicted mask coefficients of instance inside the corresponding box has a shape (n, num_protos). """ if rescale: assert img_meta.get('scale_factor') is not None multi_bboxes /= multi_bboxes.new_tensor( img_meta['scale_factor']).repeat((1, 2)) # mlvl_bboxes /= mlvl_bboxes.new_tensor(scale_factor) if self.use_sigmoid_cls: # Add a dummy background class to the backend when using sigmoid # remind that we set FG labels to [0, num_class-1] since mmdet v2.0 # BG cat_id: num_class padding = multi_scores.new_zeros(multi_scores.shape[0], 1) multi_scores = torch.cat([multi_scores, padding], dim=1) det_bboxes, det_labels, det_coeffs = fast_nms( multi_bboxes, multi_scores, multi_coeffs, cfg.score_thr, cfg.iou_thr, cfg.top_k, cfg.max_per_img) results = InstanceData() results.bboxes = det_bboxes[:, :4] results.scores = det_bboxes[:, -1] results.labels = det_labels results.coeffs = det_coeffs return results @MODELS.register_module() class YOLACTProtonet(BaseMaskHead): """YOLACT mask head used in https://arxiv.org/abs/1904.02689. This head outputs the mask prototypes for YOLACT. Args: in_channels (int): Number of channels in the input feature map. proto_channels (tuple[int]): Output channels of protonet convs. proto_kernel_sizes (tuple[int]): Kernel sizes of protonet convs. include_last_relu (bool): If keep the last relu of protonet. num_protos (int): Number of prototypes. num_classes (int): Number of categories excluding the background category. loss_mask_weight (float): Reweight the mask loss by this factor. max_masks_to_train (int): Maximum number of masks to train for each image. with_seg_branch (bool): Whether to apply a semantic segmentation branch and calculate loss during training to increase performance with no speed penalty. Defaults to True. loss_segm (:obj:`ConfigDict` or dict, optional): Config of semantic segmentation loss. train_cfg (:obj:`ConfigDict` or dict, optional): Training config of head. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of head. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. """ def __init__( self, num_classes: int, in_channels: int = 256, proto_channels: tuple = (256, 256, 256, None, 256, 32), proto_kernel_sizes: tuple = (3, 3, 3, -2, 3, 1), include_last_relu: bool = True, num_protos: int = 32, loss_mask_weight: float = 1.0, max_masks_to_train: int = 100, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, with_seg_branch: bool = True, loss_segm: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), init_cfg=dict( type='Xavier', distribution='uniform', override=dict(name='protonet')) ) -> None: super().__init__(init_cfg=init_cfg) self.in_channels = in_channels self.proto_channels = proto_channels self.proto_kernel_sizes = proto_kernel_sizes self.include_last_relu = include_last_relu # Segmentation branch self.with_seg_branch = with_seg_branch self.segm_branch = SegmentationModule( num_classes=num_classes, in_channels=in_channels) \ if with_seg_branch else None self.loss_segm = MODELS.build(loss_segm) if with_seg_branch else None self.loss_mask_weight = loss_mask_weight self.num_protos = num_protos self.num_classes = num_classes self.max_masks_to_train = max_masks_to_train self.train_cfg = train_cfg self.test_cfg = test_cfg self._init_layers() def _init_layers(self) -> None: """Initialize layers of the head.""" # Possible patterns: # ( 256, 3) -> conv # ( 256,-2) -> deconv # (None,-2) -> bilinear interpolate in_channels = self.in_channels protonets = ModuleList() for num_channels, kernel_size in zip(self.proto_channels, self.proto_kernel_sizes): if kernel_size > 0: layer = nn.Conv2d( in_channels, num_channels, kernel_size, padding=kernel_size // 2) else: if num_channels is None: layer = InterpolateModule( scale_factor=-kernel_size, mode='bilinear', align_corners=False) else: layer = nn.ConvTranspose2d( in_channels, num_channels, -kernel_size, padding=kernel_size // 2) protonets.append(layer) protonets.append(nn.ReLU(inplace=True)) in_channels = num_channels if num_channels is not None \ else in_channels if not self.include_last_relu: protonets = protonets[:-1] self.protonet = nn.Sequential(*protonets) def forward(self, x: tuple, positive_infos: InstanceList) -> tuple: """Forward feature from the upstream network to get prototypes and linearly combine the prototypes, using masks coefficients, into instance masks. Finally, crop the instance masks with given bboxes. Args: x (Tuple[Tensor]): Feature from the upstream network, which is a 4D-tensor. positive_infos (List[:obj:``InstanceData``]): Positive information that calculate from detect head. Returns: tuple: Predicted instance segmentation masks and semantic segmentation map. """ # YOLACT used single feature map to get segmentation masks single_x = x[0] # YOLACT segmentation branch, if not training or segmentation branch # is None, will not process the forward function. if self.segm_branch is not None and self.training: segm_preds = self.segm_branch(single_x) else: segm_preds = None # YOLACT mask head prototypes = self.protonet(single_x) prototypes = prototypes.permute(0, 2, 3, 1).contiguous() num_imgs = single_x.size(0) mask_pred_list = [] for idx in range(num_imgs): cur_prototypes = prototypes[idx] pos_coeffs = positive_infos[idx].coeffs # Linearly combine the prototypes with the mask coefficients mask_preds = cur_prototypes @ pos_coeffs.t() mask_preds = torch.sigmoid(mask_preds) mask_pred_list.append(mask_preds) return mask_pred_list, segm_preds def loss_by_feat(self, mask_preds: List[Tensor], segm_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], positive_infos: InstanceList, **kwargs) -> dict: """Calculate the loss based on the features extracted by the mask head. Args: mask_preds (list[Tensor]): List of predicted prototypes, each has shape (num_classes, H, W). segm_preds (Tensor): Predicted semantic segmentation map with shape (N, num_classes, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``masks``, and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of multiple images. positive_infos (List[:obj:``InstanceData``]): Information of positive samples of each image that are assigned in detection head. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert positive_infos is not None, \ 'positive_infos should not be None in `YOLACTProtonet`' losses = dict() # crop croped_mask_pred = self.crop_mask_preds(mask_preds, batch_img_metas, positive_infos) loss_mask = [] loss_segm = [] num_imgs, _, mask_h, mask_w = segm_preds.size() assert num_imgs == len(croped_mask_pred) segm_avg_factor = num_imgs * mask_h * mask_w total_pos = 0 if self.segm_branch is not None: assert segm_preds is not None for idx in range(num_imgs): img_meta = batch_img_metas[idx] (mask_preds, pos_mask_targets, segm_targets, num_pos, gt_bboxes_for_reweight) = self._get_targets_single( croped_mask_pred[idx], segm_preds[idx], batch_gt_instances[idx], positive_infos[idx]) # segmentation loss if self.with_seg_branch: if segm_targets is None: loss = segm_preds[idx].sum() * 0. else: loss = self.loss_segm( segm_preds[idx], segm_targets, avg_factor=segm_avg_factor) loss_segm.append(loss) # mask loss total_pos += num_pos if num_pos == 0 or pos_mask_targets is None: loss = mask_preds.sum() * 0. else: mask_preds = torch.clamp(mask_preds, 0, 1) loss = F.binary_cross_entropy( mask_preds, pos_mask_targets, reduction='none') * self.loss_mask_weight h, w = img_meta['img_shape'][:2] gt_bboxes_width = (gt_bboxes_for_reweight[:, 2] - gt_bboxes_for_reweight[:, 0]) / w gt_bboxes_height = (gt_bboxes_for_reweight[:, 3] - gt_bboxes_for_reweight[:, 1]) / h loss = loss.mean(dim=(1, 2)) / gt_bboxes_width / gt_bboxes_height loss = torch.sum(loss) loss_mask.append(loss) if total_pos == 0: total_pos += 1 # avoid nan loss_mask = [x / total_pos for x in loss_mask] losses.update(loss_mask=loss_mask) if self.with_seg_branch: losses.update(loss_segm=loss_segm) return losses def _get_targets_single(self, mask_preds: Tensor, segm_pred: Tensor, gt_instances: InstanceData, positive_info: InstanceData): """Compute targets for predictions of single image. Args: mask_preds (Tensor): Predicted prototypes with shape (num_classes, H, W). segm_pred (Tensor): Predicted semantic segmentation map with shape (num_classes, H, W). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes``, ``labels``, and ``masks`` attributes. positive_info (:obj:`InstanceData`): Information of positive samples that are assigned in detection head. It usually contains following keys. - pos_assigned_gt_inds (Tensor): Assigner GT indexes of positive proposals, has shape (num_pos, ) - pos_inds (Tensor): Positive index of image, has shape (num_pos, ). - coeffs (Tensor): Positive mask coefficients with shape (num_pos, num_protos). - bboxes (Tensor): Positive bboxes with shape (num_pos, 4) Returns: tuple: Usually returns a tuple containing learning targets. - mask_preds (Tensor): Positive predicted mask with shape (num_pos, mask_h, mask_w). - pos_mask_targets (Tensor): Positive mask targets with shape (num_pos, mask_h, mask_w). - segm_targets (Tensor): Semantic segmentation targets with shape (num_classes, segm_h, segm_w). - num_pos (int): Positive numbers. - gt_bboxes_for_reweight (Tensor): GT bboxes that match to the positive priors has shape (num_pos, 4). """ gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels device = gt_bboxes.device gt_masks = gt_instances.masks.to_tensor( dtype=torch.bool, device=device).float() if gt_masks.size(0) == 0: return mask_preds, None, None, 0, None # process with semantic segmentation targets if segm_pred is not None: num_classes, segm_h, segm_w = segm_pred.size() with torch.no_grad(): downsampled_masks = F.interpolate( gt_masks.unsqueeze(0), (segm_h, segm_w), mode='bilinear', align_corners=False).squeeze(0) downsampled_masks = downsampled_masks.gt(0.5).float() segm_targets = torch.zeros_like(segm_pred, requires_grad=False) for obj_idx in range(downsampled_masks.size(0)): segm_targets[gt_labels[obj_idx] - 1] = torch.max( segm_targets[gt_labels[obj_idx] - 1], downsampled_masks[obj_idx]) else: segm_targets = None # process with mask targets pos_assigned_gt_inds = positive_info.pos_assigned_gt_inds num_pos = pos_assigned_gt_inds.size(0) # Since we're producing (near) full image masks, # it'd take too much vram to backprop on every single mask. # Thus we select only a subset. if num_pos > self.max_masks_to_train: perm = torch.randperm(num_pos) select = perm[:self.max_masks_to_train] mask_preds = mask_preds[select] pos_assigned_gt_inds = pos_assigned_gt_inds[select] num_pos = self.max_masks_to_train gt_bboxes_for_reweight = gt_bboxes[pos_assigned_gt_inds] mask_h, mask_w = mask_preds.shape[-2:] gt_masks = F.interpolate( gt_masks.unsqueeze(0), (mask_h, mask_w), mode='bilinear', align_corners=False).squeeze(0) gt_masks = gt_masks.gt(0.5).float() pos_mask_targets = gt_masks[pos_assigned_gt_inds] return (mask_preds, pos_mask_targets, segm_targets, num_pos, gt_bboxes_for_reweight) def crop_mask_preds(self, mask_preds: List[Tensor], batch_img_metas: List[dict], positive_infos: InstanceList) -> list: """Crop predicted masks by zeroing out everything not in the predicted bbox. Args: mask_preds (list[Tensor]): Predicted prototypes with shape (num_classes, H, W). batch_img_metas (list[dict]): Meta information of multiple images. positive_infos (List[:obj:``InstanceData``]): Positive information that calculate from detect head. Returns: list: The cropped masks. """ croped_mask_preds = [] for img_meta, mask_preds, cur_info in zip(batch_img_metas, mask_preds, positive_infos): bboxes_for_cropping = copy.deepcopy(cur_info.bboxes) h, w = img_meta['img_shape'][:2] bboxes_for_cropping[:, 0::2] /= w bboxes_for_cropping[:, 1::2] /= h mask_preds = self.crop_single(mask_preds, bboxes_for_cropping) mask_preds = mask_preds.permute(2, 0, 1).contiguous() croped_mask_preds.append(mask_preds) return croped_mask_preds def crop_single(self, masks: Tensor, boxes: Tensor, padding: int = 1) -> Tensor: """Crop single predicted masks by zeroing out everything not in the predicted bbox. Args: masks (Tensor): Predicted prototypes, has shape [H, W, N]. boxes (Tensor): Bbox coords in relative point form with shape [N, 4]. padding (int): Image padding size. Return: Tensor: The cropped masks. """ h, w, n = masks.size() x1, x2 = self.sanitize_coordinates( boxes[:, 0], boxes[:, 2], w, padding, cast=False) y1, y2 = self.sanitize_coordinates( boxes[:, 1], boxes[:, 3], h, padding, cast=False) rows = torch.arange( w, device=masks.device, dtype=x1.dtype).view(1, -1, 1).expand(h, w, n) cols = torch.arange( h, device=masks.device, dtype=x1.dtype).view(-1, 1, 1).expand(h, w, n) masks_left = rows >= x1.view(1, 1, -1) masks_right = rows < x2.view(1, 1, -1) masks_up = cols >= y1.view(1, 1, -1) masks_down = cols < y2.view(1, 1, -1) crop_mask = masks_left * masks_right * masks_up * masks_down return masks * crop_mask.float() def sanitize_coordinates(self, x1: Tensor, x2: Tensor, img_size: int, padding: int = 0, cast: bool = True) -> tuple: """Sanitizes the input coordinates so that x1 < x2, x1 != x2, x1 >= 0, and x2 <= image_size. Also converts from relative to absolute coordinates and casts the results to long tensors. Warning: this does things in-place behind the scenes so copy if necessary. Args: x1 (Tensor): shape (N, ). x2 (Tensor): shape (N, ). img_size (int): Size of the input image. padding (int): x1 >= padding, x2 <= image_size-padding. cast (bool): If cast is false, the result won't be cast to longs. Returns: tuple: - x1 (Tensor): Sanitized _x1. - x2 (Tensor): Sanitized _x2. """ x1 = x1 * img_size x2 = x2 * img_size if cast: x1 = x1.long() x2 = x2.long() x1 = torch.min(x1, x2) x2 = torch.max(x1, x2) x1 = torch.clamp(x1 - padding, min=0) x2 = torch.clamp(x2 + padding, max=img_size) return x1, x2 def predict_by_feat(self, mask_preds: List[Tensor], segm_preds: Tensor, results_list: InstanceList, batch_img_metas: List[dict], rescale: bool = True, **kwargs) -> InstanceList: """Transform a batch of output features extracted from the head into mask results. Args: mask_preds (list[Tensor]): Predicted prototypes with shape (num_classes, H, W). results_list (List[:obj:``InstanceData``]): BBoxHead results. batch_img_metas (list[dict]): Meta information of all images. rescale (bool, optional): Whether to rescale the results. Defaults to False. Returns: list[:obj:`InstanceData`]: Processed results of multiple images.Each :obj:`InstanceData` usually contains following keys. - scores (Tensor): Classification scores, has shape (num_instance,). - labels (Tensor): Has shape (num_instances,). - masks (Tensor): Processed mask results, has shape (num_instances, h, w). """ assert len(mask_preds) == len(results_list) == len(batch_img_metas) croped_mask_pred = self.crop_mask_preds(mask_preds, batch_img_metas, results_list) for img_id in range(len(batch_img_metas)): img_meta = batch_img_metas[img_id] results = results_list[img_id] bboxes = results.bboxes mask_preds = croped_mask_pred[img_id] if bboxes.shape[0] == 0 or mask_preds.shape[0] == 0: results_list[img_id] = empty_instances( [img_meta], bboxes.device, task_type='mask', instance_results=[results])[0] else: im_mask = self._predict_by_feat_single( mask_preds=croped_mask_pred[img_id], bboxes=bboxes, img_meta=img_meta, rescale=rescale) results.masks = im_mask return results_list def _predict_by_feat_single(self, mask_preds: Tensor, bboxes: Tensor, img_meta: dict, rescale: bool, cfg: OptConfigType = None): """Transform a single image's features extracted from the head into mask results. Args: mask_preds (Tensor): Predicted prototypes, has shape [H, W, N]. bboxes (Tensor): Bbox coords in relative point form with shape [N, 4]. img_meta (dict): Meta information of each image, e.g., image size, scaling factor, etc. rescale (bool): If rescale is False, then returned masks will fit the scale of imgs[0]. cfg (dict, optional): Config used in test phase. Defaults to None. Returns: :obj:`InstanceData`: Processed results of single image. it usually contains following keys. - scores (Tensor): Classification scores, has shape (num_instance,). - labels (Tensor): Has shape (num_instances,). - masks (Tensor): Processed mask results, has shape (num_instances, h, w). """ cfg = self.test_cfg if cfg is None else cfg scale_factor = bboxes.new_tensor(img_meta['scale_factor']).repeat( (1, 2)) img_h, img_w = img_meta['ori_shape'][:2] if rescale: # in-placed rescale the bboxes scale_factor = bboxes.new_tensor(img_meta['scale_factor']).repeat( (1, 2)) bboxes /= scale_factor else: w_scale, h_scale = scale_factor[0, 0], scale_factor[0, 1] img_h = np.round(img_h * h_scale.item()).astype(np.int32) img_w = np.round(img_w * w_scale.item()).astype(np.int32) masks = F.interpolate( mask_preds.unsqueeze(0), (img_h, img_w), mode='bilinear', align_corners=False).squeeze(0) > cfg.mask_thr if cfg.mask_thr_binary < 0: # for visualization and debugging masks = (masks * 255).to(dtype=torch.uint8) return masks class SegmentationModule(BaseModule): """YOLACT segmentation branch used in `_ In mmdet v2.x `segm_loss` is calculated in YOLACTSegmHead, while in mmdet v3.x `SegmentationModule` is used to obtain the predicted semantic segmentation map and `segm_loss` is calculated in YOLACTProtonet. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__( self, num_classes: int, in_channels: int = 256, init_cfg: ConfigType = dict( type='Xavier', distribution='uniform', override=dict(name='segm_conv')) ) -> None: super().__init__(init_cfg=init_cfg) self.in_channels = in_channels self.num_classes = num_classes self._init_layers() def _init_layers(self) -> None: """Initialize layers of the head.""" self.segm_conv = nn.Conv2d( self.in_channels, self.num_classes, kernel_size=1) def forward(self, x: Tensor) -> Tensor: """Forward feature from the upstream network. Args: x (Tensor): Feature from the upstream network, which is a 4D-tensor. Returns: Tensor: Predicted semantic segmentation map with shape (N, num_classes, H, W). """ return self.segm_conv(x) class InterpolateModule(BaseModule): """This is a module version of F.interpolate. Any arguments you give it just get passed along for the ride. """ def __init__(self, *args, init_cfg=None, **kwargs) -> None: super().__init__(init_cfg=init_cfg) self.args = args self.kwargs = kwargs def forward(self, x: Tensor) -> Tensor: """Forward features from the upstream network. Args: x (Tensor): Feature from the upstream network, which is a 4D-tensor. Returns: Tensor: A 4D-tensor feature map. """ return F.interpolate(x, *self.args, **self.kwargs) ================================================ FILE: mmdet/models/dense_heads/yolo_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Copyright (c) 2019 Western Digital Corporation or its affiliates. import copy import warnings from typing import List, Optional, Sequence, Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule, is_norm from mmengine.model import bias_init_with_prob, constant_init, normal_init from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptInstanceList) from ..task_modules.samplers import PseudoSampler from ..utils import filter_scores_and_topk, images_to_levels, multi_apply from .base_dense_head import BaseDenseHead @MODELS.register_module() class YOLOV3Head(BaseDenseHead): """YOLOV3Head Paper link: https://arxiv.org/abs/1804.02767. Args: num_classes (int): The number of object classes (w/o background) in_channels (Sequence[int]): Number of input channels per scale. out_channels (Sequence[int]): The number of output channels per scale before the final 1x1 layer. Default: (1024, 512, 256). anchor_generator (:obj:`ConfigDict` or dict): Config dict for anchor generator. bbox_coder (:obj:`ConfigDict` or dict): Config of bounding box coder. featmap_strides (Sequence[int]): The stride of each scale. Should be in descending order. Defaults to (32, 16, 8). one_hot_smoother (float): Set a non-zero value to enable label-smooth Defaults to 0. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict): Dictionary to construct and config norm layer. Defaults to dict(type='BN', requires_grad=True). act_cfg (:obj:`ConfigDict` or dict): Config dict for activation layer. Defaults to dict(type='LeakyReLU', negative_slope=0.1). loss_cls (:obj:`ConfigDict` or dict): Config of classification loss. loss_conf (:obj:`ConfigDict` or dict): Config of confidence loss. loss_xy (:obj:`ConfigDict` or dict): Config of xy coordinate loss. loss_wh (:obj:`ConfigDict` or dict): Config of wh coordinate loss. train_cfg (:obj:`ConfigDict` or dict, optional): Training config of YOLOV3 head. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of YOLOV3 head. Defaults to None. """ def __init__(self, num_classes: int, in_channels: Sequence[int], out_channels: Sequence[int] = (1024, 512, 256), anchor_generator: ConfigType = dict( type='YOLOAnchorGenerator', base_sizes=[[(116, 90), (156, 198), (373, 326)], [(30, 61), (62, 45), (59, 119)], [(10, 13), (16, 30), (33, 23)]], strides=[32, 16, 8]), bbox_coder: ConfigType = dict(type='YOLOBBoxCoder'), featmap_strides: Sequence[int] = (32, 16, 8), one_hot_smoother: float = 0., conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN', requires_grad=True), act_cfg: ConfigType = dict( type='LeakyReLU', negative_slope=0.1), loss_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_conf: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_xy: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_wh: ConfigType = dict(type='MSELoss', loss_weight=1.0), train_cfg: OptConfigType = None, test_cfg: OptConfigType = None) -> None: super().__init__(init_cfg=None) # Check params assert (len(in_channels) == len(out_channels) == len(featmap_strides)) self.num_classes = num_classes self.in_channels = in_channels self.out_channels = out_channels self.featmap_strides = featmap_strides self.train_cfg = train_cfg self.test_cfg = test_cfg if self.train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) if train_cfg.get('sampler', None) is not None: self.sampler = TASK_UTILS.build( self.train_cfg['sampler'], context=self) else: self.sampler = PseudoSampler() self.one_hot_smoother = one_hot_smoother self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.act_cfg = act_cfg self.bbox_coder = TASK_UTILS.build(bbox_coder) self.prior_generator = TASK_UTILS.build(anchor_generator) self.loss_cls = MODELS.build(loss_cls) self.loss_conf = MODELS.build(loss_conf) self.loss_xy = MODELS.build(loss_xy) self.loss_wh = MODELS.build(loss_wh) self.num_base_priors = self.prior_generator.num_base_priors[0] assert len( self.prior_generator.num_base_priors) == len(featmap_strides) self._init_layers() @property def num_levels(self) -> int: """int: number of feature map levels""" return len(self.featmap_strides) @property def num_attrib(self) -> int: """int: number of attributes in pred_map, bboxes (4) + objectness (1) + num_classes""" return 5 + self.num_classes def _init_layers(self) -> None: """initialize conv layers in YOLOv3 head.""" self.convs_bridge = nn.ModuleList() self.convs_pred = nn.ModuleList() for i in range(self.num_levels): conv_bridge = ConvModule( self.in_channels[i], self.out_channels[i], 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg) conv_pred = nn.Conv2d(self.out_channels[i], self.num_base_priors * self.num_attrib, 1) self.convs_bridge.append(conv_bridge) self.convs_pred.append(conv_pred) def init_weights(self) -> None: """initialize weights.""" for m in self.modules(): if isinstance(m, nn.Conv2d): normal_init(m, mean=0, std=0.01) if is_norm(m): constant_init(m, 1) # Use prior in model initialization to improve stability for conv_pred, stride in zip(self.convs_pred, self.featmap_strides): bias = conv_pred.bias.reshape(self.num_base_priors, -1) # init objectness with prior of 8 objects per feature map # refer to https://github.com/ultralytics/yolov3 nn.init.constant_(bias.data[:, 4], bias_init_with_prob(8 / (608 / stride)**2)) nn.init.constant_(bias.data[:, 5:], bias_init_with_prob(0.01)) def forward(self, x: Tuple[Tensor, ...]) -> tuple: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple[Tensor]: A tuple of multi-level predication map, each is a 4D-tensor of shape (batch_size, 5+num_classes, height, width). """ assert len(x) == self.num_levels pred_maps = [] for i in range(self.num_levels): feat = x[i] feat = self.convs_bridge[i](feat) pred_map = self.convs_pred[i](feat) pred_maps.append(pred_map) return tuple(pred_maps), def predict_by_feat(self, pred_maps: Sequence[Tensor], batch_img_metas: Optional[List[dict]], cfg: OptConfigType = None, rescale: bool = False, with_nms: bool = True) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. It has been accelerated since PR #5991. Args: pred_maps (Sequence[Tensor]): Raw predictions for a batch of images. batch_img_metas (list[dict], Optional): Batch image meta info. Defaults to None. cfg (:obj:`ConfigDict` or dict, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(pred_maps) == self.num_levels cfg = self.test_cfg if cfg is None else cfg cfg = copy.deepcopy(cfg) num_imgs = len(batch_img_metas) featmap_sizes = [pred_map.shape[-2:] for pred_map in pred_maps] mlvl_anchors = self.prior_generator.grid_priors( featmap_sizes, device=pred_maps[0].device) flatten_preds = [] flatten_strides = [] for pred, stride in zip(pred_maps, self.featmap_strides): pred = pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, self.num_attrib) pred[..., :2].sigmoid_() flatten_preds.append(pred) flatten_strides.append( pred.new_tensor(stride).expand(pred.size(1))) flatten_preds = torch.cat(flatten_preds, dim=1) flatten_bbox_preds = flatten_preds[..., :4] flatten_objectness = flatten_preds[..., 4].sigmoid() flatten_cls_scores = flatten_preds[..., 5:].sigmoid() flatten_anchors = torch.cat(mlvl_anchors) flatten_strides = torch.cat(flatten_strides) flatten_bboxes = self.bbox_coder.decode(flatten_anchors, flatten_bbox_preds, flatten_strides.unsqueeze(-1)) results_list = [] for (bboxes, scores, objectness, img_meta) in zip(flatten_bboxes, flatten_cls_scores, flatten_objectness, batch_img_metas): # Filtering out all predictions with conf < conf_thr conf_thr = cfg.get('conf_thr', -1) if conf_thr > 0: conf_inds = objectness >= conf_thr bboxes = bboxes[conf_inds, :] scores = scores[conf_inds, :] objectness = objectness[conf_inds] score_thr = cfg.get('score_thr', 0) nms_pre = cfg.get('nms_pre', -1) scores, labels, keep_idxs, _ = filter_scores_and_topk( scores, score_thr, nms_pre) results = InstanceData( scores=scores, labels=labels, bboxes=bboxes[keep_idxs], score_factors=objectness[keep_idxs], ) results = self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta) results_list.append(results) return results_list def loss_by_feat( self, pred_maps: Sequence[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: pred_maps (list[Tensor]): Prediction map for each scale level, shape (N, num_anchors * num_attrib, H, W) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: A dictionary of loss components. """ num_imgs = len(batch_img_metas) device = pred_maps[0][0].device featmap_sizes = [ pred_maps[i].shape[-2:] for i in range(self.num_levels) ] mlvl_anchors = self.prior_generator.grid_priors( featmap_sizes, device=device) anchor_list = [mlvl_anchors for _ in range(num_imgs)] responsible_flag_list = [] for img_id in range(num_imgs): responsible_flag_list.append( self.responsible_flags(featmap_sizes, batch_gt_instances[img_id].bboxes, device)) target_maps_list, neg_maps_list = self.get_targets( anchor_list, responsible_flag_list, batch_gt_instances) losses_cls, losses_conf, losses_xy, losses_wh = multi_apply( self.loss_by_feat_single, pred_maps, target_maps_list, neg_maps_list) return dict( loss_cls=losses_cls, loss_conf=losses_conf, loss_xy=losses_xy, loss_wh=losses_wh) def loss_by_feat_single(self, pred_map: Tensor, target_map: Tensor, neg_map: Tensor) -> tuple: """Calculate the loss of a single scale level based on the features extracted by the detection head. Args: pred_map (Tensor): Raw predictions for a single level. target_map (Tensor): The Ground-Truth target for a single level. neg_map (Tensor): The negative masks for a single level. Returns: tuple: loss_cls (Tensor): Classification loss. loss_conf (Tensor): Confidence loss. loss_xy (Tensor): Regression loss of x, y coordinate. loss_wh (Tensor): Regression loss of w, h coordinate. """ num_imgs = len(pred_map) pred_map = pred_map.permute(0, 2, 3, 1).reshape(num_imgs, -1, self.num_attrib) neg_mask = neg_map.float() pos_mask = target_map[..., 4] pos_and_neg_mask = neg_mask + pos_mask pos_mask = pos_mask.unsqueeze(dim=-1) if torch.max(pos_and_neg_mask) > 1.: warnings.warn('There is overlap between pos and neg sample.') pos_and_neg_mask = pos_and_neg_mask.clamp(min=0., max=1.) pred_xy = pred_map[..., :2] pred_wh = pred_map[..., 2:4] pred_conf = pred_map[..., 4] pred_label = pred_map[..., 5:] target_xy = target_map[..., :2] target_wh = target_map[..., 2:4] target_conf = target_map[..., 4] target_label = target_map[..., 5:] loss_cls = self.loss_cls(pred_label, target_label, weight=pos_mask) loss_conf = self.loss_conf( pred_conf, target_conf, weight=pos_and_neg_mask) loss_xy = self.loss_xy(pred_xy, target_xy, weight=pos_mask) loss_wh = self.loss_wh(pred_wh, target_wh, weight=pos_mask) return loss_cls, loss_conf, loss_xy, loss_wh def get_targets(self, anchor_list: List[List[Tensor]], responsible_flag_list: List[List[Tensor]], batch_gt_instances: List[InstanceData]) -> tuple: """Compute target maps for anchors in multiple images. Args: anchor_list (list[list[Tensor]]): Multi level anchors of each image. The outer list indicates images, and the inner list corresponds to feature levels of the image. Each element of the inner list is a tensor of shape (num_total_anchors, 4). responsible_flag_list (list[list[Tensor]]): Multi level responsible flags of each image. Each element is a tensor of shape (num_total_anchors, ) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. Returns: tuple: Usually returns a tuple containing learning targets. - target_map_list (list[Tensor]): Target map of each level. - neg_map_list (list[Tensor]): Negative map of each level. """ num_imgs = len(anchor_list) # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] results = multi_apply(self._get_targets_single, anchor_list, responsible_flag_list, batch_gt_instances) all_target_maps, all_neg_maps = results assert num_imgs == len(all_target_maps) == len(all_neg_maps) target_maps_list = images_to_levels(all_target_maps, num_level_anchors) neg_maps_list = images_to_levels(all_neg_maps, num_level_anchors) return target_maps_list, neg_maps_list def _get_targets_single(self, anchors: List[Tensor], responsible_flags: List[Tensor], gt_instances: InstanceData) -> tuple: """Generate matching bounding box prior and converted GT. Args: anchors (List[Tensor]): Multi-level anchors of the image. responsible_flags (List[Tensor]): Multi-level responsible flags of anchors gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. Returns: tuple: target_map (Tensor): Predication target map of each scale level, shape (num_total_anchors, 5+num_classes) neg_map (Tensor): Negative map of each scale level, shape (num_total_anchors,) """ gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels anchor_strides = [] for i in range(len(anchors)): anchor_strides.append( torch.tensor(self.featmap_strides[i], device=gt_bboxes.device).repeat(len(anchors[i]))) concat_anchors = torch.cat(anchors) concat_responsible_flags = torch.cat(responsible_flags) anchor_strides = torch.cat(anchor_strides) assert len(anchor_strides) == len(concat_anchors) == \ len(concat_responsible_flags) pred_instances = InstanceData( priors=concat_anchors, responsible_flags=concat_responsible_flags) assign_result = self.assigner.assign(pred_instances, gt_instances) sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) target_map = concat_anchors.new_zeros( concat_anchors.size(0), self.num_attrib) target_map[sampling_result.pos_inds, :4] = self.bbox_coder.encode( sampling_result.pos_priors, sampling_result.pos_gt_bboxes, anchor_strides[sampling_result.pos_inds]) target_map[sampling_result.pos_inds, 4] = 1 gt_labels_one_hot = F.one_hot( gt_labels, num_classes=self.num_classes).float() if self.one_hot_smoother != 0: # label smooth gt_labels_one_hot = gt_labels_one_hot * ( 1 - self.one_hot_smoother ) + self.one_hot_smoother / self.num_classes target_map[sampling_result.pos_inds, 5:] = gt_labels_one_hot[ sampling_result.pos_assigned_gt_inds] neg_map = concat_anchors.new_zeros( concat_anchors.size(0), dtype=torch.uint8) neg_map[sampling_result.neg_inds] = 1 return target_map, neg_map def responsible_flags(self, featmap_sizes: List[tuple], gt_bboxes: Tensor, device: str) -> List[Tensor]: """Generate responsible anchor flags of grid cells in multiple scales. Args: featmap_sizes (List[tuple]): List of feature map sizes in multiple feature levels. gt_bboxes (Tensor): Ground truth boxes, shape (n, 4). device (str): Device where the anchors will be put on. Return: List[Tensor]: responsible flags of anchors in multiple level """ assert self.num_levels == len(featmap_sizes) multi_level_responsible_flags = [] for i in range(self.num_levels): anchor_stride = self.prior_generator.strides[i] feat_h, feat_w = featmap_sizes[i] gt_cx = ((gt_bboxes[:, 0] + gt_bboxes[:, 2]) * 0.5).to(device) gt_cy = ((gt_bboxes[:, 1] + gt_bboxes[:, 3]) * 0.5).to(device) gt_grid_x = torch.floor(gt_cx / anchor_stride[0]).long() gt_grid_y = torch.floor(gt_cy / anchor_stride[1]).long() # row major indexing gt_bboxes_grid_idx = gt_grid_y * feat_w + gt_grid_x responsible_grid = torch.zeros( feat_h * feat_w, dtype=torch.uint8, device=device) responsible_grid[gt_bboxes_grid_idx] = 1 responsible_grid = responsible_grid[:, None].expand( responsible_grid.size(0), self.prior_generator.num_base_priors[i]).contiguous().view(-1) multi_level_responsible_flags.append(responsible_grid) return multi_level_responsible_flags ================================================ FILE: mmdet/models/dense_heads/yolof_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch import torch.nn as nn from mmcv.cnn import ConvModule, is_norm from mmengine.model import bias_init_with_prob, constant_init, normal_init from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import ConfigType, InstanceList, OptInstanceList, reduce_mean from ..task_modules.prior_generators import anchor_inside_flags from ..utils import levels_to_images, multi_apply, unmap from .anchor_head import AnchorHead INF = 1e8 @MODELS.register_module() class YOLOFHead(AnchorHead): """Detection Head of `YOLOF `_ Args: num_classes (int): The number of object classes (w/o background) in_channels (list[int]): The number of input channels per scale. cls_num_convs (int): The number of convolutions of cls branch. Defaults to 2. reg_num_convs (int): The number of convolutions of reg branch. Defaults to 4. norm_cfg (:obj:`ConfigDict` or dict): Config dict for normalization layer. Defaults to ``dict(type='BN', requires_grad=True)``. """ def __init__(self, num_classes: int, in_channels: List[int], num_cls_convs: int = 2, num_reg_convs: int = 4, norm_cfg: ConfigType = dict(type='BN', requires_grad=True), **kwargs) -> None: self.num_cls_convs = num_cls_convs self.num_reg_convs = num_reg_convs self.norm_cfg = norm_cfg super().__init__( num_classes=num_classes, in_channels=in_channels, **kwargs) def _init_layers(self) -> None: cls_subnet = [] bbox_subnet = [] for i in range(self.num_cls_convs): cls_subnet.append( ConvModule( self.in_channels, self.in_channels, kernel_size=3, padding=1, norm_cfg=self.norm_cfg)) for i in range(self.num_reg_convs): bbox_subnet.append( ConvModule( self.in_channels, self.in_channels, kernel_size=3, padding=1, norm_cfg=self.norm_cfg)) self.cls_subnet = nn.Sequential(*cls_subnet) self.bbox_subnet = nn.Sequential(*bbox_subnet) self.cls_score = nn.Conv2d( self.in_channels, self.num_base_priors * self.num_classes, kernel_size=3, stride=1, padding=1) self.bbox_pred = nn.Conv2d( self.in_channels, self.num_base_priors * 4, kernel_size=3, stride=1, padding=1) self.object_pred = nn.Conv2d( self.in_channels, self.num_base_priors, kernel_size=3, stride=1, padding=1) def init_weights(self) -> None: for m in self.modules(): if isinstance(m, nn.Conv2d): normal_init(m, mean=0, std=0.01) if is_norm(m): constant_init(m, 1) # Use prior in model initialization to improve stability bias_cls = bias_init_with_prob(0.01) torch.nn.init.constant_(self.cls_score.bias, bias_cls) def forward_single(self, x: Tensor) -> Tuple[Tensor, Tensor]: """Forward feature of a single scale level. Args: x (Tensor): Features of a single scale level. Returns: tuple: normalized_cls_score (Tensor): Normalized Cls scores for a \ single scale level, the channels number is \ num_base_priors * num_classes. bbox_reg (Tensor): Box energies / deltas for a single scale \ level, the channels number is num_base_priors * 4. """ cls_score = self.cls_score(self.cls_subnet(x)) N, _, H, W = cls_score.shape cls_score = cls_score.view(N, -1, self.num_classes, H, W) reg_feat = self.bbox_subnet(x) bbox_reg = self.bbox_pred(reg_feat) objectness = self.object_pred(reg_feat) # implicit objectness objectness = objectness.view(N, -1, 1, H, W) normalized_cls_score = cls_score + objectness - torch.log( 1. + torch.clamp(cls_score.exp(), max=INF) + torch.clamp(objectness.exp(), max=INF)) normalized_cls_score = normalized_cls_score.view(N, -1, H, W) return normalized_cls_score, bbox_reg def loss_by_feat( self, cls_scores: List[Tensor], bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level has shape (N, num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: A dictionary of loss components. """ assert len(cls_scores) == 1 assert self.prior_generator.num_levels == 1 device = cls_scores[0].device featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] anchor_list, valid_flag_list = self.get_anchors( featmap_sizes, batch_img_metas, device=device) # The output level is always 1 anchor_list = [anchors[0] for anchors in anchor_list] valid_flag_list = [valid_flags[0] for valid_flags in valid_flag_list] cls_scores_list = levels_to_images(cls_scores) bbox_preds_list = levels_to_images(bbox_preds) cls_reg_targets = self.get_targets( cls_scores_list, bbox_preds_list, anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) if cls_reg_targets is None: return None (batch_labels, batch_label_weights, avg_factor, batch_bbox_weights, batch_pos_predicted_boxes, batch_target_boxes) = cls_reg_targets flatten_labels = batch_labels.reshape(-1) batch_label_weights = batch_label_weights.reshape(-1) cls_score = cls_scores[0].permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels) avg_factor = reduce_mean( torch.tensor(avg_factor, dtype=torch.float, device=device)).item() # classification loss loss_cls = self.loss_cls( cls_score, flatten_labels, batch_label_weights, avg_factor=avg_factor) # regression loss if batch_pos_predicted_boxes.shape[0] == 0: # no pos sample loss_bbox = batch_pos_predicted_boxes.sum() * 0 else: loss_bbox = self.loss_bbox( batch_pos_predicted_boxes, batch_target_boxes, batch_bbox_weights.float(), avg_factor=avg_factor) return dict(loss_cls=loss_cls, loss_bbox=loss_bbox) def get_targets(self, cls_scores_list: List[Tensor], bbox_preds_list: List[Tensor], anchor_list: List[Tensor], valid_flag_list: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None, unmap_outputs: bool = True): """Compute regression and classification targets for anchors in multiple images. Args: cls_scores_list (list[Tensor])： Classification scores of each image. each is a 4D-tensor, the shape is (h * w, num_anchors * num_classes). bbox_preds_list (list[Tensor])： Bbox preds of each image. each is a 4D-tensor, the shape is (h * w, num_anchors * 4). anchor_list (list[Tensor]): Anchors of each image. Each element of is a tensor of shape (h * w * num_anchors, 4). valid_flag_list (list[Tensor]): Valid flags of each image. Each element of is a tensor of shape (h * w * num_anchors, ) batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Returns: tuple: Usually returns a tuple containing learning targets. - batch_labels (Tensor): Label of all images. Each element \ of is a tensor of shape (batch, h * w * num_anchors) - batch_label_weights (Tensor): Label weights of all images \ of is a tensor of shape (batch, h * w * num_anchors) - num_total_pos (int): Number of positive samples in all \ images. - num_total_neg (int): Number of negative samples in all \ images. additional_returns: This function enables user-defined returns from `self._get_targets_single`. These returns are currently refined to properties at each feature map (i.e. having HxW dimension). The results will be concatenated after the end """ num_imgs = len(batch_img_metas) assert len(anchor_list) == len(valid_flag_list) == num_imgs # compute targets for each image if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None] * num_imgs results = multi_apply( self._get_targets_single, bbox_preds_list, anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore, unmap_outputs=unmap_outputs) (all_labels, all_label_weights, pos_inds, neg_inds, sampling_results_list) = results[:5] # Get `avg_factor` of all images, which calculate in `SamplingResult`. # When using sampling method, avg_factor is usually the sum of # positive and negative priors. When using `PseudoSampler`, # `avg_factor` is usually equal to the number of positive priors. avg_factor = sum( [results.avg_factor for results in sampling_results_list]) rest_results = list(results[5:]) # user-added return values batch_labels = torch.stack(all_labels, 0) batch_label_weights = torch.stack(all_label_weights, 0) res = (batch_labels, batch_label_weights, avg_factor) for i, rests in enumerate(rest_results): # user-added return values rest_results[i] = torch.cat(rests, 0) return res + tuple(rest_results) def _get_targets_single(self, bbox_preds: Tensor, flat_anchors: Tensor, valid_flags: Tensor, gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None, unmap_outputs: bool = True) -> tuple: """Compute regression and classification targets for anchors in a single image. Args: bbox_preds (Tensor): Bbox prediction of the image, which shape is (h * w ,4) flat_anchors (Tensor): Anchors of the image, which shape is (h * w * num_anchors ,4) valid_flags (Tensor): Valid flags of the image, which shape is (h * w * num_anchors,). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. unmap_outputs (bool): Whether to map outputs back to the original set of anchors. Returns: tuple: labels (Tensor): Labels of image, which shape is (h * w * num_anchors, ). label_weights (Tensor): Label weights of image, which shape is (h * w * num_anchors, ). pos_inds (Tensor): Pos index of image. neg_inds (Tensor): Neg index of image. sampling_result (obj:`SamplingResult`): Sampling result. pos_bbox_weights (Tensor): The Weight of using to calculate the bbox branch loss, which shape is (num, ). pos_predicted_boxes (Tensor): boxes predicted value of using to calculate the bbox branch loss, which shape is (num, 4). pos_target_boxes (Tensor): boxes target value of using to calculate the bbox branch loss, which shape is (num, 4). """ inside_flags = anchor_inside_flags(flat_anchors, valid_flags, img_meta['img_shape'][:2], self.train_cfg['allowed_border']) if not inside_flags.any(): raise ValueError( 'There is no valid anchor inside the image boundary. Please ' 'check the image size and anchor sizes, or set ' '``allowed_border`` to -1 to skip the condition.') # assign gt and sample anchors anchors = flat_anchors[inside_flags, :] bbox_preds = bbox_preds.reshape(-1, 4) bbox_preds = bbox_preds[inside_flags, :] # decoded bbox decoder_bbox_preds = self.bbox_coder.decode(anchors, bbox_preds) pred_instances = InstanceData( priors=anchors, decoder_priors=decoder_bbox_preds) assign_result = self.assigner.assign(pred_instances, gt_instances, gt_instances_ignore) pos_bbox_weights = assign_result.get_extra_property('pos_idx') pos_predicted_boxes = assign_result.get_extra_property( 'pos_predicted_boxes') pos_target_boxes = assign_result.get_extra_property('target_boxes') sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) num_valid_anchors = anchors.shape[0] labels = anchors.new_full((num_valid_anchors, ), self.num_classes, dtype=torch.long) label_weights = anchors.new_zeros(num_valid_anchors, dtype=torch.float) pos_inds = sampling_result.pos_inds neg_inds = sampling_result.neg_inds if len(pos_inds) > 0: labels[pos_inds] = sampling_result.pos_gt_labels if self.train_cfg['pos_weight'] <= 0: label_weights[pos_inds] = 1.0 else: label_weights[pos_inds] = self.train_cfg['pos_weight'] if len(neg_inds) > 0: label_weights[neg_inds] = 1.0 # map up to original set of anchors if unmap_outputs: num_total_anchors = flat_anchors.size(0) labels = unmap( labels, num_total_anchors, inside_flags, fill=self.num_classes) # fill bg label label_weights = unmap(label_weights, num_total_anchors, inside_flags) return (labels, label_weights, pos_inds, neg_inds, sampling_result, pos_bbox_weights, pos_predicted_boxes, pos_target_boxes) ================================================ FILE: mmdet/models/dense_heads/yolox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from typing import List, Optional, Sequence, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule from mmcv.ops.nms import batched_nms from mmengine.config import ConfigDict from mmengine.model import bias_init_with_prob from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures.bbox import bbox_xyxy_to_cxcywh from mmdet.utils import (ConfigType, OptConfigType, OptInstanceList, OptMultiConfig, reduce_mean) from ..task_modules.prior_generators import MlvlPointGenerator from ..task_modules.samplers import PseudoSampler from ..utils import multi_apply from .base_dense_head import BaseDenseHead @MODELS.register_module() class YOLOXHead(BaseDenseHead): """YOLOXHead head used in `YOLOX `_. Args: num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. feat_channels (int): Number of hidden channels in stacking convs. Defaults to 256 stacked_convs (int): Number of stacking convs of the head. Defaults to (8, 16, 32). strides (Sequence[int]): Downsample factor of each feature map. Defaults to None. use_depthwise (bool): Whether to depthwise separable convolution in blocks. Defaults to False. dcn_on_last_conv (bool): If true, use dcn in the last layer of towers. Defaults to False. conv_bias (bool or str): If specified as `auto`, it will be decided by the norm_cfg. Bias of conv will be set as True if `norm_cfg` is None, otherwise False. Defaults to "auto". conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict): Config dict for normalization layer. Defaults to dict(type='BN', momentum=0.03, eps=0.001). act_cfg (:obj:`ConfigDict` or dict): Config dict for activation layer. Defaults to None. loss_cls (:obj:`ConfigDict` or dict): Config of classification loss. loss_bbox (:obj:`ConfigDict` or dict): Config of localization loss. loss_obj (:obj:`ConfigDict` or dict): Config of objectness loss. loss_l1 (:obj:`ConfigDict` or dict): Config of L1 loss. train_cfg (:obj:`ConfigDict` or dict, optional): Training config of anchor head. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of anchor head. Defaults to None. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__( self, num_classes: int, in_channels: int, feat_channels: int = 256, stacked_convs: int = 2, strides: Sequence[int] = (8, 16, 32), use_depthwise: bool = False, dcn_on_last_conv: bool = False, conv_bias: Union[bool, str] = 'auto', conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN', momentum=0.03, eps=0.001), act_cfg: ConfigType = dict(type='Swish'), loss_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, reduction='sum', loss_weight=1.0), loss_bbox: ConfigType = dict( type='IoULoss', mode='square', eps=1e-16, reduction='sum', loss_weight=5.0), loss_obj: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, reduction='sum', loss_weight=1.0), loss_l1: ConfigType = dict( type='L1Loss', reduction='sum', loss_weight=1.0), train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: OptMultiConfig = dict( type='Kaiming', layer='Conv2d', a=math.sqrt(5), distribution='uniform', mode='fan_in', nonlinearity='leaky_relu') ) -> None: super().__init__(init_cfg=init_cfg) self.num_classes = num_classes self.cls_out_channels = num_classes self.in_channels = in_channels self.feat_channels = feat_channels self.stacked_convs = stacked_convs self.strides = strides self.use_depthwise = use_depthwise self.dcn_on_last_conv = dcn_on_last_conv assert conv_bias == 'auto' or isinstance(conv_bias, bool) self.conv_bias = conv_bias self.use_sigmoid_cls = True self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.act_cfg = act_cfg self.loss_cls: nn.Module = MODELS.build(loss_cls) self.loss_bbox: nn.Module = MODELS.build(loss_bbox) self.loss_obj: nn.Module = MODELS.build(loss_obj) self.use_l1 = False # This flag will be modified by hooks. self.loss_l1: nn.Module = MODELS.build(loss_l1) self.prior_generator = MlvlPointGenerator(strides, offset=0) self.test_cfg = test_cfg self.train_cfg = train_cfg if self.train_cfg: self.assigner = TASK_UTILS.build(self.train_cfg['assigner']) # YOLOX does not support sampling self.sampler = PseudoSampler() self._init_layers() def _init_layers(self) -> None: """Initialize heads for all level feature maps.""" self.multi_level_cls_convs = nn.ModuleList() self.multi_level_reg_convs = nn.ModuleList() self.multi_level_conv_cls = nn.ModuleList() self.multi_level_conv_reg = nn.ModuleList() self.multi_level_conv_obj = nn.ModuleList() for _ in self.strides: self.multi_level_cls_convs.append(self._build_stacked_convs()) self.multi_level_reg_convs.append(self._build_stacked_convs()) conv_cls, conv_reg, conv_obj = self._build_predictor() self.multi_level_conv_cls.append(conv_cls) self.multi_level_conv_reg.append(conv_reg) self.multi_level_conv_obj.append(conv_obj) def _build_stacked_convs(self) -> nn.Sequential: """Initialize conv layers of a single level head.""" conv = DepthwiseSeparableConvModule \ if self.use_depthwise else ConvModule stacked_convs = [] for i in range(self.stacked_convs): chn = self.in_channels if i == 0 else self.feat_channels if self.dcn_on_last_conv and i == self.stacked_convs - 1: conv_cfg = dict(type='DCNv2') else: conv_cfg = self.conv_cfg stacked_convs.append( conv( chn, self.feat_channels, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=self.norm_cfg, act_cfg=self.act_cfg, bias=self.conv_bias)) return nn.Sequential(*stacked_convs) def _build_predictor(self) -> Tuple[nn.Module, nn.Module, nn.Module]: """Initialize predictor layers of a single level head.""" conv_cls = nn.Conv2d(self.feat_channels, self.cls_out_channels, 1) conv_reg = nn.Conv2d(self.feat_channels, 4, 1) conv_obj = nn.Conv2d(self.feat_channels, 1, 1) return conv_cls, conv_reg, conv_obj def init_weights(self) -> None: """Initialize weights of the head.""" super(YOLOXHead, self).init_weights() # Use prior in model initialization to improve stability bias_init = bias_init_with_prob(0.01) for conv_cls, conv_obj in zip(self.multi_level_conv_cls, self.multi_level_conv_obj): conv_cls.bias.data.fill_(bias_init) conv_obj.bias.data.fill_(bias_init) def forward_single(self, x: Tensor, cls_convs: nn.Module, reg_convs: nn.Module, conv_cls: nn.Module, conv_reg: nn.Module, conv_obj: nn.Module) -> Tuple[Tensor, Tensor, Tensor]: """Forward feature of a single scale level.""" cls_feat = cls_convs(x) reg_feat = reg_convs(x) cls_score = conv_cls(cls_feat) bbox_pred = conv_reg(reg_feat) objectness = conv_obj(reg_feat) return cls_score, bbox_pred, objectness def forward(self, x: Tuple[Tensor]) -> Tuple[List]: """Forward features from the upstream network. Args: x (Tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: Tuple[List]: A tuple of multi-level classification scores, bbox predictions, and objectnesses. """ return multi_apply(self.forward_single, x, self.multi_level_cls_convs, self.multi_level_reg_convs, self.multi_level_conv_cls, self.multi_level_conv_reg, self.multi_level_conv_obj) def predict_by_feat(self, cls_scores: List[Tensor], bbox_preds: List[Tensor], objectnesses: Optional[List[Tensor]], batch_img_metas: Optional[List[dict]] = None, cfg: Optional[ConfigDict] = None, rescale: bool = False, with_nms: bool = True) -> List[InstanceData]: """Transform a batch of output features extracted by the head into bbox results. Args: cls_scores (list[Tensor]): Classification scores for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for all scale levels, each is a 4D-tensor, has shape (batch_size, num_priors * 4, H, W). objectnesses (list[Tensor], Optional): Score factor for all scale level, each is a 4D-tensor, has shape (batch_size, 1, H, W). batch_img_metas (list[dict], Optional): Batch image meta info. Defaults to None. cfg (ConfigDict, optional): Test / postprocessing configuration, if None, test_cfg would be used. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: list[:obj:`InstanceData`]: Object detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(cls_scores) == len(bbox_preds) == len(objectnesses) cfg = self.test_cfg if cfg is None else cfg num_imgs = len(batch_img_metas) featmap_sizes = [cls_score.shape[2:] for cls_score in cls_scores] mlvl_priors = self.prior_generator.grid_priors( featmap_sizes, dtype=cls_scores[0].dtype, device=cls_scores[0].device, with_stride=True) # flatten cls_scores, bbox_preds and objectness flatten_cls_scores = [ cls_score.permute(0, 2, 3, 1).reshape(num_imgs, -1, self.cls_out_channels) for cls_score in cls_scores ] flatten_bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, 4) for bbox_pred in bbox_preds ] flatten_objectness = [ objectness.permute(0, 2, 3, 1).reshape(num_imgs, -1) for objectness in objectnesses ] flatten_cls_scores = torch.cat(flatten_cls_scores, dim=1).sigmoid() flatten_bbox_preds = torch.cat(flatten_bbox_preds, dim=1) flatten_objectness = torch.cat(flatten_objectness, dim=1).sigmoid() flatten_priors = torch.cat(mlvl_priors) flatten_bboxes = self._bbox_decode(flatten_priors, flatten_bbox_preds) result_list = [] for img_id, img_meta in enumerate(batch_img_metas): max_scores, labels = torch.max(flatten_cls_scores[img_id], 1) valid_mask = flatten_objectness[ img_id] * max_scores >= cfg.score_thr results = InstanceData( bboxes=flatten_bboxes[img_id][valid_mask], scores=max_scores[valid_mask] * flatten_objectness[img_id][valid_mask], labels=labels[valid_mask]) result_list.append( self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta)) return result_list def _bbox_decode(self, priors: Tensor, bbox_preds: Tensor) -> Tensor: """Decode regression results (delta_x, delta_x, w, h) to bboxes (tl_x, tl_y, br_x, br_y). Args: priors (Tensor): Center proiors of an image, has shape (num_instances, 2). bbox_preds (Tensor): Box energies / deltas for all instances, has shape (batch_size, num_instances, 4). Returns: Tensor: Decoded bboxes in (tl_x, tl_y, br_x, br_y) format. Has shape (batch_size, num_instances, 4). """ xys = (bbox_preds[..., :2] * priors[:, 2:]) + priors[:, :2] whs = bbox_preds[..., 2:].exp() * priors[:, 2:] tl_x = (xys[..., 0] - whs[..., 0] / 2) tl_y = (xys[..., 1] - whs[..., 1] / 2) br_x = (xys[..., 0] + whs[..., 0] / 2) br_y = (xys[..., 1] + whs[..., 1] / 2) decoded_bboxes = torch.stack([tl_x, tl_y, br_x, br_y], -1) return decoded_bboxes def _bbox_post_process(self, results: InstanceData, cfg: ConfigDict, rescale: bool = False, with_nms: bool = True, img_meta: Optional[dict] = None) -> InstanceData: """bbox post-processing method. The boxes would be rescaled to the original image scale and do the nms operation. Usually `with_nms` is False is used for aug test. Args: results (:obj:`InstaceData`): Detection instance results, each item has shape (num_bboxes, ). cfg (mmengine.Config): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Default to False. with_nms (bool): If True, do nms before return boxes. Default to True. img_meta (dict, optional): Image meta info. Defaults to None. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ if rescale: assert img_meta.get('scale_factor') is not None results.bboxes /= results.bboxes.new_tensor( img_meta['scale_factor']).repeat((1, 2)) if with_nms and results.bboxes.numel() > 0: det_bboxes, keep_idxs = batched_nms(results.bboxes, results.scores, results.labels, cfg.nms) results = results[keep_idxs] # some nms would reweight the score, such as softnms results.scores = det_bboxes[:, -1] return results def loss_by_feat( self, cls_scores: Sequence[Tensor], bbox_preds: Sequence[Tensor], objectnesses: Sequence[Tensor], batch_gt_instances: Sequence[InstanceData], batch_img_metas: Sequence[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (Sequence[Tensor]): Box scores for each scale level, each is a 4D-tensor, the channel number is num_priors * num_classes. bbox_preds (Sequence[Tensor]): Box energies / deltas for each scale level, each is a 4D-tensor, the channel number is num_priors * 4. objectnesses (Sequence[Tensor]): Score factor for all scale level, each is a 4D-tensor, has shape (batch_size, 1, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of losses. """ num_imgs = len(batch_img_metas) if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None] * num_imgs featmap_sizes = [cls_score.shape[2:] for cls_score in cls_scores] mlvl_priors = self.prior_generator.grid_priors( featmap_sizes, dtype=cls_scores[0].dtype, device=cls_scores[0].device, with_stride=True) flatten_cls_preds = [ cls_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, self.cls_out_channels) for cls_pred in cls_scores ] flatten_bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(num_imgs, -1, 4) for bbox_pred in bbox_preds ] flatten_objectness = [ objectness.permute(0, 2, 3, 1).reshape(num_imgs, -1) for objectness in objectnesses ] flatten_cls_preds = torch.cat(flatten_cls_preds, dim=1) flatten_bbox_preds = torch.cat(flatten_bbox_preds, dim=1) flatten_objectness = torch.cat(flatten_objectness, dim=1) flatten_priors = torch.cat(mlvl_priors) flatten_bboxes = self._bbox_decode(flatten_priors, flatten_bbox_preds) (pos_masks, cls_targets, obj_targets, bbox_targets, l1_targets, num_fg_imgs) = multi_apply( self._get_targets_single, flatten_priors.unsqueeze(0).repeat(num_imgs, 1, 1), flatten_cls_preds.detach(), flatten_bboxes.detach(), flatten_objectness.detach(), batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) # The experimental results show that 'reduce_mean' can improve # performance on the COCO dataset. num_pos = torch.tensor( sum(num_fg_imgs), dtype=torch.float, device=flatten_cls_preds.device) num_total_samples = max(reduce_mean(num_pos), 1.0) pos_masks = torch.cat(pos_masks, 0) cls_targets = torch.cat(cls_targets, 0) obj_targets = torch.cat(obj_targets, 0) bbox_targets = torch.cat(bbox_targets, 0) if self.use_l1: l1_targets = torch.cat(l1_targets, 0) loss_obj = self.loss_obj(flatten_objectness.view(-1, 1), obj_targets) / num_total_samples if num_pos > 0: loss_cls = self.loss_cls( flatten_cls_preds.view(-1, self.num_classes)[pos_masks], cls_targets) / num_total_samples loss_bbox = self.loss_bbox( flatten_bboxes.view(-1, 4)[pos_masks], bbox_targets) / num_total_samples else: # Avoid cls and reg branch not participating in the gradient # propagation when there is no ground-truth in the images. # For more details, please refer to # https://github.com/open-mmlab/mmdetection/issues/7298 loss_cls = flatten_cls_preds.sum() * 0 loss_bbox = flatten_bboxes.sum() * 0 loss_dict = dict( loss_cls=loss_cls, loss_bbox=loss_bbox, loss_obj=loss_obj) if self.use_l1: if num_pos > 0: loss_l1 = self.loss_l1( flatten_bbox_preds.view(-1, 4)[pos_masks], l1_targets) / num_total_samples else: # Avoid cls and reg branch not participating in the gradient # propagation when there is no ground-truth in the images. # For more details, please refer to # https://github.com/open-mmlab/mmdetection/issues/7298 loss_l1 = flatten_bbox_preds.sum() * 0 loss_dict.update(loss_l1=loss_l1) return loss_dict @torch.no_grad() def _get_targets_single( self, priors: Tensor, cls_preds: Tensor, decoded_bboxes: Tensor, objectness: Tensor, gt_instances: InstanceData, img_meta: dict, gt_instances_ignore: Optional[InstanceData] = None) -> tuple: """Compute classification, regression, and objectness targets for priors in a single image. Args: priors (Tensor): All priors of one image, a 2D-Tensor with shape [num_priors, 4] in [cx, xy, stride_w, stride_y] format. cls_preds (Tensor): Classification predictions of one image, a 2D-Tensor with shape [num_priors, num_classes] decoded_bboxes (Tensor): Decoded bboxes predictions of one image, a 2D-Tensor with shape [num_priors, 4] in [tl_x, tl_y, br_x, br_y] format. objectness (Tensor): Objectness predictions of one image, a 1D-Tensor with shape [num_priors] gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It should includes ``bboxes`` and ``labels`` attributes. img_meta (dict): Meta information for current image. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: tuple: foreground_mask (list[Tensor]): Binary mask of foreground targets. cls_target (list[Tensor]): Classification targets of an image. obj_target (list[Tensor]): Objectness targets of an image. bbox_target (list[Tensor]): BBox targets of an image. l1_target (int): BBox L1 targets of an image. num_pos_per_img (int): Number of positive samples in an image. """ num_priors = priors.size(0) num_gts = len(gt_instances) # No target if num_gts == 0: cls_target = cls_preds.new_zeros((0, self.num_classes)) bbox_target = cls_preds.new_zeros((0, 4)) l1_target = cls_preds.new_zeros((0, 4)) obj_target = cls_preds.new_zeros((num_priors, 1)) foreground_mask = cls_preds.new_zeros(num_priors).bool() return (foreground_mask, cls_target, obj_target, bbox_target, l1_target, 0) # YOLOX uses center priors with 0.5 offset to assign targets, # but use center priors without offset to regress bboxes. offset_priors = torch.cat( [priors[:, :2] + priors[:, 2:] * 0.5, priors[:, 2:]], dim=-1) scores = cls_preds.sigmoid() * objectness.unsqueeze(1).sigmoid() pred_instances = InstanceData( bboxes=decoded_bboxes, scores=scores.sqrt_(), priors=offset_priors) assign_result = self.assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances, gt_instances_ignore=gt_instances_ignore) sampling_result = self.sampler.sample(assign_result, pred_instances, gt_instances) pos_inds = sampling_result.pos_inds num_pos_per_img = pos_inds.size(0) pos_ious = assign_result.max_overlaps[pos_inds] # IOU aware classification score cls_target = F.one_hot(sampling_result.pos_gt_labels, self.num_classes) * pos_ious.unsqueeze(-1) obj_target = torch.zeros_like(objectness).unsqueeze(-1) obj_target[pos_inds] = 1 bbox_target = sampling_result.pos_gt_bboxes l1_target = cls_preds.new_zeros((num_pos_per_img, 4)) if self.use_l1: l1_target = self._get_l1_target(l1_target, bbox_target, priors[pos_inds]) foreground_mask = torch.zeros_like(objectness).to(torch.bool) foreground_mask[pos_inds] = 1 return (foreground_mask, cls_target, obj_target, bbox_target, l1_target, num_pos_per_img) def _get_l1_target(self, l1_target: Tensor, gt_bboxes: Tensor, priors: Tensor, eps: float = 1e-8) -> Tensor: """Convert gt bboxes to center offset and log width height.""" gt_cxcywh = bbox_xyxy_to_cxcywh(gt_bboxes) l1_target[:, :2] = (gt_cxcywh[:, :2] - priors[:, :2]) / priors[:, 2:] l1_target[:, 2:] = torch.log(gt_cxcywh[:, 2:] / priors[:, 2:] + eps) return l1_target ================================================ FILE: mmdet/models/detectors/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .atss import ATSS from .autoassign import AutoAssign from .base import BaseDetector from .base_detr import DetectionTransformer from .boxinst import BoxInst from .cascade_rcnn import CascadeRCNN from .centernet import CenterNet from .condinst import CondInst from .conditional_detr import ConditionalDETR from .cornernet import CornerNet from .crowddet import CrowdDet from .d2_wrapper import Detectron2Wrapper from .dab_detr import DABDETR from .ddod import DDOD from .deformable_detr import DeformableDETR from .detr import DETR from .dino import DINO from .fast_rcnn import FastRCNN from .faster_rcnn import FasterRCNN from .fcos import FCOS from .fovea import FOVEA from .fsaf import FSAF from .gfl import GFL from .grid_rcnn import GridRCNN from .htc import HybridTaskCascade from .kd_one_stage import KnowledgeDistillationSingleStageDetector from .lad import LAD from .mask2former import Mask2Former from .mask_rcnn import MaskRCNN from .mask_scoring_rcnn import MaskScoringRCNN from .maskformer import MaskFormer from .nasfcos import NASFCOS from .paa import PAA from .panoptic_fpn import PanopticFPN from .panoptic_two_stage_segmentor import TwoStagePanopticSegmentor from .point_rend import PointRend from .queryinst import QueryInst from .reppoints_detector import RepPointsDetector from .retinanet import RetinaNet from .rpn import RPN from .rtmdet import RTMDet from .scnet import SCNet from .semi_base import SemiBaseDetector from .single_stage import SingleStageDetector from .soft_teacher import SoftTeacher from .solo import SOLO from .solov2 import SOLOv2 from .sparse_rcnn import SparseRCNN from .tood import TOOD from .trident_faster_rcnn import TridentFasterRCNN from .two_stage import TwoStageDetector from .vfnet import VFNet from .yolact import YOLACT from .yolo import YOLOV3 from .yolof import YOLOF from .yolox import YOLOX from .crosskd_single_stage import CrossKDSingleStageDetector from .crosskd_retinanet import CrossKDRetinaNet from .crosskd_gfl import CrossKDGFL from .crosskd_atss import CrossKDATSS from .crosskd_fcos import CrossKDFCOS __all__ = [ 'ATSS', 'BaseDetector', 'SingleStageDetector', 'TwoStageDetector', 'RPN', 'KnowledgeDistillationSingleStageDetector', 'FastRCNN', 'FasterRCNN', 'MaskRCNN', 'CascadeRCNN', 'HybridTaskCascade', 'RetinaNet', 'FCOS', 'GridRCNN', 'MaskScoringRCNN', 'RepPointsDetector', 'FOVEA', 'FSAF', 'NASFCOS', 'PointRend', 'GFL', 'CornerNet', 'PAA', 'YOLOV3', 'YOLACT', 'VFNet', 'DETR', 'TridentFasterRCNN', 'SparseRCNN', 'SCNet', 'SOLO', 'SOLOv2', 'DeformableDETR', 'AutoAssign', 'YOLOF', 'CenterNet', 'YOLOX', 'TwoStagePanopticSegmentor', 'PanopticFPN', 'QueryInst', 'LAD', 'TOOD', 'MaskFormer', 'DDOD', 'Mask2Former', 'SemiBaseDetector', 'SoftTeacher', 'RTMDet', 'Detectron2Wrapper', 'CrowdDet', 'CondInst', 'BoxInst', 'DetectionTransformer', 'ConditionalDETR', 'DINO', 'DABDETR', 'CrossKDGFL', 'CrossKDSingleStageDetector', 'CrossKDRetinaNet', 'CrossKDATSS', 'CrossKDFCOS' ] ================================================ FILE: mmdet/models/detectors/atss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class ATSS(SingleStageDetector): """Implementation of `ATSS `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of ATSS. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of ATSS. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/autoassign.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class AutoAssign(SingleStageDetector): """Implementation of `AutoAssign: Differentiable Label Assignment for Dense Object Detection `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone config. neck (:obj:`ConfigDict` or dict): The neck config. bbox_head (:obj:`ConfigDict` or dict): The bbox head config. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of AutoAssign. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of AutoAssign. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None): super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/base.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from typing import Dict, List, Tuple, Union import torch from mmengine.model import BaseModel from torch import Tensor from mmdet.structures import DetDataSample, OptSampleList, SampleList from mmdet.utils import InstanceList, OptConfigType, OptMultiConfig from ..utils import samplelist_boxtype2tensor ForwardResults = Union[Dict[str, torch.Tensor], List[DetDataSample], Tuple[torch.Tensor], torch.Tensor] class BaseDetector(BaseModel, metaclass=ABCMeta): """Base class for detectors. Args: data_preprocessor (dict or ConfigDict, optional): The pre-process config of :class:`BaseDataPreprocessor`. it usually includes, ``pad_size_divisor``, ``pad_value``, ``mean`` and ``std``. init_cfg (dict or ConfigDict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None): super().__init__( data_preprocessor=data_preprocessor, init_cfg=init_cfg) @property def with_neck(self) -> bool: """bool: whether the detector has a neck""" return hasattr(self, 'neck') and self.neck is not None # TODO: these properties need to be carefully handled # for both single stage & two stage detectors @property def with_shared_head(self) -> bool: """bool: whether the detector has a shared head in the RoI Head""" return hasattr(self, 'roi_head') and self.roi_head.with_shared_head @property def with_bbox(self) -> bool: """bool: whether the detector has a bbox head""" return ((hasattr(self, 'roi_head') and self.roi_head.with_bbox) or (hasattr(self, 'bbox_head') and self.bbox_head is not None)) @property def with_mask(self) -> bool: """bool: whether the detector has a mask head""" return ((hasattr(self, 'roi_head') and self.roi_head.with_mask) or (hasattr(self, 'mask_head') and self.mask_head is not None)) def forward(self, inputs: torch.Tensor, data_samples: OptSampleList = None, mode: str = 'tensor') -> ForwardResults: """The unified entry for a forward process in both training and test. The method should accept three modes: "tensor", "predict" and "loss": - "tensor": Forward the whole network and return tensor or tuple of tensor without any post-processing, same as a common nn.Module. - "predict": Forward and return the predictions, which are fully processed to a list of :obj:`DetDataSample`. - "loss": Forward and return a dict of losses according to the given inputs and data samples. Note that this method doesn't handle either back propagation or parameter update, which are supposed to be done in :meth:`train_step`. Args: inputs (torch.Tensor): The input tensor with shape (N, C, ...) in general. data_samples (list[:obj:`DetDataSample`], optional): A batch of data samples that contain annotations and predictions. Defaults to None. mode (str): Return what kind of value. Defaults to 'tensor'. Returns: The return type depends on ``mode``. - If ``mode="tensor"``, return a tensor or a tuple of tensor. - If ``mode="predict"``, return a list of :obj:`DetDataSample`. - If ``mode="loss"``, return a dict of tensor. """ if mode == 'loss': return self.loss(inputs, data_samples) elif mode == 'predict': return self.predict(inputs, data_samples) elif mode == 'tensor': return self._forward(inputs, data_samples) else: raise RuntimeError(f'Invalid mode "{mode}". ' 'Only supports loss, predict and tensor mode') @abstractmethod def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Union[dict, tuple]: """Calculate losses from a batch of inputs and data samples.""" pass @abstractmethod def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> SampleList: """Predict results from a batch of inputs and data samples with post- processing.""" pass @abstractmethod def _forward(self, batch_inputs: Tensor, batch_data_samples: OptSampleList = None): """Network forward process. Usually includes backbone, neck and head forward without any post- processing. """ pass @abstractmethod def extract_feat(self, batch_inputs: Tensor): """Extract features from images.""" pass def add_pred_to_datasample(self, data_samples: SampleList, results_list: InstanceList) -> SampleList: """Add predictions to `DetDataSample`. Args: data_samples (list[:obj:`DetDataSample`], optional): A batch of data samples that contain annotations and predictions. results_list (list[:obj:`InstanceData`]): Detection results of each image. Returns: list[:obj:`DetDataSample`]: Detection results of the input images. Each DetDataSample usually contain 'pred_instances'. And the ``pred_instances`` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ for data_sample, pred_instances in zip(data_samples, results_list): data_sample.pred_instances = pred_instances samplelist_boxtype2tensor(data_samples) return data_samples ================================================ FILE: mmdet/models/detectors/base_detr.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from typing import Dict, List, Tuple, Union from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import OptSampleList, SampleList from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .base import BaseDetector @MODELS.register_module() class DetectionTransformer(BaseDetector, metaclass=ABCMeta): r"""Base class for Detection Transformer. In Detection Transformer, an encoder is used to process output features of neck, then several queries interact with the encoder features using a decoder and do the regression and classification with the bounding box head. Args: backbone (:obj:`ConfigDict` or dict): Config of the backbone. neck (:obj:`ConfigDict` or dict, optional): Config of the neck. Defaults to None. encoder (:obj:`ConfigDict` or dict, optional): Config of the Transformer encoder. Defaults to None. decoder (:obj:`ConfigDict` or dict, optional): Config of the Transformer decoder. Defaults to None. bbox_head (:obj:`ConfigDict` or dict, optional): Config for the bounding box head module. Defaults to None. positional_encoding (:obj:`ConfigDict` or dict, optional): Config of the positional encoding module. Defaults to None. num_queries (int, optional): Number of decoder query in Transformer. Defaults to 100. train_cfg (:obj:`ConfigDict` or dict, optional): Training config of the bounding box head module. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): Testing config of the bounding box head module. Defaults to None. data_preprocessor (dict or ConfigDict, optional): The pre-process config of :class:`BaseDataPreprocessor`. it usually includes, ``pad_size_divisor``, ``pad_value``, ``mean`` and ``std``. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: OptConfigType = None, encoder: OptConfigType = None, decoder: OptConfigType = None, bbox_head: OptConfigType = None, positional_encoding: OptConfigType = None, num_queries: int = 100, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( data_preprocessor=data_preprocessor, init_cfg=init_cfg) # process args bbox_head.update(train_cfg=train_cfg) bbox_head.update(test_cfg=test_cfg) self.train_cfg = train_cfg self.test_cfg = test_cfg self.encoder = encoder self.decoder = decoder self.positional_encoding = positional_encoding self.num_queries = num_queries # init model layers self.backbone = MODELS.build(backbone) if neck is not None: self.neck = MODELS.build(neck) self.bbox_head = MODELS.build(bbox_head) self._init_layers() @abstractmethod def _init_layers(self) -> None: """Initialize layers except for backbone, neck and bbox_head.""" pass def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Union[dict, list]: """Calculate losses from a batch of inputs and data samples. Args: batch_inputs (Tensor): Input images of shape (bs, dim, H, W). These should usually be mean centered and std scaled. batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components """ img_feats = self.extract_feat(batch_inputs) head_inputs_dict = self.forward_transformer(img_feats, batch_data_samples) losses = self.bbox_head.loss( **head_inputs_dict, batch_data_samples=batch_data_samples) return losses def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList, rescale: bool = True) -> SampleList: """Predict results from a batch of inputs and data samples with post- processing. Args: batch_inputs (Tensor): Inputs, has shape (bs, dim, H, W). batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. rescale (bool): Whether to rescale the results. Defaults to True. Returns: list[:obj:`DetDataSample`]: Detection results of the input images. Each DetDataSample usually contain 'pred_instances'. And the `pred_instances` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ img_feats = self.extract_feat(batch_inputs) head_inputs_dict = self.forward_transformer(img_feats, batch_data_samples) results_list = self.bbox_head.predict( **head_inputs_dict, rescale=rescale, batch_data_samples=batch_data_samples) batch_data_samples = self.add_pred_to_datasample( batch_data_samples, results_list) return batch_data_samples def _forward( self, batch_inputs: Tensor, batch_data_samples: OptSampleList = None) -> Tuple[List[Tensor]]: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: batch_inputs (Tensor): Inputs, has shape (bs, dim, H, W). batch_data_samples (List[:obj:`DetDataSample`], optional): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Defaults to None. Returns: tuple[Tensor]: A tuple of features from ``bbox_head`` forward. """ img_feats = self.extract_feat(batch_inputs) head_inputs_dict = self.forward_transformer(img_feats, batch_data_samples) results = self.bbox_head.forward(**head_inputs_dict) return results def forward_transformer(self, img_feats: Tuple[Tensor], batch_data_samples: OptSampleList = None) -> Dict: """Forward process of Transformer, which includes four steps: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder'. We summarized the parameters flow of the existing DETR-like detector, which can be illustrated as follow: .. code:: text img_feats & batch_data_samples | V +-----------------+ | pre_transformer | +-----------------+ | | | V | +-----------------+ | | forward_encoder | | +-----------------+ | | | V | +---------------+ | | pre_decoder | | +---------------+ | | | V V | +-----------------+ | | forward_decoder | | +-----------------+ | | | V V head_inputs_dict Args: img_feats (tuple[Tensor]): Tuple of feature maps from neck. Each feature map has shape (bs, dim, H, W). batch_data_samples (list[:obj:`DetDataSample`], optional): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Defaults to None. Returns: dict: The dictionary of bbox_head function inputs, which always includes the `hidden_states` of the decoder output and may contain `references` including the initial and intermediate references. """ encoder_inputs_dict, decoder_inputs_dict = self.pre_transformer( img_feats, batch_data_samples) encoder_outputs_dict = self.forward_encoder(**encoder_inputs_dict) tmp_dec_in, head_inputs_dict = self.pre_decoder(**encoder_outputs_dict) decoder_inputs_dict.update(tmp_dec_in) decoder_outputs_dict = self.forward_decoder(**decoder_inputs_dict) head_inputs_dict.update(decoder_outputs_dict) return head_inputs_dict def extract_feat(self, batch_inputs: Tensor) -> Tuple[Tensor]: """Extract features. Args: batch_inputs (Tensor): Image tensor, has shape (bs, dim, H, W). Returns: tuple[Tensor]: Tuple of feature maps from neck. Each feature map has shape (bs, dim, H, W). """ x = self.backbone(batch_inputs) if self.with_neck: x = self.neck(x) return x @abstractmethod def pre_transformer( self, img_feats: Tuple[Tensor], batch_data_samples: OptSampleList = None) -> Tuple[Dict, Dict]: """Process image features before feeding them to the transformer. Args: img_feats (tuple[Tensor]): Tuple of feature maps from neck. Each feature map has shape (bs, dim, H, W). batch_data_samples (list[:obj:`DetDataSample`], optional): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Defaults to None. Returns: tuple[dict, dict]: The first dict contains the inputs of encoder and the second dict contains the inputs of decoder. - encoder_inputs_dict (dict): The keyword args dictionary of `self.forward_encoder()`, which includes 'feat', 'feat_mask', 'feat_pos', and other algorithm-specific arguments. - decoder_inputs_dict (dict): The keyword args dictionary of `self.forward_decoder()`, which includes 'memory_mask', and other algorithm-specific arguments. """ pass @abstractmethod def forward_encoder(self, feat: Tensor, feat_mask: Tensor, feat_pos: Tensor, **kwargs) -> Dict: """Forward with Transformer encoder. Args: feat (Tensor): Sequential features, has shape (bs, num_feat_points, dim). feat_mask (Tensor): ByteTensor, the padding mask of the features, has shape (bs, num_feat_points). feat_pos (Tensor): The positional embeddings of the features, has shape (bs, num_feat_points, dim). Returns: dict: The dictionary of encoder outputs, which includes the `memory` of the encoder output and other algorithm-specific arguments. """ pass @abstractmethod def pre_decoder(self, memory: Tensor, **kwargs) -> Tuple[Dict, Dict]: """Prepare intermediate variables before entering Transformer decoder, such as `query`, `query_pos`, and `reference_points`. Args: memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). Returns: tuple[dict, dict]: The first dict contains the inputs of decoder and the second dict contains the inputs of the bbox_head function. - decoder_inputs_dict (dict): The keyword dictionary args of `self.forward_decoder()`, which includes 'query', 'query_pos', 'memory', and other algorithm-specific arguments. - head_inputs_dict (dict): The keyword dictionary args of the bbox_head functions, which is usually empty, or includes `enc_outputs_class` and `enc_outputs_class` when the detector support 'two stage' or 'query selection' strategies. """ pass @abstractmethod def forward_decoder(self, query: Tensor, query_pos: Tensor, memory: Tensor, **kwargs) -> Dict: """Forward with Transformer decoder. Args: query (Tensor): The queries of decoder inputs, has shape (bs, num_queries, dim). query_pos (Tensor): The positional queries of decoder inputs, has shape (bs, num_queries, dim). memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). Returns: dict: The dictionary of decoder outputs, which includes the `hidden_states` of the decoder output, `references` including the initial and intermediate reference_points, and other algorithm-specific arguments. """ pass ================================================ FILE: mmdet/models/detectors/boxinst.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage_instance_seg import SingleStageInstanceSegmentor @MODELS.register_module() class BoxInst(SingleStageInstanceSegmentor): """Implementation of `BoxInst `_""" def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, mask_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, mask_head=mask_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/cascade_rcnn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .two_stage import TwoStageDetector @MODELS.register_module() class CascadeRCNN(TwoStageDetector): r"""Implementation of `Cascade R-CNN: Delving into High Quality Object Detection `_""" def __init__(self, backbone: ConfigType, neck: OptConfigType = None, rpn_head: OptConfigType = None, roi_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/centernet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class CenterNet(SingleStageDetector): """Implementation of CenterNet(Objects as Points) . """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/condinst.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage_instance_seg import SingleStageInstanceSegmentor @MODELS.register_module() class CondInst(SingleStageInstanceSegmentor): """Implementation of `CondInst `_""" def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, mask_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, mask_head=mask_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/conditional_detr.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict import torch.nn as nn from torch import Tensor from mmdet.registry import MODELS from ..layers import (ConditionalDetrTransformerDecoder, DetrTransformerEncoder, SinePositionalEncoding) from .detr import DETR @MODELS.register_module() class ConditionalDETR(DETR): r"""Implementation of `Conditional DETR for Fast Training Convergence. `_. Code is modified from the `official github repo `_. """ def _init_layers(self) -> None: """Initialize layers except for backbone, neck and bbox_head.""" self.positional_encoding = SinePositionalEncoding( **self.positional_encoding) self.encoder = DetrTransformerEncoder(**self.encoder) self.decoder = ConditionalDetrTransformerDecoder(**self.decoder) self.embed_dims = self.encoder.embed_dims # NOTE The embed_dims is typically passed from the inside out. # For example in DETR, The embed_dims is passed as # self_attn -> the first encoder layer -> encoder -> detector. self.query_embedding = nn.Embedding(self.num_queries, self.embed_dims) num_feats = self.positional_encoding.num_feats assert num_feats * 2 == self.embed_dims, \ f'embed_dims should be exactly 2 times of num_feats. ' \ f'Found {self.embed_dims} and {num_feats}.' def forward_decoder(self, query: Tensor, query_pos: Tensor, memory: Tensor, memory_mask: Tensor, memory_pos: Tensor) -> Dict: """Forward with Transformer decoder. Args: query (Tensor): The queries of decoder inputs, has shape (bs, num_queries, dim). query_pos (Tensor): The positional queries of decoder inputs, has shape (bs, num_queries, dim). memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). memory_mask (Tensor): ByteTensor, the padding mask of the memory, has shape (bs, num_feat_points). memory_pos (Tensor): The positional embeddings of memory, has shape (bs, num_feat_points, dim). Returns: dict: The dictionary of decoder outputs, which includes the `hidden_states` and `references` of the decoder output. - hidden_states (Tensor): Has shape (num_decoder_layers, bs, num_queries, dim) - references (Tensor): Has shape (bs, num_queries, 2) """ hidden_states, references = self.decoder( query=query, key=memory, query_pos=query_pos, key_pos=memory_pos, key_padding_mask=memory_mask) head_inputs_dict = dict( hidden_states=hidden_states, references=references) return head_inputs_dict ================================================ FILE: mmdet/models/detectors/cornernet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class CornerNet(SingleStageDetector): """CornerNet. This detector is the implementation of the paper `CornerNet: Detecting Objects as Paired Keypoints `_ . """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/crosskd_atss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Union import torch from torch import Tensor import torch.nn.functional as F from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import (InstanceList, OptInstanceList, OptConfigType, reduce_mean) from ..utils import multi_apply, unpack_gt_instances from .crosskd_single_stage import CrossKDSingleStageDetector @MODELS.register_module() class CrossKDATSS(CrossKDSingleStageDetector): def __init__(self, kd_cfg: OptConfigType = None, **kwargs) -> None: super().__init__(kd_cfg=kd_cfg,**kwargs) self.loss_center_kd = None if kd_cfg.get('loss_center_kd', None): self.loss_center_kd = MODELS.build(kd_cfg['loss_center_kd']) def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Union[dict, list]: """Calculate losses from a batch of inputs and data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ tea_x = self.teacher.extract_feat(batch_inputs) tea_cls_scores, tea_bbox_preds, tea_centernesses, tea_cls_hold, tea_reg_hold = \ multi_apply(self.forward_hkd_single, tea_x, self.teacher.bbox_head.scales, module=self.teacher) stu_x = self.extract_feat(batch_inputs) stu_cls_scores, stu_bbox_preds, stu_centernesses, stu_cls_hold, stu_reg_hold = \ multi_apply(self.forward_hkd_single, stu_x, self.bbox_head.scales, module=self) reused_cls_scores, reused_bbox_preds, reused_centernesses = multi_apply( self.reuse_teacher_head, tea_cls_hold, tea_reg_hold, stu_cls_hold, stu_reg_hold, self.teacher.bbox_head.scales) outputs = unpack_gt_instances(batch_data_samples) (batch_gt_instances, batch_gt_instances_ignore, batch_img_metas) = outputs losses = self.loss_by_feat(tea_cls_scores, tea_bbox_preds, tea_centernesses, tea_x, stu_cls_scores, stu_bbox_preds, stu_centernesses, stu_x, reused_cls_scores, reused_bbox_preds, reused_centernesses, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) return losses def forward_hkd_single(self, x, scale, module): cls_feat, reg_feat = x, x cls_feat_hold, reg_feat_hold = x, x for i, cls_conv in enumerate(module.bbox_head.cls_convs): cls_feat = cls_conv(cls_feat, activate=False) if i + 1 == self.reused_teacher_head_idx: cls_feat_hold = cls_feat cls_feat = cls_conv.activate(cls_feat) for i, reg_conv in enumerate(module.bbox_head.reg_convs): reg_feat = reg_conv(reg_feat, activate=False) if i + 1 == self.reused_teacher_head_idx: reg_feat_hold = reg_feat reg_feat = reg_conv.activate(reg_feat) cls_score = module.bbox_head.atss_cls(cls_feat) bbox_pred = scale(module.bbox_head.atss_reg(reg_feat)).float() centerness = module.bbox_head.atss_centerness(reg_feat) return cls_score, bbox_pred, centerness, cls_feat_hold, reg_feat_hold def reuse_teacher_head(self, tea_cls_feat, tea_reg_feat, stu_cls_feat, stu_reg_feat, scale): reused_cls_feat = self.align_scale(stu_cls_feat, tea_cls_feat) reused_reg_feat = self.align_scale(stu_reg_feat, tea_reg_feat) if self.reused_teacher_head_idx != 0: reused_cls_feat = F.relu(reused_cls_feat) reused_reg_feat = F.relu(reused_reg_feat) module = self.teacher.bbox_head for i in range(self.reused_teacher_head_idx, module.stacked_convs): reused_cls_feat = module.cls_convs[i](reused_cls_feat) reused_reg_feat = module.reg_convs[i](reused_reg_feat) reused_cls_score = module.atss_cls(reused_cls_feat) reused_bbox_pred = scale(module.atss_reg(reused_reg_feat)).float() reused_centerness = module.atss_centerness(reused_reg_feat) return reused_cls_score, reused_bbox_pred, reused_centerness def align_scale(self, stu_feat, tea_feat): N, C, H, W = stu_feat.size() # normalize student feature stu_feat = stu_feat.permute(1, 0, 2, 3).reshape(C, -1) stu_mean = stu_feat.mean(dim=-1, keepdim=True) stu_std = stu_feat.std(dim=-1, keepdim=True) stu_feat = (stu_feat - stu_mean) / (stu_std + 1e-6) # tea_feat = tea_feat.permute(1, 0, 2, 3).reshape(C, -1) tea_mean = tea_feat.mean(dim=-1, keepdim=True) tea_std = tea_feat.std(dim=-1, keepdim=True) stu_feat = stu_feat * tea_std + tea_mean return stu_feat.reshape(C, N, H, W).permute(1, 0, 2, 3) def loss_by_feat( self, tea_cls_scores: List[Tensor], tea_bbox_preds: List[Tensor], tea_centernesses: List[Tensor], tea_feats: List[Tensor], cls_scores: List[Tensor], bbox_preds: List[Tensor], centernesses: List[Tensor], feats: List[Tensor], reused_cls_scores: List[Tensor], reused_bbox_preds: List[Tensor], reused_centernesses: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Cls and quality scores for each scale level has shape (N, num_classes, H, W). bbox_preds (list[Tensor]): Box distribution logits for each scale level with shape (N, 4*(n+1), H, W), n is max value of integral set. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.bbox_head.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.bbox_head.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.bbox_head.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets avg_factor = reduce_mean( torch.tensor(avg_factor, dtype=torch.float, device=device)).item() losses_cls, losses_bbox, loss_centerness, \ bbox_avg_factor = multi_apply( self.bbox_head.loss_by_feat_single, anchor_list, cls_scores, bbox_preds, centernesses, labels_list, label_weights_list, bbox_targets_list, avg_factor=avg_factor) bbox_avg_factor = sum(bbox_avg_factor) bbox_avg_factor = reduce_mean(bbox_avg_factor).clamp_(min=1).item() losses_bbox = list(map(lambda x: x / bbox_avg_factor, losses_bbox)) losses = dict(loss_cls=losses_cls, loss_bbox=losses_bbox, loss_centerness=loss_centerness) losses_cls_kd, losses_reg_kd, losses_center_kd = multi_apply( self.pred_imitation_loss_single, labels_list, anchor_list, tea_cls_scores, tea_bbox_preds, tea_centernesses, reused_cls_scores, reused_bbox_preds, reused_centernesses, label_weights_list, avg_factor=avg_factor) losses.update(dict(loss_cls_kd=losses_cls_kd, loss_reg_kd=losses_reg_kd, losses_center_kd=losses_center_kd)) if self.with_feat_distill: losses_feat_kd = [ self.loss_feat_kd(feat, tea_feat) for feat, tea_feat in zip(feats, tea_feats) ] losses.update(loss_feat_kd=losses_feat_kd) return losses def pred_imitation_loss_single(self, labels, anchors, tea_cls_score, tea_bbox_pred, tea_centernesses, reused_cls_score, reused_bbox_pred, reused_centernesses, label_weights, avg_factor): # classification branch distillation tea_cls_score = tea_cls_score.permute(0, 2, 3, 1).reshape(-1, self.bbox_head.cls_out_channels) reused_cls_score = reused_cls_score.permute(0, 2, 3, 1).reshape(-1, self.bbox_head.cls_out_channels) label_weights = label_weights.reshape(-1) loss_cls_kd = self.loss_cls_kd( reused_cls_score, tea_cls_score, label_weights, avg_factor=avg_factor) # regression branch distillation bbox_coder = self.bbox_head.bbox_coder tea_bbox_pred = tea_bbox_pred.permute(0, 2, 3, 1).reshape(-1, bbox_coder.encode_size) reused_bbox_pred = reused_bbox_pred.permute(0, 2, 3, 1).reshape(-1, bbox_coder.encode_size) anchors = anchors.reshape(-1, anchors.size(-1)) tea_bbox_pred = bbox_coder.decode(anchors, tea_bbox_pred) reused_bbox_pred = bbox_coder.decode(anchors, reused_bbox_pred) reg_weights = tea_cls_score.max(dim=1)[0].sigmoid() reg_weights[label_weights == 0] = 0 loss_reg_kd = self.loss_reg_kd( reused_bbox_pred, tea_bbox_pred, weight=reg_weights, avg_factor=avg_factor) # centernesses branch distillation labels = labels.reshape(-1) bg_class_ind = self.bbox_head.num_classes pos_inds = ((labels >= 0) & (labels < bg_class_ind)).nonzero().squeeze(1) tea_centernesses = tea_centernesses.permute(0, 2, 3, 1).reshape(-1) reused_centernesses = reused_centernesses.permute(0, 2, 3, 1).reshape(-1) if len(pos_inds) > 0: loss_center_kd = self.loss_center_kd( reused_centernesses[pos_inds], tea_centernesses[pos_inds].sigmoid(), avg_factor=avg_factor) else: loss_center_kd = reused_centernesses.new_tensor(0.) return loss_cls_kd, loss_reg_kd, loss_center_kd ================================================ FILE: mmdet/models/detectors/crosskd_fcos.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Union import torch from torch import Tensor import torch.nn.functional as F from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import (InstanceList, OptInstanceList, reduce_mean) from ..utils import multi_apply, unpack_gt_instances from .crosskd_single_stage import CrossKDSingleStageDetector INF = 1e8 @MODELS.register_module() class CrossKDFCOS(CrossKDSingleStageDetector): def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Union[dict, list]: """Calculate losses from a batch of inputs and data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ tea_x = self.teacher.extract_feat(batch_inputs) tea_cls_scores, tea_bbox_preds, tea_centernesses, tea_cls_hold, tea_reg_hold = \ multi_apply(self.forward_hkd_single, tea_x, self.teacher.bbox_head.scales, self.teacher.bbox_head.strides, module=self.teacher) stu_x = self.extract_feat(batch_inputs) stu_cls_scores, stu_bbox_preds,stu_centernesses, stu_cls_hold, stu_reg_hold = \ multi_apply(self.forward_hkd_single, stu_x, self.bbox_head.scales, self.bbox_head.strides, module=self) reused_cls_scores, reused_bbox_preds, reused_centernesses = multi_apply( self.reuse_teacher_head, tea_cls_hold, tea_reg_hold, stu_cls_hold, stu_reg_hold, self.teacher.bbox_head.scales, self.teacher.bbox_head.strides) outputs = unpack_gt_instances(batch_data_samples) (batch_gt_instances, batch_gt_instances_ignore, batch_img_metas) = outputs losses = self.loss_by_feat(tea_cls_scores, tea_bbox_preds, tea_centernesses, tea_x, stu_cls_scores, stu_bbox_preds, stu_centernesses, stu_x, reused_cls_scores, reused_bbox_preds, reused_centernesses, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) return losses def forward_hkd_single(self, x, scale, stride, module): cls_feat, reg_feat = x, x cls_feat_hold, reg_feat_hold = x, x for i, cls_conv in enumerate(module.bbox_head.cls_convs): cls_feat = cls_conv(cls_feat, activate=False) if i + 1 == self.reused_teacher_head_idx: cls_feat_hold = cls_feat cls_feat = cls_conv.activate(cls_feat) for i, reg_conv in enumerate(module.bbox_head.reg_convs): reg_feat = reg_conv(reg_feat, activate=False) if i + 1 == self.reused_teacher_head_idx: reg_feat_hold = reg_feat reg_feat = reg_conv.activate(reg_feat) cls_score = module.bbox_head.conv_cls(cls_feat) bbox_pred = scale(module.bbox_head.conv_reg(reg_feat)).float() if module.bbox_head.centerness_on_reg: centerness = module.bbox_head.conv_centerness(reg_feat) else: centerness = module.bbox_head.conv_centerness(cls_feat) if module.bbox_head.norm_on_bbox: # bbox_pred needed for gradient computation has been modified # by F.relu(bbox_pred) when run with PyTorch 1.10. So replace # F.relu(bbox_pred) with bbox_pred.clamp(min=0) bbox_pred = bbox_pred.clamp(min=0) if not module.bbox_head.training: bbox_pred *= stride else: bbox_pred = bbox_pred.exp() return cls_score, bbox_pred, centerness, cls_feat_hold, reg_feat_hold def reuse_teacher_head(self, tea_cls_feat, tea_reg_feat, stu_cls_feat, stu_reg_feat, scale, stride): reused_cls_feat = self.align_scale(stu_cls_feat, tea_cls_feat) reused_reg_feat = self.align_scale(stu_reg_feat, tea_reg_feat) if self.reused_teacher_head_idx != 0: reused_cls_feat = F.relu(reused_cls_feat) reused_reg_feat = F.relu(reused_reg_feat) module = self.teacher.bbox_head for i in range(self.reused_teacher_head_idx, module.stacked_convs): reused_cls_feat = module.cls_convs[i](reused_cls_feat) reused_reg_feat = module.reg_convs[i](reused_reg_feat) reused_cls_score = module.conv_cls(reused_cls_feat) reused_bbox_pred = scale(module.conv_reg(reused_reg_feat)).float() if module.centerness_on_reg: reused_centerness = module.conv_centerness(reused_reg_feat) else: reused_centerness = module.conv_centerness(reused_cls_feat) if module.norm_on_bbox: # bbox_pred needed for gradient computation has been modified # by F.relu(bbox_pred) when run with PyTorch 1.10. So replace # F.relu(bbox_pred) with bbox_pred.clamp(min=0) reused_bbox_pred = reused_bbox_pred.clamp(min=0) if not module.training: reused_bbox_pred *= stride else: reused_bbox_pred = reused_bbox_pred.exp() return reused_cls_score, reused_bbox_pred, reused_centerness def align_scale(self, stu_feat, tea_feat): N, C, H, W = stu_feat.size() # normalize student feature stu_feat = stu_feat.permute(1, 0, 2, 3).reshape(C, -1) stu_mean = stu_feat.mean(dim=-1, keepdim=True) stu_std = stu_feat.std(dim=-1, keepdim=True) stu_feat = (stu_feat - stu_mean) / (stu_std + 1e-6) # tea_feat = tea_feat.permute(1, 0, 2, 3).reshape(C, -1) tea_mean = tea_feat.mean(dim=-1, keepdim=True) tea_std = tea_feat.std(dim=-1, keepdim=True) stu_feat = stu_feat * tea_std + tea_mean return stu_feat.reshape(C, N, H, W).permute(1, 0, 2, 3) def loss_by_feat( self, tea_cls_scores: List[Tensor], tea_bbox_preds: List[Tensor], tea_centernesses: List[Tensor], tea_feats: List[Tensor], cls_scores: List[Tensor], bbox_preds: List[Tensor], centernesses: List[Tensor], feats: List[Tensor], reused_cls_scores: List[Tensor], reused_bbox_preds: List[Tensor], reused_centernesses: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Cls and quality scores for each scale level has shape (N, num_classes, H, W). bbox_preds (list[Tensor]): Box distribution logits for each scale level with shape (N, 4*(n+1), H, W), n is max value of integral set. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ assert len(cls_scores) == len(bbox_preds) == len(centernesses) featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] all_level_points = self.bbox_head.prior_generator.grid_priors( featmap_sizes, dtype=bbox_preds[0].dtype, device=bbox_preds[0].device) labels, bbox_targets = self.bbox_head.get_targets(all_level_points, batch_gt_instances) num_imgs = cls_scores[0].size(0) # flatten cls_scores, bbox_preds and centerness flatten_cls_scores = [ cls_score.permute(0, 2, 3, 1).reshape(-1, self.bbox_head.cls_out_channels) for cls_score in cls_scores ] flatten_bbox_preds = [ bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) for bbox_pred in bbox_preds ] flatten_centerness = [ centerness.permute(0, 2, 3, 1).reshape(-1) for centerness in centernesses ] flatten_cls_scores = torch.cat(flatten_cls_scores) flatten_bbox_preds = torch.cat(flatten_bbox_preds) flatten_centerness = torch.cat(flatten_centerness) flatten_labels = torch.cat(labels) flatten_bbox_targets = torch.cat(bbox_targets) # repeat points to align with bbox_preds flatten_points = torch.cat( [points.repeat(num_imgs, 1) for points in all_level_points]) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = self.bbox_head.num_classes pos_inds = ((flatten_labels >= 0) & (flatten_labels < bg_class_ind)).nonzero().reshape(-1) num_pos = torch.tensor( len(pos_inds), dtype=torch.float, device=bbox_preds[0].device) num_pos = max(reduce_mean(num_pos), 1.0) loss_cls = self.bbox_head.loss_cls( flatten_cls_scores, flatten_labels, avg_factor=num_pos) pos_bbox_preds = flatten_bbox_preds[pos_inds] pos_centerness = flatten_centerness[pos_inds] pos_bbox_targets = flatten_bbox_targets[pos_inds] pos_centerness_targets = self.bbox_head.centerness_target(pos_bbox_targets) # centerness weighted iou loss pos_centerness_denorm = max( reduce_mean(pos_centerness_targets.sum().detach()), 1e-6) if len(pos_inds) > 0: pos_points = flatten_points[pos_inds] pos_decoded_bbox_preds = self.bbox_head.bbox_coder.decode( pos_points, pos_bbox_preds) pos_decoded_target_preds = self.bbox_head.bbox_coder.decode( pos_points, pos_bbox_targets) loss_bbox = self.bbox_head.loss_bbox( pos_decoded_bbox_preds, pos_decoded_target_preds, weight=pos_centerness_targets, avg_factor=pos_centerness_denorm) loss_centerness = self.bbox_head.loss_centerness( pos_centerness, pos_centerness_targets, avg_factor=num_pos) else: loss_bbox = pos_bbox_preds.sum() loss_centerness = pos_centerness.sum() # flatten tea_cls_scores, tea_bbox_preds and tea_centernesses flatten_tea_cls_scores = [ tea_cls_scores.permute(0, 2, 3, 1).reshape(-1, self.bbox_head.cls_out_channels) for tea_cls_scores in tea_cls_scores ] flatten_tea_bbox_preds = [ tea_bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) for tea_bbox_pred in tea_bbox_preds ] flatten_tea_centernesses = [ tea_centerness.permute(0, 2, 3, 1).reshape(-1, 1) for tea_centerness in tea_centernesses ] flatten_tea_cls_scores = torch.cat(flatten_tea_cls_scores) flatten_tea_bbox_preds = torch.cat(flatten_tea_bbox_preds) flatten_tea_centernesses = torch.cat(flatten_tea_centernesses) # flatten reused_cls_scores, reused_bbox_preds and reused_centernesses flatten_reused_cls_scores = [ reused_cls_score.permute(0, 2, 3, 1).reshape(-1, self.bbox_head.cls_out_channels) for reused_cls_score in reused_cls_scores ] flatten_reused_bbox_preds = [ reused_bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4) for reused_bbox_pred in reused_bbox_preds ] flatten_reused_centernesses = [ reused_centerness.permute(0, 2, 3, 1).reshape(-1, 1) for reused_centerness in reused_centernesses ] flatten_reused_cls_scores = torch.cat(flatten_reused_cls_scores) flatten_reused_bbox_preds = torch.cat(flatten_reused_bbox_preds) flatten_reused_centernesses = torch.cat(flatten_reused_centernesses) losses_cls_kd = self.loss_cls_kd(flatten_reused_cls_scores, flatten_tea_cls_scores, avg_factor=pos_centerness_denorm) flatten_tea_bbox_preds = self.bbox_head.bbox_coder.decode( flatten_points, flatten_tea_bbox_preds) flatten_reused_bbox_preds = self.bbox_head.bbox_coder.decode( flatten_points, flatten_reused_bbox_preds) reg_weights = flatten_tea_cls_scores.max(dim=1)[0].sigmoid() losses_reg_kd = self.loss_reg_kd(flatten_reused_bbox_preds, flatten_tea_bbox_preds, weight=reg_weights, avg_factor=pos_centerness_denorm) losses = dict(loss_cls=loss_cls, loss_bbox=loss_bbox, loss_centerness=loss_centerness, loss_cls_kd=losses_cls_kd, loss_reg_kd=losses_reg_kd) if self.with_feat_distill: losses_feat_kd = [ self.loss_feat_kd(feat, tea_feat) for feat, tea_feat in zip(feats, tea_feats) ] for i, loss in enumerate(losses_feat_kd): losses.update({"loss_feat_kd_{}".format(i):loss}) return losses ================================================ FILE: mmdet/models/detectors/crosskd_gfl.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Union import torch import torch.nn.functional as F from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import InstanceList, OptInstanceList, reduce_mean from ..utils import multi_apply, unpack_gt_instances from .crosskd_single_stage import CrossKDSingleStageDetector @MODELS.register_module() class CrossKDGFL(CrossKDSingleStageDetector): def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Union[dict, list]: """Calculate losses from a batch of inputs and data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ tea_x = self.teacher.extract_feat(batch_inputs) tea_cls_scores, tea_bbox_preds, tea_cls_hold, tea_reg_hold = \ multi_apply(self.forward_crosskd_single, tea_x, self.teacher.bbox_head.scales, module=self.teacher) stu_x = self.extract_feat(batch_inputs) stu_cls_scores, stu_bbox_preds, stu_cls_hold, stu_reg_hold = \ multi_apply(self.forward_crosskd_single, stu_x, self.bbox_head.scales, module=self) reused_cls_scores, reused_bbox_preds = multi_apply( self.reuse_teacher_head, tea_cls_hold, tea_reg_hold, stu_cls_hold, stu_reg_hold, self.teacher.bbox_head.scales) outputs = unpack_gt_instances(batch_data_samples) (batch_gt_instances, batch_gt_instances_ignore, batch_img_metas) = outputs losses = self.loss_by_feat(tea_cls_scores, tea_bbox_preds, tea_x, stu_cls_scores, stu_bbox_preds, stu_x, reused_cls_scores, reused_bbox_preds, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) return losses def forward_crosskd_single(self, x, scale, module): cls_feat, reg_feat = x, x cls_feat_hold, reg_feat_hold = x, x for i, cls_conv in enumerate(module.bbox_head.cls_convs): cls_feat = cls_conv(cls_feat, activate=False) if i + 1 == self.reused_teacher_head_idx: cls_feat_hold = cls_feat cls_feat = cls_conv.activate(cls_feat) for i, reg_conv in enumerate(module.bbox_head.reg_convs): reg_feat = reg_conv(reg_feat, activate=False) if i + 1 == self.reused_teacher_head_idx: reg_feat_hold = reg_feat reg_feat = reg_conv.activate(reg_feat) cls_score = module.bbox_head.gfl_cls(cls_feat) bbox_pred = scale(module.bbox_head.gfl_reg(reg_feat)).float() return cls_score, bbox_pred, cls_feat_hold, reg_feat_hold def reuse_teacher_head(self, tea_cls_feat, tea_reg_feat, stu_cls_feat, stu_reg_feat, scale): reused_cls_feat = self.align_scale(stu_cls_feat, tea_cls_feat) reused_reg_feat = self.align_scale(stu_reg_feat, tea_reg_feat) if self.reused_teacher_head_idx != 0: reused_cls_feat = F.relu(reused_cls_feat) reused_reg_feat = F.relu(reused_reg_feat) module = self.teacher.bbox_head for i in range(self.reused_teacher_head_idx, module.stacked_convs): reused_cls_feat = module.cls_convs[i](reused_cls_feat) reused_reg_feat = module.reg_convs[i](reused_reg_feat) reused_cls_score = module.gfl_cls(reused_cls_feat) reused_bbox_pred = scale(module.gfl_reg(reused_reg_feat)).float() return reused_cls_score, reused_bbox_pred def align_scale(self, stu_feat, tea_feat): N, C, H, W = stu_feat.size() # normalize student feature stu_feat = stu_feat.permute(1, 0, 2, 3).reshape(C, -1) stu_mean = stu_feat.mean(dim=-1, keepdim=True) stu_std = stu_feat.std(dim=-1, keepdim=True) stu_feat = (stu_feat - stu_mean) / (stu_std + 1e-6) # tea_feat = tea_feat.permute(1, 0, 2, 3).reshape(C, -1) tea_mean = tea_feat.mean(dim=-1, keepdim=True) tea_std = tea_feat.std(dim=-1, keepdim=True) stu_feat = stu_feat * tea_std + tea_mean return stu_feat.reshape(C, N, H, W).permute(1, 0, 2, 3) def loss_by_feat( self, tea_cls_scores: List[Tensor], tea_bbox_preds: List[Tensor], tea_feats: List[Tensor], cls_scores: List[Tensor], bbox_preds: List[Tensor], feats: List[Tensor], reused_cls_scores: List[Tensor], reused_bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Cls and quality scores for each scale level has shape (N, num_classes, H, W). bbox_preds (list[Tensor]): Box distribution logits for each scale level with shape (N, 4*(n+1), H, W), n is max value of integral set. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], Optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict[str, Tensor]: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == self.bbox_head.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.bbox_head.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.bbox_head.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets avg_factor = reduce_mean( torch.tensor(avg_factor, dtype=torch.float, device=device)).item() losses_cls, losses_bbox, losses_dfl,\ new_avg_factor = multi_apply( self.bbox_head.loss_by_feat_single, anchor_list, cls_scores, bbox_preds, labels_list, label_weights_list, bbox_targets_list, self.bbox_head.prior_generator.strides, avg_factor=avg_factor) new_avg_factor = sum(new_avg_factor) new_avg_factor = reduce_mean(new_avg_factor).clamp_(min=1).item() losses_bbox = list(map(lambda x: x / new_avg_factor, losses_bbox)) losses_dfl = list(map(lambda x: x / new_avg_factor, losses_dfl)) losses = dict( loss_cls=losses_cls, loss_bbox=losses_bbox, loss_dfl=losses_dfl) losses_cls_kd, losses_reg_kd, kd_avg_factor = multi_apply( self.pred_mimicking_loss_single, tea_cls_scores, tea_bbox_preds, reused_cls_scores, reused_bbox_preds, label_weights_list, avg_factor=avg_factor) kd_avg_factor = sum(kd_avg_factor) losses_reg_kd = list(map(lambda x: x / kd_avg_factor, losses_reg_kd)) losses.update( dict(loss_cls_kd=losses_cls_kd, loss_reg_kd=losses_reg_kd)) if self.with_feat_distill: losses_feat_kd = [ self.loss_feat_kd(feat, tea_feat) for feat, tea_feat in zip(feats, tea_feats) ] losses.update(loss_feat_kd=losses_feat_kd) return losses def pred_mimicking_loss_single(self, tea_cls_score, tea_bbox_pred, reused_cls_score, reused_bbox_pred, label_weights, avg_factor): # classification branch distillation tea_cls_score = tea_cls_score.permute(0, 2, 3, 1).reshape( -1, self.bbox_head.cls_out_channels) reused_cls_score = reused_cls_score.permute(0, 2, 3, 1).reshape( -1, self.bbox_head.cls_out_channels) label_weights = label_weights.reshape(-1) loss_cls_kd = self.loss_cls_kd( reused_cls_score, tea_cls_score, label_weights, avg_factor=avg_factor) # regression branch distillation reg_max = self.bbox_head.reg_max tea_bbox_pred = tea_bbox_pred.permute(0, 2, 3, 1).reshape(-1, reg_max + 1) reused_bbox_pred = reused_bbox_pred.permute(0, 2, 3, 1).reshape( -1, reg_max + 1) reg_weights = tea_cls_score.max(dim=1)[0].sigmoid() reg_weights[label_weights == 0] = 0 loss_reg_kd = self.loss_reg_kd( reused_bbox_pred, tea_bbox_pred, weight=reg_weights[:, None].expand(-1, 4).reshape(-1), avg_factor=4.0) return loss_cls_kd, loss_reg_kd, reg_weights.sum() ================================================ FILE: mmdet/models/detectors/crosskd_retinanet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Union import torch.nn.functional as F from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import cat_boxes from mmdet.utils import InstanceList, OptInstanceList from ..utils import images_to_levels, multi_apply, unpack_gt_instances from .crosskd_single_stage import CrossKDSingleStageDetector @MODELS.register_module() class CrossKDRetinaNet(CrossKDSingleStageDetector): def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Union[dict, list]: """Calculate losses from a batch of inputs and data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ tea_x = self.teacher.extract_feat(batch_inputs) tea_cls_scores, tea_bbox_preds, tea_cls_hold, tea_reg_hold = \ multi_apply(self.forward_crosskd_single, tea_x, module=self.teacher) stu_x = self.extract_feat(batch_inputs) stu_cls_scores, stu_bbox_preds, stu_cls_hold, stu_reg_hold = \ multi_apply(self.forward_crosskd_single, stu_x, module=self) reused_cls_scores, reused_bbox_preds = multi_apply( self.reuse_teacher_head, tea_cls_hold, tea_reg_hold, stu_cls_hold, stu_reg_hold) outputs = unpack_gt_instances(batch_data_samples) (batch_gt_instances, batch_gt_instances_ignore, batch_img_metas) = outputs losses = self.loss_by_feat(tea_cls_scores, tea_bbox_preds, tea_x, stu_cls_scores, stu_bbox_preds, stu_x, reused_cls_scores, reused_bbox_preds, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) return losses def forward_crosskd_single(self, x, module): cls_feat, reg_feat = x, x cls_feat_hold, reg_feat_hold = x, x for i, cls_conv in enumerate(module.bbox_head.cls_convs): cls_feat = cls_conv(cls_feat, activate=False) if i + 1 == self.reused_teacher_head_idx: cls_feat_hold = cls_feat cls_feat = cls_conv.activate(cls_feat) for i, reg_conv in enumerate(module.bbox_head.reg_convs): reg_feat = reg_conv(reg_feat, activate=False) if i + 1 == self.reused_teacher_head_idx: reg_feat_hold = reg_feat reg_feat = reg_conv.activate(reg_feat) cls_score = module.bbox_head.retina_cls(cls_feat) bbox_pred = module.bbox_head.retina_reg(reg_feat) return cls_score, bbox_pred, cls_feat_hold, reg_feat_hold def reuse_teacher_head(self, tea_cls_feat, tea_reg_feat, stu_cls_feat, stu_reg_feat): reused_cls_feat = self.align_scale(stu_cls_feat, tea_cls_feat) reused_reg_feat = self.align_scale(stu_reg_feat, tea_reg_feat) if self.reused_teacher_head_idx != 0: reused_cls_feat = F.relu(reused_cls_feat) reused_reg_feat = F.relu(reused_reg_feat) module = self.teacher.bbox_head for i in range(self.reused_teacher_head_idx, module.stacked_convs): reused_cls_feat = module.cls_convs[i](reused_cls_feat) reused_reg_feat = module.reg_convs[i](reused_reg_feat) reused_cls_score = module.retina_cls(reused_cls_feat) reused_bbox_pred = module.retina_reg(reused_reg_feat) return reused_cls_score, reused_bbox_pred def align_scale(self, stu_feat, tea_feat): N, C, H, W = stu_feat.size() # normalize student feature stu_feat = stu_feat.permute(1, 0, 2, 3).reshape(C, -1) stu_mean = stu_feat.mean(dim=-1, keepdim=True) stu_std = stu_feat.std(dim=-1, keepdim=True) stu_feat = (stu_feat - stu_mean) / (stu_std + 1e-6) # tea_feat = tea_feat.permute(1, 0, 2, 3).reshape(C, -1) tea_mean = tea_feat.mean(dim=-1, keepdim=True) tea_std = tea_feat.std(dim=-1, keepdim=True) stu_feat = stu_feat * tea_std + tea_mean return stu_feat.reshape(C, N, H, W).permute(1, 0, 2, 3) def loss_by_feat( self, tea_cls_scores: List[Tensor], tea_bbox_preds: List[Tensor], tea_feats: List[Tensor], cls_scores: List[Tensor], bbox_preds: List[Tensor], feats: List[Tensor], reused_cls_scores: List[Tensor], reused_bbox_preds: List[Tensor], batch_gt_instances: InstanceList, batch_img_metas: List[dict], batch_gt_instances_ignore: OptInstanceList = None) -> dict: """Calculate the loss based on the features extracted by the detection head. Args: cls_scores (list[Tensor]): Box scores for each scale level has shape (N, num_anchors * num_classes, H, W). bbox_preds (list[Tensor]): Box energies / deltas for each scale level with shape (N, num_anchors * 4, H, W). batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. batch_gt_instances_ignore (list[:obj:`InstanceData`], optional): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: dict: A dictionary of loss components. """ featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores] assert len(featmap_sizes) == \ self.bbox_head.prior_generator.num_levels device = cls_scores[0].device anchor_list, valid_flag_list = self.bbox_head.get_anchors( featmap_sizes, batch_img_metas, device=device) cls_reg_targets = self.bbox_head.get_targets( anchor_list, valid_flag_list, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore=batch_gt_instances_ignore) (labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor) = cls_reg_targets # anchor number of multi levels num_level_anchors = [anchors.size(0) for anchors in anchor_list[0]] # concat all level anchors and flags to a single tensor concat_anchor_list = [] for i in range(len(anchor_list)): concat_anchor_list.append(cat_boxes(anchor_list[i])) all_anchor_list = images_to_levels(concat_anchor_list, num_level_anchors) losses_cls, losses_bbox = multi_apply( self.bbox_head.loss_by_feat_single, cls_scores, bbox_preds, all_anchor_list, labels_list, label_weights_list, bbox_targets_list, bbox_weights_list, avg_factor=avg_factor) losses = dict(loss_cls=losses_cls, loss_bbox=losses_bbox) losses_cls_kd, losses_reg_kd = multi_apply( self.pred_mimicking_loss_single, tea_cls_scores, tea_bbox_preds, reused_cls_scores, reused_bbox_preds, all_anchor_list, label_weights_list, avg_factor=avg_factor) losses.update( dict(loss_cls_kd=losses_cls_kd, loss_reg_kd=losses_reg_kd)) if self.with_feat_distill: losses_feat_kd = [ self.loss_feat_kd(feat, tea_feat) for feat, tea_feat in zip(feats, tea_feats) ] losses.update(loss_feat_kd=losses_feat_kd) return losses def pred_mimicking_loss_single(self, tea_cls_score, tea_bbox_pred, reused_cls_score, reused_bbox_pred, anchors, label_weights, avg_factor): # classification branch distillation tea_cls_score = tea_cls_score.permute(0, 2, 3, 1).reshape( -1, self.bbox_head.cls_out_channels) reused_cls_score = reused_cls_score.permute(0, 2, 3, 1).reshape( -1, self.bbox_head.cls_out_channels) label_weights = label_weights.reshape(-1) loss_cls_kd = self.loss_cls_kd( reused_cls_score, tea_cls_score, label_weights, avg_factor=avg_factor) # regression branch distillation bbox_coder = self.bbox_head.bbox_coder tea_bbox_pred = tea_bbox_pred.permute(0, 2, 3, 1).reshape( -1, bbox_coder.encode_size) reused_bbox_pred = reused_bbox_pred.permute(0, 2, 3, 1).reshape( -1, bbox_coder.encode_size) anchors = anchors.reshape(-1, anchors.size(-1)) tea_bbox_pred = bbox_coder.decode(anchors, tea_bbox_pred) reused_bbox_pred = bbox_coder.decode(anchors, reused_bbox_pred) reg_weights = tea_cls_score.max(dim=1)[0].sigmoid() reg_weights[label_weights == 0] = 0 loss_reg_kd = self.loss_reg_kd( reused_bbox_pred, tea_bbox_pred, reg_weights, avg_factor=avg_factor) return loss_cls_kd, loss_reg_kd ================================================ FILE: mmdet/models/detectors/crosskd_single_stage.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from pathlib import Path from typing import Any, List, Optional, Union import torch import torch.nn as nn import torch.nn.functional as F from mmengine.config import Config from mmengine.runner import load_checkpoint from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import cat_boxes from mmdet.utils import (ConfigType, InstanceList, OptConfigType, OptInstanceList, reduce_mean) from ..utils import images_to_levels, multi_apply, unpack_gt_instances from .single_stage import SingleStageDetector @MODELS.register_module() class CrossKDSingleStageDetector(SingleStageDetector): r"""Implementation of `Distilling the Knowledge in a Neural Network. `_. Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. teacher_config (:obj:`ConfigDict` | dict | str | Path): Config file path or the config object of teacher model. teacher_ckpt (str, optional): Checkpoint path of teacher model. If left as None, the model will not load any weights. Defaults to True. eval_teacher (bool): Set the train mode for teacher. Defaults to True. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of ATSS. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of ATSS. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. """ def __init__( self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, teacher_config: Union[ConfigType, str, Path], teacher_ckpt: Optional[str] = None, kd_cfg: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, ) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor) # Build teacher model if isinstance(teacher_config, (str, Path)): teacher_config = Config.fromfile(teacher_config) self.teacher = MODELS.build(teacher_config['model']) if teacher_ckpt is not None: load_checkpoint(self.teacher, teacher_ckpt, map_location='cpu') # In order to reforward teacher model, # set requires_grad of teacher model to False self.freeze(self.teacher) self.loss_cls_kd = MODELS.build(kd_cfg['loss_cls_kd']) self.loss_reg_kd = MODELS.build(kd_cfg['loss_reg_kd']) self.with_feat_distill = False if kd_cfg.get('loss_feat_kd', None): self.loss_feat_kd = MODELS.build(kd_cfg['loss_feat_kd']) self.with_feat_distill = True self.reused_teacher_head_idx = kd_cfg['reused_teacher_head_idx'] @staticmethod def freeze(model: nn.Module): """Freeze the model.""" model.eval() for param in model.parameters(): param.requires_grad = False def cuda(self, device: Optional[str] = None) -> nn.Module: """Since teacher is registered as a plain object, it is necessary to put the teacher model to cuda when calling ``cuda`` function.""" self.teacher.cuda(device=device) return super().cuda(device=device) def to(self, device: Optional[str] = None) -> nn.Module: """Since teacher is registered as a plain object, it is necessary to put the teacher model to other device when calling ``to`` function.""" self.teacher.to(device=device) return super().to(device=device) def train(self, mode: bool = True) -> None: """Set the same train mode for teacher and student model.""" self.teacher.train(False) super().train(mode) def __setattr__(self, name: str, value: Any) -> None: """Set attribute, i.e. self.name = value This reloading prevent the teacher model from being registered as a nn.Module. The teacher module is registered as a plain object, so that the teacher parameters will not show up when calling ``self.parameters``, ``self.modules``, ``self.children`` methods. """ if name == 'teacher': object.__setattr__(self, name, value) else: super().__setattr__(name, value) ================================================ FILE: mmdet/models/detectors/crowddet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .two_stage import TwoStageDetector @MODELS.register_module() class CrowdDet(TwoStageDetector): """Implementation of `CrowdDet `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone config. rpn_head (:obj:`ConfigDict` or dict): The rpn config. roi_head (:obj:`ConfigDict` or dict): The roi config. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of FCOS. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of FCOS. Defaults to None. neck (:obj:`ConfigDict` or dict): The neck config. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, backbone: ConfigType, rpn_head: ConfigType, roi_head: ConfigType, train_cfg: ConfigType, test_cfg: ConfigType, neck: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, init_cfg=init_cfg, data_preprocessor=data_preprocessor) ================================================ FILE: mmdet/models/detectors/d2_wrapper.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Union from mmengine.config import ConfigDict from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import BaseBoxes from mmdet.structures.mask import BitmapMasks, PolygonMasks from mmdet.utils import ConfigType from .base import BaseDetector try: import detectron2 from detectron2.config import get_cfg from detectron2.modeling import build_model from detectron2.structures.masks import BitMasks as D2_BitMasks from detectron2.structures.masks import PolygonMasks as D2_PolygonMasks from detectron2.utils.events import EventStorage except ImportError: detectron2 = None def _to_cfgnode_list(cfg: ConfigType, config_list: list = [], father_name: str = 'MODEL') -> tuple: """Convert the key and value of mmengine.ConfigDict into a list. Args: cfg (ConfigDict): The detectron2 model config. config_list (list): A list contains the key and value of ConfigDict. Defaults to []. father_name (str): The father name add before the key. Defaults to "MODEL". Returns: tuple: - config_list: A list contains the key and value of ConfigDict. - father_name (str): The father name add before the key. Defaults to "MODEL". """ for key, value in cfg.items(): name = f'{father_name}.{key.upper()}' if isinstance(value, ConfigDict) or isinstance(value, dict): config_list, fater_name = \ _to_cfgnode_list(value, config_list, name) else: config_list.append(name) config_list.append(value) return config_list, father_name def convert_d2_pred_to_datasample(data_samples: SampleList, d2_results_list: list) -> SampleList: """Convert the Detectron2's result to DetDataSample. Args: data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. d2_results_list (list): The list of the results of Detectron2's model. Returns: list[:obj:`DetDataSample`]: Detection results of the input images. Each DetDataSample usually contain 'pred_instances'. And the ``pred_instances`` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(data_samples) == len(d2_results_list) for data_sample, d2_results in zip(data_samples, d2_results_list): d2_instance = d2_results['instances'] results = InstanceData() results.bboxes = d2_instance.pred_boxes.tensor results.scores = d2_instance.scores results.labels = d2_instance.pred_classes if d2_instance.has('pred_masks'): results.masks = d2_instance.pred_masks data_sample.pred_instances = results return data_samples @MODELS.register_module() class Detectron2Wrapper(BaseDetector): """Wrapper of a Detectron2 model. Input/output formats of this class follow MMDetection's convention, so a Detectron2 model can be trained and evaluated in MMDetection. Args: detector (:obj:`ConfigDict` or dict): The module config of Detectron2. bgr_to_rgb (bool): whether to convert image from BGR to RGB. Defaults to False. rgb_to_bgr (bool): whether to convert image from RGB to BGR. Defaults to False. """ def __init__(self, detector: ConfigType, bgr_to_rgb: bool = False, rgb_to_bgr: bool = False) -> None: if detectron2 is None: raise ImportError('Please install Detectron2 first') assert not (bgr_to_rgb and rgb_to_bgr), ( '`bgr2rgb` and `rgb2bgr` cannot be set to True at the same time') super().__init__() self._channel_conversion = rgb_to_bgr or bgr_to_rgb cfgnode_list, _ = _to_cfgnode_list(detector) self.cfg = get_cfg() self.cfg.merge_from_list(cfgnode_list) self.d2_model = build_model(self.cfg) self.storage = EventStorage() def init_weights(self) -> None: """Initialization Backbone. NOTE: The initialization of other layers are in Detectron2, if users want to change the initialization way, please change the code in Detectron2. """ from detectron2.checkpoint import DetectionCheckpointer checkpointer = DetectionCheckpointer(model=self.d2_model) checkpointer.load(self.cfg.MODEL.WEIGHTS, checkpointables=[]) def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Union[dict, tuple]: """Calculate losses from a batch of inputs and data samples. The inputs will first convert to the Detectron2 type and feed into D2 models. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ d2_batched_inputs = self._convert_to_d2_inputs( batch_inputs=batch_inputs, batch_data_samples=batch_data_samples, training=True) with self.storage as storage: # noqa losses = self.d2_model(d2_batched_inputs) # storage contains some training information, such as cls_accuracy. # you can use storage.latest() to get the detail information return losses def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> SampleList: """Predict results from a batch of inputs and data samples with post- processing. The inputs will first convert to the Detectron2 type and feed into D2 models. And the results will convert back to the MMDet type. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: list[:obj:`DetDataSample`]: Detection results of the input images. Each DetDataSample usually contain 'pred_instances'. And the ``pred_instances`` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ d2_batched_inputs = self._convert_to_d2_inputs( batch_inputs=batch_inputs, batch_data_samples=batch_data_samples, training=False) # results in detectron2 has already rescale d2_results_list = self.d2_model(d2_batched_inputs) batch_data_samples = convert_d2_pred_to_datasample( data_samples=batch_data_samples, d2_results_list=d2_results_list) return batch_data_samples def _forward(self, *args, **kwargs): """Network forward process. Usually includes backbone, neck and head forward without any post- processing. """ raise NotImplementedError( f'`_forward` is not implemented in {self.__class__.__name__}') def extract_feat(self, *args, **kwargs): """Extract features from images. `extract_feat` will not be used in obj:``Detectron2Wrapper``. """ pass def _convert_to_d2_inputs(self, batch_inputs: Tensor, batch_data_samples: SampleList, training=True) -> list: """Convert inputs type to support Detectron2's model. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. training (bool): Whether to enable training time processing. Returns: list[dict]: A list of dict, which will be fed into Detectron2's model. And the dict usually contains following keys. - image (Tensor): Image in (C, H, W) format. - instances (Instances): GT Instance. - height (int): the output height resolution of the model - width (int): the output width resolution of the model """ from detectron2.data.detection_utils import filter_empty_instances from detectron2.structures import Boxes, Instances batched_d2_inputs = [] for image, data_samples in zip(batch_inputs, batch_data_samples): d2_inputs = dict() # deal with metainfo meta_info = data_samples.metainfo d2_inputs['file_name'] = meta_info['img_path'] d2_inputs['height'], d2_inputs['width'] = meta_info['ori_shape'] d2_inputs['image_id'] = meta_info['img_id'] # deal with image if self._channel_conversion: image = image[[2, 1, 0], ...] d2_inputs['image'] = image # deal with gt_instances gt_instances = data_samples.gt_instances d2_instances = Instances(meta_info['img_shape']) gt_boxes = gt_instances.bboxes # TODO: use mmdet.structures.box.get_box_tensor after PR 8658 # has merged if isinstance(gt_boxes, BaseBoxes): gt_boxes = gt_boxes.tensor d2_instances.gt_boxes = Boxes(gt_boxes) d2_instances.gt_classes = gt_instances.labels if gt_instances.get('masks', None) is not None: gt_masks = gt_instances.masks if isinstance(gt_masks, PolygonMasks): d2_instances.gt_masks = D2_PolygonMasks(gt_masks.masks) elif isinstance(gt_masks, BitmapMasks): d2_instances.gt_masks = D2_BitMasks(gt_masks.masks) else: raise TypeError('The type of `gt_mask` can be ' '`PolygonMasks` or `BitMasks`, but get ' f'{type(gt_masks)}.') # convert to cpu and convert back to cuda to avoid # some potential error if training: device = gt_boxes.device d2_instances = filter_empty_instances( d2_instances.to('cpu')).to(device) d2_inputs['instances'] = d2_instances batched_d2_inputs.append(d2_inputs) return batched_d2_inputs ================================================ FILE: mmdet/models/detectors/dab_detr.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, Tuple from mmengine.model import uniform_init from torch import Tensor, nn from mmdet.registry import MODELS from ..layers import SinePositionalEncoding from ..layers.transformer import (DABDetrTransformerDecoder, DABDetrTransformerEncoder, inverse_sigmoid) from .detr import DETR @MODELS.register_module() class DABDETR(DETR): r"""Implementation of `DAB-DETR: Dynamic Anchor Boxes are Better Queries for DETR. `_. Code is modified from the `official github repo `_. Args: with_random_refpoints (bool): Whether to randomly initialize query embeddings and not update them during training. Defaults to False. num_patterns (int): Inspired by Anchor-DETR. Defaults to 0. """ def __init__(self, *args, with_random_refpoints: bool = False, num_patterns: int = 0, **kwargs) -> None: self.with_random_refpoints = with_random_refpoints assert isinstance(num_patterns, int), \ f'num_patterns should be int but {num_patterns}.' self.num_patterns = num_patterns super().__init__(*args, **kwargs) def _init_layers(self) -> None: """Initialize layers except for backbone, neck and bbox_head.""" self.positional_encoding = SinePositionalEncoding( **self.positional_encoding) self.encoder = DABDetrTransformerEncoder(**self.encoder) self.decoder = DABDetrTransformerDecoder(**self.decoder) self.embed_dims = self.encoder.embed_dims self.query_dim = self.decoder.query_dim self.query_embedding = nn.Embedding(self.num_queries, self.query_dim) if self.num_patterns > 0: self.patterns = nn.Embedding(self.num_patterns, self.embed_dims) num_feats = self.positional_encoding.num_feats assert num_feats * 2 == self.embed_dims, \ f'embed_dims should be exactly 2 times of num_feats. ' \ f'Found {self.embed_dims} and {num_feats}.' def init_weights(self) -> None: """Initialize weights for Transformer and other components.""" super(DABDETR, self).init_weights() if self.with_random_refpoints: uniform_init(self.query_embedding) self.query_embedding.weight.data[:, :2] = \ inverse_sigmoid(self.query_embedding.weight.data[:, :2]) self.query_embedding.weight.data[:, :2].requires_grad = False def pre_decoder(self, memory: Tensor) -> Tuple[Dict, Dict]: """Prepare intermediate variables before entering Transformer decoder, such as `query`, `query_pos`. Args: memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). Returns: tuple[dict, dict]: The first dict contains the inputs of decoder and the second dict contains the inputs of the bbox_head function. - decoder_inputs_dict (dict): The keyword args dictionary of `self.forward_decoder()`, which includes 'query', 'query_pos', 'memory' and 'reg_branches'. - head_inputs_dict (dict): The keyword args dictionary of the bbox_head functions, which is usually empty, or includes `enc_outputs_class` and `enc_outputs_class` when the detector support 'two stage' or 'query selection' strategies. """ batch_size = memory.size(0) query_pos = self.query_embedding.weight query_pos = query_pos.unsqueeze(0).repeat(batch_size, 1, 1) if self.num_patterns == 0: query = query_pos.new_zeros(batch_size, self.num_queries, self.embed_dims) else: query = self.patterns.weight[:, None, None, :]\ .repeat(1, self.num_queries, batch_size, 1)\ .view(-1, batch_size, self.embed_dims)\ .permute(1, 0, 2) query_pos = query_pos.repeat(1, self.num_patterns, 1) decoder_inputs_dict = dict( query_pos=query_pos, query=query, memory=memory) head_inputs_dict = dict() return decoder_inputs_dict, head_inputs_dict def forward_decoder(self, query: Tensor, query_pos: Tensor, memory: Tensor, memory_mask: Tensor, memory_pos: Tensor) -> Dict: """Forward with Transformer decoder. Args: query (Tensor): The queries of decoder inputs, has shape (bs, num_queries, dim). query_pos (Tensor): The positional queries of decoder inputs, has shape (bs, num_queries, dim). memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). memory_mask (Tensor): ByteTensor, the padding mask of the memory, has shape (bs, num_feat_points). memory_pos (Tensor): The positional embeddings of memory, has shape (bs, num_feat_points, dim). Returns: dict: The dictionary of decoder outputs, which includes the `hidden_states` and `references` of the decoder output. """ hidden_states, references = self.decoder( query=query, key=memory, query_pos=query_pos, key_pos=memory_pos, key_padding_mask=memory_mask, reg_branches=self.bbox_head. fc_reg # iterative refinement for anchor boxes ) head_inputs_dict = dict( hidden_states=hidden_states, references=references) return head_inputs_dict ================================================ FILE: mmdet/models/detectors/ddod.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class DDOD(SingleStageDetector): """Implementation of `DDOD `_. Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of ATSS. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of ATSS. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/deformable_detr.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from typing import Dict, Tuple import torch import torch.nn.functional as F from mmcv.cnn.bricks.transformer import MultiScaleDeformableAttention from mmengine.model import xavier_init from torch import Tensor, nn from torch.nn.init import normal_ from mmdet.registry import MODELS from mmdet.structures import OptSampleList from mmdet.utils import OptConfigType from ..layers import (DeformableDetrTransformerDecoder, DeformableDetrTransformerEncoder, SinePositionalEncoding) from .base_detr import DetectionTransformer @MODELS.register_module() class DeformableDETR(DetectionTransformer): r"""Implementation of `Deformable DETR: Deformable Transformers for End-to-End Object Detection `_ Code is modified from the `official github repo `_. Args: decoder (:obj:`ConfigDict` or dict, optional): Config of the Transformer decoder. Defaults to None. bbox_head (:obj:`ConfigDict` or dict, optional): Config for the bounding box head module. Defaults to None. with_box_refine (bool, optional): Whether to refine the references in the decoder. Defaults to `False`. as_two_stage (bool, optional): Whether to generate the proposal from the outputs of encoder. Defaults to `False`. num_feature_levels (int, optional): Number of feature levels. Defaults to 4. """ def __init__(self, *args, decoder: OptConfigType = None, bbox_head: OptConfigType = None, with_box_refine: bool = False, as_two_stage: bool = False, num_feature_levels: int = 4, **kwargs) -> None: self.with_box_refine = with_box_refine self.as_two_stage = as_two_stage self.num_feature_levels = num_feature_levels if bbox_head is not None: assert 'share_pred_layer' not in bbox_head and \ 'num_pred_layer' not in bbox_head and \ 'as_two_stage' not in bbox_head, \ 'The two keyword args `share_pred_layer`, `num_pred_layer`, ' \ 'and `as_two_stage are set in `detector.__init__()`, users ' \ 'should not set them in `bbox_head` config.' # The last prediction layer is used to generate proposal # from encode feature map when `as_two_stage` is `True`. # And all the prediction layers should share parameters # when `with_box_refine` is `True`. bbox_head['share_pred_layer'] = not with_box_refine bbox_head['num_pred_layer'] = (decoder['num_layers'] + 1) \ if self.as_two_stage else decoder['num_layers'] bbox_head['as_two_stage'] = as_two_stage super().__init__(*args, decoder=decoder, bbox_head=bbox_head, **kwargs) def _init_layers(self) -> None: """Initialize layers except for backbone, neck and bbox_head.""" self.positional_encoding = SinePositionalEncoding( **self.positional_encoding) self.encoder = DeformableDetrTransformerEncoder(**self.encoder) self.decoder = DeformableDetrTransformerDecoder(**self.decoder) self.embed_dims = self.encoder.embed_dims if not self.as_two_stage: self.query_embedding = nn.Embedding(self.num_queries, self.embed_dims * 2) # NOTE The query_embedding will be split into query and query_pos # in self.pre_decoder, hence, the embed_dims are doubled. num_feats = self.positional_encoding.num_feats assert num_feats * 2 == self.embed_dims, \ 'embed_dims should be exactly 2 times of num_feats. ' \ f'Found {self.embed_dims} and {num_feats}.' self.level_embed = nn.Parameter( torch.Tensor(self.num_feature_levels, self.embed_dims)) if self.as_two_stage: self.memory_trans_fc = nn.Linear(self.embed_dims, self.embed_dims) self.memory_trans_norm = nn.LayerNorm(self.embed_dims) self.pos_trans_fc = nn.Linear(self.embed_dims * 2, self.embed_dims * 2) self.pos_trans_norm = nn.LayerNorm(self.embed_dims * 2) else: self.reference_points_fc = nn.Linear(self.embed_dims, 2) def init_weights(self) -> None: """Initialize weights for Transformer and other components.""" super().init_weights() for coder in self.encoder, self.decoder: for p in coder.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) for m in self.modules(): if isinstance(m, MultiScaleDeformableAttention): m.init_weights() if self.as_two_stage: nn.init.xavier_uniform_(self.memory_trans_fc.weight) nn.init.xavier_uniform_(self.pos_trans_fc.weight) else: xavier_init( self.reference_points_fc, distribution='uniform', bias=0.) normal_(self.level_embed) def pre_transformer( self, mlvl_feats: Tuple[Tensor], batch_data_samples: OptSampleList = None) -> Tuple[Dict]: """Process image features before feeding them to the transformer. The forward procedure of the transformer is defined as: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder' More details can be found at `TransformerDetector.forward_transformer` in `mmdet/detector/base_detr.py`. Args: mlvl_feats (tuple[Tensor]): Multi-level features that may have different resolutions, output from neck. Each feature has shape (bs, dim, h_lvl, w_lvl), where 'lvl' means 'layer'. batch_data_samples (list[:obj:`DetDataSample`], optional): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Defaults to None. Returns: tuple[dict]: The first dict contains the inputs of encoder and the second dict contains the inputs of decoder. - encoder_inputs_dict (dict): The keyword args dictionary of `self.forward_encoder()`, which includes 'feat', 'feat_mask', and 'feat_pos'. - decoder_inputs_dict (dict): The keyword args dictionary of `self.forward_decoder()`, which includes 'memory_mask'. """ batch_size = mlvl_feats[0].size(0) # construct binary masks for the transformer. assert batch_data_samples is not None batch_input_shape = batch_data_samples[0].batch_input_shape img_shape_list = [sample.img_shape for sample in batch_data_samples] input_img_h, input_img_w = batch_input_shape masks = mlvl_feats[0].new_ones((batch_size, input_img_h, input_img_w)) for img_id in range(batch_size): img_h, img_w = img_shape_list[img_id] masks[img_id, :img_h, :img_w] = 0 # NOTE following the official DETR repo, non-zero values representing # ignored positions, while zero values means valid positions. mlvl_masks = [] mlvl_pos_embeds = [] for feat in mlvl_feats: mlvl_masks.append( F.interpolate(masks[None], size=feat.shape[-2:]).to(torch.bool).squeeze(0)) mlvl_pos_embeds.append(self.positional_encoding(mlvl_masks[-1])) feat_flatten = [] lvl_pos_embed_flatten = [] mask_flatten = [] spatial_shapes = [] for lvl, (feat, mask, pos_embed) in enumerate( zip(mlvl_feats, mlvl_masks, mlvl_pos_embeds)): batch_size, c, h, w = feat.shape # [bs, c, h_lvl, w_lvl] -> [bs, h_lvl*w_lvl, c] feat = feat.view(batch_size, c, -1).permute(0, 2, 1) pos_embed = pos_embed.view(batch_size, c, -1).permute(0, 2, 1) lvl_pos_embed = pos_embed + self.level_embed[lvl].view(1, 1, -1) # [bs, h_lvl, w_lvl] -> [bs, h_lvl*w_lvl] mask = mask.flatten(1) spatial_shape = (h, w) feat_flatten.append(feat) lvl_pos_embed_flatten.append(lvl_pos_embed) mask_flatten.append(mask) spatial_shapes.append(spatial_shape) # (bs, num_feat_points, dim) feat_flatten = torch.cat(feat_flatten, 1) lvl_pos_embed_flatten = torch.cat(lvl_pos_embed_flatten, 1) # (bs, num_feat_points), where num_feat_points = sum_lvl(h_lvl*w_lvl) mask_flatten = torch.cat(mask_flatten, 1) spatial_shapes = torch.as_tensor( # (num_level, 2) spatial_shapes, dtype=torch.long, device=feat_flatten.device) level_start_index = torch.cat(( spatial_shapes.new_zeros((1, )), # (num_level) spatial_shapes.prod(1).cumsum(0)[:-1])) valid_ratios = torch.stack( # (bs, num_level, 2) [self.get_valid_ratio(m) for m in mlvl_masks], 1) encoder_inputs_dict = dict( feat=feat_flatten, feat_mask=mask_flatten, feat_pos=lvl_pos_embed_flatten, spatial_shapes=spatial_shapes, level_start_index=level_start_index, valid_ratios=valid_ratios) decoder_inputs_dict = dict( memory_mask=mask_flatten, spatial_shapes=spatial_shapes, level_start_index=level_start_index, valid_ratios=valid_ratios) return encoder_inputs_dict, decoder_inputs_dict def forward_encoder(self, feat: Tensor, feat_mask: Tensor, feat_pos: Tensor, spatial_shapes: Tensor, level_start_index: Tensor, valid_ratios: Tensor) -> Dict: """Forward with Transformer encoder. The forward procedure of the transformer is defined as: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder' More details can be found at `TransformerDetector.forward_transformer` in `mmdet/detector/base_detr.py`. Args: feat (Tensor): Sequential features, has shape (bs, num_feat_points, dim). feat_mask (Tensor): ByteTensor, the padding mask of the features, has shape (bs, num_feat_points). feat_pos (Tensor): The positional embeddings of the features, has shape (bs, num_feat_points, dim). spatial_shapes (Tensor): Spatial shapes of features in all levels, has shape (num_levels, 2), last dimension represents (h, w). level_start_index (Tensor): The start index of each level. A tensor has shape (num_levels, ) and can be represented as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. valid_ratios (Tensor): The ratios of the valid width and the valid height relative to the width and the height of features in all levels, has shape (bs, num_levels, 2). Returns: dict: The dictionary of encoder outputs, which includes the `memory` of the encoder output. """ memory = self.encoder( query=feat, query_pos=feat_pos, key_padding_mask=feat_mask, # for self_attn spatial_shapes=spatial_shapes, level_start_index=level_start_index, valid_ratios=valid_ratios) encoder_outputs_dict = dict( memory=memory, memory_mask=feat_mask, spatial_shapes=spatial_shapes) return encoder_outputs_dict def pre_decoder(self, memory: Tensor, memory_mask: Tensor, spatial_shapes: Tensor) -> Tuple[Dict, Dict]: """Prepare intermediate variables before entering Transformer decoder, such as `query`, `query_pos`, and `reference_points`. The forward procedure of the transformer is defined as: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder' More details can be found at `TransformerDetector.forward_transformer` in `mmdet/detector/base_detr.py`. Args: memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). memory_mask (Tensor): ByteTensor, the padding mask of the memory, has shape (bs, num_feat_points). It will only be used when `as_two_stage` is `True`. spatial_shapes (Tensor): Spatial shapes of features in all levels, has shape (num_levels, 2), last dimension represents (h, w). It will only be used when `as_two_stage` is `True`. Returns: tuple[dict, dict]: The decoder_inputs_dict and head_inputs_dict. - decoder_inputs_dict (dict): The keyword dictionary args of `self.forward_decoder()`, which includes 'query', 'query_pos', 'memory', and `reference_points`. The reference_points of decoder input here are 4D boxes when `as_two_stage` is `True`, otherwise 2D points, although it has `points` in its name. The reference_points in encoder is always 2D points. - head_inputs_dict (dict): The keyword dictionary args of the bbox_head functions, which includes `enc_outputs_class` and `enc_outputs_coord`. They are both `None` when 'as_two_stage' is `False`. The dict is empty when `self.training` is `False`. """ batch_size, _, c = memory.shape if self.as_two_stage: output_memory, output_proposals = \ self.gen_encoder_output_proposals( memory, memory_mask, spatial_shapes) enc_outputs_class = self.bbox_head.cls_branches[ self.decoder.num_layers]( output_memory) enc_outputs_coord_unact = self.bbox_head.reg_branches[ self.decoder.num_layers](output_memory) + output_proposals enc_outputs_coord = enc_outputs_coord_unact.sigmoid() # We only use the first channel in enc_outputs_class as foreground, # the other (num_classes - 1) channels are actually not used. # Its targets are set to be 0s, which indicates the first # class (foreground) because we use [0, num_classes - 1] to # indicate class labels, background class is indicated by # num_classes (similar convention in RPN). # See https://github.com/open-mmlab/mmdetection/blob/master/mmdet/models/dense_heads/deformable_detr_head.py#L241 # noqa # This follows the official implementation of Deformable DETR. topk_proposals = torch.topk( enc_outputs_class[..., 0], self.num_queries, dim=1)[1] topk_coords_unact = torch.gather( enc_outputs_coord_unact, 1, topk_proposals.unsqueeze(-1).repeat(1, 1, 4)) topk_coords_unact = topk_coords_unact.detach() reference_points = topk_coords_unact.sigmoid() pos_trans_out = self.pos_trans_fc( self.get_proposal_pos_embed(topk_coords_unact)) pos_trans_out = self.pos_trans_norm(pos_trans_out) query_pos, query = torch.split(pos_trans_out, c, dim=2) else: enc_outputs_class, enc_outputs_coord = None, None query_embed = self.query_embedding.weight query_pos, query = torch.split(query_embed, c, dim=1) query_pos = query_pos.unsqueeze(0).expand(batch_size, -1, -1) query = query.unsqueeze(0).expand(batch_size, -1, -1) reference_points = self.reference_points_fc(query_pos).sigmoid() decoder_inputs_dict = dict( query=query, query_pos=query_pos, memory=memory, reference_points=reference_points) head_inputs_dict = dict( enc_outputs_class=enc_outputs_class, enc_outputs_coord=enc_outputs_coord) if self.training else dict() return decoder_inputs_dict, head_inputs_dict def forward_decoder(self, query: Tensor, query_pos: Tensor, memory: Tensor, memory_mask: Tensor, reference_points: Tensor, spatial_shapes: Tensor, level_start_index: Tensor, valid_ratios: Tensor) -> Dict: """Forward with Transformer decoder. The forward procedure of the transformer is defined as: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder' More details can be found at `TransformerDetector.forward_transformer` in `mmdet/detector/base_detr.py`. Args: query (Tensor): The queries of decoder inputs, has shape (bs, num_queries, dim). query_pos (Tensor): The positional queries of decoder inputs, has shape (bs, num_queries, dim). memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). memory_mask (Tensor): ByteTensor, the padding mask of the memory, has shape (bs, num_feat_points). reference_points (Tensor): The initial reference, has shape (bs, num_queries, 4) with the last dimension arranged as (cx, cy, w, h) when `as_two_stage` is `True`, otherwise has shape (bs, num_queries, 2) with the last dimension arranged as (cx, cy). spatial_shapes (Tensor): Spatial shapes of features in all levels, has shape (num_levels, 2), last dimension represents (h, w). level_start_index (Tensor): The start index of each level. A tensor has shape (num_levels, ) and can be represented as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. valid_ratios (Tensor): The ratios of the valid width and the valid height relative to the width and the height of features in all levels, has shape (bs, num_levels, 2). Returns: dict: The dictionary of decoder outputs, which includes the `hidden_states` of the decoder output and `references` including the initial and intermediate reference_points. """ inter_states, inter_references = self.decoder( query=query, value=memory, query_pos=query_pos, key_padding_mask=memory_mask, # for cross_attn reference_points=reference_points, spatial_shapes=spatial_shapes, level_start_index=level_start_index, valid_ratios=valid_ratios, reg_branches=self.bbox_head.reg_branches if self.with_box_refine else None) references = [reference_points, *inter_references] decoder_outputs_dict = dict( hidden_states=inter_states, references=references) return decoder_outputs_dict @staticmethod def get_valid_ratio(mask: Tensor) -> Tensor: """Get the valid radios of feature map in a level. .. code:: text |---> valid_W <---| ---+-----------------+-----+--- A | | | A | | | | | | | | | | valid_H | | | | | | | | H | | | | | V | | | | ---+-----------------+ | | | | V +-----------------------+--- |---------> W <---------| The valid_ratios are defined as: r_h = valid_H / H, r_w = valid_W / W They are the factors to re-normalize the relative coordinates of the image to the relative coordinates of the current level feature map. Args: mask (Tensor): Binary mask of a feature map, has shape (bs, H, W). Returns: Tensor: valid ratios [r_w, r_h] of a feature map, has shape (1, 2). """ _, H, W = mask.shape valid_H = torch.sum(~mask[:, :, 0], 1) valid_W = torch.sum(~mask[:, 0, :], 1) valid_ratio_h = valid_H.float() / H valid_ratio_w = valid_W.float() / W valid_ratio = torch.stack([valid_ratio_w, valid_ratio_h], -1) return valid_ratio def gen_encoder_output_proposals( self, memory: Tensor, memory_mask: Tensor, spatial_shapes: Tensor) -> Tuple[Tensor, Tensor]: """Generate proposals from encoded memory. The function will only be used when `as_two_stage` is `True`. Args: memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). memory_mask (Tensor): ByteTensor, the padding mask of the memory, has shape (bs, num_feat_points). spatial_shapes (Tensor): Spatial shapes of features in all levels, has shape (num_levels, 2), last dimension represents (h, w). Returns: tuple: A tuple of transformed memory and proposals. - output_memory (Tensor): The transformed memory for obtaining top-k proposals, has shape (bs, num_feat_points, dim). - output_proposals (Tensor): The inverse-normalized proposal, has shape (batch_size, num_keys, 4) with the last dimension arranged as (cx, cy, w, h). """ bs = memory.size(0) proposals = [] _cur = 0 # start index in the sequence of the current level for lvl, (H, W) in enumerate(spatial_shapes): mask_flatten_ = memory_mask[:, _cur:(_cur + H * W)].view(bs, H, W, 1) valid_H = torch.sum(~mask_flatten_[:, :, 0, 0], 1).unsqueeze(-1) valid_W = torch.sum(~mask_flatten_[:, 0, :, 0], 1).unsqueeze(-1) grid_y, grid_x = torch.meshgrid( torch.linspace( 0, H - 1, H, dtype=torch.float32, device=memory.device), torch.linspace( 0, W - 1, W, dtype=torch.float32, device=memory.device)) grid = torch.cat([grid_x.unsqueeze(-1), grid_y.unsqueeze(-1)], -1) scale = torch.cat([valid_W, valid_H], 1).view(bs, 1, 1, 2) grid = (grid.unsqueeze(0).expand(bs, -1, -1, -1) + 0.5) / scale wh = torch.ones_like(grid) * 0.05 * (2.0**lvl) proposal = torch.cat((grid, wh), -1).view(bs, -1, 4) proposals.append(proposal) _cur += (H * W) output_proposals = torch.cat(proposals, 1) output_proposals_valid = ((output_proposals > 0.01) & (output_proposals < 0.99)).all( -1, keepdim=True) # inverse_sigmoid output_proposals = torch.log(output_proposals / (1 - output_proposals)) output_proposals = output_proposals.masked_fill( memory_mask.unsqueeze(-1), float('inf')) output_proposals = output_proposals.masked_fill( ~output_proposals_valid, float('inf')) output_memory = memory output_memory = output_memory.masked_fill( memory_mask.unsqueeze(-1), float(0)) output_memory = output_memory.masked_fill(~output_proposals_valid, float(0)) output_memory = self.memory_trans_fc(output_memory) output_memory = self.memory_trans_norm(output_memory) # [bs, sum(hw), 2] return output_memory, output_proposals @staticmethod def get_proposal_pos_embed(proposals: Tensor, num_pos_feats: int = 128, temperature: int = 10000) -> Tensor: """Get the position embedding of the proposal. Args: proposals (Tensor): Not normalized proposals, has shape (bs, num_queries, 4) with the last dimension arranged as (cx, cy, w, h). num_pos_feats (int, optional): The feature dimension for each position along x, y, w, and h-axis. Note the final returned dimension for each position is 4 times of num_pos_feats. Default to 128. temperature (int, optional): The temperature used for scaling the position embedding. Defaults to 10000. Returns: Tensor: The position embedding of proposal, has shape (bs, num_queries, num_pos_feats * 4), with the last dimension arranged as (cx, cy, w, h) """ scale = 2 * math.pi dim_t = torch.arange( num_pos_feats, dtype=torch.float32, device=proposals.device) dim_t = temperature**(2 * (dim_t // 2) / num_pos_feats) # N, L, 4 proposals = proposals.sigmoid() * scale # N, L, 4, 128 pos = proposals[:, :, :, None] / dim_t # N, L, 4, 64, 2 pos = torch.stack((pos[:, :, :, 0::2].sin(), pos[:, :, :, 1::2].cos()), dim=4).flatten(2) return pos ================================================ FILE: mmdet/models/detectors/detr.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, Tuple import torch import torch.nn.functional as F from torch import Tensor, nn from mmdet.registry import MODELS from mmdet.structures import OptSampleList from ..layers import (DetrTransformerDecoder, DetrTransformerEncoder, SinePositionalEncoding) from .base_detr import DetectionTransformer @MODELS.register_module() class DETR(DetectionTransformer): r"""Implementation of `DETR: End-to-End Object Detection with Transformers. `_. Code is modified from the `official github repo `_. """ def _init_layers(self) -> None: """Initialize layers except for backbone, neck and bbox_head.""" self.positional_encoding = SinePositionalEncoding( **self.positional_encoding) self.encoder = DetrTransformerEncoder(**self.encoder) self.decoder = DetrTransformerDecoder(**self.decoder) self.embed_dims = self.encoder.embed_dims # NOTE The embed_dims is typically passed from the inside out. # For example in DETR, The embed_dims is passed as # self_attn -> the first encoder layer -> encoder -> detector. self.query_embedding = nn.Embedding(self.num_queries, self.embed_dims) num_feats = self.positional_encoding.num_feats assert num_feats * 2 == self.embed_dims, \ 'embed_dims should be exactly 2 times of num_feats. ' \ f'Found {self.embed_dims} and {num_feats}.' def init_weights(self) -> None: """Initialize weights for Transformer and other components.""" super().init_weights() for coder in self.encoder, self.decoder: for p in coder.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) def pre_transformer( self, img_feats: Tuple[Tensor], batch_data_samples: OptSampleList = None) -> Tuple[Dict, Dict]: """Prepare the inputs of the Transformer. The forward procedure of the transformer is defined as: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder' More details can be found at `TransformerDetector.forward_transformer` in `mmdet/detector/base_detr.py`. Args: img_feats (Tuple[Tensor]): Tuple of features output from the neck, has shape (bs, c, h, w). batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Defaults to None. Returns: tuple[dict, dict]: The first dict contains the inputs of encoder and the second dict contains the inputs of decoder. - encoder_inputs_dict (dict): The keyword args dictionary of `self.forward_encoder()`, which includes 'feat', 'feat_mask', and 'feat_pos'. - decoder_inputs_dict (dict): The keyword args dictionary of `self.forward_decoder()`, which includes 'memory_mask', and 'memory_pos'. """ feat = img_feats[-1] # NOTE img_feats contains only one feature. batch_size, feat_dim, _, _ = feat.shape # construct binary masks which for the transformer. assert batch_data_samples is not None batch_input_shape = batch_data_samples[0].batch_input_shape img_shape_list = [sample.img_shape for sample in batch_data_samples] input_img_h, input_img_w = batch_input_shape masks = feat.new_ones((batch_size, input_img_h, input_img_w)) for img_id in range(batch_size): img_h, img_w = img_shape_list[img_id] masks[img_id, :img_h, :img_w] = 0 # NOTE following the official DETR repo, non-zero values represent # ignored positions, while zero values mean valid positions. masks = F.interpolate( masks.unsqueeze(1), size=feat.shape[-2:]).to(torch.bool).squeeze(1) # [batch_size, embed_dim, h, w] pos_embed = self.positional_encoding(masks) # use `view` instead of `flatten` for dynamically exporting to ONNX # [bs, c, h, w] -> [bs, h*w, c] feat = feat.view(batch_size, feat_dim, -1).permute(0, 2, 1) pos_embed = pos_embed.view(batch_size, feat_dim, -1).permute(0, 2, 1) # [bs, h, w] -> [bs, h*w] masks = masks.view(batch_size, -1) # prepare transformer_inputs_dict encoder_inputs_dict = dict( feat=feat, feat_mask=masks, feat_pos=pos_embed) decoder_inputs_dict = dict(memory_mask=masks, memory_pos=pos_embed) return encoder_inputs_dict, decoder_inputs_dict def forward_encoder(self, feat: Tensor, feat_mask: Tensor, feat_pos: Tensor) -> Dict: """Forward with Transformer encoder. The forward procedure of the transformer is defined as: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder' More details can be found at `TransformerDetector.forward_transformer` in `mmdet/detector/base_detr.py`. Args: feat (Tensor): Sequential features, has shape (bs, num_feat_points, dim). feat_mask (Tensor): ByteTensor, the padding mask of the features, has shape (bs, num_feat_points). feat_pos (Tensor): The positional embeddings of the features, has shape (bs, num_feat_points, dim). Returns: dict: The dictionary of encoder outputs, which includes the `memory` of the encoder output. """ memory = self.encoder( query=feat, query_pos=feat_pos, key_padding_mask=feat_mask) # for self_attn encoder_outputs_dict = dict(memory=memory) return encoder_outputs_dict def pre_decoder(self, memory: Tensor) -> Tuple[Dict, Dict]: """Prepare intermediate variables before entering Transformer decoder, such as `query`, `query_pos`. The forward procedure of the transformer is defined as: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder' More details can be found at `TransformerDetector.forward_transformer` in `mmdet/detector/base_detr.py`. Args: memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). Returns: tuple[dict, dict]: The first dict contains the inputs of decoder and the second dict contains the inputs of the bbox_head function. - decoder_inputs_dict (dict): The keyword args dictionary of `self.forward_decoder()`, which includes 'query', 'query_pos', 'memory'. - head_inputs_dict (dict): The keyword args dictionary of the bbox_head functions, which is usually empty, or includes `enc_outputs_class` and `enc_outputs_class` when the detector support 'two stage' or 'query selection' strategies. """ batch_size = memory.size(0) # (bs, num_feat_points, dim) query_pos = self.query_embedding.weight # (num_queries, dim) -> (bs, num_queries, dim) query_pos = query_pos.unsqueeze(0).repeat(batch_size, 1, 1) query = torch.zeros_like(query_pos) decoder_inputs_dict = dict( query_pos=query_pos, query=query, memory=memory) head_inputs_dict = dict() return decoder_inputs_dict, head_inputs_dict def forward_decoder(self, query: Tensor, query_pos: Tensor, memory: Tensor, memory_mask: Tensor, memory_pos: Tensor) -> Dict: """Forward with Transformer decoder. The forward procedure of the transformer is defined as: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder' More details can be found at `TransformerDetector.forward_transformer` in `mmdet/detector/base_detr.py`. Args: query (Tensor): The queries of decoder inputs, has shape (bs, num_queries, dim). query_pos (Tensor): The positional queries of decoder inputs, has shape (bs, num_queries, dim). memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). memory_mask (Tensor): ByteTensor, the padding mask of the memory, has shape (bs, num_feat_points). memory_pos (Tensor): The positional embeddings of memory, has shape (bs, num_feat_points, dim). Returns: dict: The dictionary of decoder outputs, which includes the `hidden_states` of the decoder output. - hidden_states (Tensor): Has shape (num_decoder_layers, bs, num_queries, dim) """ hidden_states = self.decoder( query=query, key=memory, value=memory, query_pos=query_pos, key_pos=memory_pos, key_padding_mask=memory_mask) # for cross_attn head_inputs_dict = dict(hidden_states=hidden_states) return head_inputs_dict ================================================ FILE: mmdet/models/detectors/dino.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, Optional, Tuple import torch from torch import Tensor, nn from torch.nn.init import normal_ from mmdet.registry import MODELS from mmdet.structures import OptSampleList from mmdet.utils import OptConfigType from ..layers import (CdnQueryGenerator, DeformableDetrTransformerEncoder, DinoTransformerDecoder, SinePositionalEncoding) from .deformable_detr import DeformableDETR, MultiScaleDeformableAttention @MODELS.register_module() class DINO(DeformableDETR): r"""Implementation of `DINO: DETR with Improved DeNoising Anchor Boxes for End-to-End Object Detection `_ Code is modified from the `official github repo `_. Args: dn_cfg (:obj:`ConfigDict` or dict, optional): Config of denoising query generator. Defaults to `None`. """ def __init__(self, *args, dn_cfg: OptConfigType = None, **kwargs) -> None: super().__init__(*args, **kwargs) assert self.as_two_stage, 'as_two_stage must be True for DINO' assert self.with_box_refine, 'with_box_refine must be True for DINO' if dn_cfg is not None: assert 'num_classes' not in dn_cfg and \ 'num_queries' not in dn_cfg and \ 'hidden_dim' not in dn_cfg, \ 'The three keyword args `num_classes`, `embed_dims`, and ' \ '`num_matching_queries` are set in `detector.__init__()`, ' \ 'users should not set them in `dn_cfg` config.' dn_cfg['num_classes'] = self.bbox_head.num_classes dn_cfg['embed_dims'] = self.embed_dims dn_cfg['num_matching_queries'] = self.num_queries self.dn_query_generator = CdnQueryGenerator(**dn_cfg) def _init_layers(self) -> None: """Initialize layers except for backbone, neck and bbox_head.""" self.positional_encoding = SinePositionalEncoding( **self.positional_encoding) self.encoder = DeformableDetrTransformerEncoder(**self.encoder) self.decoder = DinoTransformerDecoder(**self.decoder) self.embed_dims = self.encoder.embed_dims self.query_embedding = nn.Embedding(self.num_queries, self.embed_dims) # NOTE In DINO, the query_embedding only contains content # queries, while in Deformable DETR, the query_embedding # contains both content and spatial queries, and in DETR, # it only contains spatial queries. num_feats = self.positional_encoding.num_feats assert num_feats * 2 == self.embed_dims, \ f'embed_dims should be exactly 2 times of num_feats. ' \ f'Found {self.embed_dims} and {num_feats}.' self.level_embed = nn.Parameter( torch.Tensor(self.num_feature_levels, self.embed_dims)) self.memory_trans_fc = nn.Linear(self.embed_dims, self.embed_dims) self.memory_trans_norm = nn.LayerNorm(self.embed_dims) def init_weights(self) -> None: """Initialize weights for Transformer and other components.""" super(DeformableDETR, self).init_weights() for coder in self.encoder, self.decoder: for p in coder.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) for m in self.modules(): if isinstance(m, MultiScaleDeformableAttention): m.init_weights() nn.init.xavier_uniform_(self.memory_trans_fc.weight) nn.init.xavier_uniform_(self.query_embedding.weight) normal_(self.level_embed) def forward_transformer( self, img_feats: Tuple[Tensor], batch_data_samples: OptSampleList = None, ) -> Dict: """Forward process of Transformer. The forward procedure of the transformer is defined as: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder' More details can be found at `TransformerDetector.forward_transformer` in `mmdet/detector/base_detr.py`. The difference is that the ground truth in `batch_data_samples` is required for the `pre_decoder` to prepare the query of DINO. Additionally, DINO inherits the `pre_transformer` method and the `forward_encoder` method of DeformableDETR. More details about the two methods can be found in `mmdet/detector/deformable_detr.py`. Args: img_feats (tuple[Tensor]): Tuple of feature maps from neck. Each feature map has shape (bs, dim, H, W). batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Defaults to None. Returns: dict: The dictionary of bbox_head function inputs, which always includes the `hidden_states` of the decoder output and may contain `references` including the initial and intermediate references. """ encoder_inputs_dict, decoder_inputs_dict = self.pre_transformer( img_feats, batch_data_samples) encoder_outputs_dict = self.forward_encoder(**encoder_inputs_dict) tmp_dec_in, head_inputs_dict = self.pre_decoder( **encoder_outputs_dict, batch_data_samples=batch_data_samples) decoder_inputs_dict.update(tmp_dec_in) decoder_outputs_dict = self.forward_decoder(**decoder_inputs_dict) head_inputs_dict.update(decoder_outputs_dict) return head_inputs_dict def pre_decoder( self, memory: Tensor, memory_mask: Tensor, spatial_shapes: Tensor, batch_data_samples: OptSampleList = None, ) -> Tuple[Dict]: """Prepare intermediate variables before entering Transformer decoder, such as `query`, `query_pos`, and `reference_points`. Args: memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). memory_mask (Tensor): ByteTensor, the padding mask of the memory, has shape (bs, num_feat_points). Will only be used when `as_two_stage` is `True`. spatial_shapes (Tensor): Spatial shapes of features in all levels. With shape (num_levels, 2), last dimension represents (h, w). Will only be used when `as_two_stage` is `True`. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Defaults to None. Returns: tuple[dict]: The decoder_inputs_dict and head_inputs_dict. - decoder_inputs_dict (dict): The keyword dictionary args of `self.forward_decoder()`, which includes 'query', 'memory', `reference_points`, and `dn_mask`. The reference points of decoder input here are 4D boxes, although it has `points` in its name. - head_inputs_dict (dict): The keyword dictionary args of the bbox_head functions, which includes `topk_score`, `topk_coords`, and `dn_meta` when `self.training` is `True`, else is empty. """ bs, _, c = memory.shape cls_out_features = self.bbox_head.cls_branches[ self.decoder.num_layers].out_features output_memory, output_proposals = self.gen_encoder_output_proposals( memory, memory_mask, spatial_shapes) enc_outputs_class = self.bbox_head.cls_branches[ self.decoder.num_layers]( output_memory) enc_outputs_coord_unact = self.bbox_head.reg_branches[ self.decoder.num_layers](output_memory) + output_proposals # NOTE The DINO selects top-k proposals according to scores of # multi-class classification, while DeformDETR, where the input # is `enc_outputs_class[..., 0]` selects according to scores of # binary classification. topk_indices = torch.topk( enc_outputs_class.max(-1)[0], k=self.num_queries, dim=1)[1] topk_score = torch.gather( enc_outputs_class, 1, topk_indices.unsqueeze(-1).repeat(1, 1, cls_out_features)) topk_coords_unact = torch.gather( enc_outputs_coord_unact, 1, topk_indices.unsqueeze(-1).repeat(1, 1, 4)) topk_coords = topk_coords_unact.sigmoid() topk_coords_unact = topk_coords_unact.detach() query = self.query_embedding.weight[:, None, :] query = query.repeat(1, bs, 1).transpose(0, 1) if self.training: dn_label_query, dn_bbox_query, dn_mask, dn_meta = \ self.dn_query_generator(batch_data_samples) query = torch.cat([dn_label_query, query], dim=1) reference_points = torch.cat([dn_bbox_query, topk_coords_unact], dim=1) else: reference_points = topk_coords_unact dn_mask, dn_meta = None, None reference_points = reference_points.sigmoid() decoder_inputs_dict = dict( query=query, memory=memory, reference_points=reference_points, dn_mask=dn_mask) # NOTE DINO calculates encoder losses on scores and coordinates # of selected top-k encoder queries, while DeformDETR is of all # encoder queries. head_inputs_dict = dict( enc_outputs_class=topk_score, enc_outputs_coord=topk_coords, dn_meta=dn_meta) if self.training else dict() return decoder_inputs_dict, head_inputs_dict def forward_decoder(self, query: Tensor, memory: Tensor, memory_mask: Tensor, reference_points: Tensor, spatial_shapes: Tensor, level_start_index: Tensor, valid_ratios: Tensor, dn_mask: Optional[Tensor] = None) -> Dict: """Forward with Transformer decoder. The forward procedure of the transformer is defined as: 'pre_transformer' -> 'encoder' -> 'pre_decoder' -> 'decoder' More details can be found at `TransformerDetector.forward_transformer` in `mmdet/detector/base_detr.py`. Args: query (Tensor): The queries of decoder inputs, has shape (bs, num_queries_total, dim), where `num_queries_total` is the sum of `num_denoising_queries` and `num_matching_queries` when `self.training` is `True`, else `num_matching_queries`. memory (Tensor): The output embeddings of the Transformer encoder, has shape (bs, num_feat_points, dim). memory_mask (Tensor): ByteTensor, the padding mask of the memory, has shape (bs, num_feat_points). reference_points (Tensor): The initial reference, has shape (bs, num_queries_total, 4) with the last dimension arranged as (cx, cy, w, h). spatial_shapes (Tensor): Spatial shapes of features in all levels, has shape (num_levels, 2), last dimension represents (h, w). level_start_index (Tensor): The start index of each level. A tensor has shape (num_levels, ) and can be represented as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. valid_ratios (Tensor): The ratios of the valid width and the valid height relative to the width and the height of features in all levels, has shape (bs, num_levels, 2). dn_mask (Tensor, optional): The attention mask to prevent information leakage from different denoising groups and matching parts, will be used as `self_attn_mask` of the `self.decoder`, has shape (num_queries_total, num_queries_total). It is `None` when `self.training` is `False`. Returns: dict: The dictionary of decoder outputs, which includes the `hidden_states` of the decoder output and `references` including the initial and intermediate reference_points. """ inter_states, references = self.decoder( query=query, value=memory, key_padding_mask=memory_mask, self_attn_mask=dn_mask, reference_points=reference_points, spatial_shapes=spatial_shapes, level_start_index=level_start_index, valid_ratios=valid_ratios, reg_branches=self.bbox_head.reg_branches) if len(query) == self.num_queries: # NOTE: This is to make sure label_embeding can be involved to # produce loss even if there is no denoising query (no ground truth # target in this GPU), otherwise, this will raise runtime error in # distributed training. inter_states[0] += \ self.dn_query_generator.label_embedding.weight[0, 0] * 0.0 decoder_outputs_dict = dict( hidden_states=inter_states, references=list(references)) return decoder_outputs_dict ================================================ FILE: mmdet/models/detectors/fast_rcnn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .two_stage import TwoStageDetector @MODELS.register_module() class FastRCNN(TwoStageDetector): """Implementation of `Fast R-CNN `_""" def __init__(self, backbone: ConfigType, roi_head: ConfigType, train_cfg: ConfigType, test_cfg: ConfigType, neck: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, init_cfg=init_cfg, data_preprocessor=data_preprocessor) ================================================ FILE: mmdet/models/detectors/faster_rcnn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .two_stage import TwoStageDetector @MODELS.register_module() class FasterRCNN(TwoStageDetector): """Implementation of `Faster R-CNN `_""" def __init__(self, backbone: ConfigType, rpn_head: ConfigType, roi_head: ConfigType, train_cfg: ConfigType, test_cfg: ConfigType, neck: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, init_cfg=init_cfg, data_preprocessor=data_preprocessor) ================================================ FILE: mmdet/models/detectors/fcos.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class FCOS(SingleStageDetector): """Implementation of `FCOS `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone config. neck (:obj:`ConfigDict` or dict): The neck config. bbox_head (:obj:`ConfigDict` or dict): The bbox head config. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of FCOS. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of FCOS. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/fovea.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class FOVEA(SingleStageDetector): """Implementation of `FoveaBox `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone config. neck (:obj:`ConfigDict` or dict): The neck config. bbox_head (:obj:`ConfigDict` or dict): The bbox head config. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of FOVEA. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of FOVEA. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/fsaf.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class FSAF(SingleStageDetector): """Implementation of `FSAF `_""" def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None): super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/gfl.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class GFL(SingleStageDetector): """Implementation of `GFL `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of GFL. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of GFL. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/grid_rcnn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .two_stage import TwoStageDetector @MODELS.register_module() class GridRCNN(TwoStageDetector): """Grid R-CNN. This detector is the implementation of: - Grid R-CNN (https://arxiv.org/abs/1811.12030) - Grid R-CNN Plus: Faster and Better (https://arxiv.org/abs/1906.05688) """ def __init__(self, backbone: ConfigType, rpn_head: ConfigType, roi_head: ConfigType, train_cfg: ConfigType, test_cfg: ConfigType, neck: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/htc.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from .cascade_rcnn import CascadeRCNN @MODELS.register_module() class HybridTaskCascade(CascadeRCNN): """Implementation of `HTC `_""" def __init__(self, **kwargs) -> None: super().__init__(**kwargs) @property def with_semantic(self) -> bool: """bool: whether the detector has a semantic head""" return self.roi_head.with_semantic ================================================ FILE: mmdet/models/detectors/kd_one_stage.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from pathlib import Path from typing import Any, Optional, Union import torch import torch.nn as nn from mmengine.config import Config from mmengine.runner import load_checkpoint from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import ConfigType, OptConfigType from .single_stage import SingleStageDetector @MODELS.register_module() class KnowledgeDistillationSingleStageDetector(SingleStageDetector): r"""Implementation of `Distilling the Knowledge in a Neural Network. `_. Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. teacher_config (:obj:`ConfigDict` | dict | str | Path): Config file path or the config object of teacher model. teacher_ckpt (str, optional): Checkpoint path of teacher model. If left as None, the model will not load any weights. Defaults to True. eval_teacher (bool): Set the train mode for teacher. Defaults to True. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of ATSS. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of ATSS. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. """ def __init__( self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, teacher_config: Union[ConfigType, str, Path], teacher_ckpt: Optional[str] = None, eval_teacher: bool = True, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, ) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor) self.eval_teacher = eval_teacher # Build teacher model if isinstance(teacher_config, (str, Path)): teacher_config = Config.fromfile(teacher_config) self.teacher_model = MODELS.build(teacher_config['model']) if teacher_ckpt is not None: load_checkpoint( self.teacher_model, teacher_ckpt, map_location='cpu') def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> dict: """ Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components. """ x = self.extract_feat(batch_inputs) with torch.no_grad(): teacher_x = self.teacher_model.extract_feat(batch_inputs) out_teacher = self.teacher_model.bbox_head(teacher_x) losses = self.bbox_head.loss(x, out_teacher, batch_data_samples) return losses def cuda(self, device: Optional[str] = None) -> nn.Module: """Since teacher_model is registered as a plain object, it is necessary to put the teacher model to cuda when calling ``cuda`` function.""" self.teacher_model.cuda(device=device) return super().cuda(device=device) def to(self, device: Optional[str] = None) -> nn.Module: """Since teacher_model is registered as a plain object, it is necessary to put the teacher model to other device when calling ``to`` function.""" self.teacher_model.to(device=device) return super().to(device=device) def train(self, mode: bool = True) -> None: """Set the same train mode for teacher and student model.""" if self.eval_teacher: self.teacher_model.train(False) else: self.teacher_model.train(mode) super().train(mode) def __setattr__(self, name: str, value: Any) -> None: """Set attribute, i.e. self.name = value This reloading prevent the teacher model from being registered as a nn.Module. The teacher module is registered as a plain object, so that the teacher parameters will not show up when calling ``self.parameters``, ``self.modules``, ``self.children`` methods. """ if name == 'teacher_model': object.__setattr__(self, name, value) else: super().__setattr__(name, value) ================================================ FILE: mmdet/models/detectors/lad.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch import torch.nn as nn from mmengine.runner import load_checkpoint from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import ConfigType, OptConfigType from ..utils.misc import unpack_gt_instances from .kd_one_stage import KnowledgeDistillationSingleStageDetector @MODELS.register_module() class LAD(KnowledgeDistillationSingleStageDetector): """Implementation of `LAD `_.""" def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, teacher_backbone: ConfigType, teacher_neck: ConfigType, teacher_bbox_head: ConfigType, teacher_ckpt: Optional[str] = None, eval_teacher: bool = True, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None) -> None: super(KnowledgeDistillationSingleStageDetector, self).__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor) self.eval_teacher = eval_teacher self.teacher_model = nn.Module() self.teacher_model.backbone = MODELS.build(teacher_backbone) if teacher_neck is not None: self.teacher_model.neck = MODELS.build(teacher_neck) teacher_bbox_head.update(train_cfg=train_cfg) teacher_bbox_head.update(test_cfg=test_cfg) self.teacher_model.bbox_head = MODELS.build(teacher_bbox_head) if teacher_ckpt is not None: load_checkpoint( self.teacher_model, teacher_ckpt, map_location='cpu') @property def with_teacher_neck(self) -> bool: """bool: whether the detector has a teacher_neck""" return hasattr(self.teacher_model, 'neck') and \ self.teacher_model.neck is not None def extract_teacher_feat(self, batch_inputs: Tensor) -> Tensor: """Directly extract teacher features from the backbone+neck.""" x = self.teacher_model.backbone(batch_inputs) if self.with_teacher_neck: x = self.teacher_model.neck(x) return x def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> dict: """ Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components. """ outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, batch_img_metas \ = outputs # get label assignment from the teacher with torch.no_grad(): x_teacher = self.extract_teacher_feat(batch_inputs) outs_teacher = self.teacher_model.bbox_head(x_teacher) label_assignment_results = \ self.teacher_model.bbox_head.get_label_assignment( *outs_teacher, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) # the student use the label assignment from the teacher to learn x = self.extract_feat(batch_inputs) losses = self.bbox_head.loss(x, label_assignment_results, batch_data_samples) return losses ================================================ FILE: mmdet/models/detectors/mask2former.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .maskformer import MaskFormer @MODELS.register_module() class Mask2Former(MaskFormer): r"""Implementation of `Masked-attention Mask Transformer for Universal Image Segmentation `_.""" def __init__(self, backbone: ConfigType, neck: OptConfigType = None, panoptic_head: OptConfigType = None, panoptic_fusion_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None): super().__init__( backbone=backbone, neck=neck, panoptic_head=panoptic_head, panoptic_fusion_head=panoptic_fusion_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/mask_rcnn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmengine.config import ConfigDict from mmdet.registry import MODELS from mmdet.utils import OptConfigType, OptMultiConfig from .two_stage import TwoStageDetector @MODELS.register_module() class MaskRCNN(TwoStageDetector): """Implementation of `Mask R-CNN `_""" def __init__(self, backbone: ConfigDict, rpn_head: ConfigDict, roi_head: ConfigDict, train_cfg: ConfigDict, test_cfg: ConfigDict, neck: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, init_cfg=init_cfg, data_preprocessor=data_preprocessor) ================================================ FILE: mmdet/models/detectors/mask_scoring_rcnn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .two_stage import TwoStageDetector @MODELS.register_module() class MaskScoringRCNN(TwoStageDetector): """Mask Scoring RCNN. https://arxiv.org/abs/1903.00241 """ def __init__(self, backbone: ConfigType, rpn_head: ConfigType, roi_head: ConfigType, train_cfg: ConfigType, test_cfg: ConfigType, neck: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/maskformer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Tuple from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class MaskFormer(SingleStageDetector): r"""Implementation of `Per-Pixel Classification is NOT All You Need for Semantic Segmentation `_.""" def __init__(self, backbone: ConfigType, neck: OptConfigType = None, panoptic_head: OptConfigType = None, panoptic_fusion_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None): super(SingleStageDetector, self).__init__( data_preprocessor=data_preprocessor, init_cfg=init_cfg) self.backbone = MODELS.build(backbone) if neck is not None: self.neck = MODELS.build(neck) panoptic_head_ = panoptic_head.deepcopy() panoptic_head_.update(train_cfg=train_cfg) panoptic_head_.update(test_cfg=test_cfg) self.panoptic_head = MODELS.build(panoptic_head_) panoptic_fusion_head_ = panoptic_fusion_head.deepcopy() panoptic_fusion_head_.update(test_cfg=test_cfg) self.panoptic_fusion_head = MODELS.build(panoptic_fusion_head_) self.num_things_classes = self.panoptic_head.num_things_classes self.num_stuff_classes = self.panoptic_head.num_stuff_classes self.num_classes = self.panoptic_head.num_classes self.train_cfg = train_cfg self.test_cfg = test_cfg def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Dict[str, Tensor]: """ Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: a dictionary of loss components """ x = self.extract_feat(batch_inputs) losses = self.panoptic_head.loss(x, batch_data_samples) return losses def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList, rescale: bool = True) -> SampleList: """Predict results from a batch of inputs and data samples with post- processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results. Defaults to True. Returns: list[:obj:`DetDataSample`]: Detection results of the input images. Each DetDataSample usually contain 'pred_instances' and `pred_panoptic_seg`. And the ``pred_instances`` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). And the ``pred_panoptic_seg`` contains the following key - sem_seg (Tensor): panoptic segmentation mask, has a shape (1, h, w). """ feats = self.extract_feat(batch_inputs) mask_cls_results, mask_pred_results = self.panoptic_head.predict( feats, batch_data_samples) results_list = self.panoptic_fusion_head.predict( mask_cls_results, mask_pred_results, batch_data_samples, rescale=rescale) results = self.add_pred_to_datasample(batch_data_samples, results_list) return results def add_pred_to_datasample(self, data_samples: SampleList, results_list: List[dict]) -> SampleList: """Add predictions to `DetDataSample`. Args: data_samples (list[:obj:`DetDataSample`], optional): A batch of data samples that contain annotations and predictions. results_list (List[dict]): Instance segmentation, segmantic segmentation and panoptic segmentation results. Returns: list[:obj:`DetDataSample`]: Detection results of the input images. Each DetDataSample usually contain 'pred_instances' and `pred_panoptic_seg`. And the ``pred_instances`` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). And the ``pred_panoptic_seg`` contains the following key - sem_seg (Tensor): panoptic segmentation mask, has a shape (1, h, w). """ for data_sample, pred_results in zip(data_samples, results_list): if 'pan_results' in pred_results: data_sample.pred_panoptic_seg = pred_results['pan_results'] if 'ins_results' in pred_results: data_sample.pred_instances = pred_results['ins_results'] assert 'sem_results' not in pred_results, 'segmantic ' \ 'segmentation results are not supported yet.' return data_samples def _forward(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Tuple[List[Tensor]]: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: tuple[List[Tensor]]: A tuple of features from ``panoptic_head`` forward. """ feats = self.extract_feat(batch_inputs) results = self.panoptic_head.forward(feats, batch_data_samples) return results ================================================ FILE: mmdet/models/detectors/nasfcos.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class NASFCOS(SingleStageDetector): """Implementation of `NAS-FCOS: Fast Neural Architecture Search for Object Detection. `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone config. neck (:obj:`ConfigDict` or dict): The neck config. bbox_head (:obj:`ConfigDict` or dict): The bbox head config. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of NASFCOS. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of NASFCOS. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/paa.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class PAA(SingleStageDetector): """Implementation of `PAA `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of PAA. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of PAA. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/panoptic_fpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .panoptic_two_stage_segmentor import TwoStagePanopticSegmentor @MODELS.register_module() class PanopticFPN(TwoStagePanopticSegmentor): r"""Implementation of `Panoptic feature pyramid networks `_""" def __init__( self, backbone: ConfigType, neck: OptConfigType = None, rpn_head: OptConfigType = None, roi_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None, # for panoptic segmentation semantic_head: OptConfigType = None, panoptic_fusion_head: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg, semantic_head=semantic_head, panoptic_fusion_head=panoptic_fusion_head) ================================================ FILE: mmdet/models/detectors/panoptic_two_stage_segmentor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import List import torch from mmengine.structures import PixelData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .two_stage import TwoStageDetector @MODELS.register_module() class TwoStagePanopticSegmentor(TwoStageDetector): """Base class of Two-stage Panoptic Segmentor. As well as the components in TwoStageDetector, Panoptic Segmentor has extra semantic_head and panoptic_fusion_head. """ def __init__( self, backbone: ConfigType, neck: OptConfigType = None, rpn_head: OptConfigType = None, roi_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None, # for panoptic segmentation semantic_head: OptConfigType = None, panoptic_fusion_head: OptConfigType = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) if semantic_head is not None: self.semantic_head = MODELS.build(semantic_head) if panoptic_fusion_head is not None: panoptic_cfg = test_cfg.panoptic if test_cfg is not None else None panoptic_fusion_head_ = panoptic_fusion_head.deepcopy() panoptic_fusion_head_.update(test_cfg=panoptic_cfg) self.panoptic_fusion_head = MODELS.build(panoptic_fusion_head_) self.num_things_classes = self.panoptic_fusion_head.\ num_things_classes self.num_stuff_classes = self.panoptic_fusion_head.\ num_stuff_classes self.num_classes = self.panoptic_fusion_head.num_classes @property def with_semantic_head(self) -> bool: """bool: whether the detector has semantic head""" return hasattr(self, 'semantic_head') and self.semantic_head is not None @property def with_panoptic_fusion_head(self) -> bool: """bool: whether the detector has panoptic fusion head""" return hasattr(self, 'panoptic_fusion_head') and \ self.panoptic_fusion_head is not None def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> dict: """ Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ x = self.extract_feat(batch_inputs) losses = dict() # RPN forward and loss if self.with_rpn: proposal_cfg = self.train_cfg.get('rpn_proposal', self.test_cfg.rpn) rpn_data_samples = copy.deepcopy(batch_data_samples) # set cat_id of gt_labels to 0 in RPN for data_sample in rpn_data_samples: data_sample.gt_instances.labels = \ torch.zeros_like(data_sample.gt_instances.labels) rpn_losses, rpn_results_list = self.rpn_head.loss_and_predict( x, rpn_data_samples, proposal_cfg=proposal_cfg) # avoid get same name with roi_head loss keys = rpn_losses.keys() for key in list(keys): if 'loss' in key and 'rpn' not in key: rpn_losses[f'rpn_{key}'] = rpn_losses.pop(key) losses.update(rpn_losses) else: # TODO: Not support currently, should have a check at Fast R-CNN assert batch_data_samples[0].get('proposals', None) is not None # use pre-defined proposals in InstanceData for the second stage # to extract ROI features. rpn_results_list = [ data_sample.proposals for data_sample in batch_data_samples ] roi_losses = self.roi_head.loss(x, rpn_results_list, batch_data_samples) losses.update(roi_losses) semantic_loss = self.semantic_head.loss(x, batch_data_samples) losses.update(semantic_loss) return losses def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList, rescale: bool = True) -> SampleList: """Predict results from a batch of inputs and data samples with post- processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results. Defaults to True. Returns: List[:obj:`DetDataSample`]: Return the packed panoptic segmentation results of input images. Each DetDataSample usually contains 'pred_panoptic_seg'. And the 'pred_panoptic_seg' has a key ``sem_seg``, which is a tensor of shape (1, h, w). """ batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] x = self.extract_feat(batch_inputs) # If there are no pre-defined proposals, use RPN to get proposals if batch_data_samples[0].get('proposals', None) is None: rpn_results_list = self.rpn_head.predict( x, batch_data_samples, rescale=False) else: rpn_results_list = [ data_sample.proposals for data_sample in batch_data_samples ] results_list = self.roi_head.predict( x, rpn_results_list, batch_data_samples, rescale=rescale) seg_preds = self.semantic_head.predict(x, batch_img_metas, rescale) results_list = self.panoptic_fusion_head.predict( results_list, seg_preds) batch_data_samples = self.add_pred_to_datasample( batch_data_samples, results_list) return batch_data_samples # TODO the code has not been verified and needs to be refactored later. def _forward(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> tuple: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). Returns: tuple: A tuple of features from ``rpn_head``, ``roi_head`` and ``semantic_head`` forward. """ results = () x = self.extract_feat(batch_inputs) rpn_outs = self.rpn_head.forward(x) results = results + (rpn_outs) # If there are no pre-defined proposals, use RPN to get proposals if batch_data_samples[0].get('proposals', None) is None: batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] rpn_results_list = self.rpn_head.predict_by_feat( *rpn_outs, batch_img_metas=batch_img_metas, rescale=False) else: # TODO: Not checked currently. rpn_results_list = [ data_sample.proposals for data_sample in batch_data_samples ] # roi_head roi_outs = self.roi_head(x, rpn_results_list) results = results + (roi_outs) # semantic_head sem_outs = self.semantic_head.forward(x) results = results + (sem_outs['seg_preds'], ) return results def add_pred_to_datasample(self, data_samples: SampleList, results_list: List[PixelData]) -> SampleList: """Add predictions to `DetDataSample`. Args: data_samples (list[:obj:`DetDataSample`]): The annotation data of every samples. results_list (List[PixelData]): Panoptic segmentation results of each image. Returns: List[:obj:`DetDataSample`]: Return the packed panoptic segmentation results of input images. Each DetDataSample usually contains 'pred_panoptic_seg'. And the 'pred_panoptic_seg' has a key ``sem_seg``, which is a tensor of shape (1, h, w). """ for data_sample, pred_panoptic_seg in zip(data_samples, results_list): data_sample.pred_panoptic_seg = pred_panoptic_seg return data_samples ================================================ FILE: mmdet/models/detectors/point_rend.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmengine.config import ConfigDict from mmdet.registry import MODELS from mmdet.utils import OptConfigType, OptMultiConfig from .two_stage import TwoStageDetector @MODELS.register_module() class PointRend(TwoStageDetector): """PointRend: Image Segmentation as Rendering This detector is the implementation of `PointRend `_. """ def __init__(self, backbone: ConfigDict, rpn_head: ConfigDict, roi_head: ConfigDict, train_cfg: ConfigDict, test_cfg: ConfigDict, neck: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, init_cfg=init_cfg, data_preprocessor=data_preprocessor) ================================================ FILE: mmdet/models/detectors/queryinst.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .sparse_rcnn import SparseRCNN @MODELS.register_module() class QueryInst(SparseRCNN): r"""Implementation of `Instances as Queries `_""" def __init__(self, backbone: ConfigType, rpn_head: ConfigType, roi_head: ConfigType, train_cfg: ConfigType, test_cfg: ConfigType, neck: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/reppoints_detector.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class RepPointsDetector(SingleStageDetector): """RepPoints: Point Set Representation for Object Detection. This detector is the implementation of: - RepPoints detector (https://arxiv.org/pdf/1904.11490) """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None): super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/retinanet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class RetinaNet(SingleStageDetector): """Implementation of `RetinaNet `_""" def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/rpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import warnings import torch from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class RPN(SingleStageDetector): """Implementation of Region Proposal Network. Args: backbone (:obj:`ConfigDict` or dict): The backbone config. neck (:obj:`ConfigDict` or dict): The neck config. bbox_head (:obj:`ConfigDict` or dict): The bbox head config. train_cfg (:obj:`ConfigDict` or dict, optional): The training config. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, rpn_head: ConfigType, train_cfg: ConfigType, test_cfg: ConfigType, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None, **kwargs) -> None: super(SingleStageDetector, self).__init__( data_preprocessor=data_preprocessor, init_cfg=init_cfg) self.backbone = MODELS.build(backbone) self.neck = MODELS.build(neck) if neck is not None else None rpn_train_cfg = train_cfg['rpn'] if train_cfg is not None else None rpn_head_num_classes = rpn_head.get('num_classes', 1) if rpn_head_num_classes != 1: warnings.warn('The `num_classes` should be 1 in RPN, but get ' f'{rpn_head_num_classes}, please set ' 'rpn_head.num_classes = 1 in your config file.') rpn_head.update(num_classes=1) rpn_head.update(train_cfg=rpn_train_cfg) rpn_head.update(test_cfg=test_cfg['rpn']) self.bbox_head = MODELS.build(rpn_head) self.train_cfg = train_cfg self.test_cfg = test_cfg def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> dict: """Calculate losses from a batch of inputs and data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components. """ x = self.extract_feat(batch_inputs) # set cat_id of gt_labels to 0 in RPN rpn_data_samples = copy.deepcopy(batch_data_samples) for data_sample in rpn_data_samples: data_sample.gt_instances.labels = \ torch.zeros_like(data_sample.gt_instances.labels) losses = self.bbox_head.loss(x, rpn_data_samples) return losses ================================================ FILE: mmdet/models/detectors/rtmdet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch from mmengine.dist import get_world_size from mmengine.logging import print_log from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class RTMDet(SingleStageDetector): """Implementation of RTMDet. Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of ATSS. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of ATSS. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. use_syncbn (bool): Whether to use SyncBatchNorm. Defaults to True. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None, use_syncbn: bool = True) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) # TODO： Waiting for mmengine support if use_syncbn and get_world_size() > 1: torch.nn.SyncBatchNorm.convert_sync_batchnorm(self) print_log('Using SyncBatchNorm()', 'current') ================================================ FILE: mmdet/models/detectors/scnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from .cascade_rcnn import CascadeRCNN @MODELS.register_module() class SCNet(CascadeRCNN): """Implementation of `SCNet `_""" def __init__(self, **kwargs) -> None: super().__init__(**kwargs) ================================================ FILE: mmdet/models/detectors/semi_base.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import Dict, List, Optional, Tuple, Union import torch import torch.nn as nn from torch import Tensor from mmdet.models.utils import (filter_gt_instances, rename_loss_dict, reweight_loss_dict) from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox_project from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .base import BaseDetector @MODELS.register_module() class SemiBaseDetector(BaseDetector): """Base class for semi-supervised detectors. Semi-supervised detectors typically consisting of a teacher model updated by exponential moving average and a student model updated by gradient descent. Args: detector (:obj:`ConfigDict` or dict): The detector config. semi_train_cfg (:obj:`ConfigDict` or dict, optional): The semi-supervised training config. semi_test_cfg (:obj:`ConfigDict` or dict, optional): The semi-supervised testing config. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, detector: ConfigType, semi_train_cfg: OptConfigType = None, semi_test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( data_preprocessor=data_preprocessor, init_cfg=init_cfg) self.student = MODELS.build(detector) self.teacher = MODELS.build(detector) self.semi_train_cfg = semi_train_cfg self.semi_test_cfg = semi_test_cfg if self.semi_train_cfg.get('freeze_teacher', True) is True: self.freeze(self.teacher) @staticmethod def freeze(model: nn.Module): """Freeze the model.""" model.eval() for param in model.parameters(): param.requires_grad = False def loss(self, multi_batch_inputs: Dict[str, Tensor], multi_batch_data_samples: Dict[str, SampleList]) -> dict: """Calculate losses from multi-branch inputs and data samples. Args: multi_batch_inputs (Dict[str, Tensor]): The dict of multi-branch input images, each value with shape (N, C, H, W). Each value should usually be mean centered and std scaled. multi_batch_data_samples (Dict[str, List[:obj:`DetDataSample`]]): The dict of multi-branch data samples. Returns: dict: A dictionary of loss components """ losses = dict() losses.update(**self.loss_by_gt_instances( multi_batch_inputs['sup'], multi_batch_data_samples['sup'])) origin_pseudo_data_samples, batch_info = self.get_pseudo_instances( multi_batch_inputs['unsup_teacher'], multi_batch_data_samples['unsup_teacher']) multi_batch_data_samples[ 'unsup_student'] = self.project_pseudo_instances( origin_pseudo_data_samples, multi_batch_data_samples['unsup_student']) losses.update(**self.loss_by_pseudo_instances( multi_batch_inputs['unsup_student'], multi_batch_data_samples['unsup_student'], batch_info)) return losses def loss_by_gt_instances(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> dict: """Calculate losses from a batch of inputs and ground-truth data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components """ losses = self.student.loss(batch_inputs, batch_data_samples) sup_weight = self.semi_train_cfg.get('sup_weight', 1.) return rename_loss_dict('sup_', reweight_loss_dict(losses, sup_weight)) def loss_by_pseudo_instances(self, batch_inputs: Tensor, batch_data_samples: SampleList, batch_info: Optional[dict] = None) -> dict: """Calculate losses from a batch of inputs and pseudo data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`, which are `pseudo_instance` or `pseudo_panoptic_seg` or `pseudo_sem_seg` in fact. batch_info (dict): Batch information of teacher model forward propagation process. Defaults to None. Returns: dict: A dictionary of loss components """ batch_data_samples = filter_gt_instances( batch_data_samples, score_thr=self.semi_train_cfg.cls_pseudo_thr) losses = self.student.loss(batch_inputs, batch_data_samples) pseudo_instances_num = sum([ len(data_samples.gt_instances) for data_samples in batch_data_samples ]) unsup_weight = self.semi_train_cfg.get( 'unsup_weight', 1.) if pseudo_instances_num > 0 else 0. return rename_loss_dict('unsup_', reweight_loss_dict(losses, unsup_weight)) @torch.no_grad() def get_pseudo_instances( self, batch_inputs: Tensor, batch_data_samples: SampleList ) -> Tuple[SampleList, Optional[dict]]: """Get pseudo instances from teacher model.""" self.teacher.eval() results_list = self.teacher.predict( batch_inputs, batch_data_samples, rescale=False) batch_info = {} for data_samples, results in zip(batch_data_samples, results_list): data_samples.gt_instances = results.pred_instances data_samples.gt_instances.bboxes = bbox_project( data_samples.gt_instances.bboxes, torch.from_numpy(data_samples.homography_matrix).inverse().to( self.data_preprocessor.device), data_samples.ori_shape) return batch_data_samples, batch_info def project_pseudo_instances(self, batch_pseudo_instances: SampleList, batch_data_samples: SampleList) -> SampleList: """Project pseudo instances.""" for pseudo_instances, data_samples in zip(batch_pseudo_instances, batch_data_samples): data_samples.gt_instances = copy.deepcopy( pseudo_instances.gt_instances) data_samples.gt_instances.bboxes = bbox_project( data_samples.gt_instances.bboxes, torch.tensor(data_samples.homography_matrix).to( self.data_preprocessor.device), data_samples.img_shape) wh_thr = self.semi_train_cfg.get('min_pseudo_bbox_wh', (1e-2, 1e-2)) return filter_gt_instances(batch_data_samples, wh_thr=wh_thr) def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> SampleList: """Predict results from a batch of inputs and data samples with post- processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results. Defaults to True. Returns: list[:obj:`DetDataSample`]: Return the detection results of the input images. The returns value is DetDataSample, which usually contain 'pred_instances'. And the ``pred_instances`` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ if self.semi_test_cfg.get('predict_on', 'teacher') == 'teacher': return self.teacher( batch_inputs, batch_data_samples, mode='predict') else: return self.student( batch_inputs, batch_data_samples, mode='predict') def _forward(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> SampleList: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). Returns: tuple: A tuple of features from ``rpn_head`` and ``roi_head`` forward. """ if self.semi_test_cfg.get('forward_on', 'teacher') == 'teacher': return self.teacher( batch_inputs, batch_data_samples, mode='tensor') else: return self.student( batch_inputs, batch_data_samples, mode='tensor') def extract_feat(self, batch_inputs: Tensor) -> Tuple[Tensor]: """Extract features. Args: batch_inputs (Tensor): Image tensor with shape (N, C, H ,W). Returns: tuple[Tensor]: Multi-level features that may have different resolutions. """ if self.semi_test_cfg.get('extract_feat_on', 'teacher') == 'teacher': return self.teacher.extract_feat(batch_inputs) else: return self.student.extract_feat(batch_inputs) def _load_from_state_dict(self, state_dict: dict, prefix: str, local_metadata: dict, strict: bool, missing_keys: Union[List[str], str], unexpected_keys: Union[List[str], str], error_msgs: Union[List[str], str]) -> None: """Add teacher and student prefixes to model parameter names.""" if not any([ 'student' in key or 'teacher' in key for key in state_dict.keys() ]): keys = list(state_dict.keys()) state_dict.update({'teacher.' + k: state_dict[k] for k in keys}) state_dict.update({'student.' + k: state_dict[k] for k in keys}) for k in keys: state_dict.pop(k) return super()._load_from_state_dict( state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs, ) ================================================ FILE: mmdet/models/detectors/single_stage.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple, Union from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import OptSampleList, SampleList from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .base import BaseDetector @MODELS.register_module() class SingleStageDetector(BaseDetector): """Base class for single-stage detectors. Single-stage detectors directly and densely predict bounding boxes on the output features of the backbone+neck. """ def __init__(self, backbone: ConfigType, neck: OptConfigType = None, bbox_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( data_preprocessor=data_preprocessor, init_cfg=init_cfg) self.backbone = MODELS.build(backbone) if neck is not None: self.neck = MODELS.build(neck) bbox_head.update(train_cfg=train_cfg) bbox_head.update(test_cfg=test_cfg) self.bbox_head = MODELS.build(bbox_head) self.train_cfg = train_cfg self.test_cfg = test_cfg def _load_from_state_dict(self, state_dict: dict, prefix: str, local_metadata: dict, strict: bool, missing_keys: Union[List[str], str], unexpected_keys: Union[List[str], str], error_msgs: Union[List[str], str]) -> None: """Exchange bbox_head key to rpn_head key when loading two-stage weights into single-stage model.""" bbox_head_prefix = prefix + '.bbox_head' if prefix else 'bbox_head' bbox_head_keys = [ k for k in state_dict.keys() if k.startswith(bbox_head_prefix) ] rpn_head_prefix = prefix + '.rpn_head' if prefix else 'rpn_head' rpn_head_keys = [ k for k in state_dict.keys() if k.startswith(rpn_head_prefix) ] if len(bbox_head_keys) == 0 and len(rpn_head_keys) != 0: for rpn_head_key in rpn_head_keys: bbox_head_key = bbox_head_prefix + \ rpn_head_key[len(rpn_head_prefix):] state_dict[bbox_head_key] = state_dict.pop(rpn_head_key) super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Union[dict, list]: """Calculate losses from a batch of inputs and data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ x = self.extract_feat(batch_inputs) losses = self.bbox_head.loss(x, batch_data_samples) return losses def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList, rescale: bool = True) -> SampleList: """Predict results from a batch of inputs and data samples with post- processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results. Defaults to True. Returns: list[:obj:`DetDataSample`]: Detection results of the input images. Each DetDataSample usually contain 'pred_instances'. And the ``pred_instances`` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ x = self.extract_feat(batch_inputs) results_list = self.bbox_head.predict( x, batch_data_samples, rescale=rescale) batch_data_samples = self.add_pred_to_datasample( batch_data_samples, results_list) return batch_data_samples def _forward( self, batch_inputs: Tensor, batch_data_samples: OptSampleList = None) -> Tuple[List[Tensor]]: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. Returns: tuple[list]: A tuple of features from ``bbox_head`` forward. """ x = self.extract_feat(batch_inputs) results = self.bbox_head.forward(x) return results def extract_feat(self, batch_inputs: Tensor) -> Tuple[Tensor]: """Extract features. Args: batch_inputs (Tensor): Image tensor with shape (N, C, H ,W). Returns: tuple[Tensor]: Multi-level features that may have different resolutions. """ x = self.backbone(batch_inputs) if self.with_neck: x = self.neck(x) return x ================================================ FILE: mmdet/models/detectors/single_stage_instance_seg.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import Tuple from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import OptSampleList, SampleList from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .base import BaseDetector INF = 1e8 @MODELS.register_module() class SingleStageInstanceSegmentor(BaseDetector): """Base class for single-stage instance segmentors.""" def __init__(self, backbone: ConfigType, neck: OptConfigType = None, bbox_head: OptConfigType = None, mask_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( data_preprocessor=data_preprocessor, init_cfg=init_cfg) self.backbone = MODELS.build(backbone) if neck is not None: self.neck = MODELS.build(neck) else: self.neck = None if bbox_head is not None: bbox_head.update(train_cfg=copy.deepcopy(train_cfg)) bbox_head.update(test_cfg=copy.deepcopy(test_cfg)) self.bbox_head = MODELS.build(bbox_head) else: self.bbox_head = None assert mask_head, f'`mask_head` must ' \ f'be implemented in {self.__class__.__name__}' mask_head.update(train_cfg=copy.deepcopy(train_cfg)) mask_head.update(test_cfg=copy.deepcopy(test_cfg)) self.mask_head = MODELS.build(mask_head) self.train_cfg = train_cfg self.test_cfg = test_cfg def extract_feat(self, batch_inputs: Tensor) -> Tuple[Tensor]: """Extract features. Args: batch_inputs (Tensor): Image tensor with shape (N, C, H ,W). Returns: tuple[Tensor]: Multi-level features that may have different resolutions. """ x = self.backbone(batch_inputs) if self.with_neck: x = self.neck(x) return x def _forward(self, batch_inputs: Tensor, batch_data_samples: OptSampleList = None, **kwargs) -> tuple: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). Returns: tuple: A tuple of features from ``bbox_head`` forward. """ outs = () # backbone x = self.extract_feat(batch_inputs) # bbox_head positive_infos = None if self.with_bbox: assert batch_data_samples is not None bbox_outs = self.bbox_head.forward(x) outs = outs + (bbox_outs, ) # It is necessary to use `bbox_head.loss` to update # `_raw_positive_infos` which will be used in `get_positive_infos` # positive_infos will be used in the following mask head. _ = self.bbox_head.loss(x, batch_data_samples, **kwargs) positive_infos = self.bbox_head.get_positive_infos() # mask_head if positive_infos is None: mask_outs = self.mask_head.forward(x) else: mask_outs = self.mask_head.forward(x, positive_infos) outs = outs + (mask_outs, ) return outs def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList, **kwargs) -> dict: """ Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ x = self.extract_feat(batch_inputs) losses = dict() positive_infos = None # CondInst and YOLACT have bbox_head if self.with_bbox: bbox_losses = self.bbox_head.loss(x, batch_data_samples, **kwargs) losses.update(bbox_losses) # get positive information from bbox head, which will be used # in the following mask head. positive_infos = self.bbox_head.get_positive_infos() mask_loss = self.mask_head.loss( x, batch_data_samples, positive_infos=positive_infos, **kwargs) # avoid loss override assert not set(mask_loss.keys()) & set(losses.keys()) losses.update(mask_loss) return losses def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList, rescale: bool = True, **kwargs) -> SampleList: """Perform forward propagation of the mask head and predict mask results on the features of the upstream network. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results. Defaults to False. Returns: list[:obj:`DetDataSample`]: Detection results of the input images. Each DetDataSample usually contain 'pred_instances'. And the ``pred_instances`` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ x = self.extract_feat(batch_inputs) if self.with_bbox: # the bbox branch does not need to be scaled to the original # image scale, because the mask branch will scale both bbox # and mask at the same time. bbox_rescale = rescale if not self.with_mask else False results_list = self.bbox_head.predict( x, batch_data_samples, rescale=bbox_rescale) else: results_list = None results_list = self.mask_head.predict( x, batch_data_samples, rescale=rescale, results_list=results_list) batch_data_samples = self.add_pred_to_datasample( batch_data_samples, results_list) return batch_data_samples ================================================ FILE: mmdet/models/detectors/soft_teacher.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import List, Optional, Tuple import torch from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.utils import (filter_gt_instances, rename_loss_dict, reweight_loss_dict) from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox2roi, bbox_project from mmdet.utils import ConfigType, InstanceList, OptConfigType, OptMultiConfig from ..utils.misc import unpack_gt_instances from .semi_base import SemiBaseDetector @MODELS.register_module() class SoftTeacher(SemiBaseDetector): r"""Implementation of `End-to-End Semi-Supervised Object Detection with Soft Teacher `_ Args: detector (:obj:`ConfigDict` or dict): The detector config. semi_train_cfg (:obj:`ConfigDict` or dict, optional): The semi-supervised training config. semi_test_cfg (:obj:`ConfigDict` or dict, optional): The semi-supervised testing config. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, detector: ConfigType, semi_train_cfg: OptConfigType = None, semi_test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( detector=detector, semi_train_cfg=semi_train_cfg, semi_test_cfg=semi_test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) def loss_by_pseudo_instances(self, batch_inputs: Tensor, batch_data_samples: SampleList, batch_info: Optional[dict] = None) -> dict: """Calculate losses from a batch of inputs and pseudo data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`, which are `pseudo_instance` or `pseudo_panoptic_seg` or `pseudo_sem_seg` in fact. batch_info (dict): Batch information of teacher model forward propagation process. Defaults to None. Returns: dict: A dictionary of loss components """ x = self.student.extract_feat(batch_inputs) losses = {} rpn_losses, rpn_results_list = self.rpn_loss_by_pseudo_instances( x, batch_data_samples) losses.update(**rpn_losses) losses.update(**self.rcnn_cls_loss_by_pseudo_instances( x, rpn_results_list, batch_data_samples, batch_info)) losses.update(**self.rcnn_reg_loss_by_pseudo_instances( x, rpn_results_list, batch_data_samples)) unsup_weight = self.semi_train_cfg.get('unsup_weight', 1.) return rename_loss_dict('unsup_', reweight_loss_dict(losses, unsup_weight)) @torch.no_grad() def get_pseudo_instances( self, batch_inputs: Tensor, batch_data_samples: SampleList ) -> Tuple[SampleList, Optional[dict]]: """Get pseudo instances from teacher model.""" assert self.teacher.with_bbox, 'Bbox head must be implemented.' x = self.teacher.extract_feat(batch_inputs) # If there are no pre-defined proposals, use RPN to get proposals if batch_data_samples[0].get('proposals', None) is None: rpn_results_list = self.teacher.rpn_head.predict( x, batch_data_samples, rescale=False) else: rpn_results_list = [ data_sample.proposals for data_sample in batch_data_samples ] results_list = self.teacher.roi_head.predict( x, rpn_results_list, batch_data_samples, rescale=False) for data_samples, results in zip(batch_data_samples, results_list): data_samples.gt_instances = results batch_data_samples = filter_gt_instances( batch_data_samples, score_thr=self.semi_train_cfg.pseudo_label_initial_score_thr) reg_uncs_list = self.compute_uncertainty_with_aug( x, batch_data_samples) for data_samples, reg_uncs in zip(batch_data_samples, reg_uncs_list): data_samples.gt_instances['reg_uncs'] = reg_uncs data_samples.gt_instances.bboxes = bbox_project( data_samples.gt_instances.bboxes, torch.from_numpy(data_samples.homography_matrix).inverse().to( self.data_preprocessor.device), data_samples.ori_shape) batch_info = { 'feat': x, 'img_shape': [], 'homography_matrix': [], 'metainfo': [] } for data_samples in batch_data_samples: batch_info['img_shape'].append(data_samples.img_shape) batch_info['homography_matrix'].append( torch.from_numpy(data_samples.homography_matrix).to( self.data_preprocessor.device)) batch_info['metainfo'].append(data_samples.metainfo) return batch_data_samples, batch_info def rpn_loss_by_pseudo_instances(self, x: Tuple[Tensor], batch_data_samples: SampleList) -> dict: """Calculate rpn loss from a batch of inputs and pseudo data samples. Args: x (tuple[Tensor]): Features from FPN. batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`, which are `pseudo_instance` or `pseudo_panoptic_seg` or `pseudo_sem_seg` in fact. Returns: dict: A dictionary of rpn loss components """ rpn_data_samples = copy.deepcopy(batch_data_samples) rpn_data_samples = filter_gt_instances( rpn_data_samples, score_thr=self.semi_train_cfg.rpn_pseudo_thr) proposal_cfg = self.student.train_cfg.get('rpn_proposal', self.student.test_cfg.rpn) # set cat_id of gt_labels to 0 in RPN for data_sample in rpn_data_samples: data_sample.gt_instances.labels = \ torch.zeros_like(data_sample.gt_instances.labels) rpn_losses, rpn_results_list = self.student.rpn_head.loss_and_predict( x, rpn_data_samples, proposal_cfg=proposal_cfg) for key in rpn_losses.keys(): if 'loss' in key and 'rpn' not in key: rpn_losses[f'rpn_{key}'] = rpn_losses.pop(key) return rpn_losses, rpn_results_list def rcnn_cls_loss_by_pseudo_instances(self, x: Tuple[Tensor], unsup_rpn_results_list: InstanceList, batch_data_samples: SampleList, batch_info: dict) -> dict: """Calculate classification loss from a batch of inputs and pseudo data samples. Args: x (tuple[Tensor]): List of multi-level img features. unsup_rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`, which are `pseudo_instance` or `pseudo_panoptic_seg` or `pseudo_sem_seg` in fact. batch_info (dict): Batch information of teacher model forward propagation process. Returns: dict[str, Tensor]: A dictionary of rcnn classification loss components """ rpn_results_list = copy.deepcopy(unsup_rpn_results_list) cls_data_samples = copy.deepcopy(batch_data_samples) cls_data_samples = filter_gt_instances( cls_data_samples, score_thr=self.semi_train_cfg.cls_pseudo_thr) outputs = unpack_gt_instances(cls_data_samples) batch_gt_instances, batch_gt_instances_ignore, _ = outputs # assign gts and sample proposals num_imgs = len(cls_data_samples) sampling_results = [] for i in range(num_imgs): # rename rpn_results.bboxes to rpn_results.priors rpn_results = rpn_results_list[i] rpn_results.priors = rpn_results.pop('bboxes') assign_result = self.student.roi_head.bbox_assigner.assign( rpn_results, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = self.student.roi_head.bbox_sampler.sample( assign_result, rpn_results, batch_gt_instances[i], feats=[lvl_feat[i][None] for lvl_feat in x]) sampling_results.append(sampling_result) selected_bboxes = [res.priors for res in sampling_results] rois = bbox2roi(selected_bboxes) bbox_results = self.student.roi_head._bbox_forward(x, rois) # cls_reg_targets is a tuple of labels, label_weights, # and bbox_targets, bbox_weights cls_reg_targets = self.student.roi_head.bbox_head.get_targets( sampling_results, self.student.train_cfg.rcnn) selected_results_list = [] for bboxes, data_samples, teacher_matrix, teacher_img_shape in zip( selected_bboxes, batch_data_samples, batch_info['homography_matrix'], batch_info['img_shape']): student_matrix = torch.tensor( data_samples.homography_matrix, device=teacher_matrix.device) homography_matrix = teacher_matrix @ student_matrix.inverse() projected_bboxes = bbox_project(bboxes, homography_matrix, teacher_img_shape) selected_results_list.append(InstanceData(bboxes=projected_bboxes)) with torch.no_grad(): results_list = self.teacher.roi_head.predict_bbox( batch_info['feat'], batch_info['metainfo'], selected_results_list, rcnn_test_cfg=None, rescale=False) bg_score = torch.cat( [results.scores[:, -1] for results in results_list]) # cls_reg_targets[0] is labels neg_inds = cls_reg_targets[ 0] == self.student.roi_head.bbox_head.num_classes # cls_reg_targets[1] is label_weights cls_reg_targets[1][neg_inds] = bg_score[neg_inds].detach() losses = self.student.roi_head.bbox_head.loss( bbox_results['cls_score'], bbox_results['bbox_pred'], rois, *cls_reg_targets) # cls_reg_targets[1] is label_weights losses['loss_cls'] = losses['loss_cls'] * len( cls_reg_targets[1]) / max(sum(cls_reg_targets[1]), 1.0) return losses def rcnn_reg_loss_by_pseudo_instances( self, x: Tuple[Tensor], unsup_rpn_results_list: InstanceList, batch_data_samples: SampleList) -> dict: """Calculate rcnn regression loss from a batch of inputs and pseudo data samples. Args: x (tuple[Tensor]): List of multi-level img features. unsup_rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`, which are `pseudo_instance` or `pseudo_panoptic_seg` or `pseudo_sem_seg` in fact. Returns: dict[str, Tensor]: A dictionary of rcnn regression loss components """ rpn_results_list = copy.deepcopy(unsup_rpn_results_list) reg_data_samples = copy.deepcopy(batch_data_samples) for data_samples in reg_data_samples: if data_samples.gt_instances.bboxes.shape[0] > 0: data_samples.gt_instances = data_samples.gt_instances[ data_samples.gt_instances.reg_uncs < self.semi_train_cfg.reg_pseudo_thr] roi_losses = self.student.roi_head.loss(x, rpn_results_list, reg_data_samples) return {'loss_bbox': roi_losses['loss_bbox']} def compute_uncertainty_with_aug( self, x: Tuple[Tensor], batch_data_samples: SampleList) -> List[Tensor]: """Compute uncertainty with augmented bboxes. Args: x (tuple[Tensor]): List of multi-level img features. batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`, which are `pseudo_instance` or `pseudo_panoptic_seg` or `pseudo_sem_seg` in fact. Returns: list[Tensor]: A list of uncertainty for pseudo bboxes. """ auged_results_list = self.aug_box(batch_data_samples, self.semi_train_cfg.jitter_times, self.semi_train_cfg.jitter_scale) # flatten auged_results_list = [ InstanceData(bboxes=auged.reshape(-1, auged.shape[-1])) for auged in auged_results_list ] self.teacher.roi_head.test_cfg = None results_list = self.teacher.roi_head.predict( x, auged_results_list, batch_data_samples, rescale=False) self.teacher.roi_head.test_cfg = self.teacher.test_cfg.rcnn reg_channel = max( [results.bboxes.shape[-1] for results in results_list]) // 4 bboxes = [ results.bboxes.reshape(self.semi_train_cfg.jitter_times, -1, results.bboxes.shape[-1]) if results.bboxes.numel() > 0 else results.bboxes.new_zeros( self.semi_train_cfg.jitter_times, 0, 4 * reg_channel).float() for results in results_list ] box_unc = [bbox.std(dim=0) for bbox in bboxes] bboxes = [bbox.mean(dim=0) for bbox in bboxes] labels = [ data_samples.gt_instances.labels for data_samples in batch_data_samples ] if reg_channel != 1: bboxes = [ bbox.reshape(bbox.shape[0], reg_channel, 4)[torch.arange(bbox.shape[0]), label] for bbox, label in zip(bboxes, labels) ] box_unc = [ unc.reshape(unc.shape[0], reg_channel, 4)[torch.arange(unc.shape[0]), label] for unc, label in zip(box_unc, labels) ] box_shape = [(bbox[:, 2:4] - bbox[:, :2]).clamp(min=1.0) for bbox in bboxes] box_unc = [ torch.mean( unc / wh[:, None, :].expand(-1, 2, 2).reshape(-1, 4), dim=-1) if wh.numel() > 0 else unc for unc, wh in zip(box_unc, box_shape) ] return box_unc @staticmethod def aug_box(batch_data_samples, times, frac): """Augment bboxes with jitter.""" def _aug_single(box): box_scale = box[:, 2:4] - box[:, :2] box_scale = ( box_scale.clamp(min=1)[:, None, :].expand(-1, 2, 2).reshape(-1, 4)) aug_scale = box_scale * frac # [n,4] offset = ( torch.randn(times, box.shape[0], 4, device=box.device) * aug_scale[None, ...]) new_box = box.clone()[None, ...].expand(times, box.shape[0], -1) + offset return new_box return [ _aug_single(data_samples.gt_instances.bboxes) for data_samples in batch_data_samples ] ================================================ FILE: mmdet/models/detectors/solo.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage_instance_seg import SingleStageInstanceSegmentor @MODELS.register_module() class SOLO(SingleStageInstanceSegmentor): """`SOLO: Segmenting Objects by Locations `_ """ def __init__(self, backbone: ConfigType, neck: OptConfigType = None, bbox_head: OptConfigType = None, mask_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None): super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, mask_head=mask_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/solov2.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage_instance_seg import SingleStageInstanceSegmentor @MODELS.register_module() class SOLOv2(SingleStageInstanceSegmentor): """`SOLOv2: Dynamic and Fast Instance Segmentation `_ """ def __init__(self, backbone: ConfigType, neck: OptConfigType = None, bbox_head: OptConfigType = None, mask_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None): super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, mask_head=mask_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/sparse_rcnn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .two_stage import TwoStageDetector @MODELS.register_module() class SparseRCNN(TwoStageDetector): r"""Implementation of `Sparse R-CNN: End-to-End Object Detection with Learnable Proposals `_""" def __init__(self, backbone: ConfigType, neck: OptConfigType = None, rpn_head: OptConfigType = None, roi_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) assert self.with_rpn, 'Sparse R-CNN and QueryInst ' \ 'do not support external proposals' ================================================ FILE: mmdet/models/detectors/tood.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class TOOD(SingleStageDetector): r"""Implementation of `TOOD: Task-aligned One-stage Object Detection. `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of TOOD. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of TOOD. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/trident_faster_rcnn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .faster_rcnn import FasterRCNN @MODELS.register_module() class TridentFasterRCNN(FasterRCNN): """Implementation of `TridentNet `_""" def __init__(self, backbone: ConfigType, rpn_head: ConfigType, roi_head: ConfigType, train_cfg: ConfigType, test_cfg: ConfigType, neck: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, rpn_head=rpn_head, roi_head=roi_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) assert self.backbone.num_branch == self.roi_head.num_branch assert self.backbone.test_branch_idx == self.roi_head.test_branch_idx self.num_branch = self.backbone.num_branch self.test_branch_idx = self.backbone.test_branch_idx def _forward(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> tuple: """copy the ``batch_data_samples`` to fit multi-branch.""" num_branch = self.num_branch \ if self.training or self.test_branch_idx == -1 else 1 trident_data_samples = batch_data_samples * num_branch return super()._forward( batch_inputs=batch_inputs, batch_data_samples=trident_data_samples) def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> dict: """copy the ``batch_data_samples`` to fit multi-branch.""" num_branch = self.num_branch \ if self.training or self.test_branch_idx == -1 else 1 trident_data_samples = batch_data_samples * num_branch return super().loss( batch_inputs=batch_inputs, batch_data_samples=trident_data_samples) def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList, rescale: bool = True) -> SampleList: """copy the ``batch_data_samples`` to fit multi-branch.""" num_branch = self.num_branch \ if self.training or self.test_branch_idx == -1 else 1 trident_data_samples = batch_data_samples * num_branch return super().predict( batch_inputs=batch_inputs, batch_data_samples=trident_data_samples, rescale=rescale) # TODO need to refactor def aug_test(self, imgs, img_metas, rescale=False): """Test with augmentations. If rescale is False, then returned bboxes and masks will fit the scale of imgs[0]. """ x = self.extract_feats(imgs) num_branch = (self.num_branch if self.test_branch_idx == -1 else 1) trident_img_metas = [img_metas * num_branch for img_metas in img_metas] proposal_list = self.rpn_head.aug_test_rpn(x, trident_img_metas) return self.roi_head.aug_test( x, proposal_list, img_metas, rescale=rescale) ================================================ FILE: mmdet/models/detectors/two_stage.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import warnings from typing import List, Tuple, Union import torch from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .base import BaseDetector @MODELS.register_module() class TwoStageDetector(BaseDetector): """Base class for two-stage detectors. Two-stage detectors typically consisting of a region proposal network and a task-specific regression head. """ def __init__(self, backbone: ConfigType, neck: OptConfigType = None, rpn_head: OptConfigType = None, roi_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( data_preprocessor=data_preprocessor, init_cfg=init_cfg) self.backbone = MODELS.build(backbone) if neck is not None: self.neck = MODELS.build(neck) if rpn_head is not None: rpn_train_cfg = train_cfg.rpn if train_cfg is not None else None rpn_head_ = rpn_head.copy() rpn_head_.update(train_cfg=rpn_train_cfg, test_cfg=test_cfg.rpn) rpn_head_num_classes = rpn_head_.get('num_classes', None) if rpn_head_num_classes is None: rpn_head_.update(num_classes=1) else: if rpn_head_num_classes != 1: warnings.warn( 'The `num_classes` should be 1 in RPN, but get ' f'{rpn_head_num_classes}, please set ' 'rpn_head.num_classes = 1 in your config file.') rpn_head_.update(num_classes=1) self.rpn_head = MODELS.build(rpn_head_) if roi_head is not None: # update train and test cfg here for now # TODO: refactor assigner & sampler rcnn_train_cfg = train_cfg.rcnn if train_cfg is not None else None roi_head.update(train_cfg=rcnn_train_cfg) roi_head.update(test_cfg=test_cfg.rcnn) self.roi_head = MODELS.build(roi_head) self.train_cfg = train_cfg self.test_cfg = test_cfg def _load_from_state_dict(self, state_dict: dict, prefix: str, local_metadata: dict, strict: bool, missing_keys: Union[List[str], str], unexpected_keys: Union[List[str], str], error_msgs: Union[List[str], str]) -> None: """Exchange bbox_head key to rpn_head key when loading single-stage weights into two-stage model.""" bbox_head_prefix = prefix + '.bbox_head' if prefix else 'bbox_head' bbox_head_keys = [ k for k in state_dict.keys() if k.startswith(bbox_head_prefix) ] rpn_head_prefix = prefix + '.rpn_head' if prefix else 'rpn_head' rpn_head_keys = [ k for k in state_dict.keys() if k.startswith(rpn_head_prefix) ] if len(bbox_head_keys) != 0 and len(rpn_head_keys) == 0: for bbox_head_key in bbox_head_keys: rpn_head_key = rpn_head_prefix + \ bbox_head_key[len(bbox_head_prefix):] state_dict[rpn_head_key] = state_dict.pop(bbox_head_key) super()._load_from_state_dict(state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys, error_msgs) @property def with_rpn(self) -> bool: """bool: whether the detector has RPN""" return hasattr(self, 'rpn_head') and self.rpn_head is not None @property def with_roi_head(self) -> bool: """bool: whether the detector has a RoI head""" return hasattr(self, 'roi_head') and self.roi_head is not None def extract_feat(self, batch_inputs: Tensor) -> Tuple[Tensor]: """Extract features. Args: batch_inputs (Tensor): Image tensor with shape (N, C, H ,W). Returns: tuple[Tensor]: Multi-level features that may have different resolutions. """ x = self.backbone(batch_inputs) if self.with_neck: x = self.neck(x) return x def _forward(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> tuple: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. Returns: tuple: A tuple of features from ``rpn_head`` and ``roi_head`` forward. """ results = () x = self.extract_feat(batch_inputs) if self.with_rpn: rpn_results_list = self.rpn_head.predict( x, batch_data_samples, rescale=False) else: assert batch_data_samples[0].get('proposals', None) is not None rpn_results_list = [ data_sample.proposals for data_sample in batch_data_samples ] roi_outs = self.roi_head.forward(x, rpn_results_list, batch_data_samples) results = results + (roi_outs, ) return results def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> dict: """Calculate losses from a batch of inputs and data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (List[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components """ x = self.extract_feat(batch_inputs) losses = dict() # RPN forward and loss if self.with_rpn: proposal_cfg = self.train_cfg.get('rpn_proposal', self.test_cfg.rpn) rpn_data_samples = copy.deepcopy(batch_data_samples) # set cat_id of gt_labels to 0 in RPN for data_sample in rpn_data_samples: data_sample.gt_instances.labels = \ torch.zeros_like(data_sample.gt_instances.labels) rpn_losses, rpn_results_list = self.rpn_head.loss_and_predict( x, rpn_data_samples, proposal_cfg=proposal_cfg) # avoid get same name with roi_head loss keys = rpn_losses.keys() for key in list(keys): if 'loss' in key and 'rpn' not in key: rpn_losses[f'rpn_{key}'] = rpn_losses.pop(key) losses.update(rpn_losses) else: assert batch_data_samples[0].get('proposals', None) is not None # use pre-defined proposals in InstanceData for the second stage # to extract ROI features. rpn_results_list = [ data_sample.proposals for data_sample in batch_data_samples ] roi_losses = self.roi_head.loss(x, rpn_results_list, batch_data_samples) losses.update(roi_losses) return losses def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList, rescale: bool = True) -> SampleList: """Predict results from a batch of inputs and data samples with post- processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results. Defaults to True. Returns: list[:obj:`DetDataSample`]: Return the detection results of the input images. The returns value is DetDataSample, which usually contain 'pred_instances'. And the ``pred_instances`` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ assert self.with_bbox, 'Bbox head must be implemented.' x = self.extract_feat(batch_inputs) # If there are no pre-defined proposals, use RPN to get proposals if batch_data_samples[0].get('proposals', None) is None: rpn_results_list = self.rpn_head.predict( x, batch_data_samples, rescale=False) else: rpn_results_list = [ data_sample.proposals for data_sample in batch_data_samples ] results_list = self.roi_head.predict( x, rpn_results_list, batch_data_samples, rescale=rescale) batch_data_samples = self.add_pred_to_datasample( batch_data_samples, results_list) return batch_data_samples ================================================ FILE: mmdet/models/detectors/vfnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class VFNet(SingleStageDetector): """Implementation of `VarifocalNet (VFNet).`_ Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of VFNet. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of VFNet. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/yolact.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage_instance_seg import SingleStageInstanceSegmentor @MODELS.register_module() class YOLACT(SingleStageInstanceSegmentor): """Implementation of `YOLACT `_""" def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, mask_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, mask_head=mask_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/yolo.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Copyright (c) 2019 Western Digital Corporation or its affiliates. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class YOLOV3(SingleStageDetector): r"""Implementation of `Yolov3: An incremental improvement `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of YOLOX. Default: None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of YOLOX. Default: None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Model preprocessing config for processing the input data. it usually includes ``to_rgb``, ``pad_size_divisor``, ``pad_value``, ``mean`` and ``std``. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/yolof.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class YOLOF(SingleStageDetector): r"""Implementation of `You Only Look One-level Feature `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone module. neck (:obj:`ConfigDict` or dict): The neck module. bbox_head (:obj:`ConfigDict` or dict): The bbox head module. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of YOLOF. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of YOLOF. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Model preprocessing config for processing the input data. it usually includes ``to_rgb``, ``pad_size_divisor``, ``pad_value``, ``mean`` and ``std``. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/detectors/yolox.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .single_stage import SingleStageDetector @MODELS.register_module() class YOLOX(SingleStageDetector): r"""Implementation of `YOLOX: Exceeding YOLO Series in 2021 `_ Args: backbone (:obj:`ConfigDict` or dict): The backbone config. neck (:obj:`ConfigDict` or dict): The neck config. bbox_head (:obj:`ConfigDict` or dict): The bbox head config. train_cfg (:obj:`ConfigDict` or dict, optional): The training config of YOLOX. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of YOLOX. Defaults to None. data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: mmdet/models/layers/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .activations import SiLU from .bbox_nms import fast_nms, multiclass_nms from .brick_wrappers import AdaptiveAvgPool2d, adaptive_avg_pool2d from .conv_upsample import ConvUpsample from .csp_layer import CSPLayer from .dropblock import DropBlock from .ema import ExpMomentumEMA from .inverted_residual import InvertedResidual from .matrix_nms import mask_matrix_nms from .msdeformattn_pixel_decoder import MSDeformAttnPixelDecoder from .normed_predictor import NormedConv2d, NormedLinear from .pixel_decoder import PixelDecoder, TransformerEncoderPixelDecoder from .positional_encoding import (LearnedPositionalEncoding, SinePositionalEncoding) from .res_layer import ResLayer, SimplifiedBasicBlock from .se_layer import ChannelAttention, DyReLU, SELayer # yapf: disable from .transformer import (MLP, AdaptivePadding, CdnQueryGenerator, ConditionalAttention, ConditionalDetrTransformerDecoder, ConditionalDetrTransformerDecoderLayer, DABDetrTransformerDecoder, DABDetrTransformerDecoderLayer, DABDetrTransformerEncoder, DeformableDetrTransformerDecoder, DeformableDetrTransformerDecoderLayer, DeformableDetrTransformerEncoder, DeformableDetrTransformerEncoderLayer, DetrTransformerDecoder, DetrTransformerDecoderLayer, DetrTransformerEncoder, DetrTransformerEncoderLayer, DinoTransformerDecoder, DynamicConv, Mask2FormerTransformerDecoder, Mask2FormerTransformerDecoderLayer, Mask2FormerTransformerEncoder, PatchEmbed, PatchMerging, coordinate_to_encoding, inverse_sigmoid, nchw_to_nlc, nlc_to_nchw) # yapf: enable __all__ = [ 'fast_nms', 'multiclass_nms', 'mask_matrix_nms', 'DropBlock', 'PixelDecoder', 'TransformerEncoderPixelDecoder', 'MSDeformAttnPixelDecoder', 'ResLayer', 'PatchMerging', 'SinePositionalEncoding', 'LearnedPositionalEncoding', 'DynamicConv', 'SimplifiedBasicBlock', 'NormedLinear', 'NormedConv2d', 'InvertedResidual', 'SELayer', 'ConvUpsample', 'CSPLayer', 'adaptive_avg_pool2d', 'AdaptiveAvgPool2d', 'PatchEmbed', 'nchw_to_nlc', 'nlc_to_nchw', 'DyReLU', 'ExpMomentumEMA', 'inverse_sigmoid', 'ChannelAttention', 'SiLU', 'MLP', 'DetrTransformerEncoderLayer', 'DetrTransformerDecoderLayer', 'DetrTransformerEncoder', 'DetrTransformerDecoder', 'DeformableDetrTransformerEncoder', 'DeformableDetrTransformerDecoder', 'DeformableDetrTransformerEncoderLayer', 'DeformableDetrTransformerDecoderLayer', 'AdaptivePadding', 'coordinate_to_encoding', 'ConditionalAttention', 'DABDetrTransformerDecoderLayer', 'DABDetrTransformerDecoder', 'DABDetrTransformerEncoder', 'ConditionalDetrTransformerDecoder', 'ConditionalDetrTransformerDecoderLayer', 'DinoTransformerDecoder', 'CdnQueryGenerator', 'Mask2FormerTransformerEncoder', 'Mask2FormerTransformerDecoderLayer', 'Mask2FormerTransformerDecoder' ] ================================================ FILE: mmdet/models/layers/activations.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn from mmengine.utils import digit_version from mmdet.registry import MODELS if digit_version(torch.__version__) >= digit_version('1.7.0'): from torch.nn import SiLU else: class SiLU(nn.Module): """Sigmoid Weighted Liner Unit.""" def __init__(self, inplace=True): super().__init__() def forward(self, inputs) -> torch.Tensor: return inputs * torch.sigmoid(inputs) MODELS.register_module(module=SiLU, name='SiLU') ================================================ FILE: mmdet/models/layers/bbox_nms.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple, Union import torch from mmcv.ops.nms import batched_nms from torch import Tensor from mmdet.structures.bbox import bbox_overlaps from mmdet.utils import ConfigType def multiclass_nms( multi_bboxes: Tensor, multi_scores: Tensor, score_thr: float, nms_cfg: ConfigType, max_num: int = -1, score_factors: Optional[Tensor] = None, return_inds: bool = False, box_dim: int = 4 ) -> Union[Tuple[Tensor, Tensor, Tensor], Tuple[Tensor, Tensor]]: """NMS for multi-class bboxes. Args: multi_bboxes (Tensor): shape (n, #class*4) or (n, 4) multi_scores (Tensor): shape (n, #class), where the last column contains scores of the background class, but this will be ignored. score_thr (float): bbox threshold, bboxes with scores lower than it will not be considered. nms_cfg (Union[:obj:`ConfigDict`, dict]): a dict that contains the arguments of nms operations. max_num (int, optional): if there are more than max_num bboxes after NMS, only top max_num will be kept. Default to -1. score_factors (Tensor, optional): The factors multiplied to scores before applying NMS. Default to None. return_inds (bool, optional): Whether return the indices of kept bboxes. Default to False. box_dim (int): The dimension of boxes. Defaults to 4. Returns: Union[Tuple[Tensor, Tensor, Tensor], Tuple[Tensor, Tensor]]: (dets, labels, indices (optional)), tensors of shape (k, 5), (k), and (k). Dets are boxes with scores. Labels are 0-based. """ num_classes = multi_scores.size(1) - 1 # exclude background category if multi_bboxes.shape[1] > box_dim: bboxes = multi_bboxes.view(multi_scores.size(0), -1, box_dim) else: bboxes = multi_bboxes[:, None].expand( multi_scores.size(0), num_classes, box_dim) scores = multi_scores[:, :-1] labels = torch.arange(num_classes, dtype=torch.long, device=scores.device) labels = labels.view(1, -1).expand_as(scores) bboxes = bboxes.reshape(-1, box_dim) scores = scores.reshape(-1) labels = labels.reshape(-1) if not torch.onnx.is_in_onnx_export(): # NonZero not supported in TensorRT # remove low scoring boxes valid_mask = scores > score_thr # multiply score_factor after threshold to preserve more bboxes, improve # mAP by 1% for YOLOv3 if score_factors is not None: # expand the shape to match original shape of score score_factors = score_factors.view(-1, 1).expand( multi_scores.size(0), num_classes) score_factors = score_factors.reshape(-1) scores = scores * score_factors if not torch.onnx.is_in_onnx_export(): # NonZero not supported in TensorRT inds = valid_mask.nonzero(as_tuple=False).squeeze(1) bboxes, scores, labels = bboxes[inds], scores[inds], labels[inds] else: # TensorRT NMS plugin has invalid output filled with -1 # add dummy data to make detection output correct. bboxes = torch.cat([bboxes, bboxes.new_zeros(1, box_dim)], dim=0) scores = torch.cat([scores, scores.new_zeros(1)], dim=0) labels = torch.cat([labels, labels.new_zeros(1)], dim=0) if bboxes.numel() == 0: if torch.onnx.is_in_onnx_export(): raise RuntimeError('[ONNX Error] Can not record NMS ' 'as it has not been executed this time') dets = torch.cat([bboxes, scores[:, None]], -1) if return_inds: return dets, labels, inds else: return dets, labels dets, keep = batched_nms(bboxes, scores, labels, nms_cfg) if max_num > 0: dets = dets[:max_num] keep = keep[:max_num] if return_inds: return dets, labels[keep], inds[keep] else: return dets, labels[keep] def fast_nms( multi_bboxes: Tensor, multi_scores: Tensor, multi_coeffs: Tensor, score_thr: float, iou_thr: float, top_k: int, max_num: int = -1 ) -> Union[Tuple[Tensor, Tensor, Tensor], Tuple[Tensor, Tensor]]: """Fast NMS in `YOLACT `_. Fast NMS allows already-removed detections to suppress other detections so that every instance can be decided to be kept or discarded in parallel, which is not possible in traditional NMS. This relaxation allows us to implement Fast NMS entirely in standard GPU-accelerated matrix operations. Args: multi_bboxes (Tensor): shape (n, #class*4) or (n, 4) multi_scores (Tensor): shape (n, #class+1), where the last column contains scores of the background class, but this will be ignored. multi_coeffs (Tensor): shape (n, #class*coeffs_dim). score_thr (float): bbox threshold, bboxes with scores lower than it will not be considered. iou_thr (float): IoU threshold to be considered as conflicted. top_k (int): if there are more than top_k bboxes before NMS, only top top_k will be kept. max_num (int): if there are more than max_num bboxes after NMS, only top max_num will be kept. If -1, keep all the bboxes. Default: -1. Returns: Union[Tuple[Tensor, Tensor, Tensor], Tuple[Tensor, Tensor]]: (dets, labels, coefficients), tensors of shape (k, 5), (k, 1), and (k, coeffs_dim). Dets are boxes with scores. Labels are 0-based. """ scores = multi_scores[:, :-1].t() # [#class, n] scores, idx = scores.sort(1, descending=True) idx = idx[:, :top_k].contiguous() scores = scores[:, :top_k] # [#class, topk] num_classes, num_dets = idx.size() boxes = multi_bboxes[idx.view(-1), :].view(num_classes, num_dets, 4) coeffs = multi_coeffs[idx.view(-1), :].view(num_classes, num_dets, -1) iou = bbox_overlaps(boxes, boxes) # [#class, topk, topk] iou.triu_(diagonal=1) iou_max, _ = iou.max(dim=1) # Now just filter out the ones higher than the threshold keep = iou_max <= iou_thr # Second thresholding introduces 0.2 mAP gain at negligible time cost keep *= scores > score_thr # Assign each kept detection to its corresponding class classes = torch.arange( num_classes, device=boxes.device)[:, None].expand_as(keep) classes = classes[keep] boxes = boxes[keep] coeffs = coeffs[keep] scores = scores[keep] # Only keep the top max_num highest scores across all classes scores, idx = scores.sort(0, descending=True) if max_num > 0: idx = idx[:max_num] scores = scores[:max_num] classes = classes[idx] boxes = boxes[idx] coeffs = coeffs[idx] cls_dets = torch.cat([boxes, scores[:, None]], dim=1) return cls_dets, classes, coeffs ================================================ FILE: mmdet/models/layers/brick_wrappers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn.bricks.wrappers import NewEmptyTensorOp, obsolete_torch_version if torch.__version__ == 'parrots': TORCH_VERSION = torch.__version__ else: # torch.__version__ could be 1.3.1+cu92, we only need the first two # for comparison TORCH_VERSION = tuple(int(x) for x in torch.__version__.split('.')[:2]) def adaptive_avg_pool2d(input, output_size): """Handle empty batch dimension to adaptive_avg_pool2d. Args: input (tensor): 4D tensor. output_size (int, tuple[int,int]): the target output size. """ if input.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 9)): if isinstance(output_size, int): output_size = [output_size, output_size] output_size = [*input.shape[:2], *output_size] empty = NewEmptyTensorOp.apply(input, output_size) return empty else: return F.adaptive_avg_pool2d(input, output_size) class AdaptiveAvgPool2d(nn.AdaptiveAvgPool2d): """Handle empty batch dimension to AdaptiveAvgPool2d.""" def forward(self, x): # PyTorch 1.9 does not support empty tensor inference yet if x.numel() == 0 and obsolete_torch_version(TORCH_VERSION, (1, 9)): output_size = self.output_size if isinstance(output_size, int): output_size = [output_size, output_size] else: output_size = [ v if v is not None else d for v, d in zip(output_size, x.size()[-2:]) ] output_size = [*x.shape[:2], *output_size] empty = NewEmptyTensorOp.apply(x, output_size) return empty return super().forward(x) ================================================ FILE: mmdet/models/layers/conv_upsample.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.model import BaseModule, ModuleList class ConvUpsample(BaseModule): """ConvUpsample performs 2x upsampling after Conv. There are several `ConvModule` layers. In the first few layers, upsampling will be applied after each layer of convolution. The number of upsampling must be no more than the number of ConvModule layers. Args: in_channels (int): Number of channels in the input feature map. inner_channels (int): Number of channels produced by the convolution. num_layers (int): Number of convolution layers. num_upsample (int | optional): Number of upsampling layer. Must be no more than num_layers. Upsampling will be applied after the first ``num_upsample`` layers of convolution. Default: ``num_layers``. conv_cfg (dict): Config dict for convolution layer. Default: None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Default: None. init_cfg (dict): Config dict for initialization. Default: None. kwargs (key word augments): Other augments used in ConvModule. """ def __init__(self, in_channels, inner_channels, num_layers=1, num_upsample=None, conv_cfg=None, norm_cfg=None, init_cfg=None, **kwargs): super(ConvUpsample, self).__init__(init_cfg) if num_upsample is None: num_upsample = num_layers assert num_upsample <= num_layers, \ f'num_upsample({num_upsample})must be no more than ' \ f'num_layers({num_layers})' self.num_layers = num_layers self.num_upsample = num_upsample self.conv = ModuleList() for i in range(num_layers): self.conv.append( ConvModule( in_channels, inner_channels, 3, padding=1, stride=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, **kwargs)) in_channels = inner_channels def forward(self, x): num_upsample = self.num_upsample for i in range(self.num_layers): x = self.conv[i](x) if num_upsample > 0: num_upsample -= 1 x = F.interpolate( x, scale_factor=2, mode='bilinear', align_corners=False) return x ================================================ FILE: mmdet/models/layers/csp_layer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule from mmengine.model import BaseModule from torch import Tensor from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from .se_layer import ChannelAttention class DarknetBottleneck(BaseModule): """The basic bottleneck block used in Darknet. Each ResBlock consists of two ConvModules and the input is added to the final output. Each ConvModule is composed of Conv, BN, and LeakyReLU. The first convLayer has filter size of 1x1 and the second one has the filter size of 3x3. Args: in_channels (int): The input channels of this Module. out_channels (int): The output channels of this Module. expansion (float): The kernel size of the convolution. Defaults to 0.5. add_identity (bool): Whether to add identity to the out. Defaults to True. use_depthwise (bool): Whether to use depthwise separable convolution. Defaults to False. conv_cfg (dict): Config dict for convolution layer. Defaults to None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Defaults to dict(type='BN'). act_cfg (dict): Config dict for activation layer. Defaults to dict(type='Swish'). """ def __init__(self, in_channels: int, out_channels: int, expansion: float = 0.5, add_identity: bool = True, use_depthwise: bool = False, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict( type='BN', momentum=0.03, eps=0.001), act_cfg: ConfigType = dict(type='Swish'), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) hidden_channels = int(out_channels * expansion) conv = DepthwiseSeparableConvModule if use_depthwise else ConvModule self.conv1 = ConvModule( in_channels, hidden_channels, 1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.conv2 = conv( hidden_channels, out_channels, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.add_identity = \ add_identity and in_channels == out_channels def forward(self, x: Tensor) -> Tensor: """Forward function.""" identity = x out = self.conv1(x) out = self.conv2(out) if self.add_identity: return out + identity else: return out class CSPNeXtBlock(BaseModule): """The basic bottleneck block used in CSPNeXt. Args: in_channels (int): The input channels of this Module. out_channels (int): The output channels of this Module. expansion (float): Expand ratio of the hidden channel. Defaults to 0.5. add_identity (bool): Whether to add identity to the out. Only works when in_channels == out_channels. Defaults to True. use_depthwise (bool): Whether to use depthwise separable convolution. Defaults to False. kernel_size (int): The kernel size of the second convolution layer. Defaults to 5. conv_cfg (dict): Config dict for convolution layer. Defaults to None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Defaults to dict(type='BN', momentum=0.03, eps=0.001). act_cfg (dict): Config dict for activation layer. Defaults to dict(type='SiLU'). init_cfg (:obj:`ConfigDict` or dict or list[dict] or list[:obj:`ConfigDict`], optional): Initialization config dict. Defaults to None. """ def __init__(self, in_channels: int, out_channels: int, expansion: float = 0.5, add_identity: bool = True, use_depthwise: bool = False, kernel_size: int = 5, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict( type='BN', momentum=0.03, eps=0.001), act_cfg: ConfigType = dict(type='SiLU'), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) hidden_channels = int(out_channels * expansion) conv = DepthwiseSeparableConvModule if use_depthwise else ConvModule self.conv1 = conv( in_channels, hidden_channels, 3, stride=1, padding=1, norm_cfg=norm_cfg, act_cfg=act_cfg) self.conv2 = DepthwiseSeparableConvModule( hidden_channels, out_channels, kernel_size, stride=1, padding=kernel_size // 2, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.add_identity = \ add_identity and in_channels == out_channels def forward(self, x: Tensor) -> Tensor: """Forward function.""" identity = x out = self.conv1(x) out = self.conv2(out) if self.add_identity: return out + identity else: return out class CSPLayer(BaseModule): """Cross Stage Partial Layer. Args: in_channels (int): The input channels of the CSP layer. out_channels (int): The output channels of the CSP layer. expand_ratio (float): Ratio to adjust the number of channels of the hidden layer. Defaults to 0.5. num_blocks (int): Number of blocks. Defaults to 1. add_identity (bool): Whether to add identity in blocks. Defaults to True. use_cspnext_block (bool): Whether to use CSPNeXt block. Defaults to False. use_depthwise (bool): Whether to use depthwise separable convolution in blocks. Defaults to False. channel_attention (bool): Whether to add channel attention in each stage. Defaults to True. conv_cfg (dict, optional): Config dict for convolution layer. Defaults to None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Defaults to dict(type='BN') act_cfg (dict): Config dict for activation layer. Defaults to dict(type='Swish') init_cfg (:obj:`ConfigDict` or dict or list[dict] or list[:obj:`ConfigDict`], optional): Initialization config dict. Defaults to None. """ def __init__(self, in_channels: int, out_channels: int, expand_ratio: float = 0.5, num_blocks: int = 1, add_identity: bool = True, use_depthwise: bool = False, use_cspnext_block: bool = False, channel_attention: bool = False, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict( type='BN', momentum=0.03, eps=0.001), act_cfg: ConfigType = dict(type='Swish'), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) block = CSPNeXtBlock if use_cspnext_block else DarknetBottleneck mid_channels = int(out_channels * expand_ratio) self.channel_attention = channel_attention self.main_conv = ConvModule( in_channels, mid_channels, 1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.short_conv = ConvModule( in_channels, mid_channels, 1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.final_conv = ConvModule( 2 * mid_channels, out_channels, 1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.blocks = nn.Sequential(*[ block( mid_channels, mid_channels, 1.0, add_identity, use_depthwise, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) for _ in range(num_blocks) ]) if channel_attention: self.attention = ChannelAttention(2 * mid_channels) def forward(self, x: Tensor) -> Tensor: """Forward function.""" x_short = self.short_conv(x) x_main = self.main_conv(x) x_main = self.blocks(x_main) x_final = torch.cat((x_main, x_short), dim=1) if self.channel_attention: x_final = self.attention(x_final) return self.final_conv(x_final) ================================================ FILE: mmdet/models/layers/dropblock.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F from mmdet.registry import MODELS eps = 1e-6 @MODELS.register_module() class DropBlock(nn.Module): """Randomly drop some regions of feature maps. Please refer to the method proposed in `DropBlock `_ for details. Args: drop_prob (float): The probability of dropping each block. block_size (int): The size of dropped blocks. warmup_iters (int): The drop probability will linearly increase from `0` to `drop_prob` during the first `warmup_iters` iterations. Default: 2000. """ def __init__(self, drop_prob, block_size, warmup_iters=2000, **kwargs): super(DropBlock, self).__init__() assert block_size % 2 == 1 assert 0 < drop_prob <= 1 assert warmup_iters >= 0 self.drop_prob = drop_prob self.block_size = block_size self.warmup_iters = warmup_iters self.iter_cnt = 0 def forward(self, x): """ Args: x (Tensor): Input feature map on which some areas will be randomly dropped. Returns: Tensor: The tensor after DropBlock layer. """ if not self.training: return x self.iter_cnt += 1 N, C, H, W = list(x.shape) gamma = self._compute_gamma((H, W)) mask_shape = (N, C, H - self.block_size + 1, W - self.block_size + 1) mask = torch.bernoulli(torch.full(mask_shape, gamma, device=x.device)) mask = F.pad(mask, [self.block_size // 2] * 4, value=0) mask = F.max_pool2d( input=mask, stride=(1, 1), kernel_size=(self.block_size, self.block_size), padding=self.block_size // 2) mask = 1 - mask x = x * mask * mask.numel() / (eps + mask.sum()) return x def _compute_gamma(self, feat_size): """Compute the value of gamma according to paper. gamma is the parameter of bernoulli distribution, which controls the number of features to drop. gamma = (drop_prob * fm_area) / (drop_area * keep_area) Args: feat_size (tuple[int, int]): The height and width of feature map. Returns: float: The value of gamma. """ gamma = (self.drop_prob * feat_size[0] * feat_size[1]) gamma /= ((feat_size[0] - self.block_size + 1) * (feat_size[1] - self.block_size + 1)) gamma /= (self.block_size**2) factor = (1.0 if self.iter_cnt > self.warmup_iters else self.iter_cnt / self.warmup_iters) return gamma * factor def extra_repr(self): return (f'drop_prob={self.drop_prob}, block_size={self.block_size}, ' f'warmup_iters={self.warmup_iters}') ================================================ FILE: mmdet/models/layers/ema.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from typing import Optional import torch import torch.nn as nn from mmengine.model import ExponentialMovingAverage from torch import Tensor from mmdet.registry import MODELS @MODELS.register_module() class ExpMomentumEMA(ExponentialMovingAverage): """Exponential moving average (EMA) with exponential momentum strategy, which is used in YOLOX. Args: model (nn.Module): The model to be averaged. momentum (float): The momentum used for updating ema parameter. Ema's parameter are updated with the formula: `averaged_param = (1-momentum) * averaged_param + momentum * source_param`. Defaults to 0.0002. gamma (int): Use a larger momentum early in training and gradually annealing to a smaller value to update the ema model smoothly. The momentum is calculated as `(1 - momentum) * exp(-(1 + steps) / gamma) + momentum`. Defaults to 2000. interval (int): Interval between two updates. Defaults to 1. device (torch.device, optional): If provided, the averaged model will be stored on the :attr:`device`. Defaults to None. update_buffers (bool): if True, it will compute running averages for both the parameters and the buffers of the model. Defaults to False. """ def __init__(self, model: nn.Module, momentum: float = 0.0002, gamma: int = 2000, interval=1, device: Optional[torch.device] = None, update_buffers: bool = False) -> None: super().__init__( model=model, momentum=momentum, interval=interval, device=device, update_buffers=update_buffers) assert gamma > 0, f'gamma must be greater than 0, but got {gamma}' self.gamma = gamma def avg_func(self, averaged_param: Tensor, source_param: Tensor, steps: int) -> None: """Compute the moving average of the parameters using the exponential momentum strategy. Args: averaged_param (Tensor): The averaged parameters. source_param (Tensor): The source parameters. steps (int): The number of times the parameters have been updated. """ momentum = (1 - self.momentum) * math.exp( -float(1 + steps) / self.gamma) + self.momentum averaged_param.mul_(1 - momentum).add_(source_param, alpha=momentum) ================================================ FILE: mmdet/models/layers/inverted_residual.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn as nn import torch.utils.checkpoint as cp from mmcv.cnn import ConvModule from mmcv.cnn.bricks import DropPath from mmengine.model import BaseModule from .se_layer import SELayer class InvertedResidual(BaseModule): """Inverted Residual Block. Args: in_channels (int): The input channels of this Module. out_channels (int): The output channels of this Module. mid_channels (int): The input channels of the depthwise convolution. kernel_size (int): The kernel size of the depthwise convolution. Default: 3. stride (int): The stride of the depthwise convolution. Default: 1. se_cfg (dict): Config dict for se layer. Default: None, which means no se layer. with_expand_conv (bool): Use expand conv or not. If set False, mid_channels must be the same with in_channels. Default: True. conv_cfg (dict): Config dict for convolution layer. Default: None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Default: dict(type='BN'). act_cfg (dict): Config dict for activation layer. Default: dict(type='ReLU'). drop_path_rate (float): stochastic depth rate. Defaults to 0. with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. Default: False. init_cfg (dict or list[dict], optional): Initialization config dict. Default: None Returns: Tensor: The output tensor. """ def __init__(self, in_channels, out_channels, mid_channels, kernel_size=3, stride=1, se_cfg=None, with_expand_conv=True, conv_cfg=None, norm_cfg=dict(type='BN'), act_cfg=dict(type='ReLU'), drop_path_rate=0., with_cp=False, init_cfg=None): super(InvertedResidual, self).__init__(init_cfg) self.with_res_shortcut = (stride == 1 and in_channels == out_channels) assert stride in [1, 2], f'stride must in [1, 2]. ' \ f'But received {stride}.' self.with_cp = with_cp self.drop_path = DropPath( drop_path_rate) if drop_path_rate > 0 else nn.Identity() self.with_se = se_cfg is not None self.with_expand_conv = with_expand_conv if self.with_se: assert isinstance(se_cfg, dict) if not self.with_expand_conv: assert mid_channels == in_channels if self.with_expand_conv: self.expand_conv = ConvModule( in_channels=in_channels, out_channels=mid_channels, kernel_size=1, stride=1, padding=0, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.depthwise_conv = ConvModule( in_channels=mid_channels, out_channels=mid_channels, kernel_size=kernel_size, stride=stride, padding=kernel_size // 2, groups=mid_channels, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) if self.with_se: self.se = SELayer(**se_cfg) self.linear_conv = ConvModule( in_channels=mid_channels, out_channels=out_channels, kernel_size=1, stride=1, padding=0, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=None) def forward(self, x): def _inner_forward(x): out = x if self.with_expand_conv: out = self.expand_conv(out) out = self.depthwise_conv(out) if self.with_se: out = self.se(out) out = self.linear_conv(out) if self.with_res_shortcut: return x + self.drop_path(out) else: return out if self.with_cp and x.requires_grad: out = cp.checkpoint(_inner_forward, x) else: out = _inner_forward(x) return out ================================================ FILE: mmdet/models/layers/matrix_nms.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch def mask_matrix_nms(masks, labels, scores, filter_thr=-1, nms_pre=-1, max_num=-1, kernel='gaussian', sigma=2.0, mask_area=None): """Matrix NMS for multi-class masks. Args: masks (Tensor): Has shape (num_instances, h, w) labels (Tensor): Labels of corresponding masks, has shape (num_instances,). scores (Tensor): Mask scores of corresponding masks, has shape (num_instances). filter_thr (float): Score threshold to filter the masks after matrix nms. Default: -1, which means do not use filter_thr. nms_pre (int): The max number of instances to do the matrix nms. Default: -1, which means do not use nms_pre. max_num (int, optional): If there are more than max_num masks after matrix, only top max_num will be kept. Default: -1, which means do not use max_num. kernel (str): 'linear' or 'gaussian'. sigma (float): std in gaussian method. mask_area (Tensor): The sum of seg_masks. Returns: tuple(Tensor): Processed mask results. - scores (Tensor): Updated scores, has shape (n,). - labels (Tensor): Remained labels, has shape (n,). - masks (Tensor): Remained masks, has shape (n, w, h). - keep_inds (Tensor): The indices number of the remaining mask in the input mask, has shape (n,). """ assert len(labels) == len(masks) == len(scores) if len(labels) == 0: return scores.new_zeros(0), labels.new_zeros(0), masks.new_zeros( 0, *masks.shape[-2:]), labels.new_zeros(0) if mask_area is None: mask_area = masks.sum((1, 2)).float() else: assert len(masks) == len(mask_area) # sort and keep top nms_pre scores, sort_inds = torch.sort(scores, descending=True) keep_inds = sort_inds if nms_pre > 0 and len(sort_inds) > nms_pre: sort_inds = sort_inds[:nms_pre] keep_inds = keep_inds[:nms_pre] scores = scores[:nms_pre] masks = masks[sort_inds] mask_area = mask_area[sort_inds] labels = labels[sort_inds] num_masks = len(labels) flatten_masks = masks.reshape(num_masks, -1).float() # inter. inter_matrix = torch.mm(flatten_masks, flatten_masks.transpose(1, 0)) expanded_mask_area = mask_area.expand(num_masks, num_masks) # Upper triangle iou matrix. iou_matrix = (inter_matrix / (expanded_mask_area + expanded_mask_area.transpose(1, 0) - inter_matrix)).triu(diagonal=1) # label_specific matrix. expanded_labels = labels.expand(num_masks, num_masks) # Upper triangle label matrix. label_matrix = (expanded_labels == expanded_labels.transpose( 1, 0)).triu(diagonal=1) # IoU compensation compensate_iou, _ = (iou_matrix * label_matrix).max(0) compensate_iou = compensate_iou.expand(num_masks, num_masks).transpose(1, 0) # IoU decay decay_iou = iou_matrix * label_matrix # Calculate the decay_coefficient if kernel == 'gaussian': decay_matrix = torch.exp(-1 * sigma * (decay_iou**2)) compensate_matrix = torch.exp(-1 * sigma * (compensate_iou**2)) decay_coefficient, _ = (decay_matrix / compensate_matrix).min(0) elif kernel == 'linear': decay_matrix = (1 - decay_iou) / (1 - compensate_iou) decay_coefficient, _ = decay_matrix.min(0) else: raise NotImplementedError( f'{kernel} kernel is not supported in matrix nms!') # update the score. scores = scores * decay_coefficient if filter_thr > 0: keep = scores >= filter_thr keep_inds = keep_inds[keep] if not keep.any(): return scores.new_zeros(0), labels.new_zeros(0), masks.new_zeros( 0, *masks.shape[-2:]), labels.new_zeros(0) masks = masks[keep] scores = scores[keep] labels = labels[keep] # sort and keep top max_num scores, sort_inds = torch.sort(scores, descending=True) keep_inds = keep_inds[sort_inds] if max_num > 0 and len(sort_inds) > max_num: sort_inds = sort_inds[:max_num] keep_inds = keep_inds[:max_num] scores = scores[:max_num] masks = masks[sort_inds] labels = labels[sort_inds] return scores, labels, masks, keep_inds ================================================ FILE: mmdet/models/layers/msdeformattn_pixel_decoder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import Conv2d, ConvModule from mmcv.cnn.bricks.transformer import MultiScaleDeformableAttention from mmengine.model import (BaseModule, ModuleList, caffe2_xavier_init, normal_init, xavier_init) from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptMultiConfig from ..task_modules.prior_generators import MlvlPointGenerator from .positional_encoding import SinePositionalEncoding from .transformer import Mask2FormerTransformerEncoder @MODELS.register_module() class MSDeformAttnPixelDecoder(BaseModule): """Pixel decoder with multi-scale deformable attention. Args: in_channels (list[int] | tuple[int]): Number of channels in the input feature maps. strides (list[int] | tuple[int]): Output strides of feature from backbone. feat_channels (int): Number of channels for feature. out_channels (int): Number of channels for output. num_outs (int): Number of output scales. norm_cfg (:obj:`ConfigDict` or dict): Config for normalization. Defaults to dict(type='GN', num_groups=32). act_cfg (:obj:`ConfigDict` or dict): Config for activation. Defaults to dict(type='ReLU'). encoder (:obj:`ConfigDict` or dict): Config for transformer encoder. Defaults to None. positional_encoding (:obj:`ConfigDict` or dict): Config for transformer encoder position encoding. Defaults to dict(num_feats=128, normalize=True). init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, in_channels: Union[List[int], Tuple[int]] = [256, 512, 1024, 2048], strides: Union[List[int], Tuple[int]] = [4, 8, 16, 32], feat_channels: int = 256, out_channels: int = 256, num_outs: int = 3, norm_cfg: ConfigType = dict(type='GN', num_groups=32), act_cfg: ConfigType = dict(type='ReLU'), encoder: ConfigType = None, positional_encoding: ConfigType = dict( num_feats=128, normalize=True), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) self.strides = strides self.num_input_levels = len(in_channels) self.num_encoder_levels = \ encoder.layer_cfg.self_attn_cfg.num_levels assert self.num_encoder_levels >= 1, \ 'num_levels in attn_cfgs must be at least one' input_conv_list = [] # from top to down (low to high resolution) for i in range(self.num_input_levels - 1, self.num_input_levels - self.num_encoder_levels - 1, -1): input_conv = ConvModule( in_channels[i], feat_channels, kernel_size=1, norm_cfg=norm_cfg, act_cfg=None, bias=True) input_conv_list.append(input_conv) self.input_convs = ModuleList(input_conv_list) self.encoder = Mask2FormerTransformerEncoder(**encoder) self.postional_encoding = SinePositionalEncoding(**positional_encoding) # high resolution to low resolution self.level_encoding = nn.Embedding(self.num_encoder_levels, feat_channels) # fpn-like structure self.lateral_convs = ModuleList() self.output_convs = ModuleList() self.use_bias = norm_cfg is None # from top to down (low to high resolution) # fpn for the rest features that didn't pass in encoder for i in range(self.num_input_levels - self.num_encoder_levels - 1, -1, -1): lateral_conv = ConvModule( in_channels[i], feat_channels, kernel_size=1, bias=self.use_bias, norm_cfg=norm_cfg, act_cfg=None) output_conv = ConvModule( feat_channels, feat_channels, kernel_size=3, stride=1, padding=1, bias=self.use_bias, norm_cfg=norm_cfg, act_cfg=act_cfg) self.lateral_convs.append(lateral_conv) self.output_convs.append(output_conv) self.mask_feature = Conv2d( feat_channels, out_channels, kernel_size=1, stride=1, padding=0) self.num_outs = num_outs self.point_generator = MlvlPointGenerator(strides) def init_weights(self) -> None: """Initialize weights.""" for i in range(0, self.num_encoder_levels): xavier_init( self.input_convs[i].conv, gain=1, bias=0, distribution='uniform') for i in range(0, self.num_input_levels - self.num_encoder_levels): caffe2_xavier_init(self.lateral_convs[i].conv, bias=0) caffe2_xavier_init(self.output_convs[i].conv, bias=0) caffe2_xavier_init(self.mask_feature, bias=0) normal_init(self.level_encoding, mean=0, std=1) for p in self.encoder.parameters(): if p.dim() > 1: nn.init.xavier_normal_(p) # init_weights defined in MultiScaleDeformableAttention for m in self.encoder.layers.modules(): if isinstance(m, MultiScaleDeformableAttention): m.init_weights() def forward(self, feats: List[Tensor]) -> Tuple[Tensor, Tensor]: """ Args: feats (list[Tensor]): Feature maps of each level. Each has shape of (batch_size, c, h, w). Returns: tuple: A tuple containing the following: - mask_feature (Tensor): shape (batch_size, c, h, w). - multi_scale_features (list[Tensor]): Multi scale \ features, each in shape (batch_size, c, h, w). """ # generate padding mask for each level, for each image batch_size = feats[0].shape[0] encoder_input_list = [] padding_mask_list = [] level_positional_encoding_list = [] spatial_shapes = [] reference_points_list = [] for i in range(self.num_encoder_levels): level_idx = self.num_input_levels - i - 1 feat = feats[level_idx] feat_projected = self.input_convs[i](feat) h, w = feat.shape[-2:] # no padding padding_mask_resized = feat.new_zeros( (batch_size, ) + feat.shape[-2:], dtype=torch.bool) pos_embed = self.postional_encoding(padding_mask_resized) level_embed = self.level_encoding.weight[i] level_pos_embed = level_embed.view(1, -1, 1, 1) + pos_embed # (h_i * w_i, 2) reference_points = self.point_generator.single_level_grid_priors( feat.shape[-2:], level_idx, device=feat.device) # normalize factor = feat.new_tensor([[w, h]]) * self.strides[level_idx] reference_points = reference_points / factor # shape (batch_size, c, h_i, w_i) -> (h_i * w_i, batch_size, c) feat_projected = feat_projected.flatten(2).permute(0, 2, 1) level_pos_embed = level_pos_embed.flatten(2).permute(0, 2, 1) padding_mask_resized = padding_mask_resized.flatten(1) encoder_input_list.append(feat_projected) padding_mask_list.append(padding_mask_resized) level_positional_encoding_list.append(level_pos_embed) spatial_shapes.append(feat.shape[-2:]) reference_points_list.append(reference_points) # shape (batch_size, total_num_queries), # total_num_queries=sum([., h_i * w_i,.]) padding_masks = torch.cat(padding_mask_list, dim=1) # shape (total_num_queries, batch_size, c) encoder_inputs = torch.cat(encoder_input_list, dim=1) level_positional_encodings = torch.cat( level_positional_encoding_list, dim=1) device = encoder_inputs.device # shape (num_encoder_levels, 2), from low # resolution to high resolution spatial_shapes = torch.as_tensor( spatial_shapes, dtype=torch.long, device=device) # shape (0, h_0*w_0, h_0*w_0+h_1*w_1, ...) level_start_index = torch.cat((spatial_shapes.new_zeros( (1, )), spatial_shapes.prod(1).cumsum(0)[:-1])) reference_points = torch.cat(reference_points_list, dim=0) reference_points = reference_points[None, :, None].repeat( batch_size, 1, self.num_encoder_levels, 1) valid_radios = reference_points.new_ones( (batch_size, self.num_encoder_levels, 2)) # shape (num_total_queries, batch_size, c) memory = self.encoder( query=encoder_inputs, query_pos=level_positional_encodings, key_padding_mask=padding_masks, spatial_shapes=spatial_shapes, reference_points=reference_points, level_start_index=level_start_index, valid_ratios=valid_radios) # (batch_size, c, num_total_queries) memory = memory.permute(0, 2, 1) # from low resolution to high resolution num_queries_per_level = [e[0] * e[1] for e in spatial_shapes] outs = torch.split(memory, num_queries_per_level, dim=-1) outs = [ x.reshape(batch_size, -1, spatial_shapes[i][0], spatial_shapes[i][1]) for i, x in enumerate(outs) ] for i in range(self.num_input_levels - self.num_encoder_levels - 1, -1, -1): x = feats[i] cur_feat = self.lateral_convs[i](x) y = cur_feat + F.interpolate( outs[-1], size=cur_feat.shape[-2:], mode='bilinear', align_corners=False) y = self.output_convs[i](y) outs.append(y) multi_scale_features = outs[:self.num_outs] mask_feature = self.mask_feature(outs[-1]) return mask_feature, multi_scale_features ================================================ FILE: mmdet/models/layers/normed_predictor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F from torch import Tensor from mmdet.registry import MODELS MODELS.register_module('Linear', module=nn.Linear) @MODELS.register_module(name='NormedLinear') class NormedLinear(nn.Linear): """Normalized Linear Layer. Args: tempeature (float, optional): Tempeature term. Defaults to 20. power (int, optional): Power term. Defaults to 1.0. eps (float, optional): The minimal value of divisor to keep numerical stability. Defaults to 1e-6. """ def __init__(self, *args, tempearture: float = 20, power: int = 1.0, eps: float = 1e-6, **kwargs) -> None: super().__init__(*args, **kwargs) self.tempearture = tempearture self.power = power self.eps = eps self.init_weights() def init_weights(self) -> None: """Initialize the weights.""" nn.init.normal_(self.weight, mean=0, std=0.01) if self.bias is not None: nn.init.constant_(self.bias, 0) def forward(self, x: Tensor) -> Tensor: """Forward function for `NormedLinear`.""" weight_ = self.weight / ( self.weight.norm(dim=1, keepdim=True).pow(self.power) + self.eps) x_ = x / (x.norm(dim=1, keepdim=True).pow(self.power) + self.eps) x_ = x_ * self.tempearture return F.linear(x_, weight_, self.bias) @MODELS.register_module(name='NormedConv2d') class NormedConv2d(nn.Conv2d): """Normalized Conv2d Layer. Args: tempeature (float, optional): Tempeature term. Defaults to 20. power (int, optional): Power term. Defaults to 1.0. eps (float, optional): The minimal value of divisor to keep numerical stability. Defaults to 1e-6. norm_over_kernel (bool, optional): Normalize over kernel. Defaults to False. """ def __init__(self, *args, tempearture: float = 20, power: int = 1.0, eps: float = 1e-6, norm_over_kernel: bool = False, **kwargs) -> None: super().__init__(*args, **kwargs) self.tempearture = tempearture self.power = power self.norm_over_kernel = norm_over_kernel self.eps = eps def forward(self, x: Tensor) -> Tensor: """Forward function for `NormedConv2d`.""" if not self.norm_over_kernel: weight_ = self.weight / ( self.weight.norm(dim=1, keepdim=True).pow(self.power) + self.eps) else: weight_ = self.weight / ( self.weight.view(self.weight.size(0), -1).norm( dim=1, keepdim=True).pow(self.power)[..., None, None] + self.eps) x_ = x / (x.norm(dim=1, keepdim=True).pow(self.power) + self.eps) x_ = x_ * self.tempearture if hasattr(self, 'conv2d_forward'): x_ = self.conv2d_forward(x_, weight_) else: if torch.__version__ >= '1.8': x_ = self._conv_forward(x_, weight_, self.bias) else: x_ = self._conv_forward(x_, weight_) return x_ ================================================ FILE: mmdet/models/layers/pixel_decoder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import Conv2d, ConvModule from mmengine.model import BaseModule, ModuleList, caffe2_xavier_init from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptMultiConfig from .positional_encoding import SinePositionalEncoding from .transformer import DetrTransformerEncoder @MODELS.register_module() class PixelDecoder(BaseModule): """Pixel decoder with a structure like fpn. Args: in_channels (list[int] | tuple[int]): Number of channels in the input feature maps. feat_channels (int): Number channels for feature. out_channels (int): Number channels for output. norm_cfg (:obj:`ConfigDict` or dict): Config for normalization. Defaults to dict(type='GN', num_groups=32). act_cfg (:obj:`ConfigDict` or dict): Config for activation. Defaults to dict(type='ReLU'). encoder (:obj:`ConfigDict` or dict): Config for transorformer encoder.Defaults to None. positional_encoding (:obj:`ConfigDict` or dict): Config for transformer encoder position encoding. Defaults to dict(type='SinePositionalEncoding', num_feats=128, normalize=True). init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, in_channels: Union[List[int], Tuple[int]], feat_channels: int, out_channels: int, norm_cfg: ConfigType = dict(type='GN', num_groups=32), act_cfg: ConfigType = dict(type='ReLU'), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) self.in_channels = in_channels self.num_inputs = len(in_channels) self.lateral_convs = ModuleList() self.output_convs = ModuleList() self.use_bias = norm_cfg is None for i in range(0, self.num_inputs - 1): lateral_conv = ConvModule( in_channels[i], feat_channels, kernel_size=1, bias=self.use_bias, norm_cfg=norm_cfg, act_cfg=None) output_conv = ConvModule( feat_channels, feat_channels, kernel_size=3, stride=1, padding=1, bias=self.use_bias, norm_cfg=norm_cfg, act_cfg=act_cfg) self.lateral_convs.append(lateral_conv) self.output_convs.append(output_conv) self.last_feat_conv = ConvModule( in_channels[-1], feat_channels, kernel_size=3, padding=1, stride=1, bias=self.use_bias, norm_cfg=norm_cfg, act_cfg=act_cfg) self.mask_feature = Conv2d( feat_channels, out_channels, kernel_size=3, stride=1, padding=1) def init_weights(self) -> None: """Initialize weights.""" for i in range(0, self.num_inputs - 2): caffe2_xavier_init(self.lateral_convs[i].conv, bias=0) caffe2_xavier_init(self.output_convs[i].conv, bias=0) caffe2_xavier_init(self.mask_feature, bias=0) caffe2_xavier_init(self.last_feat_conv, bias=0) def forward(self, feats: List[Tensor], batch_img_metas: List[dict]) -> Tuple[Tensor, Tensor]: """ Args: feats (list[Tensor]): Feature maps of each level. Each has shape of (batch_size, c, h, w). batch_img_metas (list[dict]): List of image information. Pass in for creating more accurate padding mask. Not used here. Returns: tuple[Tensor, Tensor]: a tuple containing the following: - mask_feature (Tensor): Shape (batch_size, c, h, w). - memory (Tensor): Output of last stage of backbone.\ Shape (batch_size, c, h, w). """ y = self.last_feat_conv(feats[-1]) for i in range(self.num_inputs - 2, -1, -1): x = feats[i] cur_feat = self.lateral_convs[i](x) y = cur_feat + \ F.interpolate(y, size=cur_feat.shape[-2:], mode='nearest') y = self.output_convs[i](y) mask_feature = self.mask_feature(y) memory = feats[-1] return mask_feature, memory @MODELS.register_module() class TransformerEncoderPixelDecoder(PixelDecoder): """Pixel decoder with transormer encoder inside. Args: in_channels (list[int] | tuple[int]): Number of channels in the input feature maps. feat_channels (int): Number channels for feature. out_channels (int): Number channels for output. norm_cfg (:obj:`ConfigDict` or dict): Config for normalization. Defaults to dict(type='GN', num_groups=32). act_cfg (:obj:`ConfigDict` or dict): Config for activation. Defaults to dict(type='ReLU'). encoder (:obj:`ConfigDict` or dict): Config for transformer encoder. Defaults to None. positional_encoding (:obj:`ConfigDict` or dict): Config for transformer encoder position encoding. Defaults to dict(num_feats=128, normalize=True). init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, in_channels: Union[List[int], Tuple[int]], feat_channels: int, out_channels: int, norm_cfg: ConfigType = dict(type='GN', num_groups=32), act_cfg: ConfigType = dict(type='ReLU'), encoder: ConfigType = None, positional_encoding: ConfigType = dict( num_feats=128, normalize=True), init_cfg: OptMultiConfig = None) -> None: super().__init__( in_channels=in_channels, feat_channels=feat_channels, out_channels=out_channels, norm_cfg=norm_cfg, act_cfg=act_cfg, init_cfg=init_cfg) self.last_feat_conv = None self.encoder = DetrTransformerEncoder(**encoder) self.encoder_embed_dims = self.encoder.embed_dims assert self.encoder_embed_dims == feat_channels, 'embed_dims({}) of ' \ 'tranformer encoder must equal to feat_channels({})'.format( feat_channels, self.encoder_embed_dims) self.positional_encoding = SinePositionalEncoding( **positional_encoding) self.encoder_in_proj = Conv2d( in_channels[-1], feat_channels, kernel_size=1) self.encoder_out_proj = ConvModule( feat_channels, feat_channels, kernel_size=3, stride=1, padding=1, bias=self.use_bias, norm_cfg=norm_cfg, act_cfg=act_cfg) def init_weights(self) -> None: """Initialize weights.""" for i in range(0, self.num_inputs - 2): caffe2_xavier_init(self.lateral_convs[i].conv, bias=0) caffe2_xavier_init(self.output_convs[i].conv, bias=0) caffe2_xavier_init(self.mask_feature, bias=0) caffe2_xavier_init(self.encoder_in_proj, bias=0) caffe2_xavier_init(self.encoder_out_proj.conv, bias=0) for p in self.encoder.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) def forward(self, feats: List[Tensor], batch_img_metas: List[dict]) -> Tuple[Tensor, Tensor]: """ Args: feats (list[Tensor]): Feature maps of each level. Each has shape of (batch_size, c, h, w). batch_img_metas (list[dict]): List of image information. Pass in for creating more accurate padding mask. Returns: tuple: a tuple containing the following: - mask_feature (Tensor): shape (batch_size, c, h, w). - memory (Tensor): shape (batch_size, c, h, w). """ feat_last = feats[-1] bs, c, h, w = feat_last.shape input_img_h, input_img_w = batch_img_metas[0]['batch_input_shape'] padding_mask = feat_last.new_ones((bs, input_img_h, input_img_w), dtype=torch.float32) for i in range(bs): img_h, img_w = batch_img_metas[i]['img_shape'] padding_mask[i, :img_h, :img_w] = 0 padding_mask = F.interpolate( padding_mask.unsqueeze(1), size=feat_last.shape[-2:], mode='nearest').to(torch.bool).squeeze(1) pos_embed = self.positional_encoding(padding_mask) feat_last = self.encoder_in_proj(feat_last) # (batch_size, c, h, w) -> (batch_size, num_queries, c) feat_last = feat_last.flatten(2).permute(0, 2, 1) pos_embed = pos_embed.flatten(2).permute(0, 2, 1) # (batch_size, h, w) -> (batch_size, h*w) padding_mask = padding_mask.flatten(1) memory = self.encoder( query=feat_last, query_pos=pos_embed, key_padding_mask=padding_mask) # (batch_size, num_queries, c) -> (batch_size, c, h, w) memory = memory.permute(0, 2, 1).view(bs, self.encoder_embed_dims, h, w) y = self.encoder_out_proj(memory) for i in range(self.num_inputs - 2, -1, -1): x = feats[i] cur_feat = self.lateral_convs[i](x) y = cur_feat + \ F.interpolate(y, size=cur_feat.shape[-2:], mode='nearest') y = self.output_convs[i](y) mask_feature = self.mask_feature(y) return mask_feature, memory ================================================ FILE: mmdet/models/layers/positional_encoding.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math import torch import torch.nn as nn from mmengine.model import BaseModule from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import MultiConfig, OptMultiConfig @MODELS.register_module() class SinePositionalEncoding(BaseModule): """Position encoding with sine and cosine functions. See `End-to-End Object Detection with Transformers `_ for details. Args: num_feats (int): The feature dimension for each position along x-axis or y-axis. Note the final returned dimension for each position is 2 times of this value. temperature (int, optional): The temperature used for scaling the position embedding. Defaults to 10000. normalize (bool, optional): Whether to normalize the position embedding. Defaults to False. scale (float, optional): A scale factor that scales the position embedding. The scale will be used only when `normalize` is True. Defaults to 2*pi. eps (float, optional): A value added to the denominator for numerical stability. Defaults to 1e-6. offset (float): offset add to embed when do the normalization. Defaults to 0. init_cfg (dict or list[dict], optional): Initialization config dict. Defaults to None """ def __init__(self, num_feats: int, temperature: int = 10000, normalize: bool = False, scale: float = 2 * math.pi, eps: float = 1e-6, offset: float = 0., init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) if normalize: assert isinstance(scale, (float, int)), 'when normalize is set,' \ 'scale should be provided and in float or int type, ' \ f'found {type(scale)}' self.num_feats = num_feats self.temperature = temperature self.normalize = normalize self.scale = scale self.eps = eps self.offset = offset def forward(self, mask: Tensor) -> Tensor: """Forward function for `SinePositionalEncoding`. Args: mask (Tensor): ByteTensor mask. Non-zero values representing ignored positions, while zero values means valid positions for this image. Shape [bs, h, w]. Returns: pos (Tensor): Returned position embedding with shape [bs, num_feats*2, h, w]. """ # For convenience of exporting to ONNX, it's required to convert # `masks` from bool to int. mask = mask.to(torch.int) not_mask = 1 - mask # logical_not y_embed = not_mask.cumsum(1, dtype=torch.float32) x_embed = not_mask.cumsum(2, dtype=torch.float32) if self.normalize: y_embed = (y_embed + self.offset) / \ (y_embed[:, -1:, :] + self.eps) * self.scale x_embed = (x_embed + self.offset) / \ (x_embed[:, :, -1:] + self.eps) * self.scale dim_t = torch.arange( self.num_feats, dtype=torch.float32, device=mask.device) dim_t = self.temperature**(2 * (dim_t // 2) / self.num_feats) pos_x = x_embed[:, :, :, None] / dim_t pos_y = y_embed[:, :, :, None] / dim_t # use `view` instead of `flatten` for dynamically exporting to ONNX B, H, W = mask.size() pos_x = torch.stack( (pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).view(B, H, W, -1) pos_y = torch.stack( (pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).view(B, H, W, -1) pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) return pos def __repr__(self) -> str: """str: a string that describes the module""" repr_str = self.__class__.__name__ repr_str += f'(num_feats={self.num_feats}, ' repr_str += f'temperature={self.temperature}, ' repr_str += f'normalize={self.normalize}, ' repr_str += f'scale={self.scale}, ' repr_str += f'eps={self.eps})' return repr_str @MODELS.register_module() class LearnedPositionalEncoding(BaseModule): """Position embedding with learnable embedding weights. Args: num_feats (int): The feature dimension for each position along x-axis or y-axis. The final returned dimension for each position is 2 times of this value. row_num_embed (int, optional): The dictionary size of row embeddings. Defaults to 50. col_num_embed (int, optional): The dictionary size of col embeddings. Defaults to 50. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__( self, num_feats: int, row_num_embed: int = 50, col_num_embed: int = 50, init_cfg: MultiConfig = dict(type='Uniform', layer='Embedding') ) -> None: super().__init__(init_cfg=init_cfg) self.row_embed = nn.Embedding(row_num_embed, num_feats) self.col_embed = nn.Embedding(col_num_embed, num_feats) self.num_feats = num_feats self.row_num_embed = row_num_embed self.col_num_embed = col_num_embed def forward(self, mask: Tensor) -> Tensor: """Forward function for `LearnedPositionalEncoding`. Args: mask (Tensor): ByteTensor mask. Non-zero values representing ignored positions, while zero values means valid positions for this image. Shape [bs, h, w]. Returns: pos (Tensor): Returned position embedding with shape [bs, num_feats*2, h, w]. """ h, w = mask.shape[-2:] x = torch.arange(w, device=mask.device) y = torch.arange(h, device=mask.device) x_embed = self.col_embed(x) y_embed = self.row_embed(y) pos = torch.cat( (x_embed.unsqueeze(0).repeat(h, 1, 1), y_embed.unsqueeze(1).repeat( 1, w, 1)), dim=-1).permute(2, 0, 1).unsqueeze(0).repeat(mask.shape[0], 1, 1, 1) return pos def __repr__(self) -> str: """str: a string that describes the module""" repr_str = self.__class__.__name__ repr_str += f'(num_feats={self.num_feats}, ' repr_str += f'row_num_embed={self.row_num_embed}, ' repr_str += f'col_num_embed={self.col_num_embed})' return repr_str ================================================ FILE: mmdet/models/layers/res_layer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional from mmcv.cnn import build_conv_layer, build_norm_layer from mmengine.model import BaseModule, Sequential from torch import Tensor from torch import nn as nn from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig class ResLayer(Sequential): """ResLayer to build ResNet style backbone. Args: block (nn.Module): block used to build ResLayer. inplanes (int): inplanes of block. planes (int): planes of block. num_blocks (int): number of blocks. stride (int): stride of the first block. Defaults to 1 avg_down (bool): Use AvgPool instead of stride conv when downsampling in the bottleneck. Defaults to False conv_cfg (dict): dictionary to construct and config conv layer. Defaults to None norm_cfg (dict): dictionary to construct and config norm layer. Defaults to dict(type='BN') downsample_first (bool): Downsample at the first block or last block. False for Hourglass, True for ResNet. Defaults to True """ def __init__(self, block: BaseModule, inplanes: int, planes: int, num_blocks: int, stride: int = 1, avg_down: bool = False, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN'), downsample_first: bool = True, **kwargs) -> None: self.block = block downsample = None if stride != 1 or inplanes != planes * block.expansion: downsample = [] conv_stride = stride if avg_down: conv_stride = 1 downsample.append( nn.AvgPool2d( kernel_size=stride, stride=stride, ceil_mode=True, count_include_pad=False)) downsample.extend([ build_conv_layer( conv_cfg, inplanes, planes * block.expansion, kernel_size=1, stride=conv_stride, bias=False), build_norm_layer(norm_cfg, planes * block.expansion)[1] ]) downsample = nn.Sequential(*downsample) layers = [] if downsample_first: layers.append( block( inplanes=inplanes, planes=planes, stride=stride, downsample=downsample, conv_cfg=conv_cfg, norm_cfg=norm_cfg, **kwargs)) inplanes = planes * block.expansion for _ in range(1, num_blocks): layers.append( block( inplanes=inplanes, planes=planes, stride=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, **kwargs)) else: # downsample_first=False is for HourglassModule for _ in range(num_blocks - 1): layers.append( block( inplanes=inplanes, planes=inplanes, stride=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, **kwargs)) layers.append( block( inplanes=inplanes, planes=planes, stride=stride, downsample=downsample, conv_cfg=conv_cfg, norm_cfg=norm_cfg, **kwargs)) super().__init__(*layers) class SimplifiedBasicBlock(BaseModule): """Simplified version of original basic residual block. This is used in `SCNet `_. - Norm layer is now optional - Last ReLU in forward function is removed """ expansion = 1 def __init__(self, inplanes: int, planes: int, stride: int = 1, dilation: int = 1, downsample: Optional[Sequential] = None, style: ConfigType = 'pytorch', with_cp: bool = False, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN'), dcn: OptConfigType = None, plugins: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) assert dcn is None, 'Not implemented yet.' assert plugins is None, 'Not implemented yet.' assert not with_cp, 'Not implemented yet.' self.with_norm = norm_cfg is not None with_bias = True if norm_cfg is None else False self.conv1 = build_conv_layer( conv_cfg, inplanes, planes, 3, stride=stride, padding=dilation, dilation=dilation, bias=with_bias) if self.with_norm: self.norm1_name, norm1 = build_norm_layer( norm_cfg, planes, postfix=1) self.add_module(self.norm1_name, norm1) self.conv2 = build_conv_layer( conv_cfg, planes, planes, 3, padding=1, bias=with_bias) if self.with_norm: self.norm2_name, norm2 = build_norm_layer( norm_cfg, planes, postfix=2) self.add_module(self.norm2_name, norm2) self.relu = nn.ReLU(inplace=True) self.downsample = downsample self.stride = stride self.dilation = dilation self.with_cp = with_cp @property def norm1(self) -> Optional[BaseModule]: """nn.Module: normalization layer after the first convolution layer""" return getattr(self, self.norm1_name) if self.with_norm else None @property def norm2(self) -> Optional[BaseModule]: """nn.Module: normalization layer after the second convolution layer""" return getattr(self, self.norm2_name) if self.with_norm else None def forward(self, x: Tensor) -> Tensor: """Forward function for SimplifiedBasicBlock.""" identity = x out = self.conv1(x) if self.with_norm: out = self.norm1(out) out = self.relu(out) out = self.conv2(out) if self.with_norm: out = self.norm2(out) if self.downsample is not None: identity = self.downsample(x) out += identity return out ================================================ FILE: mmdet/models/layers/se_layer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn from mmcv.cnn import ConvModule from mmengine.model import BaseModule from mmengine.utils import digit_version, is_tuple_of from torch import Tensor from mmdet.utils import MultiConfig, OptConfigType, OptMultiConfig class SELayer(BaseModule): """Squeeze-and-Excitation Module. Args: channels (int): The input (and output) channels of the SE layer. ratio (int): Squeeze ratio in SELayer, the intermediate channel will be ``int(channels/ratio)``. Defaults to 16. conv_cfg (None or dict): Config dict for convolution layer. Defaults to None, which means using conv2d. act_cfg (dict or Sequence[dict]): Config dict for activation layer. If act_cfg is a dict, two activation layers will be configurated by this dict. If act_cfg is a sequence of dicts, the first activation layer will be configurated by the first dict and the second activation layer will be configurated by the second dict. Defaults to (dict(type='ReLU'), dict(type='Sigmoid')) init_cfg (dict or list[dict], optional): Initialization config dict. Defaults to None """ def __init__(self, channels: int, ratio: int = 16, conv_cfg: OptConfigType = None, act_cfg: MultiConfig = (dict(type='ReLU'), dict(type='Sigmoid')), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) if isinstance(act_cfg, dict): act_cfg = (act_cfg, act_cfg) assert len(act_cfg) == 2 assert is_tuple_of(act_cfg, dict) self.global_avgpool = nn.AdaptiveAvgPool2d(1) self.conv1 = ConvModule( in_channels=channels, out_channels=int(channels / ratio), kernel_size=1, stride=1, conv_cfg=conv_cfg, act_cfg=act_cfg[0]) self.conv2 = ConvModule( in_channels=int(channels / ratio), out_channels=channels, kernel_size=1, stride=1, conv_cfg=conv_cfg, act_cfg=act_cfg[1]) def forward(self, x: Tensor) -> Tensor: """Forward function for SELayer.""" out = self.global_avgpool(x) out = self.conv1(out) out = self.conv2(out) return x * out class DyReLU(BaseModule): """Dynamic ReLU (DyReLU) module. See `Dynamic ReLU `_ for details. Current implementation is specialized for task-aware attention in DyHead. HSigmoid arguments in default act_cfg follow DyHead official code. https://github.com/microsoft/DynamicHead/blob/master/dyhead/dyrelu.py Args: channels (int): The input (and output) channels of DyReLU module. ratio (int): Squeeze ratio in Squeeze-and-Excitation-like module, the intermediate channel will be ``int(channels/ratio)``. Defaults to 4. conv_cfg (None or dict): Config dict for convolution layer. Defaults to None, which means using conv2d. act_cfg (dict or Sequence[dict]): Config dict for activation layer. If act_cfg is a dict, two activation layers will be configurated by this dict. If act_cfg is a sequence of dicts, the first activation layer will be configurated by the first dict and the second activation layer will be configurated by the second dict. Defaults to (dict(type='ReLU'), dict(type='HSigmoid', bias=3.0, divisor=6.0)) init_cfg (dict or list[dict], optional): Initialization config dict. Defaults to None """ def __init__(self, channels: int, ratio: int = 4, conv_cfg: OptConfigType = None, act_cfg: MultiConfig = (dict(type='ReLU'), dict( type='HSigmoid', bias=3.0, divisor=6.0)), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) if isinstance(act_cfg, dict): act_cfg = (act_cfg, act_cfg) assert len(act_cfg) == 2 assert is_tuple_of(act_cfg, dict) self.channels = channels self.expansion = 4 # for a1, b1, a2, b2 self.global_avgpool = nn.AdaptiveAvgPool2d(1) self.conv1 = ConvModule( in_channels=channels, out_channels=int(channels / ratio), kernel_size=1, stride=1, conv_cfg=conv_cfg, act_cfg=act_cfg[0]) self.conv2 = ConvModule( in_channels=int(channels / ratio), out_channels=channels * self.expansion, kernel_size=1, stride=1, conv_cfg=conv_cfg, act_cfg=act_cfg[1]) def forward(self, x: Tensor) -> Tensor: """Forward function.""" coeffs = self.global_avgpool(x) coeffs = self.conv1(coeffs) coeffs = self.conv2(coeffs) - 0.5 # value range: [-0.5, 0.5] a1, b1, a2, b2 = torch.split(coeffs, self.channels, dim=1) a1 = a1 * 2.0 + 1.0 # [-1.0, 1.0] + 1.0 a2 = a2 * 2.0 # [-1.0, 1.0] out = torch.max(x * a1 + b1, x * a2 + b2) return out class ChannelAttention(BaseModule): """Channel attention Module. Args: channels (int): The input (and output) channels of the attention layer. init_cfg (dict or list[dict], optional): Initialization config dict. Defaults to None """ def __init__(self, channels: int, init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) self.global_avgpool = nn.AdaptiveAvgPool2d(1) self.fc = nn.Conv2d(channels, channels, 1, 1, 0, bias=True) if digit_version(torch.__version__) < (1, 7, 0): self.act = nn.Hardsigmoid() else: self.act = nn.Hardsigmoid(inplace=True) def forward(self, x: Tensor) -> Tensor: """Forward function for ChannelAttention.""" with torch.cuda.amp.autocast(enabled=False): out = self.global_avgpool(x) out = self.fc(out) out = self.act(out) return x * out ================================================ FILE: mmdet/models/layers/transformer/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .conditional_detr_layers import (ConditionalDetrTransformerDecoder, ConditionalDetrTransformerDecoderLayer) from .dab_detr_layers import (DABDetrTransformerDecoder, DABDetrTransformerDecoderLayer, DABDetrTransformerEncoder) from .deformable_detr_layers import (DeformableDetrTransformerDecoder, DeformableDetrTransformerDecoderLayer, DeformableDetrTransformerEncoder, DeformableDetrTransformerEncoderLayer) from .detr_layers import (DetrTransformerDecoder, DetrTransformerDecoderLayer, DetrTransformerEncoder, DetrTransformerEncoderLayer) from .dino_layers import CdnQueryGenerator, DinoTransformerDecoder from .mask2former_layers import (Mask2FormerTransformerDecoder, Mask2FormerTransformerDecoderLayer, Mask2FormerTransformerEncoder) from .utils import (MLP, AdaptivePadding, ConditionalAttention, DynamicConv, PatchEmbed, PatchMerging, coordinate_to_encoding, inverse_sigmoid, nchw_to_nlc, nlc_to_nchw) __all__ = [ 'nlc_to_nchw', 'nchw_to_nlc', 'AdaptivePadding', 'PatchEmbed', 'PatchMerging', 'inverse_sigmoid', 'DynamicConv', 'MLP', 'DetrTransformerEncoder', 'DetrTransformerDecoder', 'DetrTransformerEncoderLayer', 'DetrTransformerDecoderLayer', 'DeformableDetrTransformerEncoder', 'DeformableDetrTransformerDecoder', 'DeformableDetrTransformerEncoderLayer', 'DeformableDetrTransformerDecoderLayer', 'coordinate_to_encoding', 'ConditionalAttention', 'DABDetrTransformerDecoderLayer', 'DABDetrTransformerDecoder', 'DABDetrTransformerEncoder', 'ConditionalDetrTransformerDecoder', 'ConditionalDetrTransformerDecoderLayer', 'DinoTransformerDecoder', 'CdnQueryGenerator', 'Mask2FormerTransformerEncoder', 'Mask2FormerTransformerDecoderLayer', 'Mask2FormerTransformerDecoder' ] ================================================ FILE: mmdet/models/layers/transformer/conditional_detr_layers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.cnn import build_norm_layer from mmcv.cnn.bricks.transformer import FFN from torch import Tensor from torch.nn import ModuleList from .detr_layers import DetrTransformerDecoder, DetrTransformerDecoderLayer from .utils import MLP, ConditionalAttention, coordinate_to_encoding class ConditionalDetrTransformerDecoder(DetrTransformerDecoder): """Decoder of Conditional DETR.""" def _init_layers(self) -> None: """Initialize decoder layers and other layers.""" self.layers = ModuleList([ ConditionalDetrTransformerDecoderLayer(**self.layer_cfg) for _ in range(self.num_layers) ]) self.embed_dims = self.layers[0].embed_dims self.post_norm = build_norm_layer(self.post_norm_cfg, self.embed_dims)[1] # conditional detr affline self.query_scale = MLP(self.embed_dims, self.embed_dims, self.embed_dims, 2) self.ref_point_head = MLP(self.embed_dims, self.embed_dims, 2, 2) # we have substitute 'qpos_proj' with 'qpos_sine_proj' except for # the first decoder layer), so 'qpos_proj' should be deleted # in other layers. for layer_id in range(self.num_layers - 1): self.layers[layer_id + 1].cross_attn.qpos_proj = None def forward(self, query: Tensor, key: Tensor = None, query_pos: Tensor = None, key_pos: Tensor = None, key_padding_mask: Tensor = None): """Forward function of decoder. Args: query (Tensor): The input query with shape (bs, num_queries, dim). key (Tensor): The input key with shape (bs, num_keys, dim) If `None`, the `query` will be used. Defaults to `None`. query_pos (Tensor): The positional encoding for `query`, with the same shape as `query`. If not `None`, it will be added to `query` before forward function. Defaults to `None`. key_pos (Tensor): The positional encoding for `key`, with the same shape as `key`. If not `None`, it will be added to `key` before forward function. If `None`, and `query_pos` has the same shape as `key`, then `query_pos` will be used as `key_pos`. Defaults to `None`. key_padding_mask (Tensor): ByteTensor with shape (bs, num_keys). Defaults to `None`. Returns: List[Tensor]: forwarded results with shape (num_decoder_layers, bs, num_queries, dim) if `return_intermediate` is True, otherwise with shape (1, bs, num_queries, dim). References with shape (bs, num_queries, 2). """ reference_unsigmoid = self.ref_point_head( query_pos) # [bs, num_queries, 2] reference = reference_unsigmoid.sigmoid() reference_xy = reference[..., :2] intermediate = [] for layer_id, layer in enumerate(self.layers): if layer_id == 0: pos_transformation = 1 else: pos_transformation = self.query_scale(query) # get sine embedding for the query reference ref_sine_embed = coordinate_to_encoding(coord_tensor=reference_xy) # apply transformation ref_sine_embed = ref_sine_embed * pos_transformation query = layer( query, key=key, query_pos=query_pos, key_pos=key_pos, key_padding_mask=key_padding_mask, ref_sine_embed=ref_sine_embed, is_first=(layer_id == 0)) if self.return_intermediate: intermediate.append(self.post_norm(query)) if self.return_intermediate: return torch.stack(intermediate), reference query = self.post_norm(query) return query.unsqueeze(0), reference class ConditionalDetrTransformerDecoderLayer(DetrTransformerDecoderLayer): """Implements decoder layer in Conditional DETR transformer.""" def _init_layers(self): """Initialize self-attention, cross-attention, FFN, and normalization.""" self.self_attn = ConditionalAttention(**self.self_attn_cfg) self.cross_attn = ConditionalAttention(**self.cross_attn_cfg) self.embed_dims = self.self_attn.embed_dims self.ffn = FFN(**self.ffn_cfg) norms_list = [ build_norm_layer(self.norm_cfg, self.embed_dims)[1] for _ in range(3) ] self.norms = ModuleList(norms_list) def forward(self, query: Tensor, key: Tensor = None, query_pos: Tensor = None, key_pos: Tensor = None, self_attn_masks: Tensor = None, cross_attn_masks: Tensor = None, key_padding_mask: Tensor = None, ref_sine_embed: Tensor = None, is_first: bool = False): """ Args: query (Tensor): The input query, has shape (bs, num_queries, dim) key (Tensor, optional): The input key, has shape (bs, num_keys, dim). If `None`, the `query` will be used. Defaults to `None`. query_pos (Tensor, optional): The positional encoding for `query`, has the same shape as `query`. If not `None`, it will be added to `query` before forward function. Defaults to `None`. ref_sine_embed (Tensor): The positional encoding for query in cross attention, with the same shape as `x`. Defaults to None. key_pos (Tensor, optional): The positional encoding for `key`, has the same shape as `key`. If not None, it will be added to `key` before forward function. If None, and `query_pos` has the same shape as `key`, then `query_pos` will be used for `key_pos`. Defaults to None. self_attn_masks (Tensor, optional): ByteTensor mask, has shape (num_queries, num_keys), Same in `nn.MultiheadAttention. forward`. Defaults to None. cross_attn_masks (Tensor, optional): ByteTensor mask, has shape (num_queries, num_keys), Same in `nn.MultiheadAttention. forward`. Defaults to None. key_padding_mask (Tensor, optional): ByteTensor, has shape (bs, num_keys). Defaults to None. is_first (bool): A indicator to tell whether the current layer is the first layer of the decoder. Defaults to False. Returns: Tensor: Forwarded results, has shape (bs, num_queries, dim). """ query = self.self_attn( query=query, key=query, query_pos=query_pos, key_pos=query_pos, attn_mask=self_attn_masks) query = self.norms[0](query) query = self.cross_attn( query=query, key=key, query_pos=query_pos, key_pos=key_pos, attn_mask=cross_attn_masks, key_padding_mask=key_padding_mask, ref_sine_embed=ref_sine_embed, is_first=is_first) query = self.norms[1](query) query = self.ffn(query) query = self.norms[2](query) return query ================================================ FILE: mmdet/models/layers/transformer/dab_detr_layers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List import torch import torch.nn as nn from mmcv.cnn import build_norm_layer from mmcv.cnn.bricks.transformer import FFN from mmengine.model import ModuleList from torch import Tensor from .detr_layers import (DetrTransformerDecoder, DetrTransformerDecoderLayer, DetrTransformerEncoder, DetrTransformerEncoderLayer) from .utils import (MLP, ConditionalAttention, coordinate_to_encoding, inverse_sigmoid) class DABDetrTransformerDecoderLayer(DetrTransformerDecoderLayer): """Implements decoder layer in DAB-DETR transformer.""" def _init_layers(self): """Initialize self-attention, cross-attention, FFN, normalization and others.""" self.self_attn = ConditionalAttention(**self.self_attn_cfg) self.cross_attn = ConditionalAttention(**self.cross_attn_cfg) self.embed_dims = self.self_attn.embed_dims self.ffn = FFN(**self.ffn_cfg) norms_list = [ build_norm_layer(self.norm_cfg, self.embed_dims)[1] for _ in range(3) ] self.norms = ModuleList(norms_list) self.keep_query_pos = self.cross_attn.keep_query_pos def forward(self, query: Tensor, key: Tensor, query_pos: Tensor, key_pos: Tensor, ref_sine_embed: Tensor = None, self_attn_masks: Tensor = None, cross_attn_masks: Tensor = None, key_padding_mask: Tensor = None, is_first: bool = False, **kwargs) -> Tensor: """ Args: query (Tensor): The input query with shape [bs, num_queries, dim]. key (Tensor): The key tensor with shape [bs, num_keys, dim]. query_pos (Tensor): The positional encoding for query in self attention, with the same shape as `x`. key_pos (Tensor): The positional encoding for `key`, with the same shape as `key`. ref_sine_embed (Tensor): The positional encoding for query in cross attention, with the same shape as `x`. Defaults to None. self_attn_masks (Tensor): ByteTensor mask with shape [num_queries, num_keys]. Same in `nn.MultiheadAttention.forward`. Defaults to None. cross_attn_masks (Tensor): ByteTensor mask with shape [num_queries, num_keys]. Same in `nn.MultiheadAttention.forward`. Defaults to None. key_padding_mask (Tensor): ByteTensor with shape [bs, num_keys]. Defaults to None. is_first (bool): A indicator to tell whether the current layer is the first layer of the decoder. Defaults to False. Returns: Tensor: forwarded results with shape [bs, num_queries, dim]. """ query = self.self_attn( query=query, key=query, query_pos=query_pos, key_pos=query_pos, attn_mask=self_attn_masks, **kwargs) query = self.norms[0](query) query = self.cross_attn( query=query, key=key, query_pos=query_pos, key_pos=key_pos, ref_sine_embed=ref_sine_embed, attn_mask=cross_attn_masks, key_padding_mask=key_padding_mask, is_first=is_first, **kwargs) query = self.norms[1](query) query = self.ffn(query) query = self.norms[2](query) return query class DABDetrTransformerDecoder(DetrTransformerDecoder): """Decoder of DAB-DETR. Args: query_dim (int): The last dimension of query pos, 4 for anchor format, 2 for point format. Defaults to 4. query_scale_type (str): Type of transformation applied to content query. Defaults to `cond_elewise`. with_modulated_hw_attn (bool): Whether to inject h&w info during cross conditional attention. Defaults to True. """ def __init__(self, *args, query_dim: int = 4, query_scale_type: str = 'cond_elewise', with_modulated_hw_attn: bool = True, **kwargs): self.query_dim = query_dim self.query_scale_type = query_scale_type self.with_modulated_hw_attn = with_modulated_hw_attn super().__init__(*args, **kwargs) def _init_layers(self): """Initialize decoder layers and other layers.""" assert self.query_dim in [2, 4], \ f'{"dab-detr only supports anchor prior or reference point prior"}' assert self.query_scale_type in [ 'cond_elewise', 'cond_scalar', 'fix_elewise' ] self.layers = ModuleList([ DABDetrTransformerDecoderLayer(**self.layer_cfg) for _ in range(self.num_layers) ]) embed_dims = self.layers[0].embed_dims self.embed_dims = embed_dims self.post_norm = build_norm_layer(self.post_norm_cfg, embed_dims)[1] if self.query_scale_type == 'cond_elewise': self.query_scale = MLP(embed_dims, embed_dims, embed_dims, 2) elif self.query_scale_type == 'cond_scalar': self.query_scale = MLP(embed_dims, embed_dims, 1, 2) elif self.query_scale_type == 'fix_elewise': self.query_scale = nn.Embedding(self.num_layers, embed_dims) else: raise NotImplementedError('Unknown query_scale_type: {}'.format( self.query_scale_type)) self.ref_point_head = MLP(self.query_dim // 2 * embed_dims, embed_dims, embed_dims, 2) if self.with_modulated_hw_attn and self.query_dim == 4: self.ref_anchor_head = MLP(embed_dims, embed_dims, 2, 2) self.keep_query_pos = self.layers[0].keep_query_pos if not self.keep_query_pos: for layer_id in range(self.num_layers - 1): self.layers[layer_id + 1].cross_attn.qpos_proj = None def forward(self, query: Tensor, key: Tensor, query_pos: Tensor, key_pos: Tensor, reg_branches: nn.Module, key_padding_mask: Tensor = None, **kwargs) -> List[Tensor]: """Forward function of decoder. Args: query (Tensor): The input query with shape (bs, num_queries, dim). key (Tensor): The input key with shape (bs, num_keys, dim). query_pos (Tensor): The positional encoding for `query`, with the same shape as `query`. key_pos (Tensor): The positional encoding for `key`, with the same shape as `key`. reg_branches (nn.Module): The regression branch for dynamically updating references in each layer. key_padding_mask (Tensor): ByteTensor with shape (bs, num_keys). Defaults to `None`. Returns: List[Tensor]: forwarded results with shape (num_decoder_layers, bs, num_queries, dim) if `return_intermediate` is True, otherwise with shape (1, bs, num_queries, dim). references with shape (num_decoder_layers, bs, num_queries, 2/4). """ output = query unsigmoid_references = query_pos reference_points = unsigmoid_references.sigmoid() intermediate_reference_points = [reference_points] intermediate = [] for layer_id, layer in enumerate(self.layers): obj_center = reference_points[..., :self.query_dim] ref_sine_embed = coordinate_to_encoding( coord_tensor=obj_center, num_feats=self.embed_dims // 2) query_pos = self.ref_point_head( ref_sine_embed) # [bs, nq, 2c] -> [bs, nq, c] # For the first decoder layer, do not apply transformation if self.query_scale_type != 'fix_elewise': if layer_id == 0: pos_transformation = 1 else: pos_transformation = self.query_scale(output) else: pos_transformation = self.query_scale.weight[layer_id] # apply transformation ref_sine_embed = ref_sine_embed[ ..., :self.embed_dims] * pos_transformation # modulated height and weight attention if self.with_modulated_hw_attn: assert obj_center.size(-1) == 4 ref_hw = self.ref_anchor_head(output).sigmoid() ref_sine_embed[..., self.embed_dims // 2:] *= \ (ref_hw[..., 0] / obj_center[..., 2]).unsqueeze(-1) ref_sine_embed[..., : self.embed_dims // 2] *= \ (ref_hw[..., 1] / obj_center[..., 3]).unsqueeze(-1) output = layer( output, key, query_pos=query_pos, ref_sine_embed=ref_sine_embed, key_pos=key_pos, key_padding_mask=key_padding_mask, is_first=(layer_id == 0), **kwargs) # iter update tmp_reg_preds = reg_branches(output) tmp_reg_preds[..., :self.query_dim] += inverse_sigmoid( reference_points) new_reference_points = tmp_reg_preds[ ..., :self.query_dim].sigmoid() if layer_id != self.num_layers - 1: intermediate_reference_points.append(new_reference_points) reference_points = new_reference_points.detach() if self.return_intermediate: intermediate.append(self.post_norm(output)) output = self.post_norm(output) if self.return_intermediate: return [ torch.stack(intermediate), torch.stack(intermediate_reference_points), ] else: return [ output.unsqueeze(0), torch.stack(intermediate_reference_points) ] class DABDetrTransformerEncoder(DetrTransformerEncoder): """Encoder of DAB-DETR.""" def _init_layers(self): """Initialize encoder layers.""" self.layers = ModuleList([ DetrTransformerEncoderLayer(**self.layer_cfg) for _ in range(self.num_layers) ]) embed_dims = self.layers[0].embed_dims self.embed_dims = embed_dims self.query_scale = MLP(embed_dims, embed_dims, embed_dims, 2) def forward(self, query: Tensor, query_pos: Tensor, key_padding_mask: Tensor, **kwargs): """Forward function of encoder. Args: query (Tensor): Input queries of encoder, has shape (bs, num_queries, dim). query_pos (Tensor): The positional embeddings of the queries, has shape (bs, num_feat_points, dim). key_padding_mask (Tensor): ByteTensor, the key padding mask of the queries, has shape (bs, num_feat_points). Returns: Tensor: With shape (num_queries, bs, dim). """ for layer in self.layers: pos_scales = self.query_scale(query) query = layer( query, query_pos=query_pos * pos_scales, key_padding_mask=key_padding_mask, **kwargs) return query ================================================ FILE: mmdet/models/layers/transformer/deformable_detr_layers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple, Union import torch from mmcv.cnn import build_norm_layer from mmcv.cnn.bricks.transformer import FFN, MultiheadAttention from mmcv.ops import MultiScaleDeformableAttention from mmengine.model import ModuleList from torch import Tensor, nn from .detr_layers import (DetrTransformerDecoder, DetrTransformerDecoderLayer, DetrTransformerEncoder, DetrTransformerEncoderLayer) from .utils import inverse_sigmoid class DeformableDetrTransformerEncoder(DetrTransformerEncoder): """Transformer encoder of Deformable DETR.""" def _init_layers(self) -> None: """Initialize encoder layers.""" self.layers = ModuleList([ DeformableDetrTransformerEncoderLayer(**self.layer_cfg) for _ in range(self.num_layers) ]) self.embed_dims = self.layers[0].embed_dims def forward(self, query: Tensor, query_pos: Tensor, key_padding_mask: Tensor, spatial_shapes: Tensor, level_start_index: Tensor, valid_ratios: Tensor, **kwargs) -> Tensor: """Forward function of Transformer encoder. Args: query (Tensor): The input query, has shape (bs, num_queries, dim). query_pos (Tensor): The positional encoding for query, has shape (bs, num_queries, dim). key_padding_mask (Tensor): The `key_padding_mask` of `self_attn` input. ByteTensor, has shape (bs, num_queries). spatial_shapes (Tensor): Spatial shapes of features in all levels, has shape (num_levels, 2), last dimension represents (h, w). level_start_index (Tensor): The start index of each level. A tensor has shape (num_levels, ) and can be represented as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. valid_ratios (Tensor): The ratios of the valid width and the valid height relative to the width and the height of features in all levels, has shape (bs, num_levels, 2). Returns: Tensor: Output queries of Transformer encoder, which is also called 'encoder output embeddings' or 'memory', has shape (bs, num_queries, dim) """ reference_points = self.get_encoder_reference_points( spatial_shapes, valid_ratios, device=query.device) for layer in self.layers: query = layer( query=query, query_pos=query_pos, key_padding_mask=key_padding_mask, spatial_shapes=spatial_shapes, level_start_index=level_start_index, valid_ratios=valid_ratios, reference_points=reference_points, **kwargs) return query @staticmethod def get_encoder_reference_points( spatial_shapes: Tensor, valid_ratios: Tensor, device: Union[torch.device, str]) -> Tensor: """Get the reference points used in encoder. Args: spatial_shapes (Tensor): Spatial shapes of features in all levels, has shape (num_levels, 2), last dimension represents (h, w). valid_ratios (Tensor): The ratios of the valid width and the valid height relative to the width and the height of features in all levels, has shape (bs, num_levels, 2). device (obj:`device` or str): The device acquired by the `reference_points`. Returns: Tensor: Reference points used in decoder, has shape (bs, length, num_levels, 2). """ reference_points_list = [] for lvl, (H, W) in enumerate(spatial_shapes): ref_y, ref_x = torch.meshgrid( torch.linspace( 0.5, H - 0.5, H, dtype=torch.float32, device=device), torch.linspace( 0.5, W - 0.5, W, dtype=torch.float32, device=device)) ref_y = ref_y.reshape(-1)[None] / ( valid_ratios[:, None, lvl, 1] * H) ref_x = ref_x.reshape(-1)[None] / ( valid_ratios[:, None, lvl, 0] * W) ref = torch.stack((ref_x, ref_y), -1) reference_points_list.append(ref) reference_points = torch.cat(reference_points_list, 1) # [bs, sum(hw), num_level, 2] reference_points = reference_points[:, :, None] * valid_ratios[:, None] return reference_points class DeformableDetrTransformerDecoder(DetrTransformerDecoder): """Transformer Decoder of Deformable DETR.""" def _init_layers(self) -> None: """Initialize decoder layers.""" self.layers = ModuleList([ DeformableDetrTransformerDecoderLayer(**self.layer_cfg) for _ in range(self.num_layers) ]) self.embed_dims = self.layers[0].embed_dims if self.post_norm_cfg is not None: raise ValueError('There is not post_norm in ' f'{self._get_name()}') def forward(self, query: Tensor, query_pos: Tensor, value: Tensor, key_padding_mask: Tensor, reference_points: Tensor, spatial_shapes: Tensor, level_start_index: Tensor, valid_ratios: Tensor, reg_branches: Optional[nn.Module] = None, **kwargs) -> Tuple[Tensor]: """Forward function of Transformer decoder. Args: query (Tensor): The input queries, has shape (bs, num_queries, dim). query_pos (Tensor): The input positional query, has shape (bs, num_queries, dim). It will be added to `query` before forward function. value (Tensor): The input values, has shape (bs, num_value, dim). key_padding_mask (Tensor): The `key_padding_mask` of `cross_attn` input. ByteTensor, has shape (bs, num_value). reference_points (Tensor): The initial reference, has shape (bs, num_queries, 4) with the last dimension arranged as (cx, cy, w, h) when `as_two_stage` is `True`, otherwise has shape (bs, num_queries, 2) with the last dimension arranged as (cx, cy). spatial_shapes (Tensor): Spatial shapes of features in all levels, has shape (num_levels, 2), last dimension represents (h, w). level_start_index (Tensor): The start index of each level. A tensor has shape (num_levels, ) and can be represented as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. valid_ratios (Tensor): The ratios of the valid width and the valid height relative to the width and the height of features in all levels, has shape (bs, num_levels, 2). reg_branches: (obj:`nn.ModuleList`, optional): Used for refining the regression results. Only would be passed when `with_box_refine` is `True`, otherwise would be `None`. Returns: tuple[Tensor]: Outputs of Deformable Transformer Decoder. - output (Tensor): Output embeddings of the last decoder, has shape (num_queries, bs, embed_dims) when `return_intermediate` is `False`. Otherwise, Intermediate output embeddings of all decoder layers, has shape (num_decoder_layers, num_queries, bs, embed_dims). - reference_points (Tensor): The reference of the last decoder layer, has shape (bs, num_queries, 4) when `return_intermediate` is `False`. Otherwise, Intermediate references of all decoder layers, has shape (num_decoder_layers, bs, num_queries, 4). The coordinates are arranged as (cx, cy, w, h) """ output = query intermediate = [] intermediate_reference_points = [] for layer_id, layer in enumerate(self.layers): if reference_points.shape[-1] == 4: reference_points_input = \ reference_points[:, :, None] * \ torch.cat([valid_ratios, valid_ratios], -1)[:, None] else: assert reference_points.shape[-1] == 2 reference_points_input = \ reference_points[:, :, None] * \ valid_ratios[:, None] output = layer( output, query_pos=query_pos, value=value, key_padding_mask=key_padding_mask, spatial_shapes=spatial_shapes, level_start_index=level_start_index, valid_ratios=valid_ratios, reference_points=reference_points_input, **kwargs) if reg_branches is not None: tmp_reg_preds = reg_branches[layer_id](output) if reference_points.shape[-1] == 4: new_reference_points = tmp_reg_preds + inverse_sigmoid( reference_points) new_reference_points = new_reference_points.sigmoid() else: assert reference_points.shape[-1] == 2 new_reference_points = tmp_reg_preds new_reference_points[..., :2] = tmp_reg_preds[ ..., :2] + inverse_sigmoid(reference_points) new_reference_points = new_reference_points.sigmoid() reference_points = new_reference_points.detach() if self.return_intermediate: intermediate.append(output) intermediate_reference_points.append(reference_points) if self.return_intermediate: return torch.stack(intermediate), torch.stack( intermediate_reference_points) return output, reference_points class DeformableDetrTransformerEncoderLayer(DetrTransformerEncoderLayer): """Encoder layer of Deformable DETR.""" def _init_layers(self) -> None: """Initialize self_attn, ffn, and norms.""" self.self_attn = MultiScaleDeformableAttention(**self.self_attn_cfg) self.embed_dims = self.self_attn.embed_dims self.ffn = FFN(**self.ffn_cfg) norms_list = [ build_norm_layer(self.norm_cfg, self.embed_dims)[1] for _ in range(2) ] self.norms = ModuleList(norms_list) class DeformableDetrTransformerDecoderLayer(DetrTransformerDecoderLayer): """Decoder layer of Deformable DETR.""" def _init_layers(self) -> None: """Initialize self_attn, cross-attn, ffn, and norms.""" self.self_attn = MultiheadAttention(**self.self_attn_cfg) self.cross_attn = MultiScaleDeformableAttention(**self.cross_attn_cfg) self.embed_dims = self.self_attn.embed_dims self.ffn = FFN(**self.ffn_cfg) norms_list = [ build_norm_layer(self.norm_cfg, self.embed_dims)[1] for _ in range(3) ] self.norms = ModuleList(norms_list) ================================================ FILE: mmdet/models/layers/transformer/detr_layers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Union import torch from mmcv.cnn import build_norm_layer from mmcv.cnn.bricks.transformer import FFN, MultiheadAttention from mmengine import ConfigDict from mmengine.model import BaseModule, ModuleList from torch import Tensor from mmdet.utils import ConfigType, OptConfigType class DetrTransformerEncoder(BaseModule): """Encoder of DETR. Args: num_layers (int): Number of encoder layers. layer_cfg (:obj:`ConfigDict` or dict): the config of each encoder layer. All the layers will share the same config. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, num_layers: int, layer_cfg: ConfigType, init_cfg: OptConfigType = None) -> None: super().__init__(init_cfg=init_cfg) self.num_layers = num_layers self.layer_cfg = layer_cfg self._init_layers() def _init_layers(self) -> None: """Initialize encoder layers.""" self.layers = ModuleList([ DetrTransformerEncoderLayer(**self.layer_cfg) for _ in range(self.num_layers) ]) self.embed_dims = self.layers[0].embed_dims def forward(self, query: Tensor, query_pos: Tensor, key_padding_mask: Tensor, **kwargs) -> Tensor: """Forward function of encoder. Args: query (Tensor): Input queries of encoder, has shape (bs, num_queries, dim). query_pos (Tensor): The positional embeddings of the queries, has shape (bs, num_queries, dim). key_padding_mask (Tensor): The `key_padding_mask` of `self_attn` input. ByteTensor, has shape (bs, num_queries). Returns: Tensor: Has shape (bs, num_queries, dim) if `batch_first` is `True`, otherwise (num_queries, bs, dim). """ for layer in self.layers: query = layer(query, query_pos, key_padding_mask, **kwargs) return query class DetrTransformerDecoder(BaseModule): """Decoder of DETR. Args: num_layers (int): Number of decoder layers. layer_cfg (:obj:`ConfigDict` or dict): the config of each encoder layer. All the layers will share the same config. post_norm_cfg (:obj:`ConfigDict` or dict, optional): Config of the post normalization layer. Defaults to `LN`. return_intermediate (bool, optional): Whether to return outputs of intermediate layers. Defaults to `True`, init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, num_layers: int, layer_cfg: ConfigType, post_norm_cfg: OptConfigType = dict(type='LN'), return_intermediate: bool = True, init_cfg: Union[dict, ConfigDict] = None) -> None: super().__init__(init_cfg=init_cfg) self.layer_cfg = layer_cfg self.num_layers = num_layers self.post_norm_cfg = post_norm_cfg self.return_intermediate = return_intermediate self._init_layers() def _init_layers(self) -> None: """Initialize decoder layers.""" self.layers = ModuleList([ DetrTransformerDecoderLayer(**self.layer_cfg) for _ in range(self.num_layers) ]) self.embed_dims = self.layers[0].embed_dims self.post_norm = build_norm_layer(self.post_norm_cfg, self.embed_dims)[1] def forward(self, query: Tensor, key: Tensor, value: Tensor, query_pos: Tensor, key_pos: Tensor, key_padding_mask: Tensor, **kwargs) -> Tensor: """Forward function of decoder Args: query (Tensor): The input query, has shape (bs, num_queries, dim). key (Tensor): The input key, has shape (bs, num_keys, dim). value (Tensor): The input value with the same shape as `key`. query_pos (Tensor): The positional encoding for `query`, with the same shape as `query`. key_pos (Tensor): The positional encoding for `key`, with the same shape as `key`. key_padding_mask (Tensor): The `key_padding_mask` of `cross_attn` input. ByteTensor, has shape (bs, num_value). Returns: Tensor: The forwarded results will have shape (num_decoder_layers, bs, num_queries, dim) if `return_intermediate` is `True` else (1, bs, num_queries, dim). """ intermediate = [] for layer in self.layers: query = layer( query, key=key, value=value, query_pos=query_pos, key_pos=key_pos, key_padding_mask=key_padding_mask, **kwargs) if self.return_intermediate: intermediate.append(self.post_norm(query)) query = self.post_norm(query) if self.return_intermediate: return torch.stack(intermediate) return query.unsqueeze(0) class DetrTransformerEncoderLayer(BaseModule): """Implements encoder layer in DETR transformer. Args: self_attn_cfg (:obj:`ConfigDict` or dict, optional): Config for self attention. ffn_cfg (:obj:`ConfigDict` or dict, optional): Config for FFN. norm_cfg (:obj:`ConfigDict` or dict, optional): Config for normalization layers. All the layers will share the same config. Defaults to `LN`. init_cfg (:obj:`ConfigDict` or dict, optional): Config to control the initialization. Defaults to None. """ def __init__(self, self_attn_cfg: OptConfigType = dict( embed_dims=256, num_heads=8, dropout=0.0), ffn_cfg: OptConfigType = dict( embed_dims=256, feedforward_channels=1024, num_fcs=2, ffn_drop=0., act_cfg=dict(type='ReLU', inplace=True)), norm_cfg: OptConfigType = dict(type='LN'), init_cfg: OptConfigType = None) -> None: super().__init__(init_cfg=init_cfg) self.self_attn_cfg = self_attn_cfg if 'batch_first' not in self.self_attn_cfg: self.self_attn_cfg['batch_first'] = True else: assert self.self_attn_cfg['batch_first'] is True, 'First \ dimension of all DETRs in mmdet is `batch`, \ please set `batch_first` flag.' self.ffn_cfg = ffn_cfg self.norm_cfg = norm_cfg self._init_layers() def _init_layers(self) -> None: """Initialize self-attention, FFN, and normalization.""" self.self_attn = MultiheadAttention(**self.self_attn_cfg) self.embed_dims = self.self_attn.embed_dims self.ffn = FFN(**self.ffn_cfg) norms_list = [ build_norm_layer(self.norm_cfg, self.embed_dims)[1] for _ in range(2) ] self.norms = ModuleList(norms_list) def forward(self, query: Tensor, query_pos: Tensor, key_padding_mask: Tensor, **kwargs) -> Tensor: """Forward function of an encoder layer. Args: query (Tensor): The input query, has shape (bs, num_queries, dim). query_pos (Tensor): The positional encoding for query, with the same shape as `query`. key_padding_mask (Tensor): The `key_padding_mask` of `self_attn` input. ByteTensor. has shape (bs, num_queries). Returns: Tensor: forwarded results, has shape (bs, num_queries, dim). """ query = self.self_attn( query=query, key=query, value=query, query_pos=query_pos, key_pos=query_pos, key_padding_mask=key_padding_mask, **kwargs) query = self.norms[0](query) query = self.ffn(query) query = self.norms[1](query) return query class DetrTransformerDecoderLayer(BaseModule): """Implements decoder layer in DETR transformer. Args: self_attn_cfg (:obj:`ConfigDict` or dict, optional): Config for self attention. cross_attn_cfg (:obj:`ConfigDict` or dict, optional): Config for cross attention. ffn_cfg (:obj:`ConfigDict` or dict, optional): Config for FFN. norm_cfg (:obj:`ConfigDict` or dict, optional): Config for normalization layers. All the layers will share the same config. Defaults to `LN`. init_cfg (:obj:`ConfigDict` or dict, optional): Config to control the initialization. Defaults to None. """ def __init__(self, self_attn_cfg: OptConfigType = dict( embed_dims=256, num_heads=8, dropout=0.0, batch_first=True), cross_attn_cfg: OptConfigType = dict( embed_dims=256, num_heads=8, dropout=0.0, batch_first=True), ffn_cfg: OptConfigType = dict( embed_dims=256, feedforward_channels=1024, num_fcs=2, ffn_drop=0., act_cfg=dict(type='ReLU', inplace=True), ), norm_cfg: OptConfigType = dict(type='LN'), init_cfg: OptConfigType = None) -> None: super().__init__(init_cfg=init_cfg) self.self_attn_cfg = self_attn_cfg self.cross_attn_cfg = cross_attn_cfg if 'batch_first' not in self.self_attn_cfg: self.self_attn_cfg['batch_first'] = True else: assert self.self_attn_cfg['batch_first'] is True, 'First \ dimension of all DETRs in mmdet is `batch`, \ please set `batch_first` flag.' if 'batch_first' not in self.cross_attn_cfg: self.cross_attn_cfg['batch_first'] = True else: assert self.cross_attn_cfg['batch_first'] is True, 'First \ dimension of all DETRs in mmdet is `batch`, \ please set `batch_first` flag.' self.ffn_cfg = ffn_cfg self.norm_cfg = norm_cfg self._init_layers() def _init_layers(self) -> None: """Initialize self-attention, FFN, and normalization.""" self.self_attn = MultiheadAttention(**self.self_attn_cfg) self.cross_attn = MultiheadAttention(**self.cross_attn_cfg) self.embed_dims = self.self_attn.embed_dims self.ffn = FFN(**self.ffn_cfg) norms_list = [ build_norm_layer(self.norm_cfg, self.embed_dims)[1] for _ in range(3) ] self.norms = ModuleList(norms_list) def forward(self, query: Tensor, key: Tensor = None, value: Tensor = None, query_pos: Tensor = None, key_pos: Tensor = None, self_attn_mask: Tensor = None, cross_attn_mask: Tensor = None, key_padding_mask: Tensor = None, **kwargs) -> Tensor: """ Args: query (Tensor): The input query, has shape (bs, num_queries, dim). key (Tensor, optional): The input key, has shape (bs, num_keys, dim). If `None`, the `query` will be used. Defaults to `None`. value (Tensor, optional): The input value, has the same shape as `key`, as in `nn.MultiheadAttention.forward`. If `None`, the `key` will be used. Defaults to `None`. query_pos (Tensor, optional): The positional encoding for `query`, has the same shape as `query`. If not `None`, it will be added to `query` before forward function. Defaults to `None`. key_pos (Tensor, optional): The positional encoding for `key`, has the same shape as `key`. If not `None`, it will be added to `key` before forward function. If None, and `query_pos` has the same shape as `key`, then `query_pos` will be used for `key_pos`. Defaults to None. self_attn_mask (Tensor, optional): ByteTensor mask, has shape (num_queries, num_keys), as in `nn.MultiheadAttention.forward`. Defaults to None. cross_attn_mask (Tensor, optional): ByteTensor mask, has shape (num_queries, num_keys), as in `nn.MultiheadAttention.forward`. Defaults to None. key_padding_mask (Tensor, optional): The `key_padding_mask` of `self_attn` input. ByteTensor, has shape (bs, num_value). Defaults to None. Returns: Tensor: forwarded results, has shape (bs, num_queries, dim). """ query = self.self_attn( query=query, key=query, value=query, query_pos=query_pos, key_pos=query_pos, attn_mask=self_attn_mask, **kwargs) query = self.norms[0](query) query = self.cross_attn( query=query, key=key, value=value, query_pos=query_pos, key_pos=key_pos, attn_mask=cross_attn_mask, key_padding_mask=key_padding_mask, **kwargs) query = self.norms[1](query) query = self.ffn(query) query = self.norms[2](query) return query ================================================ FILE: mmdet/models/layers/transformer/dino_layers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings from typing import Tuple, Union import torch from mmengine.model import BaseModule from torch import Tensor, nn from mmdet.structures import SampleList from mmdet.structures.bbox import bbox_xyxy_to_cxcywh from mmdet.utils import OptConfigType from .deformable_detr_layers import DeformableDetrTransformerDecoder from .utils import MLP, coordinate_to_encoding, inverse_sigmoid class DinoTransformerDecoder(DeformableDetrTransformerDecoder): """Transformer encoder of DINO.""" def _init_layers(self) -> None: """Initialize decoder layers.""" super()._init_layers() self.ref_point_head = MLP(self.embed_dims * 2, self.embed_dims, self.embed_dims, 2) self.norm = nn.LayerNorm(self.embed_dims) def forward(self, query: Tensor, value: Tensor, key_padding_mask: Tensor, self_attn_mask: Tensor, reference_points: Tensor, spatial_shapes: Tensor, level_start_index: Tensor, valid_ratios: Tensor, reg_branches: nn.ModuleList, **kwargs) -> Tensor: """Forward function of Transformer encoder. Args: query (Tensor): The input query, has shape (num_queries, bs, dim). value (Tensor): The input values, has shape (num_value, bs, dim). key_padding_mask (Tensor): The `key_padding_mask` of `self_attn` input. ByteTensor, has shape (num_queries, bs). self_attn_mask (Tensor): The attention mask to prevent information leakage from different denoising groups and matching parts, has shape (num_queries_total, num_queries_total). It is `None` when `self.training` is `False`. reference_points (Tensor): The initial reference, has shape (bs, num_queries, 4) with the last dimension arranged as (cx, cy, w, h). spatial_shapes (Tensor): Spatial shapes of features in all levels, has shape (num_levels, 2), last dimension represents (h, w). level_start_index (Tensor): The start index of each level. A tensor has shape (num_levels, ) and can be represented as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. valid_ratios (Tensor): The ratios of the valid width and the valid height relative to the width and the height of features in all levels, has shape (bs, num_levels, 2). reg_branches: (obj:`nn.ModuleList`): Used for refining the regression results. Returns: Tensor: Output queries of Transformer encoder, which is also called 'encoder output embeddings' or 'memory', has shape (num_queries, bs, dim) """ intermediate = [] intermediate_reference_points = [reference_points] for lid, layer in enumerate(self.layers): if reference_points.shape[-1] == 4: reference_points_input = \ reference_points[:, :, None] * torch.cat( [valid_ratios, valid_ratios], -1)[:, None] else: assert reference_points.shape[-1] == 2 reference_points_input = \ reference_points[:, :, None] * valid_ratios[:, None] query_sine_embed = coordinate_to_encoding( reference_points_input[:, :, 0, :]) query_pos = self.ref_point_head(query_sine_embed) query = layer( query, query_pos=query_pos, value=value, key_padding_mask=key_padding_mask, self_attn_mask=self_attn_mask, spatial_shapes=spatial_shapes, level_start_index=level_start_index, valid_ratios=valid_ratios, reference_points=reference_points_input, **kwargs) if reg_branches is not None: tmp = reg_branches[lid](query) assert reference_points.shape[-1] == 4 new_reference_points = tmp + inverse_sigmoid( reference_points, eps=1e-3) new_reference_points = new_reference_points.sigmoid() reference_points = new_reference_points.detach() if self.return_intermediate: intermediate.append(self.norm(query)) intermediate_reference_points.append(new_reference_points) # NOTE this is for the "Look Forward Twice" module, # in the DeformDETR, reference_points was appended. if self.return_intermediate: return torch.stack(intermediate), torch.stack( intermediate_reference_points) return query, reference_points class CdnQueryGenerator(BaseModule): """Implement query generator of the Contrastive denoising (CDN) proposed in `DINO: DETR with Improved DeNoising Anchor Boxes for End-to-End Object Detection `_ Code is modified from the `official github repo `_. Args: num_classes (int): Number of object classes. embed_dims (int): The embedding dimensions of the generated queries. num_matching_queries (int): The queries number of the matching part. Used for generating dn_mask. label_noise_scale (float): The scale of label noise, defaults to 0.5. box_noise_scale (float): The scale of box noise, defaults to 1.0. group_cfg (:obj:`ConfigDict` or dict, optional): The config of the denoising queries grouping, includes `dynamic`, `num_dn_queries`, and `num_groups`. Two grouping strategies, 'static dn groups' and 'dynamic dn groups', are supported. When `dynamic` is `False`, the `num_groups` should be set, and the number of denoising query groups will always be `num_groups`. When `dynamic` is `True`, the `num_dn_queries` should be set, and the group number will be dynamic to ensure that the denoising queries number will not exceed `num_dn_queries` to prevent large fluctuations of memory. Defaults to `None`. """ def __init__(self, num_classes: int, embed_dims: int, num_matching_queries: int, label_noise_scale: float = 0.5, box_noise_scale: float = 1.0, group_cfg: OptConfigType = None) -> None: super().__init__() self.num_classes = num_classes self.embed_dims = embed_dims self.num_matching_queries = num_matching_queries self.label_noise_scale = label_noise_scale self.box_noise_scale = box_noise_scale # prepare grouping strategy group_cfg = {} if group_cfg is None else group_cfg self.dynamic_dn_groups = group_cfg.get('dynamic', True) if self.dynamic_dn_groups: if 'num_dn_queries' not in group_cfg: warnings.warn("'num_dn_queries' should be set when using " 'dynamic dn groups, use 100 as default.') self.num_dn_queries = group_cfg.get('num_dn_queries', 100) assert isinstance(self.num_dn_queries, int), \ f'Expected the num_dn_queries to have type int, but got ' \ f'{self.num_dn_queries}({type(self.num_dn_queries)}). ' else: assert 'num_groups' in group_cfg, \ 'num_groups should be set when using static dn groups' self.num_groups = group_cfg['num_groups'] assert isinstance(self.num_groups, int), \ f'Expected the num_groups to have type int, but got ' \ f'{self.num_groups}({type(self.num_groups)}). ' # NOTE The original repo of DINO set the num_embeddings 92 for coco, # 91 (0~90) of which represents target classes and the 92 (91) # indicates `Unknown` class. However, the embedding of `unknown` class # is not used in the original DINO. # TODO: num_classes + 1 or num_classes ? self.label_embedding = nn.Embedding(self.num_classes, self.embed_dims) def __call__(self, batch_data_samples: SampleList) -> tuple: """Generate contrastive denoising (cdn) queries with ground truth. Descriptions of the Number Values in code and comments: - num_target_total: the total target number of the input batch samples. - max_num_target: the max target number of the input batch samples. - num_noisy_targets: the total targets number after adding noise, i.e., num_target_total * num_groups * 2. - num_denoising_queries: the length of the output batched queries, i.e., max_num_target * num_groups * 2. NOTE The format of input bboxes in batch_data_samples is unnormalized (x, y, x, y), and the output bbox queries are embedded by normalized (cx, cy, w, h) format bboxes going through inverse_sigmoid. Args: batch_data_samples (list[:obj:`DetDataSample`]): List of the batch data samples, each includes `gt_instance` which has attributes `bboxes` and `labels`. The `bboxes` has unnormalized coordinate format (x, y, x, y). Returns: tuple: The outputs of the dn query generator. - dn_label_query (Tensor): The output content queries for denoising part, has shape (bs, num_denoising_queries, dim), where `num_denoising_queries = max_num_target * num_groups * 2`. - dn_bbox_query (Tensor): The output reference bboxes as positions of queries for denoising part, which are embedded by normalized (cx, cy, w, h) format bboxes going through inverse_sigmoid, has shape (bs, num_denoising_queries, 4) with the last dimension arranged as (cx, cy, w, h). - attn_mask (Tensor): The attention mask to prevent information leakage from different denoising groups and matching parts, will be used as `self_attn_mask` of the `decoder`, has shape (num_queries_total, num_queries_total), where `num_queries_total` is the sum of `num_denoising_queries` and `num_matching_queries`. - dn_meta (Dict[str, int]): The dictionary saves information about group collation, including 'num_denoising_queries' and 'num_denoising_groups'. It will be used for split outputs of denoising and matching parts and loss calculation. """ # normalize bbox and collate ground truth (gt) gt_labels_list = [] gt_bboxes_list = [] for sample in batch_data_samples: img_h, img_w = sample.img_shape bboxes = sample.gt_instances.bboxes factor = bboxes.new_tensor([img_w, img_h, img_w, img_h]).unsqueeze(0) bboxes_normalized = bboxes / factor gt_bboxes_list.append(bboxes_normalized) gt_labels_list.append(sample.gt_instances.labels) gt_labels = torch.cat(gt_labels_list) # (num_target_total, 4) gt_bboxes = torch.cat(gt_bboxes_list) num_target_list = [len(bboxes) for bboxes in gt_bboxes_list] max_num_target = max(num_target_list) num_groups = self.get_num_groups(max_num_target) dn_label_query = self.generate_dn_label_query(gt_labels, num_groups) dn_bbox_query = self.generate_dn_bbox_query(gt_bboxes, num_groups) # The `batch_idx` saves the batch index of the corresponding sample # for each target, has shape (num_target_total). batch_idx = torch.cat([ torch.full_like(t.long(), i) for i, t in enumerate(gt_labels_list) ]) dn_label_query, dn_bbox_query = self.collate_dn_queries( dn_label_query, dn_bbox_query, batch_idx, len(batch_data_samples), num_groups) attn_mask = self.generate_dn_mask( max_num_target, num_groups, device=dn_label_query.device) dn_meta = dict( num_denoising_queries=int(max_num_target * 2 * num_groups), num_denoising_groups=num_groups) return dn_label_query, dn_bbox_query, attn_mask, dn_meta def get_num_groups(self, max_num_target: int = None) -> int: """Calculate denoising query groups number. Two grouping strategies, 'static dn groups' and 'dynamic dn groups', are supported. When `self.dynamic_dn_groups` is `False`, the number of denoising query groups will always be `self.num_groups`. When `self.dynamic_dn_groups` is `True`, the group number will be dynamic, ensuring the denoising queries number will not exceed `self.num_dn_queries` to prevent large fluctuations of memory. NOTE The `num_group` is shared for different samples in a batch. When the target numbers in the samples varies, the denoising queries of the samples containing fewer targets are padded to the max length. Args: max_num_target (int, optional): The max target number of the batch samples. It will only be used when `self.dynamic_dn_groups` is `True`. Defaults to `None`. Returns: int: The denoising group number of the current batch. """ if self.dynamic_dn_groups: assert max_num_target is not None, \ 'group_queries should be provided when using ' \ 'dynamic dn groups' if max_num_target == 0: num_groups = 1 else: num_groups = self.num_dn_queries // max_num_target else: num_groups = self.num_groups if num_groups < 1: num_groups = 1 return int(num_groups) def generate_dn_label_query(self, gt_labels: Tensor, num_groups: int) -> Tensor: """Generate noisy labels and their query embeddings. The strategy for generating noisy labels is: Randomly choose labels of `self.label_noise_scale * 0.5` proportion and override each of them with a random object category label. NOTE Not add noise to all labels. Besides, the `self.label_noise_scale * 0.5` arg is the ratio of the chosen positions, which is higher than the actual proportion of noisy labels, because the labels to override may be correct. And the gap becomes larger as the number of target categories decreases. The users should notice this and modify the scale arg or the corresponding logic according to specific dataset. Args: gt_labels (Tensor): The concatenated gt labels of all samples in the batch, has shape (num_target_total, ) where `num_target_total = sum(num_target_list)`. num_groups (int): The number of denoising query groups. Returns: Tensor: The query embeddings of noisy labels, has shape (num_noisy_targets, embed_dims), where `num_noisy_targets = num_target_total * num_groups * 2`. """ assert self.label_noise_scale > 0 gt_labels_expand = gt_labels.repeat(2 * num_groups, 1).view(-1) # Note `* 2` # noqa p = torch.rand_like(gt_labels_expand.float()) chosen_indice = torch.nonzero(p < (self.label_noise_scale * 0.5)).view( -1) # Note `* 0.5` new_labels = torch.randint_like(chosen_indice, 0, self.num_classes) noisy_labels_expand = gt_labels_expand.scatter(0, chosen_indice, new_labels) dn_label_query = self.label_embedding(noisy_labels_expand) return dn_label_query def generate_dn_bbox_query(self, gt_bboxes: Tensor, num_groups: int) -> Tensor: """Generate noisy bboxes and their query embeddings. The strategy for generating noisy bboxes is as follow: .. code:: text +--------------------+ | negative | | +----------+ | | | positive | | | | +-----|----+------------+ | | | | | | | +----+-----+ | | | | | | +---------+----------+ | | | | gt bbox | | | | +---------+----------+ | | | | | | +----+-----+ | | | | | | | +-------------|--- +----+ | | | | positive | | | +----------+ | | negative | +--------------------+ The random noise is added to the top-left and down-right point positions, hence, normalized (x, y, x, y) format of bboxes are required. The noisy bboxes of positive queries have the points both within the inner square, while those of negative queries have the points both between the inner and outer squares. Besides, the length of outer square is twice as long as that of the inner square, i.e., self.box_noise_scale * w_or_h / 2. NOTE The noise is added to all the bboxes. Moreover, there is still unconsidered case when one point is within the positive square and the others is between the inner and outer squares. Args: gt_bboxes (Tensor): The concatenated gt bboxes of all samples in the batch, has shape (num_target_total, 4) with the last dimension arranged as (cx, cy, w, h) where `num_target_total = sum(num_target_list)`. num_groups (int): The number of denoising query groups. Returns: Tensor: The output noisy bboxes, which are embedded by normalized (cx, cy, w, h) format bboxes going through inverse_sigmoid, has shape (num_noisy_targets, 4) with the last dimension arranged as (cx, cy, w, h), where `num_noisy_targets = num_target_total * num_groups * 2`. """ assert self.box_noise_scale > 0 device = gt_bboxes.device # expand gt_bboxes as groups gt_bboxes_expand = gt_bboxes.repeat(2 * num_groups, 1) # xyxy # obtain index of negative queries in gt_bboxes_expand positive_idx = torch.arange( len(gt_bboxes), dtype=torch.long, device=device) positive_idx = positive_idx.unsqueeze(0).repeat(num_groups, 1) positive_idx += 2 * len(gt_bboxes) * torch.arange( num_groups, dtype=torch.long, device=device)[:, None] positive_idx = positive_idx.flatten() negative_idx = positive_idx + len(gt_bboxes) # determine the sign of each element in the random part of the added # noise to be positive or negative randomly. rand_sign = torch.randint_like( gt_bboxes_expand, low=0, high=2, dtype=torch.float32) * 2.0 - 1.0 # [low, high), 1 or -1, randomly # calculate the random part of the added noise rand_part = torch.rand_like(gt_bboxes_expand) # [0, 1) rand_part[negative_idx] += 1.0 # pos: [0, 1); neg: [1, 2) rand_part *= rand_sign # pos: (-1, 1); neg: (-2, -1] U [1, 2) # add noise to the bboxes bboxes_whwh = bbox_xyxy_to_cxcywh(gt_bboxes_expand)[:, 2:].repeat(1, 2) noisy_bboxes_expand = gt_bboxes_expand + torch.mul( rand_part, bboxes_whwh) * self.box_noise_scale / 2 # xyxy noisy_bboxes_expand = noisy_bboxes_expand.clamp(min=0.0, max=1.0) noisy_bboxes_expand = bbox_xyxy_to_cxcywh(noisy_bboxes_expand) dn_bbox_query = inverse_sigmoid(noisy_bboxes_expand, eps=1e-3) return dn_bbox_query def collate_dn_queries(self, input_label_query: Tensor, input_bbox_query: Tensor, batch_idx: Tensor, batch_size: int, num_groups: int) -> Tuple[Tensor]: """Collate generated queries to obtain batched dn queries. The strategy for query collation is as follow: .. code:: text input_queries (num_target_total, query_dim) P_A1 P_B1 P_B2 N_A1 N_B1 N_B2 P'A1 P'B1 P'B2 N'A1 N'B1 N'B2 |________ group1 ________| |________ group2 ________| | V P_A1 Pad0 N_A1 Pad0 P'A1 Pad0 N'A1 Pad0 P_B1 P_B2 N_B1 N_B2 P'B1 P'B2 N'B1 N'B2 |____ group1 ____| |____ group2 ____| batched_queries (batch_size, max_num_target, query_dim) where query_dim is 4 for bbox and self.embed_dims for label. Notation: _-group 1; '-group 2; A-Sample1(has 1 target); B-sample2(has 2 targets) Args: input_label_query (Tensor): The generated label queries of all targets, has shape (num_target_total, embed_dims) where `num_target_total = sum(num_target_list)`. input_bbox_query (Tensor): The generated bbox queries of all targets, has shape (num_target_total, 4) with the last dimension arranged as (cx, cy, w, h). batch_idx (Tensor): The batch index of the corresponding sample for each target, has shape (num_target_total). batch_size (int): The size of the input batch. num_groups (int): The number of denoising query groups. Returns: tuple[Tensor]: Output batched label and bbox queries. - batched_label_query (Tensor): The output batched label queries, has shape (batch_size, max_num_target, embed_dims). - batched_bbox_query (Tensor): The output batched bbox queries, has shape (batch_size, max_num_target, 4) with the last dimension arranged as (cx, cy, w, h). """ device = input_label_query.device num_target_list = [ torch.sum(batch_idx == idx) for idx in range(batch_size) ] max_num_target = max(num_target_list) num_denoising_queries = int(max_num_target * 2 * num_groups) map_query_index = torch.cat([ torch.arange(num_target, device=device) for num_target in num_target_list ]) map_query_index = torch.cat([ map_query_index + max_num_target * i for i in range(2 * num_groups) ]).long() batch_idx_expand = batch_idx.repeat(2 * num_groups, 1).view(-1) mapper = (batch_idx_expand, map_query_index) batched_label_query = torch.zeros( batch_size, num_denoising_queries, self.embed_dims, device=device) batched_bbox_query = torch.zeros( batch_size, num_denoising_queries, 4, device=device) batched_label_query[mapper] = input_label_query batched_bbox_query[mapper] = input_bbox_query return batched_label_query, batched_bbox_query def generate_dn_mask(self, max_num_target: int, num_groups: int, device: Union[torch.device, str]) -> Tensor: """Generate attention mask to prevent information leakage from different denoising groups and matching parts. .. code:: text 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 1 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 1 1 1 1 1 1 1 1 0 0 0 0 0 max_num_target |_| |_________| num_matching_queries |_____________| num_denoising_queries 1 -> True (Masked), means 'can not see'. 0 -> False (UnMasked), means 'can see'. Args: max_num_target (int): The max target number of the input batch samples. num_groups (int): The number of denoising query groups. device (obj:`device` or str): The device of generated mask. Returns: Tensor: The attention mask to prevent information leakage from different denoising groups and matching parts, will be used as `self_attn_mask` of the `decoder`, has shape (num_queries_total, num_queries_total), where `num_queries_total` is the sum of `num_denoising_queries` and `num_matching_queries`. """ num_denoising_queries = int(max_num_target * 2 * num_groups) num_queries_total = num_denoising_queries + self.num_matching_queries attn_mask = torch.zeros( num_queries_total, num_queries_total, device=device, dtype=torch.bool) # Make the matching part cannot see the denoising groups attn_mask[num_denoising_queries:, :num_denoising_queries] = True # Make the denoising groups cannot see each other for i in range(num_groups): # Mask rows of one group per step. row_scope = slice(max_num_target * 2 * i, max_num_target * 2 * (i + 1)) left_scope = slice(max_num_target * 2 * i) right_scope = slice(max_num_target * 2 * (i + 1), num_denoising_queries) attn_mask[row_scope, right_scope] = True attn_mask[row_scope, left_scope] = True return attn_mask ================================================ FILE: mmdet/models/layers/transformer/mask2former_layers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmcv.cnn import build_norm_layer from mmengine.model import ModuleList from torch import Tensor from .deformable_detr_layers import DeformableDetrTransformerEncoder from .detr_layers import DetrTransformerDecoder, DetrTransformerDecoderLayer class Mask2FormerTransformerEncoder(DeformableDetrTransformerEncoder): """Encoder in PixelDecoder of Mask2Former.""" def forward(self, query: Tensor, query_pos: Tensor, key_padding_mask: Tensor, spatial_shapes: Tensor, level_start_index: Tensor, valid_ratios: Tensor, reference_points: Tensor, **kwargs) -> Tensor: """Forward function of Transformer encoder. Args: query (Tensor): The input query, has shape (bs, num_queries, dim). query_pos (Tensor): The positional encoding for query, has shape (bs, num_queries, dim). If not None, it will be added to the `query` before forward function. Defaults to None. key_padding_mask (Tensor): The `key_padding_mask` of `self_attn` input. ByteTensor, has shape (bs, num_queries). spatial_shapes (Tensor): Spatial shapes of features in all levels, has shape (num_levels, 2), last dimension represents (h, w). level_start_index (Tensor): The start index of each level. A tensor has shape (num_levels, ) and can be represented as [0, h_0*w_0, h_0*w_0+h_1*w_1, ...]. valid_ratios (Tensor): The ratios of the valid width and the valid height relative to the width and the height of features in all levels, has shape (bs, num_levels, 2). reference_points (Tensor): The initial reference, has shape (bs, num_queries, 2) with the last dimension arranged as (cx, cy). Returns: Tensor: Output queries of Transformer encoder, which is also called 'encoder output embeddings' or 'memory', has shape (bs, num_queries, dim) """ for layer in self.layers: query = layer( query=query, query_pos=query_pos, key_padding_mask=key_padding_mask, spatial_shapes=spatial_shapes, level_start_index=level_start_index, valid_ratios=valid_ratios, reference_points=reference_points, **kwargs) return query class Mask2FormerTransformerDecoder(DetrTransformerDecoder): """Decoder of Mask2Former.""" def _init_layers(self) -> None: """Initialize decoder layers.""" self.layers = ModuleList([ Mask2FormerTransformerDecoderLayer(**self.layer_cfg) for _ in range(self.num_layers) ]) self.embed_dims = self.layers[0].embed_dims self.post_norm = build_norm_layer(self.post_norm_cfg, self.embed_dims)[1] class Mask2FormerTransformerDecoderLayer(DetrTransformerDecoderLayer): """Implements decoder layer in Mask2Former transformer.""" def forward(self, query: Tensor, key: Tensor = None, value: Tensor = None, query_pos: Tensor = None, key_pos: Tensor = None, self_attn_mask: Tensor = None, cross_attn_mask: Tensor = None, key_padding_mask: Tensor = None, **kwargs) -> Tensor: """ Args: query (Tensor): The input query, has shape (bs, num_queries, dim). key (Tensor, optional): The input key, has shape (bs, num_keys, dim). If `None`, the `query` will be used. Defaults to `None`. value (Tensor, optional): The input value, has the same shape as `key`, as in `nn.MultiheadAttention.forward`. If `None`, the `key` will be used. Defaults to `None`. query_pos (Tensor, optional): The positional encoding for `query`, has the same shape as `query`. If not `None`, it will be added to `query` before forward function. Defaults to `None`. key_pos (Tensor, optional): The positional encoding for `key`, has the same shape as `key`. If not `None`, it will be added to `key` before forward function. If None, and `query_pos` has the same shape as `key`, then `query_pos` will be used for `key_pos`. Defaults to None. self_attn_mask (Tensor, optional): ByteTensor mask, has shape (num_queries, num_keys), as in `nn.MultiheadAttention.forward`. Defaults to None. cross_attn_mask (Tensor, optional): ByteTensor mask, has shape (num_queries, num_keys), as in `nn.MultiheadAttention.forward`. Defaults to None. key_padding_mask (Tensor, optional): The `key_padding_mask` of `self_attn` input. ByteTensor, has shape (bs, num_value). Defaults to None. Returns: Tensor: forwarded results, has shape (bs, num_queries, dim). """ query = self.cross_attn( query=query, key=key, value=value, query_pos=query_pos, key_pos=key_pos, attn_mask=cross_attn_mask, key_padding_mask=key_padding_mask, **kwargs) query = self.norms[0](query) query = self.self_attn( query=query, key=query, value=query, query_pos=query_pos, key_pos=query_pos, attn_mask=self_attn_mask, **kwargs) query = self.norms[1](query) query = self.ffn(query) query = self.norms[2](query) return query ================================================ FILE: mmdet/models/layers/transformer/utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math import warnings from typing import Optional, Sequence, Tuple, Union import torch import torch.nn.functional as F from mmcv.cnn import (Linear, build_activation_layer, build_conv_layer, build_norm_layer) from mmcv.cnn.bricks.drop import Dropout from mmengine.model import BaseModule, ModuleList from mmengine.utils import to_2tuple from torch import Tensor, nn from mmdet.registry import MODELS from mmdet.utils import OptConfigType, OptMultiConfig def nlc_to_nchw(x: Tensor, hw_shape: Sequence[int]) -> Tensor: """Convert [N, L, C] shape tensor to [N, C, H, W] shape tensor. Args: x (Tensor): The input tensor of shape [N, L, C] before conversion. hw_shape (Sequence[int]): The height and width of output feature map. Returns: Tensor: The output tensor of shape [N, C, H, W] after conversion. """ H, W = hw_shape assert len(x.shape) == 3 B, L, C = x.shape assert L == H * W, 'The seq_len does not match H, W' return x.transpose(1, 2).reshape(B, C, H, W).contiguous() def nchw_to_nlc(x): """Flatten [N, C, H, W] shape tensor to [N, L, C] shape tensor. Args: x (Tensor): The input tensor of shape [N, C, H, W] before conversion. Returns: Tensor: The output tensor of shape [N, L, C] after conversion. """ assert len(x.shape) == 4 return x.flatten(2).transpose(1, 2).contiguous() def coordinate_to_encoding(coord_tensor: Tensor, num_feats: int = 128, temperature: int = 10000, scale: float = 2 * math.pi): """Convert coordinate tensor to positional encoding. Args: coord_tensor (Tensor): Coordinate tensor to be converted to positional encoding. With the last dimension as 2 or 4. num_feats (int, optional): The feature dimension for each position along x-axis or y-axis. Note the final returned dimension for each position is 2 times of this value. Defaults to 128. temperature (int, optional): The temperature used for scaling the position embedding. Defaults to 10000. scale (float, optional): A scale factor that scales the position embedding. The scale will be used only when `normalize` is True. Defaults to 2*pi. Returns: Tensor: Returned encoded positional tensor. """ dim_t = torch.arange( num_feats, dtype=torch.float32, device=coord_tensor.device) dim_t = temperature**(2 * (dim_t // 2) / num_feats) x_embed = coord_tensor[..., 0] * scale y_embed = coord_tensor[..., 1] * scale pos_x = x_embed[..., None] / dim_t pos_y = y_embed[..., None] / dim_t pos_x = torch.stack((pos_x[..., 0::2].sin(), pos_x[..., 1::2].cos()), dim=-1).flatten(2) pos_y = torch.stack((pos_y[..., 0::2].sin(), pos_y[..., 1::2].cos()), dim=-1).flatten(2) if coord_tensor.size(-1) == 2: pos = torch.cat((pos_y, pos_x), dim=-1) elif coord_tensor.size(-1) == 4: w_embed = coord_tensor[..., 2] * scale pos_w = w_embed[..., None] / dim_t pos_w = torch.stack((pos_w[..., 0::2].sin(), pos_w[..., 1::2].cos()), dim=-1).flatten(2) h_embed = coord_tensor[..., 3] * scale pos_h = h_embed[..., None] / dim_t pos_h = torch.stack((pos_h[..., 0::2].sin(), pos_h[..., 1::2].cos()), dim=-1).flatten(2) pos = torch.cat((pos_y, pos_x, pos_w, pos_h), dim=-1) else: raise ValueError('Unknown pos_tensor shape(-1):{}'.format( coord_tensor.size(-1))) return pos def inverse_sigmoid(x: Tensor, eps: float = 1e-5) -> Tensor: """Inverse function of sigmoid. Args: x (Tensor): The tensor to do the inverse. eps (float): EPS avoid numerical overflow. Defaults 1e-5. Returns: Tensor: The x has passed the inverse function of sigmoid, has the same shape with input. """ x = x.clamp(min=0, max=1) x1 = x.clamp(min=eps) x2 = (1 - x).clamp(min=eps) return torch.log(x1 / x2) class AdaptivePadding(nn.Module): """Applies padding to input (if needed) so that input can get fully covered by filter you specified. It support two modes "same" and "corner". The "same" mode is same with "SAME" padding mode in TensorFlow, pad zero around input. The "corner" mode would pad zero to bottom right. Args: kernel_size (int | tuple): Size of the kernel: stride (int | tuple): Stride of the filter. Default: 1: dilation (int | tuple): Spacing between kernel elements. Default: 1 padding (str): Support "same" and "corner", "corner" mode would pad zero to bottom right, and "same" mode would pad zero around input. Default: "corner". Example: >>> kernel_size = 16 >>> stride = 16 >>> dilation = 1 >>> input = torch.rand(1, 1, 15, 17) >>> adap_pad = AdaptivePadding( >>> kernel_size=kernel_size, >>> stride=stride, >>> dilation=dilation, >>> padding="corner") >>> out = adap_pad(input) >>> assert (out.shape[2], out.shape[3]) == (16, 32) >>> input = torch.rand(1, 1, 16, 17) >>> out = adap_pad(input) >>> assert (out.shape[2], out.shape[3]) == (16, 32) """ def __init__(self, kernel_size=1, stride=1, dilation=1, padding='corner'): super(AdaptivePadding, self).__init__() assert padding in ('same', 'corner') kernel_size = to_2tuple(kernel_size) stride = to_2tuple(stride) padding = to_2tuple(padding) dilation = to_2tuple(dilation) self.padding = padding self.kernel_size = kernel_size self.stride = stride self.dilation = dilation def get_pad_shape(self, input_shape): input_h, input_w = input_shape kernel_h, kernel_w = self.kernel_size stride_h, stride_w = self.stride output_h = math.ceil(input_h / stride_h) output_w = math.ceil(input_w / stride_w) pad_h = max((output_h - 1) * stride_h + (kernel_h - 1) * self.dilation[0] + 1 - input_h, 0) pad_w = max((output_w - 1) * stride_w + (kernel_w - 1) * self.dilation[1] + 1 - input_w, 0) return pad_h, pad_w def forward(self, x): pad_h, pad_w = self.get_pad_shape(x.size()[-2:]) if pad_h > 0 or pad_w > 0: if self.padding == 'corner': x = F.pad(x, [0, pad_w, 0, pad_h]) elif self.padding == 'same': x = F.pad(x, [ pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2 ]) return x class PatchEmbed(BaseModule): """Image to Patch Embedding. We use a conv layer to implement PatchEmbed. Args: in_channels (int): The num of input channels. Default: 3 embed_dims (int): The dimensions of embedding. Default: 768 conv_type (str): The config dict for embedding conv layer type selection. Default: "Conv2d. kernel_size (int): The kernel_size of embedding conv. Default: 16. stride (int): The slide stride of embedding conv. Default: None (Would be set as `kernel_size`). padding (int | tuple | string ): The padding length of embedding conv. When it is a string, it means the mode of adaptive padding, support "same" and "corner" now. Default: "corner". dilation (int): The dilation rate of embedding conv. Default: 1. bias (bool): Bias of embed conv. Default: True. norm_cfg (dict, optional): Config dict for normalization layer. Default: None. input_size (int | tuple | None): The size of input, which will be used to calculate the out size. Only work when `dynamic_size` is False. Default: None. init_cfg (`mmengine.ConfigDict`, optional): The Config for initialization. Default: None. """ def __init__(self, in_channels: int = 3, embed_dims: int = 768, conv_type: str = 'Conv2d', kernel_size: int = 16, stride: int = 16, padding: Union[int, tuple, str] = 'corner', dilation: int = 1, bias: bool = True, norm_cfg: OptConfigType = None, input_size: Union[int, tuple] = None, init_cfg: OptConfigType = None) -> None: super(PatchEmbed, self).__init__(init_cfg=init_cfg) self.embed_dims = embed_dims if stride is None: stride = kernel_size kernel_size = to_2tuple(kernel_size) stride = to_2tuple(stride) dilation = to_2tuple(dilation) if isinstance(padding, str): self.adap_padding = AdaptivePadding( kernel_size=kernel_size, stride=stride, dilation=dilation, padding=padding) # disable the padding of conv padding = 0 else: self.adap_padding = None padding = to_2tuple(padding) self.projection = build_conv_layer( dict(type=conv_type), in_channels=in_channels, out_channels=embed_dims, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) if norm_cfg is not None: self.norm = build_norm_layer(norm_cfg, embed_dims)[1] else: self.norm = None if input_size: input_size = to_2tuple(input_size) # `init_out_size` would be used outside to # calculate the num_patches # when `use_abs_pos_embed` outside self.init_input_size = input_size if self.adap_padding: pad_h, pad_w = self.adap_padding.get_pad_shape(input_size) input_h, input_w = input_size input_h = input_h + pad_h input_w = input_w + pad_w input_size = (input_h, input_w) # https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html h_out = (input_size[0] + 2 * padding[0] - dilation[0] * (kernel_size[0] - 1) - 1) // stride[0] + 1 w_out = (input_size[1] + 2 * padding[1] - dilation[1] * (kernel_size[1] - 1) - 1) // stride[1] + 1 self.init_out_size = (h_out, w_out) else: self.init_input_size = None self.init_out_size = None def forward(self, x: Tensor) -> Tuple[Tensor, Tuple[int]]: """ Args: x (Tensor): Has shape (B, C, H, W). In most case, C is 3. Returns: tuple: Contains merged results and its spatial shape. - x (Tensor): Has shape (B, out_h * out_w, embed_dims) - out_size (tuple[int]): Spatial shape of x, arrange as (out_h, out_w). """ if self.adap_padding: x = self.adap_padding(x) x = self.projection(x) out_size = (x.shape[2], x.shape[3]) x = x.flatten(2).transpose(1, 2) if self.norm is not None: x = self.norm(x) return x, out_size class PatchMerging(BaseModule): """Merge patch feature map. This layer groups feature map by kernel_size, and applies norm and linear layers to the grouped feature map. Our implementation uses `nn.Unfold` to merge patch, which is about 25% faster than original implementation. Instead, we need to modify pretrained models for compatibility. Args: in_channels (int): The num of input channels. to gets fully covered by filter and stride you specified.. Default: True. out_channels (int): The num of output channels. kernel_size (int | tuple, optional): the kernel size in the unfold layer. Defaults to 2. stride (int | tuple, optional): the stride of the sliding blocks in the unfold layer. Default: None. (Would be set as `kernel_size`) padding (int | tuple | string ): The padding length of embedding conv. When it is a string, it means the mode of adaptive padding, support "same" and "corner" now. Default: "corner". dilation (int | tuple, optional): dilation parameter in the unfold layer. Default: 1. bias (bool, optional): Whether to add bias in linear layer or not. Defaults: False. norm_cfg (dict, optional): Config dict for normalization layer. Default: dict(type='LN'). init_cfg (dict, optional): The extra config for initialization. Default: None. """ def __init__(self, in_channels: int, out_channels: int, kernel_size: Optional[Union[int, tuple]] = 2, stride: Optional[Union[int, tuple]] = None, padding: Union[int, tuple, str] = 'corner', dilation: Optional[Union[int, tuple]] = 1, bias: Optional[bool] = False, norm_cfg: OptConfigType = dict(type='LN'), init_cfg: OptConfigType = None) -> None: super().__init__(init_cfg=init_cfg) self.in_channels = in_channels self.out_channels = out_channels if stride: stride = stride else: stride = kernel_size kernel_size = to_2tuple(kernel_size) stride = to_2tuple(stride) dilation = to_2tuple(dilation) if isinstance(padding, str): self.adap_padding = AdaptivePadding( kernel_size=kernel_size, stride=stride, dilation=dilation, padding=padding) # disable the padding of unfold padding = 0 else: self.adap_padding = None padding = to_2tuple(padding) self.sampler = nn.Unfold( kernel_size=kernel_size, dilation=dilation, padding=padding, stride=stride) sample_dim = kernel_size[0] * kernel_size[1] * in_channels if norm_cfg is not None: self.norm = build_norm_layer(norm_cfg, sample_dim)[1] else: self.norm = None self.reduction = nn.Linear(sample_dim, out_channels, bias=bias) def forward(self, x: Tensor, input_size: Tuple[int]) -> Tuple[Tensor, Tuple[int]]: """ Args: x (Tensor): Has shape (B, H*W, C_in). input_size (tuple[int]): The spatial shape of x, arrange as (H, W). Default: None. Returns: tuple: Contains merged results and its spatial shape. - x (Tensor): Has shape (B, Merged_H * Merged_W, C_out) - out_size (tuple[int]): Spatial shape of x, arrange as (Merged_H, Merged_W). """ B, L, C = x.shape assert isinstance(input_size, Sequence), f'Expect ' \ f'input_size is ' \ f'`Sequence` ' \ f'but get {input_size}' H, W = input_size assert L == H * W, 'input feature has wrong size' x = x.view(B, H, W, C).permute([0, 3, 1, 2]) # B, C, H, W # Use nn.Unfold to merge patch. About 25% faster than original method, # but need to modify pretrained model for compatibility if self.adap_padding: x = self.adap_padding(x) H, W = x.shape[-2:] x = self.sampler(x) # if kernel_size=2 and stride=2, x should has shape (B, 4*C, H/2*W/2) out_h = (H + 2 * self.sampler.padding[0] - self.sampler.dilation[0] * (self.sampler.kernel_size[0] - 1) - 1) // self.sampler.stride[0] + 1 out_w = (W + 2 * self.sampler.padding[1] - self.sampler.dilation[1] * (self.sampler.kernel_size[1] - 1) - 1) // self.sampler.stride[1] + 1 output_size = (out_h, out_w) x = x.transpose(1, 2) # B, H/2*W/2, 4*C x = self.norm(x) if self.norm else x x = self.reduction(x) return x, output_size class ConditionalAttention(BaseModule): """A wrapper of conditional attention, dropout and residual connection. Args: embed_dims (int): The embedding dimension. num_heads (int): Parallel attention heads. attn_drop (float): A Dropout layer on attn_output_weights. Default: 0.0. proj_drop: A Dropout layer after `nn.MultiheadAttention`. Default: 0.0. cross_attn (bool): Whether the attention module is for cross attention. Default: False keep_query_pos (bool): Whether to transform query_pos before cross attention. Default: False. batch_first (bool): When it is True, Key, Query and Value are shape of (batch, n, embed_dim), otherwise (n, batch, embed_dim). Default: True. init_cfg (obj:`mmcv.ConfigDict`): The Config for initialization. Default: None. """ def __init__(self, embed_dims: int, num_heads: int, attn_drop: float = 0., proj_drop: float = 0., cross_attn: bool = False, keep_query_pos: bool = False, batch_first: bool = True, init_cfg: OptMultiConfig = None): super().__init__(init_cfg=init_cfg) assert batch_first is True, 'Set `batch_first`\ to False is NOT supported in ConditionalAttention. \ First dimension of all DETRs in mmdet is `batch`, \ please set `batch_first` to True.' self.cross_attn = cross_attn self.keep_query_pos = keep_query_pos self.embed_dims = embed_dims self.num_heads = num_heads self.attn_drop = Dropout(attn_drop) self.proj_drop = Dropout(proj_drop) self._init_layers() def _init_layers(self): """Initialize layers for qkv projection.""" embed_dims = self.embed_dims self.qcontent_proj = Linear(embed_dims, embed_dims) self.qpos_proj = Linear(embed_dims, embed_dims) self.kcontent_proj = Linear(embed_dims, embed_dims) self.kpos_proj = Linear(embed_dims, embed_dims) self.v_proj = Linear(embed_dims, embed_dims) if self.cross_attn: self.qpos_sine_proj = Linear(embed_dims, embed_dims) self.out_proj = Linear(embed_dims, embed_dims) nn.init.constant_(self.out_proj.bias, 0.) def forward_attn(self, query: Tensor, key: Tensor, value: Tensor, attn_mask: Tensor = None, key_padding_mask: Tensor = None) -> Tuple[Tensor]: """Forward process for `ConditionalAttention`. Args: query (Tensor): The input query with shape [bs, num_queries, embed_dims]. key (Tensor): The key tensor with shape [bs, num_keys, embed_dims]. If None, the `query` will be used. Defaults to None. value (Tensor): The value tensor with same shape as `key`. Same in `nn.MultiheadAttention.forward`. Defaults to None. If None, the `key` will be used. attn_mask (Tensor): ByteTensor mask with shape [num_queries, num_keys]. Same in `nn.MultiheadAttention.forward`. Defaults to None. key_padding_mask (Tensor): ByteTensor with shape [bs, num_keys]. Defaults to None. Returns: Tuple[Tensor]: Attention outputs of shape :math:`(N, L, E)`, where :math:`N` is the batch size, :math:`L` is the target sequence length , and :math:`E` is the embedding dimension `embed_dim`. Attention weights per head of shape :math:` (num_heads, L, S)`. where :math:`N` is batch size, :math:`L` is target sequence length, and :math:`S` is the source sequence length. """ assert key.size(1) == value.size(1), \ f'{"key, value must have the same sequence length"}' assert query.size(0) == key.size(0) == value.size(0), \ f'{"batch size must be equal for query, key, value"}' assert query.size(2) == key.size(2), \ f'{"q_dims, k_dims must be equal"}' assert value.size(2) == self.embed_dims, \ f'{"v_dims must be equal to embed_dims"}' bs, tgt_len, hidden_dims = query.size() _, src_len, _ = key.size() head_dims = hidden_dims // self.num_heads v_head_dims = self.embed_dims // self.num_heads assert head_dims * self.num_heads == hidden_dims, \ f'{"hidden_dims must be divisible by num_heads"}' scaling = float(head_dims)**-0.5 q = query * scaling k = key v = value if attn_mask is not None: assert attn_mask.dtype == torch.float32 or \ attn_mask.dtype == torch.float64 or \ attn_mask.dtype == torch.float16 or \ attn_mask.dtype == torch.uint8 or \ attn_mask.dtype == torch.bool, \ 'Only float, byte, and bool types are supported for \ attn_mask' if attn_mask.dtype == torch.uint8: warnings.warn('Byte tensor for attn_mask is deprecated.\ Use bool tensor instead.') attn_mask = attn_mask.to(torch.bool) if attn_mask.dim() == 2: attn_mask = attn_mask.unsqueeze(0) if list(attn_mask.size()) != [1, query.size(1), key.size(1)]: raise RuntimeError( 'The size of the 2D attn_mask is not correct.') elif attn_mask.dim() == 3: if list(attn_mask.size()) != [ bs * self.num_heads, query.size(1), key.size(1) ]: raise RuntimeError( 'The size of the 3D attn_mask is not correct.') else: raise RuntimeError( "attn_mask's dimension {} is not supported".format( attn_mask.dim())) # attn_mask's dim is 3 now. if key_padding_mask is not None and key_padding_mask.dtype == int: key_padding_mask = key_padding_mask.to(torch.bool) q = q.contiguous().view(bs, tgt_len, self.num_heads, head_dims).permute(0, 2, 1, 3).flatten(0, 1) if k is not None: k = k.contiguous().view(bs, src_len, self.num_heads, head_dims).permute(0, 2, 1, 3).flatten(0, 1) if v is not None: v = v.contiguous().view(bs, src_len, self.num_heads, v_head_dims).permute(0, 2, 1, 3).flatten(0, 1) if key_padding_mask is not None: assert key_padding_mask.size(0) == bs assert key_padding_mask.size(1) == src_len attn_output_weights = torch.bmm(q, k.transpose(1, 2)) assert list(attn_output_weights.size()) == [ bs * self.num_heads, tgt_len, src_len ] if attn_mask is not None: if attn_mask.dtype == torch.bool: attn_output_weights.masked_fill_(attn_mask, float('-inf')) else: attn_output_weights += attn_mask if key_padding_mask is not None: attn_output_weights = attn_output_weights.view( bs, self.num_heads, tgt_len, src_len) attn_output_weights = attn_output_weights.masked_fill( key_padding_mask.unsqueeze(1).unsqueeze(2), float('-inf'), ) attn_output_weights = attn_output_weights.view( bs * self.num_heads, tgt_len, src_len) attn_output_weights = F.softmax( attn_output_weights - attn_output_weights.max(dim=-1, keepdim=True)[0], dim=-1) attn_output_weights = self.attn_drop(attn_output_weights) attn_output = torch.bmm(attn_output_weights, v) assert list( attn_output.size()) == [bs * self.num_heads, tgt_len, v_head_dims] attn_output = attn_output.view(bs, self.num_heads, tgt_len, v_head_dims).permute(0, 2, 1, 3).flatten(2) attn_output = self.out_proj(attn_output) # average attention weights over heads attn_output_weights = attn_output_weights.view(bs, self.num_heads, tgt_len, src_len) return attn_output, attn_output_weights.sum(dim=1) / self.num_heads def forward(self, query: Tensor, key: Tensor, query_pos: Tensor = None, ref_sine_embed: Tensor = None, key_pos: Tensor = None, attn_mask: Tensor = None, key_padding_mask: Tensor = None, is_first: bool = False) -> Tensor: """Forward function for `ConditionalAttention`. Args: query (Tensor): The input query with shape [bs, num_queries, embed_dims]. key (Tensor): The key tensor with shape [bs, num_keys, embed_dims]. If None, the `query` will be used. Defaults to None. query_pos (Tensor): The positional encoding for query in self attention, with the same shape as `x`. If not None, it will be added to `x` before forward function. Defaults to None. query_sine_embed (Tensor): The positional encoding for query in cross attention, with the same shape as `x`. If not None, it will be added to `x` before forward function. Defaults to None. key_pos (Tensor): The positional encoding for `key`, with the same shape as `key`. Defaults to None. If not None, it will be added to `key` before forward function. If None, and `query_pos` has the same shape as `key`, then `query_pos` will be used for `key_pos`. Defaults to None. attn_mask (Tensor): ByteTensor mask with shape [num_queries, num_keys]. Same in `nn.MultiheadAttention.forward`. Defaults to None. key_padding_mask (Tensor): ByteTensor with shape [bs, num_keys]. Defaults to None. is_first (bool): A indicator to tell whether the current layer is the first layer of the decoder. Defaults to False. Returns: Tensor: forwarded results with shape [bs, num_queries, embed_dims]. """ if self.cross_attn: q_content = self.qcontent_proj(query) k_content = self.kcontent_proj(key) v = self.v_proj(key) bs, nq, c = q_content.size() _, hw, _ = k_content.size() k_pos = self.kpos_proj(key_pos) if is_first or self.keep_query_pos: q_pos = self.qpos_proj(query_pos) q = q_content + q_pos k = k_content + k_pos else: q = q_content k = k_content q = q.view(bs, nq, self.num_heads, c // self.num_heads) query_sine_embed = self.qpos_sine_proj(ref_sine_embed) query_sine_embed = query_sine_embed.view(bs, nq, self.num_heads, c // self.num_heads) q = torch.cat([q, query_sine_embed], dim=3).view(bs, nq, 2 * c) k = k.view(bs, hw, self.num_heads, c // self.num_heads) k_pos = k_pos.view(bs, hw, self.num_heads, c // self.num_heads) k = torch.cat([k, k_pos], dim=3).view(bs, hw, 2 * c) ca_output = self.forward_attn( query=q, key=k, value=v, attn_mask=attn_mask, key_padding_mask=key_padding_mask)[0] query = query + self.proj_drop(ca_output) else: q_content = self.qcontent_proj(query) q_pos = self.qpos_proj(query_pos) k_content = self.kcontent_proj(query) k_pos = self.kpos_proj(query_pos) v = self.v_proj(query) q = q_content if q_pos is None else q_content + q_pos k = k_content if k_pos is None else k_content + k_pos sa_output = self.forward_attn( query=q, key=k, value=v, attn_mask=attn_mask, key_padding_mask=key_padding_mask)[0] query = query + self.proj_drop(sa_output) return query class MLP(BaseModule): """Very simple multi-layer perceptron (also called FFN) with relu. Mostly used in DETR series detectors. Args: input_dim (int): Feature dim of the input tensor. hidden_dim (int): Feature dim of the hidden layer. output_dim (int): Feature dim of the output tensor. num_layers (int): Number of FFN layers. As the last layer of MLP only contains FFN (Linear). """ def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_layers: int) -> None: super().__init__() self.num_layers = num_layers h = [hidden_dim] * (num_layers - 1) self.layers = ModuleList( Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim])) def forward(self, x: Tensor) -> Tensor: """Forward function of MLP. Args: x (Tensor): The input feature, has shape (num_queries, bs, input_dim). Returns: Tensor: The output feature, has shape (num_queries, bs, output_dim). """ for i, layer in enumerate(self.layers): x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x) return x @MODELS.register_module() class DynamicConv(BaseModule): """Implements Dynamic Convolution. This module generate parameters for each sample and use bmm to implement 1*1 convolution. Code is modified from the `official github repo `_ . Args: in_channels (int): The input feature channel. Defaults to 256. feat_channels (int): The inner feature channel. Defaults to 64. out_channels (int, optional): The output feature channel. When not specified, it will be set to `in_channels` by default input_feat_shape (int): The shape of input feature. Defaults to 7. with_proj (bool): Project two-dimentional feature to one-dimentional feature. Default to True. act_cfg (dict): The activation config for DynamicConv. norm_cfg (dict): Config dict for normalization layer. Default layer normalization. init_cfg (obj:`mmengine.ConfigDict`): The Config for initialization. Default: None. """ def __init__(self, in_channels: int = 256, feat_channels: int = 64, out_channels: Optional[int] = None, input_feat_shape: int = 7, with_proj: bool = True, act_cfg: OptConfigType = dict(type='ReLU', inplace=True), norm_cfg: OptConfigType = dict(type='LN'), init_cfg: OptConfigType = None) -> None: super(DynamicConv, self).__init__(init_cfg) self.in_channels = in_channels self.feat_channels = feat_channels self.out_channels_raw = out_channels self.input_feat_shape = input_feat_shape self.with_proj = with_proj self.act_cfg = act_cfg self.norm_cfg = norm_cfg self.out_channels = out_channels if out_channels else in_channels self.num_params_in = self.in_channels * self.feat_channels self.num_params_out = self.out_channels * self.feat_channels self.dynamic_layer = nn.Linear( self.in_channels, self.num_params_in + self.num_params_out) self.norm_in = build_norm_layer(norm_cfg, self.feat_channels)[1] self.norm_out = build_norm_layer(norm_cfg, self.out_channels)[1] self.activation = build_activation_layer(act_cfg) num_output = self.out_channels * input_feat_shape**2 if self.with_proj: self.fc_layer = nn.Linear(num_output, self.out_channels) self.fc_norm = build_norm_layer(norm_cfg, self.out_channels)[1] def forward(self, param_feature: Tensor, input_feature: Tensor) -> Tensor: """Forward function for `DynamicConv`. Args: param_feature (Tensor): The feature can be used to generate the parameter, has shape (num_all_proposals, in_channels). input_feature (Tensor): Feature that interact with parameters, has shape (num_all_proposals, in_channels, H, W). Returns: Tensor: The output feature has shape (num_all_proposals, out_channels). """ input_feature = input_feature.flatten(2).permute(2, 0, 1) input_feature = input_feature.permute(1, 0, 2) parameters = self.dynamic_layer(param_feature) param_in = parameters[:, :self.num_params_in].view( -1, self.in_channels, self.feat_channels) param_out = parameters[:, -self.num_params_out:].view( -1, self.feat_channels, self.out_channels) # input_feature has shape (num_all_proposals, H*W, in_channels) # param_in has shape (num_all_proposals, in_channels, feat_channels) # feature has shape (num_all_proposals, H*W, feat_channels) features = torch.bmm(input_feature, param_in) features = self.norm_in(features) features = self.activation(features) # param_out has shape (batch_size, feat_channels, out_channels) features = torch.bmm(features, param_out) features = self.norm_out(features) features = self.activation(features) if self.with_proj: features = features.flatten(1) features = self.fc_layer(features) features = self.fc_norm(features) features = self.activation(features) return features ================================================ FILE: mmdet/models/losses/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .accuracy import Accuracy, accuracy from .ae_loss import AssociativeEmbeddingLoss from .balanced_l1_loss import BalancedL1Loss, balanced_l1_loss from .cross_entropy_loss import (CrossEntropyLoss, binary_cross_entropy, cross_entropy, mask_cross_entropy) from .dice_loss import DiceLoss from .focal_loss import FocalLoss, sigmoid_focal_loss from .gaussian_focal_loss import GaussianFocalLoss from .gfocal_loss import DistributionFocalLoss, QualityFocalLoss from .ghm_loss import GHMC, GHMR from .iou_loss import (BoundedIoULoss, CIoULoss, DIoULoss, EIoULoss, GIoULoss, IoULoss, bounded_iou_loss, iou_loss) from .kd_loss import KnowledgeDistillationKLDivLoss, KDQualityFocalLoss from .mse_loss import MSELoss, mse_loss from .pisa_loss import carl_loss, isr_p from .seesaw_loss import SeesawLoss from .smooth_l1_loss import L1Loss, SmoothL1Loss, l1_loss, smooth_l1_loss from .utils import reduce_loss, weight_reduce_loss, weighted_loss from .varifocal_loss import VarifocalLoss from .pkd_loss import PKDLoss __all__ = [ 'accuracy', 'Accuracy', 'cross_entropy', 'binary_cross_entropy', 'mask_cross_entropy', 'CrossEntropyLoss', 'sigmoid_focal_loss', 'FocalLoss', 'smooth_l1_loss', 'SmoothL1Loss', 'balanced_l1_loss', 'BalancedL1Loss', 'mse_loss', 'MSELoss', 'iou_loss', 'bounded_iou_loss', 'IoULoss', 'BoundedIoULoss', 'GIoULoss', 'DIoULoss', 'CIoULoss', 'EIoULoss', 'GHMC', 'GHMR', 'reduce_loss', 'weight_reduce_loss', 'weighted_loss', 'L1Loss', 'l1_loss', 'isr_p', 'carl_loss', 'AssociativeEmbeddingLoss', 'GaussianFocalLoss', 'QualityFocalLoss', 'DistributionFocalLoss', 'VarifocalLoss', 'KnowledgeDistillationKLDivLoss', 'SeesawLoss', 'DiceLoss', 'KDQualityFocalLoss', 'PKDLoss' ] ================================================ FILE: mmdet/models/losses/accuracy.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn as nn def accuracy(pred, target, topk=1, thresh=None): """Calculate accuracy according to the prediction and target. Args: pred (torch.Tensor): The model prediction, shape (N, num_class) target (torch.Tensor): The target of each prediction, shape (N, ) topk (int | tuple[int], optional): If the predictions in ``topk`` matches the target, the predictions will be regarded as correct ones. Defaults to 1. thresh (float, optional): If not None, predictions with scores under this threshold are considered incorrect. Default to None. Returns: float | tuple[float]: If the input ``topk`` is a single integer, the function will return a single float as accuracy. If ``topk`` is a tuple containing multiple integers, the function will return a tuple containing accuracies of each ``topk`` number. """ assert isinstance(topk, (int, tuple)) if isinstance(topk, int): topk = (topk, ) return_single = True else: return_single = False maxk = max(topk) if pred.size(0) == 0: accu = [pred.new_tensor(0.) for i in range(len(topk))] return accu[0] if return_single else accu assert pred.ndim == 2 and target.ndim == 1 assert pred.size(0) == target.size(0) assert maxk <= pred.size(1), \ f'maxk {maxk} exceeds pred dimension {pred.size(1)}' pred_value, pred_label = pred.topk(maxk, dim=1) pred_label = pred_label.t() # transpose to shape (maxk, N) correct = pred_label.eq(target.view(1, -1).expand_as(pred_label)) if thresh is not None: # Only prediction values larger than thresh are counted as correct correct = correct & (pred_value > thresh).t() res = [] for k in topk: correct_k = correct[:k].reshape(-1).float().sum(0, keepdim=True) res.append(correct_k.mul_(100.0 / pred.size(0))) return res[0] if return_single else res class Accuracy(nn.Module): def __init__(self, topk=(1, ), thresh=None): """Module to calculate the accuracy. Args: topk (tuple, optional): The criterion used to calculate the accuracy. Defaults to (1,). thresh (float, optional): If not None, predictions with scores under this threshold are considered incorrect. Default to None. """ super().__init__() self.topk = topk self.thresh = thresh def forward(self, pred, target): """Forward function to calculate accuracy. Args: pred (torch.Tensor): Prediction of models. target (torch.Tensor): Target for each prediction. Returns: tuple[float]: The accuracies under different topk criterions. """ return accuracy(pred, target, self.topk, self.thresh) ================================================ FILE: mmdet/models/losses/ae_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F from mmdet.registry import MODELS def ae_loss_per_image(tl_preds, br_preds, match): """Associative Embedding Loss in one image. Associative Embedding Loss including two parts: pull loss and push loss. Pull loss makes embedding vectors from same object closer to each other. Push loss distinguish embedding vector from different objects, and makes the gap between them is large enough. During computing, usually there are 3 cases: - no object in image: both pull loss and push loss will be 0. - one object in image: push loss will be 0 and pull loss is computed by the two corner of the only object. - more than one objects in image: pull loss is computed by corner pairs from each object, push loss is computed by each object with all other objects. We use confusion matrix with 0 in diagonal to compute the push loss. Args: tl_preds (tensor): Embedding feature map of left-top corner. br_preds (tensor): Embedding feature map of bottim-right corner. match (list): Downsampled coordinates pair of each ground truth box. """ tl_list, br_list, me_list = [], [], [] if len(match) == 0: # no object in image pull_loss = tl_preds.sum() * 0. push_loss = tl_preds.sum() * 0. else: for m in match: [tl_y, tl_x], [br_y, br_x] = m tl_e = tl_preds[:, tl_y, tl_x].view(-1, 1) br_e = br_preds[:, br_y, br_x].view(-1, 1) tl_list.append(tl_e) br_list.append(br_e) me_list.append((tl_e + br_e) / 2.0) tl_list = torch.cat(tl_list) br_list = torch.cat(br_list) me_list = torch.cat(me_list) assert tl_list.size() == br_list.size() # N is object number in image, M is dimension of embedding vector N, M = tl_list.size() pull_loss = (tl_list - me_list).pow(2) + (br_list - me_list).pow(2) pull_loss = pull_loss.sum() / N margin = 1 # exp setting of CornerNet, details in section 3.3 of paper # confusion matrix of push loss conf_mat = me_list.expand((N, N, M)).permute(1, 0, 2) - me_list conf_weight = 1 - torch.eye(N).type_as(me_list) conf_mat = conf_weight * (margin - conf_mat.sum(-1).abs()) if N > 1: # more than one object in current image push_loss = F.relu(conf_mat).sum() / (N * (N - 1)) else: push_loss = tl_preds.sum() * 0. return pull_loss, push_loss @MODELS.register_module() class AssociativeEmbeddingLoss(nn.Module): """Associative Embedding Loss. More details can be found in `Associative Embedding `_ and `CornerNet `_ . Code is modified from `kp_utils.py `_ # noqa: E501 Args: pull_weight (float): Loss weight for corners from same object. push_weight (float): Loss weight for corners from different object. """ def __init__(self, pull_weight=0.25, push_weight=0.25): super(AssociativeEmbeddingLoss, self).__init__() self.pull_weight = pull_weight self.push_weight = push_weight def forward(self, pred, target, match): """Forward function.""" batch = pred.size(0) pull_all, push_all = 0.0, 0.0 for i in range(batch): pull, push = ae_loss_per_image(pred[i], target[i], match[i]) pull_all += self.pull_weight * pull push_all += self.push_weight * push return pull_all, push_all ================================================ FILE: mmdet/models/losses/balanced_l1_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch import torch.nn as nn from mmdet.registry import MODELS from .utils import weighted_loss @weighted_loss def balanced_l1_loss(pred, target, beta=1.0, alpha=0.5, gamma=1.5, reduction='mean'): """Calculate balanced L1 loss. Please see the `Libra R-CNN `_ Args: pred (torch.Tensor): The prediction with shape (N, 4). target (torch.Tensor): The learning target of the prediction with shape (N, 4). beta (float): The loss is a piecewise function of prediction and target and ``beta`` serves as a threshold for the difference between the prediction and target. Defaults to 1.0. alpha (float): The denominator ``alpha`` in the balanced L1 loss. Defaults to 0.5. gamma (float): The ``gamma`` in the balanced L1 loss. Defaults to 1.5. reduction (str, optional): The method that reduces the loss to a scalar. Options are "none", "mean" and "sum". Returns: torch.Tensor: The calculated loss """ assert beta > 0 if target.numel() == 0: return pred.sum() * 0 assert pred.size() == target.size() diff = torch.abs(pred - target) b = np.e**(gamma / alpha) - 1 loss = torch.where( diff < beta, alpha / b * (b * diff + 1) * torch.log(b * diff / beta + 1) - alpha * diff, gamma * diff + gamma / b - alpha * beta) return loss @MODELS.register_module() class BalancedL1Loss(nn.Module): """Balanced L1 Loss. arXiv: https://arxiv.org/pdf/1904.02701.pdf (CVPR 2019) Args: alpha (float): The denominator ``alpha`` in the balanced L1 loss. Defaults to 0.5. gamma (float): The ``gamma`` in the balanced L1 loss. Defaults to 1.5. beta (float, optional): The loss is a piecewise function of prediction and target. ``beta`` serves as a threshold for the difference between the prediction and target. Defaults to 1.0. reduction (str, optional): The method that reduces the loss to a scalar. Options are "none", "mean" and "sum". loss_weight (float, optional): The weight of the loss. Defaults to 1.0 """ def __init__(self, alpha=0.5, gamma=1.5, beta=1.0, reduction='mean', loss_weight=1.0): super(BalancedL1Loss, self).__init__() self.alpha = alpha self.gamma = gamma self.beta = beta self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred, target, weight=None, avg_factor=None, reduction_override=None, **kwargs): """Forward function of loss. Args: pred (torch.Tensor): The prediction with shape (N, 4). target (torch.Tensor): The learning target of the prediction with shape (N, 4). weight (torch.Tensor, optional): Sample-wise loss weight with shape (N, ). avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Options are "none", "mean" and "sum". Returns: torch.Tensor: The calculated loss """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) loss_bbox = self.loss_weight * balanced_l1_loss( pred, target, weight, alpha=self.alpha, gamma=self.gamma, beta=self.beta, reduction=reduction, avg_factor=avg_factor, **kwargs) return loss_bbox ================================================ FILE: mmdet/models/losses/cross_entropy_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings import torch import torch.nn as nn import torch.nn.functional as F from mmdet.registry import MODELS from .utils import weight_reduce_loss def cross_entropy(pred, label, weight=None, reduction='mean', avg_factor=None, class_weight=None, ignore_index=-100, avg_non_ignore=False): """Calculate the CrossEntropy loss. Args: pred (torch.Tensor): The prediction with shape (N, C), C is the number of classes. label (torch.Tensor): The learning label of the prediction. weight (torch.Tensor, optional): Sample-wise loss weight. reduction (str, optional): The method used to reduce the loss. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. class_weight (list[float], optional): The weight for each class. ignore_index (int | None): The label index to be ignored. If None, it will be set to default value. Default: -100. avg_non_ignore (bool): The flag decides to whether the loss is only averaged over non-ignored targets. Default: False. Returns: torch.Tensor: The calculated loss """ # The default value of ignore_index is the same as F.cross_entropy ignore_index = -100 if ignore_index is None else ignore_index # element-wise losses loss = F.cross_entropy( pred, label, weight=class_weight, reduction='none', ignore_index=ignore_index) # average loss over non-ignored elements # pytorch's official cross_entropy average loss over non-ignored elements # refer to https://github.com/pytorch/pytorch/blob/56b43f4fec1f76953f15a627694d4bba34588969/torch/nn/functional.py#L2660 # noqa if (avg_factor is None) and avg_non_ignore and reduction == 'mean': avg_factor = label.numel() - (label == ignore_index).sum().item() # apply weights and do the reduction if weight is not None: weight = weight.float() loss = weight_reduce_loss( loss, weight=weight, reduction=reduction, avg_factor=avg_factor) return loss def _expand_onehot_labels(labels, label_weights, label_channels, ignore_index): """Expand onehot labels to match the size of prediction.""" bin_labels = labels.new_full((labels.size(0), label_channels), 0) valid_mask = (labels >= 0) & (labels != ignore_index) inds = torch.nonzero( valid_mask & (labels < label_channels), as_tuple=False) if inds.numel() > 0: bin_labels[inds, labels[inds]] = 1 valid_mask = valid_mask.view(-1, 1).expand(labels.size(0), label_channels).float() if label_weights is None: bin_label_weights = valid_mask else: bin_label_weights = label_weights.view(-1, 1).repeat(1, label_channels) bin_label_weights *= valid_mask return bin_labels, bin_label_weights, valid_mask def binary_cross_entropy(pred, label, weight=None, reduction='mean', avg_factor=None, class_weight=None, ignore_index=-100, avg_non_ignore=False): """Calculate the binary CrossEntropy loss. Args: pred (torch.Tensor): The prediction with shape (N, 1) or (N, ). When the shape of pred is (N, 1), label will be expanded to one-hot format, and when the shape of pred is (N, ), label will not be expanded to one-hot format. label (torch.Tensor): The learning label of the prediction, with shape (N, ). weight (torch.Tensor, optional): Sample-wise loss weight. reduction (str, optional): The method used to reduce the loss. Options are "none", "mean" and "sum". avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. class_weight (list[float], optional): The weight for each class. ignore_index (int | None): The label index to be ignored. If None, it will be set to default value. Default: -100. avg_non_ignore (bool): The flag decides to whether the loss is only averaged over non-ignored targets. Default: False. Returns: torch.Tensor: The calculated loss. """ # The default value of ignore_index is the same as F.cross_entropy ignore_index = -100 if ignore_index is None else ignore_index if pred.dim() != label.dim(): label, weight, valid_mask = _expand_onehot_labels( label, weight, pred.size(-1), ignore_index) else: # should mask out the ignored elements valid_mask = ((label >= 0) & (label != ignore_index)).float() if weight is not None: # The inplace writing method will have a mismatched broadcast # shape error if the weight and valid_mask dimensions # are inconsistent such as (B,N,1) and (B,N,C). weight = weight * valid_mask else: weight = valid_mask # average loss over non-ignored elements if (avg_factor is None) and avg_non_ignore and reduction == 'mean': avg_factor = valid_mask.sum().item() # weighted element-wise losses weight = weight.float() loss = F.binary_cross_entropy_with_logits( pred, label.float(), pos_weight=class_weight, reduction='none') # do the reduction for the weighted loss loss = weight_reduce_loss( loss, weight, reduction=reduction, avg_factor=avg_factor) return loss def mask_cross_entropy(pred, target, label, reduction='mean', avg_factor=None, class_weight=None, ignore_index=None, **kwargs): """Calculate the CrossEntropy loss for masks. Args: pred (torch.Tensor): The prediction with shape (N, C, *), C is the number of classes. The trailing * indicates arbitrary shape. target (torch.Tensor): The learning label of the prediction. label (torch.Tensor): ``label`` indicates the class label of the mask corresponding object. This will be used to select the mask in the of the class which the object belongs to when the mask prediction if not class-agnostic. reduction (str, optional): The method used to reduce the loss. Options are "none", "mean" and "sum". avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. class_weight (list[float], optional): The weight for each class. ignore_index (None): Placeholder, to be consistent with other loss. Default: None. Returns: torch.Tensor: The calculated loss Example: >>> N, C = 3, 11 >>> H, W = 2, 2 >>> pred = torch.randn(N, C, H, W) * 1000 >>> target = torch.rand(N, H, W) >>> label = torch.randint(0, C, size=(N,)) >>> reduction = 'mean' >>> avg_factor = None >>> class_weights = None >>> loss = mask_cross_entropy(pred, target, label, reduction, >>> avg_factor, class_weights) >>> assert loss.shape == (1,) """ assert ignore_index is None, 'BCE loss does not support ignore_index' # TODO: handle these two reserved arguments assert reduction == 'mean' and avg_factor is None num_rois = pred.size()[0] inds = torch.arange(0, num_rois, dtype=torch.long, device=pred.device) pred_slice = pred[inds, label].squeeze(1) return F.binary_cross_entropy_with_logits( pred_slice, target, weight=class_weight, reduction='mean')[None] @MODELS.register_module() class CrossEntropyLoss(nn.Module): def __init__(self, use_sigmoid=False, use_mask=False, reduction='mean', class_weight=None, ignore_index=None, loss_weight=1.0, avg_non_ignore=False): """CrossEntropyLoss. Args: use_sigmoid (bool, optional): Whether the prediction uses sigmoid of softmax. Defaults to False. use_mask (bool, optional): Whether to use mask cross entropy loss. Defaults to False. reduction (str, optional): . Defaults to 'mean'. Options are "none", "mean" and "sum". class_weight (list[float], optional): Weight of each class. Defaults to None. ignore_index (int | None): The label index to be ignored. Defaults to None. loss_weight (float, optional): Weight of the loss. Defaults to 1.0. avg_non_ignore (bool): The flag decides to whether the loss is only averaged over non-ignored targets. Default: False. """ super(CrossEntropyLoss, self).__init__() assert (use_sigmoid is False) or (use_mask is False) self.use_sigmoid = use_sigmoid self.use_mask = use_mask self.reduction = reduction self.loss_weight = loss_weight self.class_weight = class_weight self.ignore_index = ignore_index self.avg_non_ignore = avg_non_ignore if ((ignore_index is not None) and not self.avg_non_ignore and self.reduction == 'mean'): warnings.warn( 'Default ``avg_non_ignore`` is False, if you would like to ' 'ignore the certain label and average loss over non-ignore ' 'labels, which is the same with PyTorch official ' 'cross_entropy, set ``avg_non_ignore=True``.') if self.use_sigmoid: self.cls_criterion = binary_cross_entropy elif self.use_mask: self.cls_criterion = mask_cross_entropy else: self.cls_criterion = cross_entropy def extra_repr(self): """Extra repr.""" s = f'avg_non_ignore={self.avg_non_ignore}' return s def forward(self, cls_score, label, weight=None, avg_factor=None, reduction_override=None, ignore_index=None, **kwargs): """Forward function. Args: cls_score (torch.Tensor): The prediction. label (torch.Tensor): The learning label of the prediction. weight (torch.Tensor, optional): Sample-wise loss weight. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The method used to reduce the loss. Options are "none", "mean" and "sum". ignore_index (int | None): The label index to be ignored. If not None, it will override the default value. Default: None. Returns: torch.Tensor: The calculated loss. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if ignore_index is None: ignore_index = self.ignore_index if self.class_weight is not None: class_weight = cls_score.new_tensor( self.class_weight, device=cls_score.device) else: class_weight = None loss_cls = self.loss_weight * self.cls_criterion( cls_score, label, weight, class_weight=class_weight, reduction=reduction, avg_factor=avg_factor, ignore_index=ignore_index, avg_non_ignore=self.avg_non_ignore, **kwargs) return loss_cls ================================================ FILE: mmdet/models/losses/dice_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn from mmdet.registry import MODELS from .utils import weight_reduce_loss def dice_loss(pred, target, weight=None, eps=1e-3, reduction='mean', naive_dice=False, avg_factor=None): """Calculate dice loss, there are two forms of dice loss is supported: - the one proposed in `V-Net: Fully Convolutional Neural Networks for Volumetric Medical Image Segmentation `_. - the dice loss in which the power of the number in the denominator is the first power instead of the second power. Args: pred (torch.Tensor): The prediction, has a shape (n, *) target (torch.Tensor): The learning label of the prediction, shape (n, *), same shape of pred. weight (torch.Tensor, optional): The weight of loss for each prediction, has a shape (n,). Defaults to None. eps (float): Avoid dividing by zero. Default: 1e-3. reduction (str, optional): The method used to reduce the loss into a scalar. Defaults to 'mean'. Options are "none", "mean" and "sum". naive_dice (bool, optional): If false, use the dice loss defined in the V-Net paper, otherwise, use the naive dice loss in which the power of the number in the denominator is the first power instead of the second power.Defaults to False. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. """ input = pred.flatten(1) target = target.flatten(1).float() a = torch.sum(input * target, 1) if naive_dice: b = torch.sum(input, 1) c = torch.sum(target, 1) d = (2 * a + eps) / (b + c + eps) else: b = torch.sum(input * input, 1) + eps c = torch.sum(target * target, 1) + eps d = (2 * a) / (b + c) loss = 1 - d if weight is not None: assert weight.ndim == loss.ndim assert len(weight) == len(pred) loss = weight_reduce_loss(loss, weight, reduction, avg_factor) return loss @MODELS.register_module() class DiceLoss(nn.Module): def __init__(self, use_sigmoid=True, activate=True, reduction='mean', naive_dice=False, loss_weight=1.0, eps=1e-3): """Compute dice loss. Args: use_sigmoid (bool, optional): Whether to the prediction is used for sigmoid or softmax. Defaults to True. activate (bool): Whether to activate the predictions inside, this will disable the inside sigmoid operation. Defaults to True. reduction (str, optional): The method used to reduce the loss. Options are "none", "mean" and "sum". Defaults to 'mean'. naive_dice (bool, optional): If false, use the dice loss defined in the V-Net paper, otherwise, use the naive dice loss in which the power of the number in the denominator is the first power instead of the second power. Defaults to False. loss_weight (float, optional): Weight of loss. Defaults to 1.0. eps (float): Avoid dividing by zero. Defaults to 1e-3. """ super(DiceLoss, self).__init__() self.use_sigmoid = use_sigmoid self.reduction = reduction self.naive_dice = naive_dice self.loss_weight = loss_weight self.eps = eps self.activate = activate def forward(self, pred, target, weight=None, reduction_override=None, avg_factor=None): """Forward function. Args: pred (torch.Tensor): The prediction, has a shape (n, *). target (torch.Tensor): The label of the prediction, shape (n, *), same shape of pred. weight (torch.Tensor, optional): The weight of loss for each prediction, has a shape (n,). Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Options are "none", "mean" and "sum". Returns: torch.Tensor: The calculated loss """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if self.activate: if self.use_sigmoid: pred = pred.sigmoid() else: raise NotImplementedError loss = self.loss_weight * dice_loss( pred, target, weight, eps=self.eps, reduction=reduction, naive_dice=self.naive_dice, avg_factor=avg_factor) return loss ================================================ FILE: mmdet/models/losses/focal_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F from mmcv.ops import sigmoid_focal_loss as _sigmoid_focal_loss from mmdet.registry import MODELS from .utils import weight_reduce_loss # This method is only for debugging def py_sigmoid_focal_loss(pred, target, weight=None, gamma=2.0, alpha=0.25, reduction='mean', avg_factor=None): """PyTorch version of `Focal Loss `_. Args: pred (torch.Tensor): The prediction with shape (N, C), C is the number of classes target (torch.Tensor): The learning label of the prediction. weight (torch.Tensor, optional): Sample-wise loss weight. gamma (float, optional): The gamma for calculating the modulating factor. Defaults to 2.0. alpha (float, optional): A balanced form for Focal Loss. Defaults to 0.25. reduction (str, optional): The method used to reduce the loss into a scalar. Defaults to 'mean'. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. """ pred_sigmoid = pred.sigmoid() target = target.type_as(pred) pt = (1 - pred_sigmoid) * target + pred_sigmoid * (1 - target) focal_weight = (alpha * target + (1 - alpha) * (1 - target)) * pt.pow(gamma) loss = F.binary_cross_entropy_with_logits( pred, target, reduction='none') * focal_weight if weight is not None: if weight.shape != loss.shape: if weight.size(0) == loss.size(0): # For most cases, weight is of shape (num_priors, ), # which means it does not have the second axis num_class weight = weight.view(-1, 1) else: # Sometimes, weight per anchor per class is also needed. e.g. # in FSAF. But it may be flattened of shape # (num_priors x num_class, ), while loss is still of shape # (num_priors, num_class). assert weight.numel() == loss.numel() weight = weight.view(loss.size(0), -1) assert weight.ndim == loss.ndim loss = weight_reduce_loss(loss, weight, reduction, avg_factor) return loss def py_focal_loss_with_prob(pred, target, weight=None, gamma=2.0, alpha=0.25, reduction='mean', avg_factor=None): """PyTorch version of `Focal Loss `_. Different from `py_sigmoid_focal_loss`, this function accepts probability as input. Args: pred (torch.Tensor): The prediction probability with shape (N, C), C is the number of classes. target (torch.Tensor): The learning label of the prediction. The target shape support (N,C) or (N,), (N,C) means one-hot form. weight (torch.Tensor, optional): Sample-wise loss weight. gamma (float, optional): The gamma for calculating the modulating factor. Defaults to 2.0. alpha (float, optional): A balanced form for Focal Loss. Defaults to 0.25. reduction (str, optional): The method used to reduce the loss into a scalar. Defaults to 'mean'. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. """ if pred.dim() != target.dim(): num_classes = pred.size(1) target = F.one_hot(target, num_classes=num_classes + 1) target = target[:, :num_classes] target = target.type_as(pred) pt = (1 - pred) * target + pred * (1 - target) focal_weight = (alpha * target + (1 - alpha) * (1 - target)) * pt.pow(gamma) loss = F.binary_cross_entropy( pred, target, reduction='none') * focal_weight if weight is not None: if weight.shape != loss.shape: if weight.size(0) == loss.size(0): # For most cases, weight is of shape (num_priors, ), # which means it does not have the second axis num_class weight = weight.view(-1, 1) else: # Sometimes, weight per anchor per class is also needed. e.g. # in FSAF. But it may be flattened of shape # (num_priors x num_class, ), while loss is still of shape # (num_priors, num_class). assert weight.numel() == loss.numel() weight = weight.view(loss.size(0), -1) assert weight.ndim == loss.ndim loss = weight_reduce_loss(loss, weight, reduction, avg_factor) return loss def sigmoid_focal_loss(pred, target, weight=None, gamma=2.0, alpha=0.25, reduction='mean', avg_factor=None): r"""A wrapper of cuda version `Focal Loss `_. Args: pred (torch.Tensor): The prediction with shape (N, C), C is the number of classes. target (torch.Tensor): The learning label of the prediction. weight (torch.Tensor, optional): Sample-wise loss weight. gamma (float, optional): The gamma for calculating the modulating factor. Defaults to 2.0. alpha (float, optional): A balanced form for Focal Loss. Defaults to 0.25. reduction (str, optional): The method used to reduce the loss into a scalar. Defaults to 'mean'. Options are "none", "mean" and "sum". avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. """ # Function.apply does not accept keyword arguments, so the decorator # "weighted_loss" is not applicable loss = _sigmoid_focal_loss(pred.contiguous(), target.contiguous(), gamma, alpha, None, 'none') if weight is not None: if weight.shape != loss.shape: if weight.size(0) == loss.size(0): # For most cases, weight is of shape (num_priors, ), # which means it does not have the second axis num_class weight = weight.view(-1, 1) else: # Sometimes, weight per anchor per class is also needed. e.g. # in FSAF. But it may be flattened of shape # (num_priors x num_class, ), while loss is still of shape # (num_priors, num_class). assert weight.numel() == loss.numel() weight = weight.view(loss.size(0), -1) assert weight.ndim == loss.ndim loss = weight_reduce_loss(loss, weight, reduction, avg_factor) return loss @MODELS.register_module() class FocalLoss(nn.Module): def __init__(self, use_sigmoid=True, gamma=2.0, alpha=0.25, reduction='mean', loss_weight=1.0, activated=False): """`Focal Loss `_ Args: use_sigmoid (bool, optional): Whether to the prediction is used for sigmoid or softmax. Defaults to True. gamma (float, optional): The gamma for calculating the modulating factor. Defaults to 2.0. alpha (float, optional): A balanced form for Focal Loss. Defaults to 0.25. reduction (str, optional): The method used to reduce the loss into a scalar. Defaults to 'mean'. Options are "none", "mean" and "sum". loss_weight (float, optional): Weight of loss. Defaults to 1.0. activated (bool, optional): Whether the input is activated. If True, it means the input has been activated and can be treated as probabilities. Else, it should be treated as logits. Defaults to False. """ super(FocalLoss, self).__init__() assert use_sigmoid is True, 'Only sigmoid focal loss supported now.' self.use_sigmoid = use_sigmoid self.gamma = gamma self.alpha = alpha self.reduction = reduction self.loss_weight = loss_weight self.activated = activated def forward(self, pred, target, weight=None, avg_factor=None, reduction_override=None): """Forward function. Args: pred (torch.Tensor): The prediction. target (torch.Tensor): The learning label of the prediction. The target shape support (N,C) or (N,), (N,C) means one-hot form. weight (torch.Tensor, optional): The weight of loss for each prediction. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Options are "none", "mean" and "sum". Returns: torch.Tensor: The calculated loss """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if self.use_sigmoid: if self.activated: calculate_loss_func = py_focal_loss_with_prob else: if pred.dim() == target.dim(): # this means that target is already in One-Hot form. calculate_loss_func = py_sigmoid_focal_loss elif torch.cuda.is_available() and pred.is_cuda: calculate_loss_func = sigmoid_focal_loss else: num_classes = pred.size(1) target = F.one_hot(target, num_classes=num_classes + 1) target = target[:, :num_classes] calculate_loss_func = py_sigmoid_focal_loss loss_cls = self.loss_weight * calculate_loss_func( pred, target, weight, gamma=self.gamma, alpha=self.alpha, reduction=reduction, avg_factor=avg_factor) else: raise NotImplementedError return loss_cls ================================================ FILE: mmdet/models/losses/gaussian_focal_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Union import torch.nn as nn from torch import Tensor from mmdet.registry import MODELS from .utils import weight_reduce_loss, weighted_loss @weighted_loss def gaussian_focal_loss(pred: Tensor, gaussian_target: Tensor, alpha: float = 2.0, gamma: float = 4.0, pos_weight: float = 1.0, neg_weight: float = 1.0) -> Tensor: """`Focal Loss `_ for targets in gaussian distribution. Args: pred (torch.Tensor): The prediction. gaussian_target (torch.Tensor): The learning target of the prediction in gaussian distribution. alpha (float, optional): A balanced form for Focal Loss. Defaults to 2.0. gamma (float, optional): The gamma for calculating the modulating factor. Defaults to 4.0. pos_weight(float): Positive sample loss weight. Defaults to 1.0. neg_weight(float): Negative sample loss weight. Defaults to 1.0. """ eps = 1e-12 pos_weights = gaussian_target.eq(1) neg_weights = (1 - gaussian_target).pow(gamma) pos_loss = -(pred + eps).log() * (1 - pred).pow(alpha) * pos_weights neg_loss = -(1 - pred + eps).log() * pred.pow(alpha) * neg_weights return pos_weight * pos_loss + neg_weight * neg_loss def gaussian_focal_loss_with_pos_inds( pred: Tensor, gaussian_target: Tensor, pos_inds: Tensor, pos_labels: Tensor, alpha: float = 2.0, gamma: float = 4.0, pos_weight: float = 1.0, neg_weight: float = 1.0, reduction: str = 'mean', avg_factor: Optional[Union[int, float]] = None) -> Tensor: """`Focal Loss `_ for targets in gaussian distribution. Note: The index with a value of 1 in ``gaussian_target`` in the ``gaussian_focal_loss`` function is a positive sample, but in ``gaussian_focal_loss_with_pos_inds`` the positive sample is passed in through the ``pos_inds`` parameter. Args: pred (torch.Tensor): The prediction. The shape is (N, num_classes). gaussian_target (torch.Tensor): The learning target of the prediction in gaussian distribution. The shape is (N, num_classes). pos_inds (torch.Tensor): The positive sample index. The shape is (M, ). pos_labels (torch.Tensor): The label corresponding to the positive sample index. The shape is (M, ). alpha (float, optional): A balanced form for Focal Loss. Defaults to 2.0. gamma (float, optional): The gamma for calculating the modulating factor. Defaults to 4.0. pos_weight(float): Positive sample loss weight. Defaults to 1.0. neg_weight(float): Negative sample loss weight. Defaults to 1.0. reduction (str): Options are "none", "mean" and "sum". Defaults to 'mean`. avg_factor (int, float, optional): Average factor that is used to average the loss. Defaults to None. """ eps = 1e-12 neg_weights = (1 - gaussian_target).pow(gamma) pos_pred_pix = pred[pos_inds] pos_pred = pos_pred_pix.gather(1, pos_labels.unsqueeze(1)) pos_loss = -(pos_pred + eps).log() * (1 - pos_pred).pow(alpha) pos_loss = weight_reduce_loss(pos_loss, None, reduction, avg_factor) neg_loss = -(1 - pred + eps).log() * pred.pow(alpha) * neg_weights neg_loss = weight_reduce_loss(neg_loss, None, reduction, avg_factor) return pos_weight * pos_loss + neg_weight * neg_loss @MODELS.register_module() class GaussianFocalLoss(nn.Module): """GaussianFocalLoss is a variant of focal loss. More details can be found in the `paper `_ Code is modified from `kp_utils.py `_ # noqa: E501 Please notice that the target in GaussianFocalLoss is a gaussian heatmap, not 0/1 binary target. Args: alpha (float): Power of prediction. gamma (float): Power of target for negative samples. reduction (str): Options are "none", "mean" and "sum". loss_weight (float): Loss weight of current loss. pos_weight(float): Positive sample loss weight. Defaults to 1.0. neg_weight(float): Negative sample loss weight. Defaults to 1.0. """ def __init__(self, alpha: float = 2.0, gamma: float = 4.0, reduction: str = 'mean', loss_weight: float = 1.0, pos_weight: float = 1.0, neg_weight: float = 1.0) -> None: super().__init__() self.alpha = alpha self.gamma = gamma self.reduction = reduction self.loss_weight = loss_weight self.pos_weight = pos_weight self.neg_weight = neg_weight def forward(self, pred: Tensor, target: Tensor, pos_inds: Optional[Tensor] = None, pos_labels: Optional[Tensor] = None, weight: Optional[Tensor] = None, avg_factor: Optional[Union[int, float]] = None, reduction_override: Optional[str] = None) -> Tensor: """Forward function. If you want to manually determine which positions are positive samples, you can set the pos_index and pos_label parameter. Currently, only the CenterNet update version uses the parameter. Args: pred (torch.Tensor): The prediction. The shape is (N, num_classes). target (torch.Tensor): The learning target of the prediction in gaussian distribution. The shape is (N, num_classes). pos_inds (torch.Tensor): The positive sample index. Defaults to None. pos_labels (torch.Tensor): The label corresponding to the positive sample index. Defaults to None. weight (torch.Tensor, optional): The weight of loss for each prediction. Defaults to None. avg_factor (int, float, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if pos_inds is not None: assert pos_labels is not None # Only used by centernet update version loss_reg = self.loss_weight * gaussian_focal_loss_with_pos_inds( pred, target, pos_inds, pos_labels, alpha=self.alpha, gamma=self.gamma, pos_weight=self.pos_weight, neg_weight=self.neg_weight, reduction=reduction, avg_factor=avg_factor) else: loss_reg = self.loss_weight * gaussian_focal_loss( pred, target, weight, alpha=self.alpha, gamma=self.gamma, pos_weight=self.pos_weight, neg_weight=self.neg_weight, reduction=reduction, avg_factor=avg_factor) return loss_reg ================================================ FILE: mmdet/models/losses/gfocal_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from functools import partial import torch import torch.nn as nn import torch.nn.functional as F from mmdet.models.losses.utils import weighted_loss from mmdet.registry import MODELS @weighted_loss def quality_focal_loss(pred, target, beta=2.0): r"""Quality Focal Loss (QFL) is from `Generalized Focal Loss: Learning Qualified and Distributed Bounding Boxes for Dense Object Detection `_. Args: pred (torch.Tensor): Predicted joint representation of classification and quality (IoU) estimation with shape (N, C), C is the number of classes. target (tuple([torch.Tensor])): Target category label with shape (N,) and target quality label with shape (N,). beta (float): The beta parameter for calculating the modulating factor. Defaults to 2.0. Returns: torch.Tensor: Loss tensor with shape (N,). """ assert len(target) == 2, """target for QFL must be a tuple of two elements, including category label and quality label, respectively""" # label denotes the category id, score denotes the quality score label, score = target # negatives are supervised by 0 quality score pred_sigmoid = pred.sigmoid() scale_factor = pred_sigmoid zerolabel = scale_factor.new_zeros(pred.shape) loss = F.binary_cross_entropy_with_logits( pred, zerolabel, reduction='none') * scale_factor.pow(beta) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = pred.size(1) pos = ((label >= 0) & (label < bg_class_ind)).nonzero().squeeze(1) pos_label = label[pos].long() # positives are supervised by bbox quality (IoU) score scale_factor = score[pos] - pred_sigmoid[pos, pos_label] loss[pos, pos_label] = F.binary_cross_entropy_with_logits( pred[pos, pos_label], score[pos], reduction='none') * scale_factor.abs().pow(beta) loss = loss.sum(dim=1, keepdim=False) return loss @weighted_loss def quality_focal_loss_tensor_target(pred, target, beta=2.0, activated=False): """`QualityFocal Loss `_ Args: pred (torch.Tensor): The prediction with shape (N, C), C is the number of classes target (torch.Tensor): The learning target of the iou-aware classification score with shape (N, C), C is the number of classes. beta (float): The beta parameter for calculating the modulating factor. Defaults to 2.0. activated (bool): Whether the input is activated. If True, it means the input has been activated and can be treated as probabilities. Else, it should be treated as logits. Defaults to False. """ # pred and target should be of the same size assert pred.size() == target.size() if activated: pred_sigmoid = pred loss_function = F.binary_cross_entropy else: pred_sigmoid = pred.sigmoid() loss_function = F.binary_cross_entropy_with_logits scale_factor = pred_sigmoid target = target.type_as(pred) zerolabel = scale_factor.new_zeros(pred.shape) loss = loss_function( pred, zerolabel, reduction='none') * scale_factor.pow(beta) pos = (target != 0) scale_factor = target[pos] - pred_sigmoid[pos] loss[pos] = loss_function( pred[pos], target[pos], reduction='none') * scale_factor.abs().pow(beta) loss = loss.sum(dim=1, keepdim=False) return loss @weighted_loss def quality_focal_loss_with_prob(pred, target, beta=2.0): r"""Quality Focal Loss (QFL) is from `Generalized Focal Loss: Learning Qualified and Distributed Bounding Boxes for Dense Object Detection `_. Different from `quality_focal_loss`, this function accepts probability as input. Args: pred (torch.Tensor): Predicted joint representation of classification and quality (IoU) estimation with shape (N, C), C is the number of classes. target (tuple([torch.Tensor])): Target category label with shape (N,) and target quality label with shape (N,). beta (float): The beta parameter for calculating the modulating factor. Defaults to 2.0. Returns: torch.Tensor: Loss tensor with shape (N,). """ assert len(target) == 2, """target for QFL must be a tuple of two elements, including category label and quality label, respectively""" # label denotes the category id, score denotes the quality score label, score = target # negatives are supervised by 0 quality score pred_sigmoid = pred scale_factor = pred_sigmoid zerolabel = scale_factor.new_zeros(pred.shape) loss = F.binary_cross_entropy( pred, zerolabel, reduction='none') * scale_factor.pow(beta) # FG cat_id: [0, num_classes -1], BG cat_id: num_classes bg_class_ind = pred.size(1) pos = ((label >= 0) & (label < bg_class_ind)).nonzero().squeeze(1) pos_label = label[pos].long() # positives are supervised by bbox quality (IoU) score scale_factor = score[pos] - pred_sigmoid[pos, pos_label] loss[pos, pos_label] = F.binary_cross_entropy( pred[pos, pos_label], score[pos], reduction='none') * scale_factor.abs().pow(beta) loss = loss.sum(dim=1, keepdim=False) return loss @weighted_loss def distribution_focal_loss(pred, label): r"""Distribution Focal Loss (DFL) is from `Generalized Focal Loss: Learning Qualified and Distributed Bounding Boxes for Dense Object Detection `_. Args: pred (torch.Tensor): Predicted general distribution of bounding boxes (before softmax) with shape (N, n+1), n is the max value of the integral set `{0, ..., n}` in paper. label (torch.Tensor): Target distance label for bounding boxes with shape (N,). Returns: torch.Tensor: Loss tensor with shape (N,). """ dis_left = label.long() dis_right = dis_left + 1 weight_left = dis_right.float() - label weight_right = label - dis_left.float() loss = F.cross_entropy(pred, dis_left, reduction='none') * weight_left \ + F.cross_entropy(pred, dis_right, reduction='none') * weight_right return loss @MODELS.register_module() class QualityFocalLoss(nn.Module): r"""Quality Focal Loss (QFL) is a variant of `Generalized Focal Loss: Learning Qualified and Distributed Bounding Boxes for Dense Object Detection `_. Args: use_sigmoid (bool): Whether sigmoid operation is conducted in QFL. Defaults to True. beta (float): The beta parameter for calculating the modulating factor. Defaults to 2.0. reduction (str): Options are "none", "mean" and "sum". loss_weight (float): Loss weight of current loss. activated (bool, optional): Whether the input is activated. If True, it means the input has been activated and can be treated as probabilities. Else, it should be treated as logits. Defaults to False. """ def __init__(self, use_sigmoid=True, beta=2.0, reduction='mean', loss_weight=1.0, activated=False): super(QualityFocalLoss, self).__init__() assert use_sigmoid is True, 'Only sigmoid in QFL supported now.' self.use_sigmoid = use_sigmoid self.beta = beta self.reduction = reduction self.loss_weight = loss_weight self.activated = activated def forward(self, pred, target, weight=None, avg_factor=None, reduction_override=None): """Forward function. Args: pred (torch.Tensor): Predicted joint representation of classification and quality (IoU) estimation with shape (N, C), C is the number of classes. target (Union(tuple([torch.Tensor]),Torch.Tensor)): The type is tuple, it should be included Target category label with shape (N,) and target quality label with shape (N,).The type is torch.Tensor, the target should be one-hot form with soft weights. weight (torch.Tensor, optional): The weight of loss for each prediction. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if self.use_sigmoid: if self.activated: calculate_loss_func = quality_focal_loss_with_prob else: calculate_loss_func = quality_focal_loss if isinstance(target, torch.Tensor): # the target shape with (N,C) or (N,C,...), which means # the target is one-hot form with soft weights. calculate_loss_func = partial( quality_focal_loss_tensor_target, activated=self.activated) loss_cls = self.loss_weight * calculate_loss_func( pred, target, weight, beta=self.beta, reduction=reduction, avg_factor=avg_factor) else: raise NotImplementedError return loss_cls @MODELS.register_module() class DistributionFocalLoss(nn.Module): r"""Distribution Focal Loss (DFL) is a variant of `Generalized Focal Loss: Learning Qualified and Distributed Bounding Boxes for Dense Object Detection `_. Args: reduction (str): Options are `'none'`, `'mean'` and `'sum'`. loss_weight (float): Loss weight of current loss. """ def __init__(self, reduction='mean', loss_weight=1.0): super(DistributionFocalLoss, self).__init__() self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred, target, weight=None, avg_factor=None, reduction_override=None): """Forward function. Args: pred (torch.Tensor): Predicted general distribution of bounding boxes (before softmax) with shape (N, n+1), n is the max value of the integral set `{0, ..., n}` in paper. target (torch.Tensor): Target distance label for bounding boxes with shape (N,). weight (torch.Tensor, optional): The weight of loss for each prediction. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) loss_cls = self.loss_weight * distribution_focal_loss( pred, target, weight, reduction=reduction, avg_factor=avg_factor) return loss_cls ================================================ FILE: mmdet/models/losses/ghm_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F from mmdet.registry import MODELS from .utils import weight_reduce_loss def _expand_onehot_labels(labels, label_weights, label_channels): bin_labels = labels.new_full((labels.size(0), label_channels), 0) inds = torch.nonzero( (labels >= 0) & (labels < label_channels), as_tuple=False).squeeze() if inds.numel() > 0: bin_labels[inds, labels[inds]] = 1 bin_label_weights = label_weights.view(-1, 1).expand( label_weights.size(0), label_channels) return bin_labels, bin_label_weights # TODO: code refactoring to make it consistent with other losses @MODELS.register_module() class GHMC(nn.Module): """GHM Classification Loss. Details of the theorem can be viewed in the paper `Gradient Harmonized Single-stage Detector `_. Args: bins (int): Number of the unit regions for distribution calculation. momentum (float): The parameter for moving average. use_sigmoid (bool): Can only be true for BCE based loss now. loss_weight (float): The weight of the total GHM-C loss. reduction (str): Options are "none", "mean" and "sum". Defaults to "mean" """ def __init__(self, bins=10, momentum=0, use_sigmoid=True, loss_weight=1.0, reduction='mean'): super(GHMC, self).__init__() self.bins = bins self.momentum = momentum edges = torch.arange(bins + 1).float() / bins self.register_buffer('edges', edges) self.edges[-1] += 1e-6 if momentum > 0: acc_sum = torch.zeros(bins) self.register_buffer('acc_sum', acc_sum) self.use_sigmoid = use_sigmoid if not self.use_sigmoid: raise NotImplementedError self.loss_weight = loss_weight self.reduction = reduction def forward(self, pred, target, label_weight, reduction_override=None, **kwargs): """Calculate the GHM-C loss. Args: pred (float tensor of size [batch_num, class_num]): The direct prediction of classification fc layer. target (float tensor of size [batch_num, class_num]): Binary class target for each sample. label_weight (float tensor of size [batch_num, class_num]): the value is 1 if the sample is valid and 0 if ignored. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Returns: The gradient harmonized loss. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) # the target should be binary class label if pred.dim() != target.dim(): target, label_weight = _expand_onehot_labels( target, label_weight, pred.size(-1)) target, label_weight = target.float(), label_weight.float() edges = self.edges mmt = self.momentum weights = torch.zeros_like(pred) # gradient length g = torch.abs(pred.sigmoid().detach() - target) valid = label_weight > 0 tot = max(valid.float().sum().item(), 1.0) n = 0 # n valid bins for i in range(self.bins): inds = (g >= edges[i]) & (g < edges[i + 1]) & valid num_in_bin = inds.sum().item() if num_in_bin > 0: if mmt > 0: self.acc_sum[i] = mmt * self.acc_sum[i] \ + (1 - mmt) * num_in_bin weights[inds] = tot / self.acc_sum[i] else: weights[inds] = tot / num_in_bin n += 1 if n > 0: weights = weights / n loss = F.binary_cross_entropy_with_logits( pred, target, reduction='none') loss = weight_reduce_loss( loss, weights, reduction=reduction, avg_factor=tot) return loss * self.loss_weight # TODO: code refactoring to make it consistent with other losses @MODELS.register_module() class GHMR(nn.Module): """GHM Regression Loss. Details of the theorem can be viewed in the paper `Gradient Harmonized Single-stage Detector `_. Args: mu (float): The parameter for the Authentic Smooth L1 loss. bins (int): Number of the unit regions for distribution calculation. momentum (float): The parameter for moving average. loss_weight (float): The weight of the total GHM-R loss. reduction (str): Options are "none", "mean" and "sum". Defaults to "mean" """ def __init__(self, mu=0.02, bins=10, momentum=0, loss_weight=1.0, reduction='mean'): super(GHMR, self).__init__() self.mu = mu self.bins = bins edges = torch.arange(bins + 1).float() / bins self.register_buffer('edges', edges) self.edges[-1] = 1e3 self.momentum = momentum if momentum > 0: acc_sum = torch.zeros(bins) self.register_buffer('acc_sum', acc_sum) self.loss_weight = loss_weight self.reduction = reduction # TODO: support reduction parameter def forward(self, pred, target, label_weight, avg_factor=None, reduction_override=None): """Calculate the GHM-R loss. Args: pred (float tensor of size [batch_num, 4 (* class_num)]): The prediction of box regression layer. Channel number can be 4 or 4 * class_num depending on whether it is class-agnostic. target (float tensor of size [batch_num, 4 (* class_num)]): The target regression values with the same size of pred. label_weight (float tensor of size [batch_num, 4 (* class_num)]): The weight of each sample, 0 if ignored. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Returns: The gradient harmonized loss. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) mu = self.mu edges = self.edges mmt = self.momentum # ASL1 loss diff = pred - target loss = torch.sqrt(diff * diff + mu * mu) - mu # gradient length g = torch.abs(diff / torch.sqrt(mu * mu + diff * diff)).detach() weights = torch.zeros_like(g) valid = label_weight > 0 tot = max(label_weight.float().sum().item(), 1.0) n = 0 # n: valid bins for i in range(self.bins): inds = (g >= edges[i]) & (g < edges[i + 1]) & valid num_in_bin = inds.sum().item() if num_in_bin > 0: n += 1 if mmt > 0: self.acc_sum[i] = mmt * self.acc_sum[i] \ + (1 - mmt) * num_in_bin weights[inds] = tot / self.acc_sum[i] else: weights[inds] = tot / num_in_bin if n > 0: weights /= n loss = weight_reduce_loss( loss, weights, reduction=reduction, avg_factor=tot) return loss * self.loss_weight ================================================ FILE: mmdet/models/losses/iou_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math import warnings from typing import Optional import torch import torch.nn as nn from torch import Tensor from mmdet.registry import MODELS from mmdet.structures.bbox import bbox_overlaps from .utils import weighted_loss @weighted_loss def iou_loss(pred: Tensor, target: Tensor, linear: bool = False, mode: str = 'log', eps: float = 1e-6) -> Tensor: """IoU loss. Computing the IoU loss between a set of predicted bboxes and target bboxes. The loss is calculated as negative log of IoU. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): Corresponding gt bboxes, shape (n, 4). linear (bool, optional): If True, use linear scale of loss instead of log scale. Default: False. mode (str): Loss scaling mode, including "linear", "square", and "log". Default: 'log' eps (float): Epsilon to avoid log(0). Return: Tensor: Loss tensor. """ assert mode in ['linear', 'square', 'log'] if linear: mode = 'linear' warnings.warn('DeprecationWarning: Setting "linear=True" in ' 'iou_loss is deprecated, please use "mode=`linear`" ' 'instead.') ious = bbox_overlaps(pred, target, is_aligned=True).clamp(min=eps) if mode == 'linear': loss = 1 - ious elif mode == 'square': loss = 1 - ious**2 elif mode == 'log': loss = -ious.log() else: raise NotImplementedError return loss @weighted_loss def bounded_iou_loss(pred: Tensor, target: Tensor, beta: float = 0.2, eps: float = 1e-3) -> Tensor: """BIoULoss. This is an implementation of paper `Improving Object Localization with Fitness NMS and Bounded IoU Loss. `_. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): Corresponding gt bboxes, shape (n, 4). beta (float, optional): Beta parameter in smoothl1. eps (float, optional): Epsilon to avoid NaN values. Return: Tensor: Loss tensor. """ pred_ctrx = (pred[:, 0] + pred[:, 2]) * 0.5 pred_ctry = (pred[:, 1] + pred[:, 3]) * 0.5 pred_w = pred[:, 2] - pred[:, 0] pred_h = pred[:, 3] - pred[:, 1] with torch.no_grad(): target_ctrx = (target[:, 0] + target[:, 2]) * 0.5 target_ctry = (target[:, 1] + target[:, 3]) * 0.5 target_w = target[:, 2] - target[:, 0] target_h = target[:, 3] - target[:, 1] dx = target_ctrx - pred_ctrx dy = target_ctry - pred_ctry loss_dx = 1 - torch.max( (target_w - 2 * dx.abs()) / (target_w + 2 * dx.abs() + eps), torch.zeros_like(dx)) loss_dy = 1 - torch.max( (target_h - 2 * dy.abs()) / (target_h + 2 * dy.abs() + eps), torch.zeros_like(dy)) loss_dw = 1 - torch.min(target_w / (pred_w + eps), pred_w / (target_w + eps)) loss_dh = 1 - torch.min(target_h / (pred_h + eps), pred_h / (target_h + eps)) # view(..., -1) does not work for empty tensor loss_comb = torch.stack([loss_dx, loss_dy, loss_dw, loss_dh], dim=-1).flatten(1) loss = torch.where(loss_comb < beta, 0.5 * loss_comb * loss_comb / beta, loss_comb - 0.5 * beta) return loss @weighted_loss def giou_loss(pred: Tensor, target: Tensor, eps: float = 1e-7) -> Tensor: r"""`Generalized Intersection over Union: A Metric and A Loss for Bounding Box Regression `_. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): Corresponding gt bboxes, shape (n, 4). eps (float): Epsilon to avoid log(0). Return: Tensor: Loss tensor. """ gious = bbox_overlaps(pred, target, mode='giou', is_aligned=True, eps=eps) loss = 1 - gious return loss @weighted_loss def diou_loss(pred: Tensor, target: Tensor, eps: float = 1e-7) -> Tensor: r"""Implementation of `Distance-IoU Loss: Faster and Better Learning for Bounding Box Regression https://arxiv.org/abs/1911.08287`_. Code is modified from https://github.com/Zzh-tju/DIoU. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): Corresponding gt bboxes, shape (n, 4). eps (float): Epsilon to avoid log(0). Return: Tensor: Loss tensor. """ # overlap lt = torch.max(pred[:, :2], target[:, :2]) rb = torch.min(pred[:, 2:], target[:, 2:]) wh = (rb - lt).clamp(min=0) overlap = wh[:, 0] * wh[:, 1] # union ap = (pred[:, 2] - pred[:, 0]) * (pred[:, 3] - pred[:, 1]) ag = (target[:, 2] - target[:, 0]) * (target[:, 3] - target[:, 1]) union = ap + ag - overlap + eps # IoU ious = overlap / union # enclose area enclose_x1y1 = torch.min(pred[:, :2], target[:, :2]) enclose_x2y2 = torch.max(pred[:, 2:], target[:, 2:]) enclose_wh = (enclose_x2y2 - enclose_x1y1).clamp(min=0) cw = enclose_wh[:, 0] ch = enclose_wh[:, 1] c2 = cw**2 + ch**2 + eps b1_x1, b1_y1 = pred[:, 0], pred[:, 1] b1_x2, b1_y2 = pred[:, 2], pred[:, 3] b2_x1, b2_y1 = target[:, 0], target[:, 1] b2_x2, b2_y2 = target[:, 2], target[:, 3] left = ((b2_x1 + b2_x2) - (b1_x1 + b1_x2))**2 / 4 right = ((b2_y1 + b2_y2) - (b1_y1 + b1_y2))**2 / 4 rho2 = left + right # DIoU dious = ious - rho2 / c2 loss = 1 - dious return loss @weighted_loss def ciou_loss(pred: Tensor, target: Tensor, eps: float = 1e-7) -> Tensor: r"""`Implementation of paper `Enhancing Geometric Factors into Model Learning and Inference for Object Detection and Instance Segmentation `_. Code is modified from https://github.com/Zzh-tju/CIoU. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): Corresponding gt bboxes, shape (n, 4). eps (float): Epsilon to avoid log(0). Return: Tensor: Loss tensor. """ # overlap lt = torch.max(pred[:, :2], target[:, :2]) rb = torch.min(pred[:, 2:], target[:, 2:]) wh = (rb - lt).clamp(min=0) overlap = wh[:, 0] * wh[:, 1] # union ap = (pred[:, 2] - pred[:, 0]) * (pred[:, 3] - pred[:, 1]) ag = (target[:, 2] - target[:, 0]) * (target[:, 3] - target[:, 1]) union = ap + ag - overlap + eps # IoU ious = overlap / union # enclose area enclose_x1y1 = torch.min(pred[:, :2], target[:, :2]) enclose_x2y2 = torch.max(pred[:, 2:], target[:, 2:]) enclose_wh = (enclose_x2y2 - enclose_x1y1).clamp(min=0) cw = enclose_wh[:, 0] ch = enclose_wh[:, 1] c2 = cw**2 + ch**2 + eps b1_x1, b1_y1 = pred[:, 0], pred[:, 1] b1_x2, b1_y2 = pred[:, 2], pred[:, 3] b2_x1, b2_y1 = target[:, 0], target[:, 1] b2_x2, b2_y2 = target[:, 2], target[:, 3] w1, h1 = b1_x2 - b1_x1, b1_y2 - b1_y1 + eps w2, h2 = b2_x2 - b2_x1, b2_y2 - b2_y1 + eps left = ((b2_x1 + b2_x2) - (b1_x1 + b1_x2))**2 / 4 right = ((b2_y1 + b2_y2) - (b1_y1 + b1_y2))**2 / 4 rho2 = left + right factor = 4 / math.pi**2 v = factor * torch.pow(torch.atan(w2 / h2) - torch.atan(w1 / h1), 2) with torch.no_grad(): alpha = (ious > 0.5).float() * v / (1 - ious + v) # CIoU cious = ious - (rho2 / c2 + alpha * v) loss = 1 - cious.clamp(min=-1.0, max=1.0) return loss @weighted_loss def eiou_loss(pred: Tensor, target: Tensor, smooth_point: float = 0.1, eps: float = 1e-7) -> Tensor: r"""Implementation of paper `Extended-IoU Loss: A Systematic IoU-Related Method: Beyond Simplified Regression for Better Localization `_ Code is modified from https://github.com//ShiqiYu/libfacedetection.train. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): Corresponding gt bboxes, shape (n, 4). smooth_point (float): hyperparameter, default is 0.1. eps (float): Epsilon to avoid log(0). Return: Tensor: Loss tensor. """ px1, py1, px2, py2 = pred[:, 0], pred[:, 1], pred[:, 2], pred[:, 3] tx1, ty1, tx2, ty2 = target[:, 0], target[:, 1], target[:, 2], target[:, 3] # extent top left ex1 = torch.min(px1, tx1) ey1 = torch.min(py1, ty1) # intersection coordinates ix1 = torch.max(px1, tx1) iy1 = torch.max(py1, ty1) ix2 = torch.min(px2, tx2) iy2 = torch.min(py2, ty2) # extra xmin = torch.min(ix1, ix2) ymin = torch.min(iy1, iy2) xmax = torch.max(ix1, ix2) ymax = torch.max(iy1, iy2) # Intersection intersection = (ix2 - ex1) * (iy2 - ey1) + (xmin - ex1) * (ymin - ey1) - ( ix1 - ex1) * (ymax - ey1) - (xmax - ex1) * ( iy1 - ey1) # Union union = (px2 - px1) * (py2 - py1) + (tx2 - tx1) * ( ty2 - ty1) - intersection + eps # IoU ious = 1 - (intersection / union) # Smooth-EIoU smooth_sign = (ious < smooth_point).detach().float() loss = 0.5 * smooth_sign * (ious**2) / smooth_point + (1 - smooth_sign) * ( ious - 0.5 * smooth_point) return loss @MODELS.register_module() class IoULoss(nn.Module): """IoULoss. Computing the IoU loss between a set of predicted bboxes and target bboxes. Args: linear (bool): If True, use linear scale of loss else determined by mode. Default: False. eps (float): Epsilon to avoid log(0). reduction (str): Options are "none", "mean" and "sum". loss_weight (float): Weight of loss. mode (str): Loss scaling mode, including "linear", "square", and "log". Default: 'log' """ def __init__(self, linear: bool = False, eps: float = 1e-6, reduction: str = 'mean', loss_weight: float = 1.0, mode: str = 'log') -> None: super().__init__() assert mode in ['linear', 'square', 'log'] if linear: mode = 'linear' warnings.warn('DeprecationWarning: Setting "linear=True" in ' 'IOULoss is deprecated, please use "mode=`linear`" ' 'instead.') self.mode = mode self.linear = linear self.eps = eps self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None, **kwargs) -> Tensor: """Forward function. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): The learning target of the prediction, shape (n, 4). weight (Tensor, optional): The weight of loss for each prediction. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Options are "none", "mean" and "sum". Return: Tensor: Loss tensor. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if (weight is not None) and (not torch.any(weight > 0)) and ( reduction != 'none'): if pred.dim() == weight.dim() + 1: weight = weight.unsqueeze(1) return (pred * weight).sum() # 0 if weight is not None and weight.dim() > 1: # TODO: remove this in the future # reduce the weight of shape (n, 4) to (n,) to match the # iou_loss of shape (n,) assert weight.shape == pred.shape weight = weight.mean(-1) loss = self.loss_weight * iou_loss( pred, target, weight, mode=self.mode, eps=self.eps, reduction=reduction, avg_factor=avg_factor, **kwargs) return loss @MODELS.register_module() class BoundedIoULoss(nn.Module): """BIoULoss. This is an implementation of paper `Improving Object Localization with Fitness NMS and Bounded IoU Loss. `_. Args: beta (float, optional): Beta parameter in smoothl1. eps (float, optional): Epsilon to avoid NaN values. reduction (str): Options are "none", "mean" and "sum". loss_weight (float): Weight of loss. """ def __init__(self, beta: float = 0.2, eps: float = 1e-3, reduction: str = 'mean', loss_weight: float = 1.0) -> None: super().__init__() self.beta = beta self.eps = eps self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None, **kwargs) -> Tensor: """Forward function. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): The learning target of the prediction, shape (n, 4). weight (Optional[Tensor], optional): The weight of loss for each prediction. Defaults to None. avg_factor (Optional[int], optional): Average factor that is used to average the loss. Defaults to None. reduction_override (Optional[str], optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Options are "none", "mean" and "sum". Returns: Tensor: Loss tensor. """ if weight is not None and not torch.any(weight > 0): if pred.dim() == weight.dim() + 1: weight = weight.unsqueeze(1) return (pred * weight).sum() # 0 assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) loss = self.loss_weight * bounded_iou_loss( pred, target, weight, beta=self.beta, eps=self.eps, reduction=reduction, avg_factor=avg_factor, **kwargs) return loss @MODELS.register_module() class GIoULoss(nn.Module): r"""`Generalized Intersection over Union: A Metric and A Loss for Bounding Box Regression `_. Args: eps (float): Epsilon to avoid log(0). reduction (str): Options are "none", "mean" and "sum". loss_weight (float): Weight of loss. """ def __init__(self, eps: float = 1e-6, reduction: str = 'mean', loss_weight: float = 1.0) -> None: super().__init__() self.eps = eps self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None, **kwargs) -> Tensor: """Forward function. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): The learning target of the prediction, shape (n, 4). weight (Optional[Tensor], optional): The weight of loss for each prediction. Defaults to None. avg_factor (Optional[int], optional): Average factor that is used to average the loss. Defaults to None. reduction_override (Optional[str], optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Options are "none", "mean" and "sum". Returns: Tensor: Loss tensor. """ if weight is not None and not torch.any(weight > 0): if pred.dim() == weight.dim() + 1: weight = weight.unsqueeze(1) return (pred * weight).sum() # 0 assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if weight is not None and weight.dim() > 1: # TODO: remove this in the future # reduce the weight of shape (n, 4) to (n,) to match the # giou_loss of shape (n,) assert weight.shape == pred.shape weight = weight.mean(-1) loss = self.loss_weight * giou_loss( pred, target, weight, eps=self.eps, reduction=reduction, avg_factor=avg_factor, **kwargs) return loss @MODELS.register_module() class DIoULoss(nn.Module): r"""Implementation of `Distance-IoU Loss: Faster and Better Learning for Bounding Box Regression https://arxiv.org/abs/1911.08287`_. Code is modified from https://github.com/Zzh-tju/DIoU. Args: eps (float): Epsilon to avoid log(0). reduction (str): Options are "none", "mean" and "sum". loss_weight (float): Weight of loss. """ def __init__(self, eps: float = 1e-6, reduction: str = 'mean', loss_weight: float = 1.0) -> None: super().__init__() self.eps = eps self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None, **kwargs) -> Tensor: """Forward function. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): The learning target of the prediction, shape (n, 4). weight (Optional[Tensor], optional): The weight of loss for each prediction. Defaults to None. avg_factor (Optional[int], optional): Average factor that is used to average the loss. Defaults to None. reduction_override (Optional[str], optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Options are "none", "mean" and "sum". Returns: Tensor: Loss tensor. """ if weight is not None and not torch.any(weight > 0): if pred.dim() == weight.dim() + 1: weight = weight.unsqueeze(1) return (pred * weight).sum() # 0 assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if weight is not None and weight.dim() > 1: # TODO: remove this in the future # reduce the weight of shape (n, 4) to (n,) to match the # giou_loss of shape (n,) assert weight.shape == pred.shape weight = weight.mean(-1) loss = self.loss_weight * diou_loss( pred, target, weight, eps=self.eps, reduction=reduction, avg_factor=avg_factor, **kwargs) return loss @MODELS.register_module() class CIoULoss(nn.Module): r"""`Implementation of paper `Enhancing Geometric Factors into Model Learning and Inference for Object Detection and Instance Segmentation `_. Code is modified from https://github.com/Zzh-tju/CIoU. Args: eps (float): Epsilon to avoid log(0). reduction (str): Options are "none", "mean" and "sum". loss_weight (float): Weight of loss. """ def __init__(self, eps: float = 1e-6, reduction: str = 'mean', loss_weight: float = 1.0) -> None: super().__init__() self.eps = eps self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None, **kwargs) -> Tensor: """Forward function. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): The learning target of the prediction, shape (n, 4). weight (Optional[Tensor], optional): The weight of loss for each prediction. Defaults to None. avg_factor (Optional[int], optional): Average factor that is used to average the loss. Defaults to None. reduction_override (Optional[str], optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Options are "none", "mean" and "sum". Returns: Tensor: Loss tensor. """ if weight is not None and not torch.any(weight > 0): if pred.dim() == weight.dim() + 1: weight = weight.unsqueeze(1) return (pred * weight).sum() # 0 assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if weight is not None and weight.dim() > 1: # TODO: remove this in the future # reduce the weight of shape (n, 4) to (n,) to match the # giou_loss of shape (n,) assert weight.shape == pred.shape weight = weight.mean(-1) loss = self.loss_weight * ciou_loss( pred, target, weight, eps=self.eps, reduction=reduction, avg_factor=avg_factor, **kwargs) return loss @MODELS.register_module() class EIoULoss(nn.Module): r"""Implementation of paper `Extended-IoU Loss: A Systematic IoU-Related Method: Beyond Simplified Regression for Better Localization `_ Code is modified from https://github.com//ShiqiYu/libfacedetection.train. Args: eps (float): Epsilon to avoid log(0). reduction (str): Options are "none", "mean" and "sum". loss_weight (float): Weight of loss. smooth_point (float): hyperparameter, default is 0.1. """ def __init__(self, eps: float = 1e-6, reduction: str = 'mean', loss_weight: float = 1.0, smooth_point: float = 0.1) -> None: super().__init__() self.eps = eps self.reduction = reduction self.loss_weight = loss_weight self.smooth_point = smooth_point def forward(self, pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None, **kwargs) -> Tensor: """Forward function. Args: pred (Tensor): Predicted bboxes of format (x1, y1, x2, y2), shape (n, 4). target (Tensor): The learning target of the prediction, shape (n, 4). weight (Optional[Tensor], optional): The weight of loss for each prediction. Defaults to None. avg_factor (Optional[int], optional): Average factor that is used to average the loss. Defaults to None. reduction_override (Optional[str], optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Options are "none", "mean" and "sum". Returns: Tensor: Loss tensor. """ if weight is not None and not torch.any(weight > 0): if pred.dim() == weight.dim() + 1: weight = weight.unsqueeze(1) return (pred * weight).sum() # 0 assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if weight is not None and weight.dim() > 1: assert weight.shape == pred.shape weight = weight.mean(-1) loss = self.loss_weight * eiou_loss( pred, target, weight, smooth_point=self.smooth_point, eps=self.eps, reduction=reduction, avg_factor=avg_factor, **kwargs) return loss ================================================ FILE: mmdet/models/losses/kd_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F from mmdet.registry import MODELS from .utils import weight_reduce_loss, weighted_loss @weighted_loss def knowledge_distillation_kl_div_loss(pred, soft_label, T, class_reduction='mean', detach_target=True): r"""Loss function for knowledge distilling using KL divergence. Args: pred (Tensor): Predicted logits with shape (N, n + 1). soft_label (Tensor): Target logits with shape (N, N + 1). T (int): Temperature for distillation. detach_target (bool): Remove soft_label from automatic differentiation Returns: torch.Tensor: Loss tensor with shape (N,). """ assert pred.size() == soft_label.size() target = F.softmax(soft_label / T, dim=1) if detach_target: target = target.detach() kd_loss = F.kl_div( F.log_softmax(pred / T, dim=1), target, reduction='none') if class_reduction == 'mean': kd_loss = kd_loss.mean(1) elif class_reduction == 'sum': kd_loss = kd_loss.sum(1) else: raise NotImplementedError kd_loss = kd_loss * (T * T) return kd_loss def kd_quality_focal_loss(pred, target, weight=None, beta=1, reduction='mean', avg_factor=None): num_classes = pred.size(1) if weight is not None: weight = weight[:, None].repeat(1, num_classes) target = target.detach().sigmoid() loss = F.binary_cross_entropy_with_logits(pred, target, reduction='none') focal_weight = torch.abs(pred.sigmoid() - target).pow(beta) loss = loss * focal_weight loss = weight_reduce_loss(loss, weight, reduction, avg_factor) return loss @MODELS.register_module() class KnowledgeDistillationKLDivLoss(nn.Module): """Loss function for knowledge distilling using KL divergence. Args: reduction (str): Options are `'none'`, `'mean'` and `'sum'`. loss_weight (float): Loss weight of current loss. T (int): Temperature for distillation. """ def __init__(self, class_reduction='mean', reduction='mean', loss_weight=1.0, T=10): super(KnowledgeDistillationKLDivLoss, self).__init__() assert T >= 1 self.class_reduction = class_reduction self.reduction = reduction self.loss_weight = loss_weight self.T = T def forward(self, pred, soft_label, weight=None, avg_factor=None, reduction_override=None): """Forward function. Args: pred (Tensor): Predicted logits with shape (N, n + 1). soft_label (Tensor): Target logits with shape (N, N + 1). weight (torch.Tensor, optional): The weight of loss for each prediction. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) loss_kd = self.loss_weight * knowledge_distillation_kl_div_loss( pred, soft_label, weight, class_reduction=self.class_reduction, reduction=reduction, avg_factor=avg_factor, T=self.T) return loss_kd @MODELS.register_module() class KDQualityFocalLoss(nn.Module): def __init__(self, use_sigmoid=True, beta=1.0, reduction='mean', loss_weight=1.0): super(KDQualityFocalLoss, self).__init__() assert use_sigmoid is True, 'Only sigmoid in QFL supported now.' self.use_sigmoid = use_sigmoid self.beta = beta self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred, target, weight=None, avg_factor=None, reduction_override=None): """Forward function. Args: pred (torch.Tensor): Predicted joint representation of classification and quality (IoU) estimation with shape (N, C), C is the number of classes. target (tuple([torch.Tensor])): Target category label with shape (N,) and target quality label with shape (N,). weight (torch.Tensor, optional): The weight of loss for each prediction. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if self.use_sigmoid: loss = self.loss_weight * kd_quality_focal_loss( pred, target, weight, beta=self.beta, reduction=reduction, avg_factor=avg_factor) else: raise NotImplementedError return loss ================================================ FILE: mmdet/models/losses/mse_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch.nn as nn import torch.nn.functional as F from torch import Tensor from mmdet.registry import MODELS from .utils import weighted_loss @weighted_loss def mse_loss(pred: Tensor, target: Tensor) -> Tensor: """A Wrapper of MSE loss. Args: pred (Tensor): The prediction. target (Tensor): The learning target of the prediction. Returns: Tensor: loss Tensor """ return F.mse_loss(pred, target, reduction='none') @MODELS.register_module() class MSELoss(nn.Module): """MSELoss. Args: reduction (str, optional): The method that reduces the loss to a scalar. Options are "none", "mean" and "sum". loss_weight (float, optional): The weight of the loss. Defaults to 1.0 """ def __init__(self, reduction: str = 'mean', loss_weight: float = 1.0) -> None: super().__init__() self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None) -> Tensor: """Forward function of loss. Args: pred (Tensor): The prediction. target (Tensor): The learning target of the prediction. weight (Tensor, optional): Weight of the loss for each prediction. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Returns: Tensor: The calculated loss. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) loss = self.loss_weight * mse_loss( pred, target, weight, reduction=reduction, avg_factor=avg_factor) return loss ================================================ FILE: mmdet/models/losses/pisa_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch import torch.nn as nn from torch import Tensor from mmdet.structures.bbox import bbox_overlaps from ..task_modules.coders import BaseBBoxCoder from ..task_modules.samplers import SamplingResult def isr_p(cls_score: Tensor, bbox_pred: Tensor, bbox_targets: Tuple[Tensor], rois: Tensor, sampling_results: List[SamplingResult], loss_cls: nn.Module, bbox_coder: BaseBBoxCoder, k: float = 2, bias: float = 0, num_class: int = 80) -> tuple: """Importance-based Sample Reweighting (ISR_P), positive part. Args: cls_score (Tensor): Predicted classification scores. bbox_pred (Tensor): Predicted bbox deltas. bbox_targets (tuple[Tensor]): A tuple of bbox targets, the are labels, label_weights, bbox_targets, bbox_weights, respectively. rois (Tensor): Anchors (single_stage) in shape (n, 4) or RoIs (two_stage) in shape (n, 5). sampling_results (:obj:`SamplingResult`): Sampling results. loss_cls (:obj:`nn.Module`): Classification loss func of the head. bbox_coder (:obj:`BaseBBoxCoder`): BBox coder of the head. k (float): Power of the non-linear mapping. Defaults to 2. bias (float): Shift of the non-linear mapping. Defaults to 0. num_class (int): Number of classes, defaults to 80. Return: tuple([Tensor]): labels, imp_based_label_weights, bbox_targets, bbox_target_weights """ labels, label_weights, bbox_targets, bbox_weights = bbox_targets pos_label_inds = ((labels >= 0) & (labels < num_class)).nonzero().reshape(-1) pos_labels = labels[pos_label_inds] # if no positive samples, return the original targets num_pos = float(pos_label_inds.size(0)) if num_pos == 0: return labels, label_weights, bbox_targets, bbox_weights # merge pos_assigned_gt_inds of per image to a single tensor gts = list() last_max_gt = 0 for i in range(len(sampling_results)): gt_i = sampling_results[i].pos_assigned_gt_inds gts.append(gt_i + last_max_gt) if len(gt_i) != 0: last_max_gt = gt_i.max() + 1 gts = torch.cat(gts) assert len(gts) == num_pos cls_score = cls_score.detach() bbox_pred = bbox_pred.detach() # For single stage detectors, rois here indicate anchors, in shape (N, 4) # For two stage detectors, rois are in shape (N, 5) if rois.size(-1) == 5: pos_rois = rois[pos_label_inds][:, 1:] else: pos_rois = rois[pos_label_inds] if bbox_pred.size(-1) > 4: bbox_pred = bbox_pred.view(bbox_pred.size(0), -1, 4) pos_delta_pred = bbox_pred[pos_label_inds, pos_labels].view(-1, 4) else: pos_delta_pred = bbox_pred[pos_label_inds].view(-1, 4) # compute iou of the predicted bbox and the corresponding GT pos_delta_target = bbox_targets[pos_label_inds].view(-1, 4) pos_bbox_pred = bbox_coder.decode(pos_rois, pos_delta_pred) target_bbox_pred = bbox_coder.decode(pos_rois, pos_delta_target) ious = bbox_overlaps(pos_bbox_pred, target_bbox_pred, is_aligned=True) pos_imp_weights = label_weights[pos_label_inds] # Two steps to compute IoU-HLR. Samples are first sorted by IoU locally, # then sorted again within the same-rank group max_l_num = pos_labels.bincount().max() for label in pos_labels.unique(): l_inds = (pos_labels == label).nonzero().view(-1) l_gts = gts[l_inds] for t in l_gts.unique(): t_inds = l_inds[l_gts == t] t_ious = ious[t_inds] _, t_iou_rank_idx = t_ious.sort(descending=True) _, t_iou_rank = t_iou_rank_idx.sort() ious[t_inds] += max_l_num - t_iou_rank.float() l_ious = ious[l_inds] _, l_iou_rank_idx = l_ious.sort(descending=True) _, l_iou_rank = l_iou_rank_idx.sort() # IoU-HLR # linearly map HLR to label weights pos_imp_weights[l_inds] *= (max_l_num - l_iou_rank.float()) / max_l_num pos_imp_weights = (bias + pos_imp_weights * (1 - bias)).pow(k) # normalize to make the new weighted loss value equal to the original loss pos_loss_cls = loss_cls( cls_score[pos_label_inds], pos_labels, reduction_override='none') if pos_loss_cls.dim() > 1: ori_pos_loss_cls = pos_loss_cls * label_weights[pos_label_inds][:, None] new_pos_loss_cls = pos_loss_cls * pos_imp_weights[:, None] else: ori_pos_loss_cls = pos_loss_cls * label_weights[pos_label_inds] new_pos_loss_cls = pos_loss_cls * pos_imp_weights pos_loss_cls_ratio = ori_pos_loss_cls.sum() / new_pos_loss_cls.sum() pos_imp_weights = pos_imp_weights * pos_loss_cls_ratio label_weights[pos_label_inds] = pos_imp_weights bbox_targets = labels, label_weights, bbox_targets, bbox_weights return bbox_targets def carl_loss(cls_score: Tensor, labels: Tensor, bbox_pred: Tensor, bbox_targets: Tensor, loss_bbox: nn.Module, k: float = 1, bias: float = 0.2, avg_factor: Optional[int] = None, sigmoid: bool = False, num_class: int = 80) -> dict: """Classification-Aware Regression Loss (CARL). Args: cls_score (Tensor): Predicted classification scores. labels (Tensor): Targets of classification. bbox_pred (Tensor): Predicted bbox deltas. bbox_targets (Tensor): Target of bbox regression. loss_bbox (func): Regression loss func of the head. bbox_coder (obj): BBox coder of the head. k (float): Power of the non-linear mapping. Defaults to 1. bias (float): Shift of the non-linear mapping. Defaults to 0.2. avg_factor (int, optional): Average factor used in regression loss. sigmoid (bool): Activation of the classification score. num_class (int): Number of classes, defaults to 80. Return: dict: CARL loss dict. """ pos_label_inds = ((labels >= 0) & (labels < num_class)).nonzero().reshape(-1) if pos_label_inds.numel() == 0: return dict(loss_carl=cls_score.sum()[None] * 0.) pos_labels = labels[pos_label_inds] # multiply pos_cls_score with the corresponding bbox weight # and remain gradient if sigmoid: pos_cls_score = cls_score.sigmoid()[pos_label_inds, pos_labels] else: pos_cls_score = cls_score.softmax(-1)[pos_label_inds, pos_labels] carl_loss_weights = (bias + (1 - bias) * pos_cls_score).pow(k) # normalize carl_loss_weight to make its sum equal to num positive num_pos = float(pos_cls_score.size(0)) weight_ratio = num_pos / carl_loss_weights.sum() carl_loss_weights *= weight_ratio if avg_factor is None: avg_factor = bbox_targets.size(0) # if is class agnostic, bbox pred is in shape (N, 4) # otherwise, bbox pred is in shape (N, #classes, 4) if bbox_pred.size(-1) > 4: bbox_pred = bbox_pred.view(bbox_pred.size(0), -1, 4) pos_bbox_preds = bbox_pred[pos_label_inds, pos_labels] else: pos_bbox_preds = bbox_pred[pos_label_inds] ori_loss_reg = loss_bbox( pos_bbox_preds, bbox_targets[pos_label_inds], reduction_override='none') / avg_factor loss_carl = (ori_loss_reg * carl_loss_weights[:, None]).sum() return dict(loss_carl=loss_carl[None]) ================================================ FILE: mmdet/models/losses/pkd_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F from mmdet.registry import MODELS from .utils import weighted_loss def norm(feat: torch.Tensor) -> torch.Tensor: """Normalize the feature maps to have zero mean and unit variances. Args: feat (torch.Tensor): The original feature map with shape (N, C, H, W). """ assert len(feat.shape) == 4 N, C, H, W = feat.shape feat = feat.permute(1, 0, 2, 3).reshape(C, -1) mean = feat.mean(dim=-1, keepdim=True) std = feat.std(dim=-1, keepdim=True) feat = (feat - mean) / (std + 1e-6) return feat.reshape(C, N, H, W).permute(1, 0, 2, 3) @weighted_loss def pkd_loss(pred, target): pred = norm(pred) target = norm(target) return F.mse_loss(pred, target, reduction='none') / 2 @MODELS.register_module() class PKDLoss(nn.Module): def __init__(self, reduction='mean', loss_weight=1.0): super(PKDLoss, self).__init__() self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred, target, weight=None, avg_factor=None, reduction_override=None) -> torch.Tensor: assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) loss = self.loss_weight * pkd_loss( pred, target, weight, reduction=reduction, avg_factor=avg_factor) return loss ================================================ FILE: mmdet/models/losses/seesaw_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, Optional, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F from torch import Tensor from mmdet.registry import MODELS from .accuracy import accuracy from .cross_entropy_loss import cross_entropy from .utils import weight_reduce_loss def seesaw_ce_loss(cls_score: Tensor, labels: Tensor, label_weights: Tensor, cum_samples: Tensor, num_classes: int, p: float, q: float, eps: float, reduction: str = 'mean', avg_factor: Optional[int] = None) -> Tensor: """Calculate the Seesaw CrossEntropy loss. Args: cls_score (Tensor): The prediction with shape (N, C), C is the number of classes. labels (Tensor): The learning label of the prediction. label_weights (Tensor): Sample-wise loss weight. cum_samples (Tensor): Cumulative samples for each category. num_classes (int): The number of classes. p (float): The ``p`` in the mitigation factor. q (float): The ``q`` in the compenstation factor. eps (float): The minimal value of divisor to smooth the computation of compensation factor reduction (str, optional): The method used to reduce the loss. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. Returns: Tensor: The calculated loss """ assert cls_score.size(-1) == num_classes assert len(cum_samples) == num_classes onehot_labels = F.one_hot(labels, num_classes) seesaw_weights = cls_score.new_ones(onehot_labels.size()) # mitigation factor if p > 0: sample_ratio_matrix = cum_samples[None, :].clamp( min=1) / cum_samples[:, None].clamp(min=1) index = (sample_ratio_matrix < 1.0).float() sample_weights = sample_ratio_matrix.pow(p) * index + (1 - index) mitigation_factor = sample_weights[labels.long(), :] seesaw_weights = seesaw_weights * mitigation_factor # compensation factor if q > 0: scores = F.softmax(cls_score.detach(), dim=1) self_scores = scores[ torch.arange(0, len(scores)).to(scores.device).long(), labels.long()] score_matrix = scores / self_scores[:, None].clamp(min=eps) index = (score_matrix > 1.0).float() compensation_factor = score_matrix.pow(q) * index + (1 - index) seesaw_weights = seesaw_weights * compensation_factor cls_score = cls_score + (seesaw_weights.log() * (1 - onehot_labels)) loss = F.cross_entropy(cls_score, labels, weight=None, reduction='none') if label_weights is not None: label_weights = label_weights.float() loss = weight_reduce_loss( loss, weight=label_weights, reduction=reduction, avg_factor=avg_factor) return loss @MODELS.register_module() class SeesawLoss(nn.Module): """ Seesaw Loss for Long-Tailed Instance Segmentation (CVPR 2021) arXiv: https://arxiv.org/abs/2008.10032 Args: use_sigmoid (bool, optional): Whether the prediction uses sigmoid of softmax. Only False is supported. p (float, optional): The ``p`` in the mitigation factor. Defaults to 0.8. q (float, optional): The ``q`` in the compenstation factor. Defaults to 2.0. num_classes (int, optional): The number of classes. Default to 1203 for LVIS v1 dataset. eps (float, optional): The minimal value of divisor to smooth the computation of compensation factor reduction (str, optional): The method that reduces the loss to a scalar. Options are "none", "mean" and "sum". loss_weight (float, optional): The weight of the loss. Defaults to 1.0 return_dict (bool, optional): Whether return the losses as a dict. Default to True. """ def __init__(self, use_sigmoid: bool = False, p: float = 0.8, q: float = 2.0, num_classes: int = 1203, eps: float = 1e-2, reduction: str = 'mean', loss_weight: float = 1.0, return_dict: bool = True) -> None: super().__init__() assert not use_sigmoid self.use_sigmoid = False self.p = p self.q = q self.num_classes = num_classes self.eps = eps self.reduction = reduction self.loss_weight = loss_weight self.return_dict = return_dict # 0 for pos, 1 for neg self.cls_criterion = seesaw_ce_loss # cumulative samples for each category self.register_buffer( 'cum_samples', torch.zeros(self.num_classes + 1, dtype=torch.float)) # custom output channels of the classifier self.custom_cls_channels = True # custom activation of cls_score self.custom_activation = True # custom accuracy of the classsifier self.custom_accuracy = True def _split_cls_score(self, cls_score: Tensor) -> Tuple[Tensor, Tensor]: """split cls_score. Args: cls_score (Tensor): The prediction with shape (N, C + 2). Returns: Tuple[Tensor, Tensor]: The score for classes and objectness, respectively """ # split cls_score to cls_score_classes and cls_score_objectness assert cls_score.size(-1) == self.num_classes + 2 cls_score_classes = cls_score[..., :-2] cls_score_objectness = cls_score[..., -2:] return cls_score_classes, cls_score_objectness def get_cls_channels(self, num_classes: int) -> int: """Get custom classification channels. Args: num_classes (int): The number of classes. Returns: int: The custom classification channels. """ assert num_classes == self.num_classes return num_classes + 2 def get_activation(self, cls_score: Tensor) -> Tensor: """Get custom activation of cls_score. Args: cls_score (Tensor): The prediction with shape (N, C + 2). Returns: Tensor: The custom activation of cls_score with shape (N, C + 1). """ cls_score_classes, cls_score_objectness = self._split_cls_score( cls_score) score_classes = F.softmax(cls_score_classes, dim=-1) score_objectness = F.softmax(cls_score_objectness, dim=-1) score_pos = score_objectness[..., [0]] score_neg = score_objectness[..., [1]] score_classes = score_classes * score_pos scores = torch.cat([score_classes, score_neg], dim=-1) return scores def get_accuracy(self, cls_score: Tensor, labels: Tensor) -> Dict[str, Tensor]: """Get custom accuracy w.r.t. cls_score and labels. Args: cls_score (Tensor): The prediction with shape (N, C + 2). labels (Tensor): The learning label of the prediction. Returns: Dict [str, Tensor]: The accuracy for objectness and classes, respectively. """ pos_inds = labels < self.num_classes obj_labels = (labels == self.num_classes).long() cls_score_classes, cls_score_objectness = self._split_cls_score( cls_score) acc_objectness = accuracy(cls_score_objectness, obj_labels) acc_classes = accuracy(cls_score_classes[pos_inds], labels[pos_inds]) acc = dict() acc['acc_objectness'] = acc_objectness acc['acc_classes'] = acc_classes return acc def forward( self, cls_score: Tensor, labels: Tensor, label_weights: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None ) -> Union[Tensor, Dict[str, Tensor]]: """Forward function. Args: cls_score (Tensor): The prediction with shape (N, C + 2). labels (Tensor): The learning label of the prediction. label_weights (Tensor, optional): Sample-wise loss weight. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction (str, optional): The method used to reduce the loss. Options are "none", "mean" and "sum". Returns: Tensor | Dict [str, Tensor]: if return_dict == False: The calculated loss | if return_dict == True: The dict of calculated losses for objectness and classes, respectively. """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) assert cls_score.size(-1) == self.num_classes + 2 pos_inds = labels < self.num_classes # 0 for pos, 1 for neg obj_labels = (labels == self.num_classes).long() # accumulate the samples for each category unique_labels = labels.unique() for u_l in unique_labels: inds_ = labels == u_l.item() self.cum_samples[u_l] += inds_.sum() if label_weights is not None: label_weights = label_weights.float() else: label_weights = labels.new_ones(labels.size(), dtype=torch.float) cls_score_classes, cls_score_objectness = self._split_cls_score( cls_score) # calculate loss_cls_classes (only need pos samples) if pos_inds.sum() > 0: loss_cls_classes = self.loss_weight * self.cls_criterion( cls_score_classes[pos_inds], labels[pos_inds], label_weights[pos_inds], self.cum_samples[:self.num_classes], self.num_classes, self.p, self.q, self.eps, reduction, avg_factor) else: loss_cls_classes = cls_score_classes[pos_inds].sum() # calculate loss_cls_objectness loss_cls_objectness = self.loss_weight * cross_entropy( cls_score_objectness, obj_labels, label_weights, reduction, avg_factor) if self.return_dict: loss_cls = dict() loss_cls['loss_cls_objectness'] = loss_cls_objectness loss_cls['loss_cls_classes'] = loss_cls_classes else: loss_cls = loss_cls_classes + loss_cls_objectness return loss_cls ================================================ FILE: mmdet/models/losses/smooth_l1_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch import torch.nn as nn from torch import Tensor from mmdet.registry import MODELS from .utils import weighted_loss @weighted_loss def smooth_l1_loss(pred: Tensor, target: Tensor, beta: float = 1.0) -> Tensor: """Smooth L1 loss. Args: pred (Tensor): The prediction. target (Tensor): The learning target of the prediction. beta (float, optional): The threshold in the piecewise function. Defaults to 1.0. Returns: Tensor: Calculated loss """ assert beta > 0 if target.numel() == 0: return pred.sum() * 0 assert pred.size() == target.size() diff = torch.abs(pred - target) loss = torch.where(diff < beta, 0.5 * diff * diff / beta, diff - 0.5 * beta) return loss @weighted_loss def l1_loss(pred: Tensor, target: Tensor) -> Tensor: """L1 loss. Args: pred (Tensor): The prediction. target (Tensor): The learning target of the prediction. Returns: Tensor: Calculated loss """ if target.numel() == 0: return pred.sum() * 0 assert pred.size() == target.size() loss = torch.abs(pred - target) return loss @MODELS.register_module() class SmoothL1Loss(nn.Module): """Smooth L1 loss. Args: beta (float, optional): The threshold in the piecewise function. Defaults to 1.0. reduction (str, optional): The method to reduce the loss. Options are "none", "mean" and "sum". Defaults to "mean". loss_weight (float, optional): The weight of loss. """ def __init__(self, beta: float = 1.0, reduction: str = 'mean', loss_weight: float = 1.0) -> None: super().__init__() self.beta = beta self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None, **kwargs) -> Tensor: """Forward function. Args: pred (Tensor): The prediction. target (Tensor): The learning target of the prediction. weight (Tensor, optional): The weight of loss for each prediction. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Returns: Tensor: Calculated loss """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) loss_bbox = self.loss_weight * smooth_l1_loss( pred, target, weight, beta=self.beta, reduction=reduction, avg_factor=avg_factor, **kwargs) return loss_bbox @MODELS.register_module() class L1Loss(nn.Module): """L1 loss. Args: reduction (str, optional): The method to reduce the loss. Options are "none", "mean" and "sum". loss_weight (float, optional): The weight of loss. """ def __init__(self, reduction: str = 'mean', loss_weight: float = 1.0) -> None: super().__init__() self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None) -> Tensor: """Forward function. Args: pred (Tensor): The prediction. target (Tensor): The learning target of the prediction. weight (Tensor, optional): The weight of loss for each prediction. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Defaults to None. Returns: Tensor: Calculated loss """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) loss_bbox = self.loss_weight * l1_loss( pred, target, weight, reduction=reduction, avg_factor=avg_factor) return loss_bbox ================================================ FILE: mmdet/models/losses/utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import functools from typing import Callable, Optional import torch import torch.nn.functional as F from torch import Tensor def reduce_loss(loss: Tensor, reduction: str) -> Tensor: """Reduce loss as specified. Args: loss (Tensor): Elementwise loss tensor. reduction (str): Options are "none", "mean" and "sum". Return: Tensor: Reduced loss tensor. """ reduction_enum = F._Reduction.get_enum(reduction) # none: 0, elementwise_mean:1, sum: 2 if reduction_enum == 0: return loss elif reduction_enum == 1: return loss.mean() elif reduction_enum == 2: return loss.sum() def weight_reduce_loss(loss: Tensor, weight: Optional[Tensor] = None, reduction: str = 'mean', avg_factor: Optional[float] = None) -> Tensor: """Apply element-wise weight and reduce loss. Args: loss (Tensor): Element-wise loss. weight (Optional[Tensor], optional): Element-wise weights. Defaults to None. reduction (str, optional): Same as built-in losses of PyTorch. Defaults to 'mean'. avg_factor (Optional[float], optional): Average factor when computing the mean of losses. Defaults to None. Returns: Tensor: Processed loss values. """ # if weight is specified, apply element-wise weight if weight is not None: loss = loss * weight # if avg_factor is not specified, just reduce the loss if avg_factor is None: loss = reduce_loss(loss, reduction) else: # if reduction is mean, then average the loss by avg_factor if reduction == 'mean': # Avoid causing ZeroDivisionError when avg_factor is 0.0, # i.e., all labels of an image belong to ignore index. eps = torch.finfo(torch.float32).eps loss = loss.sum() / (avg_factor + eps) # if reduction is 'none', then do nothing, otherwise raise an error elif reduction != 'none': raise ValueError('avg_factor can not be used with reduction="sum"') return loss def weighted_loss(loss_func: Callable) -> Callable: """Create a weighted version of a given loss function. To use this decorator, the loss function must have the signature like `loss_func(pred, target, **kwargs)`. The function only needs to compute element-wise loss without any reduction. This decorator will add weight and reduction arguments to the function. The decorated function will have the signature like `loss_func(pred, target, weight=None, reduction='mean', avg_factor=None, **kwargs)`. :Example: >>> import torch >>> @weighted_loss >>> def l1_loss(pred, target): >>> return (pred - target).abs() >>> pred = torch.Tensor([0, 2, 3]) >>> target = torch.Tensor([1, 1, 1]) >>> weight = torch.Tensor([1, 0, 1]) >>> l1_loss(pred, target) tensor(1.3333) >>> l1_loss(pred, target, weight) tensor(1.) >>> l1_loss(pred, target, reduction='none') tensor([1., 1., 2.]) >>> l1_loss(pred, target, weight, avg_factor=2) tensor(1.5000) """ @functools.wraps(loss_func) def wrapper(pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, reduction: str = 'mean', avg_factor: Optional[int] = None, **kwargs) -> Tensor: """ Args: pred (Tensor): The prediction. target (Tensor): Target bboxes. weight (Optional[Tensor], optional): The weight of loss for each prediction. Defaults to None. reduction (str, optional): Options are "none", "mean" and "sum". Defaults to 'mean'. avg_factor (Optional[int], optional): Average factor that is used to average the loss. Defaults to None. Returns: Tensor: Loss tensor. """ # get element-wise loss loss = loss_func(pred, target, **kwargs) loss = weight_reduce_loss(loss, weight, reduction, avg_factor) return loss return wrapper ================================================ FILE: mmdet/models/losses/varifocal_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch.nn as nn import torch.nn.functional as F from torch import Tensor from mmdet.registry import MODELS from .utils import weight_reduce_loss def varifocal_loss(pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, alpha: float = 0.75, gamma: float = 2.0, iou_weighted: bool = True, reduction: str = 'mean', avg_factor: Optional[int] = None) -> Tensor: """`Varifocal Loss `_ Args: pred (Tensor): The prediction with shape (N, C), C is the number of classes. target (Tensor): The learning target of the iou-aware classification score with shape (N, C), C is the number of classes. weight (Tensor, optional): The weight of loss for each prediction. Defaults to None. alpha (float, optional): A balance factor for the negative part of Varifocal Loss, which is different from the alpha of Focal Loss. Defaults to 0.75. gamma (float, optional): The gamma for calculating the modulating factor. Defaults to 2.0. iou_weighted (bool, optional): Whether to weight the loss of the positive example with the iou target. Defaults to True. reduction (str, optional): The method used to reduce the loss into a scalar. Defaults to 'mean'. Options are "none", "mean" and "sum". avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. Returns: Tensor: Loss tensor. """ # pred and target should be of the same size assert pred.size() == target.size() pred_sigmoid = pred.sigmoid() target = target.type_as(pred) if iou_weighted: focal_weight = target * (target > 0.0).float() + \ alpha * (pred_sigmoid - target).abs().pow(gamma) * \ (target <= 0.0).float() else: focal_weight = (target > 0.0).float() + \ alpha * (pred_sigmoid - target).abs().pow(gamma) * \ (target <= 0.0).float() loss = F.binary_cross_entropy_with_logits( pred, target, reduction='none') * focal_weight loss = weight_reduce_loss(loss, weight, reduction, avg_factor) return loss @MODELS.register_module() class VarifocalLoss(nn.Module): def __init__(self, use_sigmoid: bool = True, alpha: float = 0.75, gamma: float = 2.0, iou_weighted: bool = True, reduction: str = 'mean', loss_weight: float = 1.0) -> None: """`Varifocal Loss `_ Args: use_sigmoid (bool, optional): Whether the prediction is used for sigmoid or softmax. Defaults to True. alpha (float, optional): A balance factor for the negative part of Varifocal Loss, which is different from the alpha of Focal Loss. Defaults to 0.75. gamma (float, optional): The gamma for calculating the modulating factor. Defaults to 2.0. iou_weighted (bool, optional): Whether to weight the loss of the positive examples with the iou target. Defaults to True. reduction (str, optional): The method used to reduce the loss into a scalar. Defaults to 'mean'. Options are "none", "mean" and "sum". loss_weight (float, optional): Weight of loss. Defaults to 1.0. """ super().__init__() assert use_sigmoid is True, \ 'Only sigmoid varifocal loss supported now.' assert alpha >= 0.0 self.use_sigmoid = use_sigmoid self.alpha = alpha self.gamma = gamma self.iou_weighted = iou_weighted self.reduction = reduction self.loss_weight = loss_weight def forward(self, pred: Tensor, target: Tensor, weight: Optional[Tensor] = None, avg_factor: Optional[int] = None, reduction_override: Optional[str] = None) -> Tensor: """Forward function. Args: pred (Tensor): The prediction with shape (N, C), C is the number of classes. target (Tensor): The learning target of the iou-aware classification score with shape (N, C), C is the number of classes. weight (Tensor, optional): The weight of loss for each prediction. Defaults to None. avg_factor (int, optional): Average factor that is used to average the loss. Defaults to None. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Options are "none", "mean" and "sum". Returns: Tensor: The calculated loss """ assert reduction_override in (None, 'none', 'mean', 'sum') reduction = ( reduction_override if reduction_override else self.reduction) if self.use_sigmoid: loss_cls = self.loss_weight * varifocal_loss( pred, target, weight, alpha=self.alpha, gamma=self.gamma, iou_weighted=self.iou_weighted, reduction=reduction, avg_factor=avg_factor) else: raise NotImplementedError return loss_cls ================================================ FILE: mmdet/models/necks/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .bfp import BFP from .channel_mapper import ChannelMapper from .cspnext_pafpn import CSPNeXtPAFPN from .ct_resnet_neck import CTResNetNeck from .dilated_encoder import DilatedEncoder from .dyhead import DyHead from .fpg import FPG from .fpn import FPN from .fpn_carafe import FPN_CARAFE from .hrfpn import HRFPN from .nas_fpn import NASFPN from .nasfcos_fpn import NASFCOS_FPN from .pafpn import PAFPN from .rfp import RFP from .ssd_neck import SSDNeck from .ssh import SSH from .yolo_neck import YOLOV3Neck from .yolox_pafpn import YOLOXPAFPN __all__ = [ 'FPN', 'BFP', 'ChannelMapper', 'HRFPN', 'NASFPN', 'FPN_CARAFE', 'PAFPN', 'NASFCOS_FPN', 'RFP', 'YOLOV3Neck', 'FPG', 'DilatedEncoder', 'CTResNetNeck', 'SSDNeck', 'YOLOXPAFPN', 'DyHead', 'CSPNeXtPAFPN', 'SSH' ] ================================================ FILE: mmdet/models/necks/bfp.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple import torch.nn.functional as F from mmcv.cnn import ConvModule from mmcv.cnn.bricks import NonLocal2d from mmengine.model import BaseModule from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import OptConfigType, OptMultiConfig @MODELS.register_module() class BFP(BaseModule): """BFP (Balanced Feature Pyramids) BFP takes multi-level features as inputs and gather them into a single one, then refine the gathered feature and scatter the refined results to multi-level features. This module is used in Libra R-CNN (CVPR 2019), see the paper `Libra R-CNN: Towards Balanced Learning for Object Detection `_ for details. Args: in_channels (int): Number of input channels (feature maps of all levels should have the same channels). num_levels (int): Number of input feature levels. refine_level (int): Index of integration and refine level of BSF in multi-level features from bottom to top. refine_type (str): Type of the refine op, currently support [None, 'conv', 'non_local']. conv_cfg (:obj:`ConfigDict` or dict, optional): The config dict for convolution layers. norm_cfg (:obj:`ConfigDict` or dict, optional): The config dict for normalization layers. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or dict], optional): Initialization config dict. """ def __init__( self, in_channels: int, num_levels: int, refine_level: int = 2, refine_type: str = None, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, init_cfg: OptMultiConfig = dict( type='Xavier', layer='Conv2d', distribution='uniform') ) -> None: super().__init__(init_cfg=init_cfg) assert refine_type in [None, 'conv', 'non_local'] self.in_channels = in_channels self.num_levels = num_levels self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.refine_level = refine_level self.refine_type = refine_type assert 0 <= self.refine_level < self.num_levels if self.refine_type == 'conv': self.refine = ConvModule( self.in_channels, self.in_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg) elif self.refine_type == 'non_local': self.refine = NonLocal2d( self.in_channels, reduction=1, use_scale=False, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg) def forward(self, inputs: Tuple[Tensor]) -> Tuple[Tensor]: """Forward function.""" assert len(inputs) == self.num_levels # step 1: gather multi-level features by resize and average feats = [] gather_size = inputs[self.refine_level].size()[2:] for i in range(self.num_levels): if i < self.refine_level: gathered = F.adaptive_max_pool2d( inputs[i], output_size=gather_size) else: gathered = F.interpolate( inputs[i], size=gather_size, mode='nearest') feats.append(gathered) bsf = sum(feats) / len(feats) # step 2: refine gathered features if self.refine_type is not None: bsf = self.refine(bsf) # step 3: scatter refined features to multi-levels by a residual path outs = [] for i in range(self.num_levels): out_size = inputs[i].size()[2:] if i < self.refine_level: residual = F.interpolate(bsf, size=out_size, mode='nearest') else: residual = F.adaptive_max_pool2d(bsf, output_size=out_size) outs.append(residual + inputs[i]) return tuple(outs) ================================================ FILE: mmdet/models/necks/channel_mapper.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import torch.nn as nn from mmcv.cnn import ConvModule from mmengine.model import BaseModule from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import OptConfigType, OptMultiConfig @MODELS.register_module() class ChannelMapper(BaseModule): """Channel Mapper to reduce/increase channels of backbone features. This is used to reduce/increase channels of backbone features. Args: in_channels (List[int]): Number of input channels per scale. out_channels (int): Number of output channels (used at each scale). kernel_size (int, optional): kernel_size for reducing channels (used at each scale). Default: 3. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Default: None. norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for normalization layer. Default: None. act_cfg (:obj:`ConfigDict` or dict, optional): Config dict for activation layer in ConvModule. Default: dict(type='ReLU'). num_outs (int, optional): Number of output feature maps. There would be extra_convs when num_outs larger than the length of in_channels. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or dict], optional): Initialization config dict. Example: >>> import torch >>> in_channels = [2, 3, 5, 7] >>> scales = [340, 170, 84, 43] >>> inputs = [torch.rand(1, c, s, s) ... for c, s in zip(in_channels, scales)] >>> self = ChannelMapper(in_channels, 11, 3).eval() >>> outputs = self.forward(inputs) >>> for i in range(len(outputs)): ... print(f'outputs[{i}].shape = {outputs[i].shape}') outputs[0].shape = torch.Size([1, 11, 340, 340]) outputs[1].shape = torch.Size([1, 11, 170, 170]) outputs[2].shape = torch.Size([1, 11, 84, 84]) outputs[3].shape = torch.Size([1, 11, 43, 43]) """ def __init__( self, in_channels: List[int], out_channels: int, kernel_size: int = 3, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, act_cfg: OptConfigType = dict(type='ReLU'), num_outs: int = None, init_cfg: OptMultiConfig = dict( type='Xavier', layer='Conv2d', distribution='uniform') ) -> None: super().__init__(init_cfg=init_cfg) assert isinstance(in_channels, list) self.extra_convs = None if num_outs is None: num_outs = len(in_channels) self.convs = nn.ModuleList() for in_channel in in_channels: self.convs.append( ConvModule( in_channel, out_channels, kernel_size, padding=(kernel_size - 1) // 2, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) if num_outs > len(in_channels): self.extra_convs = nn.ModuleList() for i in range(len(in_channels), num_outs): if i == len(in_channels): in_channel = in_channels[-1] else: in_channel = out_channels self.extra_convs.append( ConvModule( in_channel, out_channels, 3, stride=2, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) def forward(self, inputs: Tuple[Tensor]) -> Tuple[Tensor]: """Forward function.""" assert len(inputs) == len(self.convs) outs = [self.convs[i](inputs[i]) for i in range(len(inputs))] if self.extra_convs: for i in range(len(self.extra_convs)): if i == 0: outs.append(self.extra_convs[0](inputs[-1])) else: outs.append(self.extra_convs[i](outs[-1])) return tuple(outs) ================================================ FILE: mmdet/models/necks/cspnext_pafpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from typing import Sequence, Tuple import torch import torch.nn as nn from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule from mmengine.model import BaseModule from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptMultiConfig from ..layers import CSPLayer @MODELS.register_module() class CSPNeXtPAFPN(BaseModule): """Path Aggregation Network with CSPNeXt blocks. Args: in_channels (Sequence[int]): Number of input channels per scale. out_channels (int): Number of output channels (used at each scale) num_csp_blocks (int): Number of bottlenecks in CSPLayer. Defaults to 3. use_depthwise (bool): Whether to use depthwise separable convolution in blocks. Defaults to False. expand_ratio (float): Ratio to adjust the number of channels of the hidden layer. Default: 0.5 upsample_cfg (dict): Config dict for interpolate layer. Default: `dict(scale_factor=2, mode='nearest')` conv_cfg (dict, optional): Config dict for convolution layer. Default: None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Default: dict(type='BN') act_cfg (dict): Config dict for activation layer. Default: dict(type='Swish') init_cfg (dict or list[dict], optional): Initialization config dict. Default: None. """ def __init__( self, in_channels: Sequence[int], out_channels: int, num_csp_blocks: int = 3, use_depthwise: bool = False, expand_ratio: float = 0.5, upsample_cfg: ConfigType = dict(scale_factor=2, mode='nearest'), conv_cfg: bool = None, norm_cfg: ConfigType = dict(type='BN', momentum=0.03, eps=0.001), act_cfg: ConfigType = dict(type='Swish'), init_cfg: OptMultiConfig = dict( type='Kaiming', layer='Conv2d', a=math.sqrt(5), distribution='uniform', mode='fan_in', nonlinearity='leaky_relu') ) -> None: super().__init__(init_cfg) self.in_channels = in_channels self.out_channels = out_channels conv = DepthwiseSeparableConvModule if use_depthwise else ConvModule # build top-down blocks self.upsample = nn.Upsample(**upsample_cfg) self.reduce_layers = nn.ModuleList() self.top_down_blocks = nn.ModuleList() for idx in range(len(in_channels) - 1, 0, -1): self.reduce_layers.append( ConvModule( in_channels[idx], in_channels[idx - 1], 1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) self.top_down_blocks.append( CSPLayer( in_channels[idx - 1] * 2, in_channels[idx - 1], num_blocks=num_csp_blocks, add_identity=False, use_depthwise=use_depthwise, use_cspnext_block=True, expand_ratio=expand_ratio, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) # build bottom-up blocks self.downsamples = nn.ModuleList() self.bottom_up_blocks = nn.ModuleList() for idx in range(len(in_channels) - 1): self.downsamples.append( conv( in_channels[idx], in_channels[idx], 3, stride=2, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) self.bottom_up_blocks.append( CSPLayer( in_channels[idx] * 2, in_channels[idx + 1], num_blocks=num_csp_blocks, add_identity=False, use_depthwise=use_depthwise, use_cspnext_block=True, expand_ratio=expand_ratio, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) self.out_convs = nn.ModuleList() for i in range(len(in_channels)): self.out_convs.append( conv( in_channels[i], out_channels, 3, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) def forward(self, inputs: Tuple[Tensor, ...]) -> Tuple[Tensor, ...]: """ Args: inputs (tuple[Tensor]): input features. Returns: tuple[Tensor]: YOLOXPAFPN features. """ assert len(inputs) == len(self.in_channels) # top-down path inner_outs = [inputs[-1]] for idx in range(len(self.in_channels) - 1, 0, -1): feat_heigh = inner_outs[0] feat_low = inputs[idx - 1] feat_heigh = self.reduce_layers[len(self.in_channels) - 1 - idx]( feat_heigh) inner_outs[0] = feat_heigh upsample_feat = self.upsample(feat_heigh) inner_out = self.top_down_blocks[len(self.in_channels) - 1 - idx]( torch.cat([upsample_feat, feat_low], 1)) inner_outs.insert(0, inner_out) # bottom-up path outs = [inner_outs[0]] for idx in range(len(self.in_channels) - 1): feat_low = outs[-1] feat_height = inner_outs[idx + 1] downsample_feat = self.downsamples[idx](feat_low) out = self.bottom_up_blocks[idx]( torch.cat([downsample_feat, feat_height], 1)) outs.append(out) # out convs for idx, conv in enumerate(self.out_convs): outs[idx] = conv(outs[idx]) return tuple(outs) ================================================ FILE: mmdet/models/necks/ct_resnet_neck.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from typing import Sequence, Tuple import torch import torch.nn as nn from mmcv.cnn import ConvModule from mmengine.model import BaseModule from mmdet.registry import MODELS from mmdet.utils import OptMultiConfig @MODELS.register_module() class CTResNetNeck(BaseModule): """The neck used in `CenterNet `_ for object classification and box regression. Args: in_channels (int): Number of input channels. num_deconv_filters (tuple[int]): Number of filters per stage. num_deconv_kernels (tuple[int]): Number of kernels per stage. use_dcn (bool): If True, use DCNv2. Defaults to True. init_cfg (:obj:`ConfigDict` or dict or list[dict] or list[:obj:`ConfigDict`], optional): Initialization config dict. """ def __init__(self, in_channels: int, num_deconv_filters: Tuple[int, ...], num_deconv_kernels: Tuple[int, ...], use_dcn: bool = True, init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) assert len(num_deconv_filters) == len(num_deconv_kernels) self.fp16_enabled = False self.use_dcn = use_dcn self.in_channels = in_channels self.deconv_layers = self._make_deconv_layer(num_deconv_filters, num_deconv_kernels) def _make_deconv_layer( self, num_deconv_filters: Tuple[int, ...], num_deconv_kernels: Tuple[int, ...]) -> nn.Sequential: """use deconv layers to upsample backbone's output.""" layers = [] for i in range(len(num_deconv_filters)): feat_channels = num_deconv_filters[i] conv_module = ConvModule( self.in_channels, feat_channels, 3, padding=1, conv_cfg=dict(type='DCNv2') if self.use_dcn else None, norm_cfg=dict(type='BN')) layers.append(conv_module) upsample_module = ConvModule( feat_channels, feat_channels, num_deconv_kernels[i], stride=2, padding=1, conv_cfg=dict(type='deconv'), norm_cfg=dict(type='BN')) layers.append(upsample_module) self.in_channels = feat_channels return nn.Sequential(*layers) def init_weights(self) -> None: """Initialize the parameters.""" for m in self.modules(): if isinstance(m, nn.ConvTranspose2d): # In order to be consistent with the source code, # reset the ConvTranspose2d initialization parameters m.reset_parameters() # Simulated bilinear upsampling kernel w = m.weight.data f = math.ceil(w.size(2) / 2) c = (2 * f - 1 - f % 2) / (2. * f) for i in range(w.size(2)): for j in range(w.size(3)): w[0, 0, i, j] = \ (1 - math.fabs(i / f - c)) * ( 1 - math.fabs(j / f - c)) for c in range(1, w.size(0)): w[c, 0, :, :] = w[0, 0, :, :] elif isinstance(m, nn.BatchNorm2d): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0) # self.use_dcn is False elif not self.use_dcn and isinstance(m, nn.Conv2d): # In order to be consistent with the source code, # reset the Conv2d initialization parameters m.reset_parameters() def forward(self, x: Sequence[torch.Tensor]) -> Tuple[torch.Tensor]: """model forward.""" assert isinstance(x, (list, tuple)) outs = self.deconv_layers(x[-1]) return outs, ================================================ FILE: mmdet/models/necks/dilated_encoder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn as nn from mmcv.cnn import ConvModule, is_norm from mmengine.model import caffe2_xavier_init, constant_init, normal_init from torch.nn import BatchNorm2d from mmdet.registry import MODELS class Bottleneck(nn.Module): """Bottleneck block for DilatedEncoder used in `YOLOF. `. The Bottleneck contains three ConvLayers and one residual connection. Args: in_channels (int): The number of input channels. mid_channels (int): The number of middle output channels. dilation (int): Dilation rate. norm_cfg (dict): Dictionary to construct and config norm layer. """ def __init__(self, in_channels, mid_channels, dilation, norm_cfg=dict(type='BN', requires_grad=True)): super(Bottleneck, self).__init__() self.conv1 = ConvModule( in_channels, mid_channels, 1, norm_cfg=norm_cfg) self.conv2 = ConvModule( mid_channels, mid_channels, 3, padding=dilation, dilation=dilation, norm_cfg=norm_cfg) self.conv3 = ConvModule( mid_channels, in_channels, 1, norm_cfg=norm_cfg) def forward(self, x): identity = x out = self.conv1(x) out = self.conv2(out) out = self.conv3(out) out = out + identity return out @MODELS.register_module() class DilatedEncoder(nn.Module): """Dilated Encoder for YOLOF `. This module contains two types of components: - the original FPN lateral convolution layer and fpn convolution layer, which are 1x1 conv + 3x3 conv - the dilated residual block Args: in_channels (int): The number of input channels. out_channels (int): The number of output channels. block_mid_channels (int): The number of middle block output channels num_residual_blocks (int): The number of residual blocks. block_dilations (list): The list of residual blocks dilation. """ def __init__(self, in_channels, out_channels, block_mid_channels, num_residual_blocks, block_dilations): super(DilatedEncoder, self).__init__() self.in_channels = in_channels self.out_channels = out_channels self.block_mid_channels = block_mid_channels self.num_residual_blocks = num_residual_blocks self.block_dilations = block_dilations self._init_layers() def _init_layers(self): self.lateral_conv = nn.Conv2d( self.in_channels, self.out_channels, kernel_size=1) self.lateral_norm = BatchNorm2d(self.out_channels) self.fpn_conv = nn.Conv2d( self.out_channels, self.out_channels, kernel_size=3, padding=1) self.fpn_norm = BatchNorm2d(self.out_channels) encoder_blocks = [] for i in range(self.num_residual_blocks): dilation = self.block_dilations[i] encoder_blocks.append( Bottleneck( self.out_channels, self.block_mid_channels, dilation=dilation)) self.dilated_encoder_blocks = nn.Sequential(*encoder_blocks) def init_weights(self): caffe2_xavier_init(self.lateral_conv) caffe2_xavier_init(self.fpn_conv) for m in [self.lateral_norm, self.fpn_norm]: constant_init(m, 1) for m in self.dilated_encoder_blocks.modules(): if isinstance(m, nn.Conv2d): normal_init(m, mean=0, std=0.01) if is_norm(m): constant_init(m, 1) def forward(self, feature): out = self.lateral_norm(self.lateral_conv(feature[-1])) out = self.fpn_norm(self.fpn_conv(out)) return self.dilated_encoder_blocks(out), ================================================ FILE: mmdet/models/necks/dyhead.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import build_activation_layer, build_norm_layer from mmcv.ops.modulated_deform_conv import ModulatedDeformConv2d from mmengine.model import BaseModule, constant_init, normal_init from mmdet.registry import MODELS from ..layers import DyReLU # Reference: # https://github.com/microsoft/DynamicHead # https://github.com/jshilong/SEPC class DyDCNv2(nn.Module): """ModulatedDeformConv2d with normalization layer used in DyHead. This module cannot be configured with `conv_cfg=dict(type='DCNv2')` because DyHead calculates offset and mask from middle-level feature. Args: in_channels (int): Number of input channels. out_channels (int): Number of output channels. stride (int | tuple[int], optional): Stride of the convolution. Default: 1. norm_cfg (dict, optional): Config dict for normalization layer. Default: dict(type='GN', num_groups=16, requires_grad=True). """ def __init__(self, in_channels, out_channels, stride=1, norm_cfg=dict(type='GN', num_groups=16, requires_grad=True)): super().__init__() self.with_norm = norm_cfg is not None bias = not self.with_norm self.conv = ModulatedDeformConv2d( in_channels, out_channels, 3, stride=stride, padding=1, bias=bias) if self.with_norm: self.norm = build_norm_layer(norm_cfg, out_channels)[1] def forward(self, x, offset, mask): """Forward function.""" x = self.conv(x.contiguous(), offset, mask) if self.with_norm: x = self.norm(x) return x class DyHeadBlock(nn.Module): """DyHead Block with three types of attention. HSigmoid arguments in default act_cfg follow official code, not paper. https://github.com/microsoft/DynamicHead/blob/master/dyhead/dyrelu.py Args: in_channels (int): Number of input channels. out_channels (int): Number of output channels. zero_init_offset (bool, optional): Whether to use zero init for `spatial_conv_offset`. Default: True. act_cfg (dict, optional): Config dict for the last activation layer of scale-aware attention. Default: dict(type='HSigmoid', bias=3.0, divisor=6.0). """ def __init__(self, in_channels, out_channels, zero_init_offset=True, act_cfg=dict(type='HSigmoid', bias=3.0, divisor=6.0)): super().__init__() self.zero_init_offset = zero_init_offset # (offset_x, offset_y, mask) * kernel_size_y * kernel_size_x self.offset_and_mask_dim = 3 * 3 * 3 self.offset_dim = 2 * 3 * 3 self.spatial_conv_high = DyDCNv2(in_channels, out_channels) self.spatial_conv_mid = DyDCNv2(in_channels, out_channels) self.spatial_conv_low = DyDCNv2(in_channels, out_channels, stride=2) self.spatial_conv_offset = nn.Conv2d( in_channels, self.offset_and_mask_dim, 3, padding=1) self.scale_attn_module = nn.Sequential( nn.AdaptiveAvgPool2d(1), nn.Conv2d(out_channels, 1, 1), nn.ReLU(inplace=True), build_activation_layer(act_cfg)) self.task_attn_module = DyReLU(out_channels) self._init_weights() def _init_weights(self): for m in self.modules(): if isinstance(m, nn.Conv2d): normal_init(m, 0, 0.01) if self.zero_init_offset: constant_init(self.spatial_conv_offset, 0) def forward(self, x): """Forward function.""" outs = [] for level in range(len(x)): # calculate offset and mask of DCNv2 from middle-level feature offset_and_mask = self.spatial_conv_offset(x[level]) offset = offset_and_mask[:, :self.offset_dim, :, :] mask = offset_and_mask[:, self.offset_dim:, :, :].sigmoid() mid_feat = self.spatial_conv_mid(x[level], offset, mask) sum_feat = mid_feat * self.scale_attn_module(mid_feat) summed_levels = 1 if level > 0: low_feat = self.spatial_conv_low(x[level - 1], offset, mask) sum_feat += low_feat * self.scale_attn_module(low_feat) summed_levels += 1 if level < len(x) - 1: # this upsample order is weird, but faster than natural order # https://github.com/microsoft/DynamicHead/issues/25 high_feat = F.interpolate( self.spatial_conv_high(x[level + 1], offset, mask), size=x[level].shape[-2:], mode='bilinear', align_corners=True) sum_feat += high_feat * self.scale_attn_module(high_feat) summed_levels += 1 outs.append(self.task_attn_module(sum_feat / summed_levels)) return outs @MODELS.register_module() class DyHead(BaseModule): """DyHead neck consisting of multiple DyHead Blocks. See `Dynamic Head: Unifying Object Detection Heads with Attentions `_ for details. Args: in_channels (int): Number of input channels. out_channels (int): Number of output channels. num_blocks (int, optional): Number of DyHead Blocks. Default: 6. zero_init_offset (bool, optional): Whether to use zero init for `spatial_conv_offset`. Default: True. init_cfg (dict or list[dict], optional): Initialization config dict. Default: None. """ def __init__(self, in_channels, out_channels, num_blocks=6, zero_init_offset=True, init_cfg=None): assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' super().__init__(init_cfg=init_cfg) self.in_channels = in_channels self.out_channels = out_channels self.num_blocks = num_blocks self.zero_init_offset = zero_init_offset dyhead_blocks = [] for i in range(num_blocks): in_channels = self.in_channels if i == 0 else self.out_channels dyhead_blocks.append( DyHeadBlock( in_channels, self.out_channels, zero_init_offset=zero_init_offset)) self.dyhead_blocks = nn.Sequential(*dyhead_blocks) def forward(self, inputs): """Forward function.""" assert isinstance(inputs, (tuple, list)) outs = self.dyhead_blocks(inputs) return tuple(outs) ================================================ FILE: mmdet/models/necks/fpg.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.model import BaseModule from mmdet.registry import MODELS class Transition(BaseModule): """Base class for transition. Args: in_channels (int): Number of input channels. out_channels (int): Number of output channels. """ def __init__(self, in_channels, out_channels, init_cfg=None): super().__init__(init_cfg) self.in_channels = in_channels self.out_channels = out_channels def forward(x): pass class UpInterpolationConv(Transition): """A transition used for up-sampling. Up-sample the input by interpolation then refines the feature by a convolution layer. Args: in_channels (int): Number of input channels. out_channels (int): Number of output channels. scale_factor (int): Up-sampling factor. Default: 2. mode (int): Interpolation mode. Default: nearest. align_corners (bool): Whether align corners when interpolation. Default: None. kernel_size (int): Kernel size for the conv. Default: 3. """ def __init__(self, in_channels, out_channels, scale_factor=2, mode='nearest', align_corners=None, kernel_size=3, init_cfg=None, **kwargs): super().__init__(in_channels, out_channels, init_cfg) self.mode = mode self.scale_factor = scale_factor self.align_corners = align_corners self.conv = ConvModule( in_channels, out_channels, kernel_size, padding=(kernel_size - 1) // 2, **kwargs) def forward(self, x): x = F.interpolate( x, scale_factor=self.scale_factor, mode=self.mode, align_corners=self.align_corners) x = self.conv(x) return x class LastConv(Transition): """A transition used for refining the output of the last stage. Args: in_channels (int): Number of input channels. out_channels (int): Number of output channels. num_inputs (int): Number of inputs of the FPN features. kernel_size (int): Kernel size for the conv. Default: 3. """ def __init__(self, in_channels, out_channels, num_inputs, kernel_size=3, init_cfg=None, **kwargs): super().__init__(in_channels, out_channels, init_cfg) self.num_inputs = num_inputs self.conv_out = ConvModule( in_channels, out_channels, kernel_size, padding=(kernel_size - 1) // 2, **kwargs) def forward(self, inputs): assert len(inputs) == self.num_inputs return self.conv_out(inputs[-1]) @MODELS.register_module() class FPG(BaseModule): """FPG. Implementation of `Feature Pyramid Grids (FPG) `_. This implementation only gives the basic structure stated in the paper. But users can implement different type of transitions to fully explore the the potential power of the structure of FPG. Args: in_channels (int): Number of input channels (feature maps of all levels should have the same channels). out_channels (int): Number of output channels (used at each scale) num_outs (int): Number of output scales. stack_times (int): The number of times the pyramid architecture will be stacked. paths (list[str]): Specify the path order of each stack level. Each element in the list should be either 'bu' (bottom-up) or 'td' (top-down). inter_channels (int): Number of inter channels. same_up_trans (dict): Transition that goes down at the same stage. same_down_trans (dict): Transition that goes up at the same stage. across_lateral_trans (dict): Across-pathway same-stage across_down_trans (dict): Across-pathway bottom-up connection. across_up_trans (dict): Across-pathway top-down connection. across_skip_trans (dict): Across-pathway skip connection. output_trans (dict): Transition that trans the output of the last stage. start_level (int): Index of the start input backbone level used to build the feature pyramid. Default: 0. end_level (int): Index of the end input backbone level (exclusive) to build the feature pyramid. Default: -1, which means the last level. add_extra_convs (bool): It decides whether to add conv layers on top of the original feature maps. Default to False. If True, its actual mode is specified by `extra_convs_on_inputs`. norm_cfg (dict): Config dict for normalization layer. Default: None. init_cfg (dict or list[dict], optional): Initialization config dict. """ transition_types = { 'conv': ConvModule, 'interpolation_conv': UpInterpolationConv, 'last_conv': LastConv, } def __init__(self, in_channels, out_channels, num_outs, stack_times, paths, inter_channels=None, same_down_trans=None, same_up_trans=dict( type='conv', kernel_size=3, stride=2, padding=1), across_lateral_trans=dict(type='conv', kernel_size=1), across_down_trans=dict(type='conv', kernel_size=3), across_up_trans=None, across_skip_trans=dict(type='identity'), output_trans=dict(type='last_conv', kernel_size=3), start_level=0, end_level=-1, add_extra_convs=False, norm_cfg=None, skip_inds=None, init_cfg=[ dict(type='Caffe2Xavier', layer='Conv2d'), dict( type='Constant', layer=[ '_BatchNorm', '_InstanceNorm', 'GroupNorm', 'LayerNorm' ], val=1.0) ]): super(FPG, self).__init__(init_cfg) assert isinstance(in_channels, list) self.in_channels = in_channels self.out_channels = out_channels self.num_ins = len(in_channels) self.num_outs = num_outs if inter_channels is None: self.inter_channels = [out_channels for _ in range(num_outs)] elif isinstance(inter_channels, int): self.inter_channels = [inter_channels for _ in range(num_outs)] else: assert isinstance(inter_channels, list) assert len(inter_channels) == num_outs self.inter_channels = inter_channels self.stack_times = stack_times self.paths = paths assert isinstance(paths, list) and len(paths) == stack_times for d in paths: assert d in ('bu', 'td') self.same_down_trans = same_down_trans self.same_up_trans = same_up_trans self.across_lateral_trans = across_lateral_trans self.across_down_trans = across_down_trans self.across_up_trans = across_up_trans self.output_trans = output_trans self.across_skip_trans = across_skip_trans self.with_bias = norm_cfg is None # skip inds must be specified if across skip trans is not None if self.across_skip_trans is not None: skip_inds is not None self.skip_inds = skip_inds assert len(self.skip_inds[0]) <= self.stack_times if end_level == -1 or end_level == self.num_ins - 1: self.backbone_end_level = self.num_ins assert num_outs >= self.num_ins - start_level else: # if end_level is not the last level, no extra level is allowed self.backbone_end_level = end_level + 1 assert end_level < self.num_ins assert num_outs == end_level - start_level + 1 self.start_level = start_level self.end_level = end_level self.add_extra_convs = add_extra_convs # build lateral 1x1 convs to reduce channels self.lateral_convs = nn.ModuleList() for i in range(self.start_level, self.backbone_end_level): l_conv = nn.Conv2d(self.in_channels[i], self.inter_channels[i - self.start_level], 1) self.lateral_convs.append(l_conv) extra_levels = num_outs - self.backbone_end_level + self.start_level self.extra_downsamples = nn.ModuleList() for i in range(extra_levels): if self.add_extra_convs: fpn_idx = self.backbone_end_level - self.start_level + i extra_conv = nn.Conv2d( self.inter_channels[fpn_idx - 1], self.inter_channels[fpn_idx], 3, stride=2, padding=1) self.extra_downsamples.append(extra_conv) else: self.extra_downsamples.append(nn.MaxPool2d(1, stride=2)) self.fpn_transitions = nn.ModuleList() # stack times for s in range(self.stack_times): stage_trans = nn.ModuleList() # num of feature levels for i in range(self.num_outs): # same, across_lateral, across_down, across_up trans = nn.ModuleDict() if s in self.skip_inds[i]: stage_trans.append(trans) continue # build same-stage down trans (used in bottom-up paths) if i == 0 or self.same_up_trans is None: same_up_trans = None else: same_up_trans = self.build_trans( self.same_up_trans, self.inter_channels[i - 1], self.inter_channels[i]) trans['same_up'] = same_up_trans # build same-stage up trans (used in top-down paths) if i == self.num_outs - 1 or self.same_down_trans is None: same_down_trans = None else: same_down_trans = self.build_trans( self.same_down_trans, self.inter_channels[i + 1], self.inter_channels[i]) trans['same_down'] = same_down_trans # build across lateral trans across_lateral_trans = self.build_trans( self.across_lateral_trans, self.inter_channels[i], self.inter_channels[i]) trans['across_lateral'] = across_lateral_trans # build across down trans if i == self.num_outs - 1 or self.across_down_trans is None: across_down_trans = None else: across_down_trans = self.build_trans( self.across_down_trans, self.inter_channels[i + 1], self.inter_channels[i]) trans['across_down'] = across_down_trans # build across up trans if i == 0 or self.across_up_trans is None: across_up_trans = None else: across_up_trans = self.build_trans( self.across_up_trans, self.inter_channels[i - 1], self.inter_channels[i]) trans['across_up'] = across_up_trans if self.across_skip_trans is None: across_skip_trans = None else: across_skip_trans = self.build_trans( self.across_skip_trans, self.inter_channels[i - 1], self.inter_channels[i]) trans['across_skip'] = across_skip_trans # build across_skip trans stage_trans.append(trans) self.fpn_transitions.append(stage_trans) self.output_transition = nn.ModuleList() # output levels for i in range(self.num_outs): trans = self.build_trans( self.output_trans, self.inter_channels[i], self.out_channels, num_inputs=self.stack_times + 1) self.output_transition.append(trans) self.relu = nn.ReLU(inplace=True) def build_trans(self, cfg, in_channels, out_channels, **extra_args): cfg_ = cfg.copy() trans_type = cfg_.pop('type') trans_cls = self.transition_types[trans_type] return trans_cls(in_channels, out_channels, **cfg_, **extra_args) def fuse(self, fuse_dict): out = None for item in fuse_dict.values(): if item is not None: if out is None: out = item else: out = out + item return out def forward(self, inputs): assert len(inputs) == len(self.in_channels) # build all levels from original feature maps feats = [ lateral_conv(inputs[i + self.start_level]) for i, lateral_conv in enumerate(self.lateral_convs) ] for downsample in self.extra_downsamples: feats.append(downsample(feats[-1])) outs = [feats] for i in range(self.stack_times): current_outs = outs[-1] next_outs = [] direction = self.paths[i] for j in range(self.num_outs): if i in self.skip_inds[j]: next_outs.append(outs[-1][j]) continue # feature level if direction == 'td': lvl = self.num_outs - j - 1 else: lvl = j # get transitions if direction == 'td': same_trans = self.fpn_transitions[i][lvl]['same_down'] else: same_trans = self.fpn_transitions[i][lvl]['same_up'] across_lateral_trans = self.fpn_transitions[i][lvl][ 'across_lateral'] across_down_trans = self.fpn_transitions[i][lvl]['across_down'] across_up_trans = self.fpn_transitions[i][lvl]['across_up'] across_skip_trans = self.fpn_transitions[i][lvl]['across_skip'] # init output to_fuse = dict( same=None, lateral=None, across_up=None, across_down=None) # same downsample/upsample if same_trans is not None: to_fuse['same'] = same_trans(next_outs[-1]) # across lateral if across_lateral_trans is not None: to_fuse['lateral'] = across_lateral_trans( current_outs[lvl]) # across downsample if lvl > 0 and across_up_trans is not None: to_fuse['across_up'] = across_up_trans(current_outs[lvl - 1]) # across upsample if (lvl < self.num_outs - 1 and across_down_trans is not None): to_fuse['across_down'] = across_down_trans( current_outs[lvl + 1]) if across_skip_trans is not None: to_fuse['across_skip'] = across_skip_trans(outs[0][lvl]) x = self.fuse(to_fuse) next_outs.append(x) if direction == 'td': outs.append(next_outs[::-1]) else: outs.append(next_outs) # output trans final_outs = [] for i in range(self.num_outs): lvl_out_list = [] for s in range(len(outs)): lvl_out_list.append(outs[s][i]) lvl_out = self.output_transition[i](lvl_out_list) final_outs.append(lvl_out) return final_outs ================================================ FILE: mmdet/models/necks/fpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple, Union import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.model import BaseModule from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import ConfigType, MultiConfig, OptConfigType @MODELS.register_module() class FPN(BaseModule): r"""Feature Pyramid Network. This is an implementation of paper `Feature Pyramid Networks for Object Detection `_. Args: in_channels (list[int]): Number of input channels per scale. out_channels (int): Number of output channels (used at each scale). num_outs (int): Number of output scales. start_level (int): Index of the start input backbone level used to build the feature pyramid. Defaults to 0. end_level (int): Index of the end input backbone level (exclusive) to build the feature pyramid. Defaults to -1, which means the last level. add_extra_convs (bool | str): If bool, it decides whether to add conv layers on top of the original feature maps. Defaults to False. If True, it is equivalent to `add_extra_convs='on_input'`. If str, it specifies the source feature map of the extra convs. Only the following options are allowed - 'on_input': Last feat map of neck inputs (i.e. backbone feature). - 'on_lateral': Last feature map after lateral convs. - 'on_output': The last output feature map after fpn convs. relu_before_extra_convs (bool): Whether to apply relu before the extra conv. Defaults to False. no_norm_on_lateral (bool): Whether to apply norm on lateral. Defaults to False. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for normalization layer. Defaults to None. act_cfg (:obj:`ConfigDict` or dict, optional): Config dict for activation layer in ConvModule. Defaults to None. upsample_cfg (:obj:`ConfigDict` or dict, optional): Config dict for interpolate layer. Defaults to dict(mode='nearest'). init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict]): Initialization config dict. Example: >>> import torch >>> in_channels = [2, 3, 5, 7] >>> scales = [340, 170, 84, 43] >>> inputs = [torch.rand(1, c, s, s) ... for c, s in zip(in_channels, scales)] >>> self = FPN(in_channels, 11, len(in_channels)).eval() >>> outputs = self.forward(inputs) >>> for i in range(len(outputs)): ... print(f'outputs[{i}].shape = {outputs[i].shape}') outputs[0].shape = torch.Size([1, 11, 340, 340]) outputs[1].shape = torch.Size([1, 11, 170, 170]) outputs[2].shape = torch.Size([1, 11, 84, 84]) outputs[3].shape = torch.Size([1, 11, 43, 43]) """ def __init__( self, in_channels: List[int], out_channels: int, num_outs: int, start_level: int = 0, end_level: int = -1, add_extra_convs: Union[bool, str] = False, relu_before_extra_convs: bool = False, no_norm_on_lateral: bool = False, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, act_cfg: OptConfigType = None, upsample_cfg: ConfigType = dict(mode='nearest'), init_cfg: MultiConfig = dict( type='Xavier', layer='Conv2d', distribution='uniform') ) -> None: super().__init__(init_cfg=init_cfg) assert isinstance(in_channels, list) self.in_channels = in_channels self.out_channels = out_channels self.num_ins = len(in_channels) self.num_outs = num_outs self.relu_before_extra_convs = relu_before_extra_convs self.no_norm_on_lateral = no_norm_on_lateral self.fp16_enabled = False self.upsample_cfg = upsample_cfg.copy() if end_level == -1 or end_level == self.num_ins - 1: self.backbone_end_level = self.num_ins assert num_outs >= self.num_ins - start_level else: # if end_level is not the last level, no extra level is allowed self.backbone_end_level = end_level + 1 assert end_level < self.num_ins assert num_outs == end_level - start_level + 1 self.start_level = start_level self.end_level = end_level self.add_extra_convs = add_extra_convs assert isinstance(add_extra_convs, (str, bool)) if isinstance(add_extra_convs, str): # Extra_convs_source choices: 'on_input', 'on_lateral', 'on_output' assert add_extra_convs in ('on_input', 'on_lateral', 'on_output') elif add_extra_convs: # True self.add_extra_convs = 'on_input' self.lateral_convs = nn.ModuleList() self.fpn_convs = nn.ModuleList() for i in range(self.start_level, self.backbone_end_level): l_conv = ConvModule( in_channels[i], out_channels, 1, conv_cfg=conv_cfg, norm_cfg=norm_cfg if not self.no_norm_on_lateral else None, act_cfg=act_cfg, inplace=False) fpn_conv = ConvModule( out_channels, out_channels, 3, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg, inplace=False) self.lateral_convs.append(l_conv) self.fpn_convs.append(fpn_conv) # add extra conv layers (e.g., RetinaNet) extra_levels = num_outs - self.backbone_end_level + self.start_level if self.add_extra_convs and extra_levels >= 1: for i in range(extra_levels): if i == 0 and self.add_extra_convs == 'on_input': in_channels = self.in_channels[self.backbone_end_level - 1] else: in_channels = out_channels extra_fpn_conv = ConvModule( in_channels, out_channels, 3, stride=2, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg, inplace=False) self.fpn_convs.append(extra_fpn_conv) def forward(self, inputs: Tuple[Tensor]) -> tuple: """Forward function. Args: inputs (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Feature maps, each is a 4D-tensor. """ assert len(inputs) == len(self.in_channels) # build laterals laterals = [ lateral_conv(inputs[i + self.start_level]) for i, lateral_conv in enumerate(self.lateral_convs) ] # build top-down path used_backbone_levels = len(laterals) for i in range(used_backbone_levels - 1, 0, -1): # In some cases, fixing `scale factor` (e.g. 2) is preferred, but # it cannot co-exist with `size` in `F.interpolate`. if 'scale_factor' in self.upsample_cfg: # fix runtime error of "+=" inplace operation in PyTorch 1.10 laterals[i - 1] = laterals[i - 1] + F.interpolate( laterals[i], **self.upsample_cfg) else: prev_shape = laterals[i - 1].shape[2:] laterals[i - 1] = laterals[i - 1] + F.interpolate( laterals[i], size=prev_shape, **self.upsample_cfg) # build outputs # part 1: from original levels outs = [ self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels) ] # part 2: add extra levels if self.num_outs > len(outs): # use max pool to get more levels on top of outputs # (e.g., Faster R-CNN, Mask R-CNN) if not self.add_extra_convs: for i in range(self.num_outs - used_backbone_levels): outs.append(F.max_pool2d(outs[-1], 1, stride=2)) # add conv layers on top of original feature maps (RetinaNet) else: if self.add_extra_convs == 'on_input': extra_source = inputs[self.backbone_end_level - 1] elif self.add_extra_convs == 'on_lateral': extra_source = laterals[-1] elif self.add_extra_convs == 'on_output': extra_source = outs[-1] else: raise NotImplementedError outs.append(self.fpn_convs[used_backbone_levels](extra_source)) for i in range(used_backbone_levels + 1, self.num_outs): if self.relu_before_extra_convs: outs.append(self.fpn_convs[i](F.relu(outs[-1]))) else: outs.append(self.fpn_convs[i](outs[-1])) return tuple(outs) ================================================ FILE: mmdet/models/necks/fpn_carafe.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn as nn from mmcv.cnn import ConvModule, build_upsample_layer from mmcv.ops.carafe import CARAFEPack from mmengine.model import BaseModule, ModuleList, xavier_init from mmdet.registry import MODELS @MODELS.register_module() class FPN_CARAFE(BaseModule): """FPN_CARAFE is a more flexible implementation of FPN. It allows more choice for upsample methods during the top-down pathway. It can reproduce the performance of ICCV 2019 paper CARAFE: Content-Aware ReAssembly of FEatures Please refer to https://arxiv.org/abs/1905.02188 for more details. Args: in_channels (list[int]): Number of channels for each input feature map. out_channels (int): Output channels of feature pyramids. num_outs (int): Number of output stages. start_level (int): Start level of feature pyramids. (Default: 0) end_level (int): End level of feature pyramids. (Default: -1 indicates the last level). norm_cfg (dict): Dictionary to construct and config norm layer. activate (str): Type of activation function in ConvModule (Default: None indicates w/o activation). order (dict): Order of components in ConvModule. upsample (str): Type of upsample layer. upsample_cfg (dict): Dictionary to construct and config upsample layer. init_cfg (dict or list[dict], optional): Initialization config dict. Default: None """ def __init__(self, in_channels, out_channels, num_outs, start_level=0, end_level=-1, norm_cfg=None, act_cfg=None, order=('conv', 'norm', 'act'), upsample_cfg=dict( type='carafe', up_kernel=5, up_group=1, encoder_kernel=3, encoder_dilation=1), init_cfg=None): assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' super(FPN_CARAFE, self).__init__(init_cfg) assert isinstance(in_channels, list) self.in_channels = in_channels self.out_channels = out_channels self.num_ins = len(in_channels) self.num_outs = num_outs self.norm_cfg = norm_cfg self.act_cfg = act_cfg self.with_bias = norm_cfg is None self.upsample_cfg = upsample_cfg.copy() self.upsample = self.upsample_cfg.get('type') self.relu = nn.ReLU(inplace=False) self.order = order assert order in [('conv', 'norm', 'act'), ('act', 'conv', 'norm')] assert self.upsample in [ 'nearest', 'bilinear', 'deconv', 'pixel_shuffle', 'carafe', None ] if self.upsample in ['deconv', 'pixel_shuffle']: assert hasattr( self.upsample_cfg, 'upsample_kernel') and self.upsample_cfg.upsample_kernel > 0 self.upsample_kernel = self.upsample_cfg.pop('upsample_kernel') if end_level == -1 or end_level == self.num_ins - 1: self.backbone_end_level = self.num_ins assert num_outs >= self.num_ins - start_level else: # if end_level is not the last level, no extra level is allowed self.backbone_end_level = end_level + 1 assert end_level < self.num_ins assert num_outs == end_level - start_level + 1 self.start_level = start_level self.end_level = end_level self.lateral_convs = ModuleList() self.fpn_convs = ModuleList() self.upsample_modules = ModuleList() for i in range(self.start_level, self.backbone_end_level): l_conv = ConvModule( in_channels[i], out_channels, 1, norm_cfg=norm_cfg, bias=self.with_bias, act_cfg=act_cfg, inplace=False, order=self.order) fpn_conv = ConvModule( out_channels, out_channels, 3, padding=1, norm_cfg=self.norm_cfg, bias=self.with_bias, act_cfg=act_cfg, inplace=False, order=self.order) if i != self.backbone_end_level - 1: upsample_cfg_ = self.upsample_cfg.copy() if self.upsample == 'deconv': upsample_cfg_.update( in_channels=out_channels, out_channels=out_channels, kernel_size=self.upsample_kernel, stride=2, padding=(self.upsample_kernel - 1) // 2, output_padding=(self.upsample_kernel - 1) // 2) elif self.upsample == 'pixel_shuffle': upsample_cfg_.update( in_channels=out_channels, out_channels=out_channels, scale_factor=2, upsample_kernel=self.upsample_kernel) elif self.upsample == 'carafe': upsample_cfg_.update(channels=out_channels, scale_factor=2) else: # suppress warnings align_corners = (None if self.upsample == 'nearest' else False) upsample_cfg_.update( scale_factor=2, mode=self.upsample, align_corners=align_corners) upsample_module = build_upsample_layer(upsample_cfg_) self.upsample_modules.append(upsample_module) self.lateral_convs.append(l_conv) self.fpn_convs.append(fpn_conv) # add extra conv layers (e.g., RetinaNet) extra_out_levels = ( num_outs - self.backbone_end_level + self.start_level) if extra_out_levels >= 1: for i in range(extra_out_levels): in_channels = ( self.in_channels[self.backbone_end_level - 1] if i == 0 else out_channels) extra_l_conv = ConvModule( in_channels, out_channels, 3, stride=2, padding=1, norm_cfg=norm_cfg, bias=self.with_bias, act_cfg=act_cfg, inplace=False, order=self.order) if self.upsample == 'deconv': upsampler_cfg_ = dict( in_channels=out_channels, out_channels=out_channels, kernel_size=self.upsample_kernel, stride=2, padding=(self.upsample_kernel - 1) // 2, output_padding=(self.upsample_kernel - 1) // 2) elif self.upsample == 'pixel_shuffle': upsampler_cfg_ = dict( in_channels=out_channels, out_channels=out_channels, scale_factor=2, upsample_kernel=self.upsample_kernel) elif self.upsample == 'carafe': upsampler_cfg_ = dict( channels=out_channels, scale_factor=2, **self.upsample_cfg) else: # suppress warnings align_corners = (None if self.upsample == 'nearest' else False) upsampler_cfg_ = dict( scale_factor=2, mode=self.upsample, align_corners=align_corners) upsampler_cfg_['type'] = self.upsample upsample_module = build_upsample_layer(upsampler_cfg_) extra_fpn_conv = ConvModule( out_channels, out_channels, 3, padding=1, norm_cfg=self.norm_cfg, bias=self.with_bias, act_cfg=act_cfg, inplace=False, order=self.order) self.upsample_modules.append(upsample_module) self.fpn_convs.append(extra_fpn_conv) self.lateral_convs.append(extra_l_conv) # default init_weights for conv(msra) and norm in ConvModule def init_weights(self): """Initialize the weights of module.""" super(FPN_CARAFE, self).init_weights() for m in self.modules(): if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)): xavier_init(m, distribution='uniform') for m in self.modules(): if isinstance(m, CARAFEPack): m.init_weights() def slice_as(self, src, dst): """Slice ``src`` as ``dst`` Note: ``src`` should have the same or larger size than ``dst``. Args: src (torch.Tensor): Tensors to be sliced. dst (torch.Tensor): ``src`` will be sliced to have the same size as ``dst``. Returns: torch.Tensor: Sliced tensor. """ assert (src.size(2) >= dst.size(2)) and (src.size(3) >= dst.size(3)) if src.size(2) == dst.size(2) and src.size(3) == dst.size(3): return src else: return src[:, :, :dst.size(2), :dst.size(3)] def tensor_add(self, a, b): """Add tensors ``a`` and ``b`` that might have different sizes.""" if a.size() == b.size(): c = a + b else: c = a + self.slice_as(b, a) return c def forward(self, inputs): """Forward function.""" assert len(inputs) == len(self.in_channels) # build laterals laterals = [] for i, lateral_conv in enumerate(self.lateral_convs): if i <= self.backbone_end_level - self.start_level: input = inputs[min(i + self.start_level, len(inputs) - 1)] else: input = laterals[-1] lateral = lateral_conv(input) laterals.append(lateral) # build top-down path for i in range(len(laterals) - 1, 0, -1): if self.upsample is not None: upsample_feat = self.upsample_modules[i - 1](laterals[i]) else: upsample_feat = laterals[i] laterals[i - 1] = self.tensor_add(laterals[i - 1], upsample_feat) # build outputs num_conv_outs = len(self.fpn_convs) outs = [] for i in range(num_conv_outs): out = self.fpn_convs[i](laterals[i]) outs.append(out) return tuple(outs) ================================================ FILE: mmdet/models/necks/hrfpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.model import BaseModule from torch.utils.checkpoint import checkpoint from mmdet.registry import MODELS @MODELS.register_module() class HRFPN(BaseModule): """HRFPN (High Resolution Feature Pyramids) paper: `High-Resolution Representations for Labeling Pixels and Regions `_. Args: in_channels (list): number of channels for each branch. out_channels (int): output channels of feature pyramids. num_outs (int): number of output stages. pooling_type (str): pooling for generating feature pyramids from {MAX, AVG}. conv_cfg (dict): dictionary to construct and config conv layer. norm_cfg (dict): dictionary to construct and config norm layer. with_cp (bool): Use checkpoint or not. Using checkpoint will save some memory while slowing down the training speed. stride (int): stride of 3x3 convolutional layers init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, in_channels, out_channels, num_outs=5, pooling_type='AVG', conv_cfg=None, norm_cfg=None, with_cp=False, stride=1, init_cfg=dict(type='Caffe2Xavier', layer='Conv2d')): super(HRFPN, self).__init__(init_cfg) assert isinstance(in_channels, list) self.in_channels = in_channels self.out_channels = out_channels self.num_ins = len(in_channels) self.num_outs = num_outs self.with_cp = with_cp self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.reduction_conv = ConvModule( sum(in_channels), out_channels, kernel_size=1, conv_cfg=self.conv_cfg, act_cfg=None) self.fpn_convs = nn.ModuleList() for i in range(self.num_outs): self.fpn_convs.append( ConvModule( out_channels, out_channels, kernel_size=3, padding=1, stride=stride, conv_cfg=self.conv_cfg, act_cfg=None)) if pooling_type == 'MAX': self.pooling = F.max_pool2d else: self.pooling = F.avg_pool2d def forward(self, inputs): """Forward function.""" assert len(inputs) == self.num_ins outs = [inputs[0]] for i in range(1, self.num_ins): outs.append( F.interpolate(inputs[i], scale_factor=2**i, mode='bilinear')) out = torch.cat(outs, dim=1) if out.requires_grad and self.with_cp: out = checkpoint(self.reduction_conv, out) else: out = self.reduction_conv(out) outs = [out] for i in range(1, self.num_outs): outs.append(self.pooling(out, kernel_size=2**i, stride=2**i)) outputs = [] for i in range(self.num_outs): if outs[i].requires_grad and self.with_cp: tmp_out = checkpoint(self.fpn_convs[i], outs[i]) else: tmp_out = self.fpn_convs[i](outs[i]) outputs.append(tmp_out) return tuple(outputs) ================================================ FILE: mmdet/models/necks/nas_fpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import torch.nn as nn from mmcv.cnn import ConvModule from mmcv.ops.merge_cells import GlobalPoolingCell, SumCell from mmengine.model import BaseModule, ModuleList from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import MultiConfig, OptConfigType @MODELS.register_module() class NASFPN(BaseModule): """NAS-FPN. Implementation of `NAS-FPN: Learning Scalable Feature Pyramid Architecture for Object Detection `_ Args: in_channels (List[int]): Number of input channels per scale. out_channels (int): Number of output channels (used at each scale) num_outs (int): Number of output scales. stack_times (int): The number of times the pyramid architecture will be stacked. start_level (int): Index of the start input backbone level used to build the feature pyramid. Defaults to 0. end_level (int): Index of the end input backbone level (exclusive) to build the feature pyramid. Defaults to -1, which means the last level. norm_cfg (:obj:`ConfigDict` or dict, optional): Config dict for normalization layer. Defaults to None. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict]): Initialization config dict. """ def __init__( self, in_channels: List[int], out_channels: int, num_outs: int, stack_times: int, start_level: int = 0, end_level: int = -1, norm_cfg: OptConfigType = None, init_cfg: MultiConfig = dict(type='Caffe2Xavier', layer='Conv2d') ) -> None: super().__init__(init_cfg=init_cfg) assert isinstance(in_channels, list) self.in_channels = in_channels self.out_channels = out_channels self.num_ins = len(in_channels) # num of input feature levels self.num_outs = num_outs # num of output feature levels self.stack_times = stack_times self.norm_cfg = norm_cfg if end_level == -1 or end_level == self.num_ins - 1: self.backbone_end_level = self.num_ins assert num_outs >= self.num_ins - start_level else: # if end_level is not the last level, no extra level is allowed self.backbone_end_level = end_level + 1 assert end_level < self.num_ins assert num_outs == end_level - start_level + 1 self.start_level = start_level self.end_level = end_level # add lateral connections self.lateral_convs = nn.ModuleList() for i in range(self.start_level, self.backbone_end_level): l_conv = ConvModule( in_channels[i], out_channels, 1, norm_cfg=norm_cfg, act_cfg=None) self.lateral_convs.append(l_conv) # add extra downsample layers (stride-2 pooling or conv) extra_levels = num_outs - self.backbone_end_level + self.start_level self.extra_downsamples = nn.ModuleList() for i in range(extra_levels): extra_conv = ConvModule( out_channels, out_channels, 1, norm_cfg=norm_cfg, act_cfg=None) self.extra_downsamples.append( nn.Sequential(extra_conv, nn.MaxPool2d(2, 2))) # add NAS FPN connections self.fpn_stages = ModuleList() for _ in range(self.stack_times): stage = nn.ModuleDict() # gp(p6, p4) -> p4_1 stage['gp_64_4'] = GlobalPoolingCell( in_channels=out_channels, out_channels=out_channels, out_norm_cfg=norm_cfg) # sum(p4_1, p4) -> p4_2 stage['sum_44_4'] = SumCell( in_channels=out_channels, out_channels=out_channels, out_norm_cfg=norm_cfg) # sum(p4_2, p3) -> p3_out stage['sum_43_3'] = SumCell( in_channels=out_channels, out_channels=out_channels, out_norm_cfg=norm_cfg) # sum(p3_out, p4_2) -> p4_out stage['sum_34_4'] = SumCell( in_channels=out_channels, out_channels=out_channels, out_norm_cfg=norm_cfg) # sum(p5, gp(p4_out, p3_out)) -> p5_out stage['gp_43_5'] = GlobalPoolingCell(with_out_conv=False) stage['sum_55_5'] = SumCell( in_channels=out_channels, out_channels=out_channels, out_norm_cfg=norm_cfg) # sum(p7, gp(p5_out, p4_2)) -> p7_out stage['gp_54_7'] = GlobalPoolingCell(with_out_conv=False) stage['sum_77_7'] = SumCell( in_channels=out_channels, out_channels=out_channels, out_norm_cfg=norm_cfg) # gp(p7_out, p5_out) -> p6_out stage['gp_75_6'] = GlobalPoolingCell( in_channels=out_channels, out_channels=out_channels, out_norm_cfg=norm_cfg) self.fpn_stages.append(stage) def forward(self, inputs: Tuple[Tensor]) -> tuple: """Forward function. Args: inputs (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: Feature maps, each is a 4D-tensor. """ # build P3-P5 feats = [ lateral_conv(inputs[i + self.start_level]) for i, lateral_conv in enumerate(self.lateral_convs) ] # build P6-P7 on top of P5 for downsample in self.extra_downsamples: feats.append(downsample(feats[-1])) p3, p4, p5, p6, p7 = feats for stage in self.fpn_stages: # gp(p6, p4) -> p4_1 p4_1 = stage['gp_64_4'](p6, p4, out_size=p4.shape[-2:]) # sum(p4_1, p4) -> p4_2 p4_2 = stage['sum_44_4'](p4_1, p4, out_size=p4.shape[-2:]) # sum(p4_2, p3) -> p3_out p3 = stage['sum_43_3'](p4_2, p3, out_size=p3.shape[-2:]) # sum(p3_out, p4_2) -> p4_out p4 = stage['sum_34_4'](p3, p4_2, out_size=p4.shape[-2:]) # sum(p5, gp(p4_out, p3_out)) -> p5_out p5_tmp = stage['gp_43_5'](p4, p3, out_size=p5.shape[-2:]) p5 = stage['sum_55_5'](p5, p5_tmp, out_size=p5.shape[-2:]) # sum(p7, gp(p5_out, p4_2)) -> p7_out p7_tmp = stage['gp_54_7'](p5, p4_2, out_size=p7.shape[-2:]) p7 = stage['sum_77_7'](p7, p7_tmp, out_size=p7.shape[-2:]) # gp(p7_out, p5_out) -> p6_out p6 = stage['gp_75_6'](p7, p5, out_size=p6.shape[-2:]) return p3, p4, p5, p6, p7 ================================================ FILE: mmdet/models/necks/nasfcos_fpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmcv.ops.merge_cells import ConcatCell from mmengine.model import BaseModule, caffe2_xavier_init from mmdet.registry import MODELS @MODELS.register_module() class NASFCOS_FPN(BaseModule): """FPN structure in NASFPN. Implementation of paper `NAS-FCOS: Fast Neural Architecture Search for Object Detection `_ Args: in_channels (List[int]): Number of input channels per scale. out_channels (int): Number of output channels (used at each scale) num_outs (int): Number of output scales. start_level (int): Index of the start input backbone level used to build the feature pyramid. Default: 0. end_level (int): Index of the end input backbone level (exclusive) to build the feature pyramid. Default: -1, which means the last level. add_extra_convs (bool): It decides whether to add conv layers on top of the original feature maps. Default to False. If True, its actual mode is specified by `extra_convs_on_inputs`. conv_cfg (dict): dictionary to construct and config conv layer. norm_cfg (dict): dictionary to construct and config norm layer. init_cfg (dict or list[dict], optional): Initialization config dict. Default: None """ def __init__(self, in_channels, out_channels, num_outs, start_level=1, end_level=-1, add_extra_convs=False, conv_cfg=None, norm_cfg=None, init_cfg=None): assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' super(NASFCOS_FPN, self).__init__(init_cfg) assert isinstance(in_channels, list) self.in_channels = in_channels self.out_channels = out_channels self.num_ins = len(in_channels) self.num_outs = num_outs self.norm_cfg = norm_cfg self.conv_cfg = conv_cfg if end_level == -1 or end_level == self.num_ins - 1: self.backbone_end_level = self.num_ins assert num_outs >= self.num_ins - start_level else: # if end_level is not the last level, no extra level is allowed self.backbone_end_level = end_level + 1 assert end_level < self.num_ins assert num_outs == end_level - start_level + 1 self.start_level = start_level self.end_level = end_level self.add_extra_convs = add_extra_convs self.adapt_convs = nn.ModuleList() for i in range(self.start_level, self.backbone_end_level): adapt_conv = ConvModule( in_channels[i], out_channels, 1, stride=1, padding=0, bias=False, norm_cfg=dict(type='BN'), act_cfg=dict(type='ReLU', inplace=False)) self.adapt_convs.append(adapt_conv) # C2 is omitted according to the paper extra_levels = num_outs - self.backbone_end_level + self.start_level def build_concat_cell(with_input1_conv, with_input2_conv): cell_conv_cfg = dict( kernel_size=1, padding=0, bias=False, groups=out_channels) return ConcatCell( in_channels=out_channels, out_channels=out_channels, with_out_conv=True, out_conv_cfg=cell_conv_cfg, out_norm_cfg=dict(type='BN'), out_conv_order=('norm', 'act', 'conv'), with_input1_conv=with_input1_conv, with_input2_conv=with_input2_conv, input_conv_cfg=conv_cfg, input_norm_cfg=norm_cfg, upsample_mode='nearest') # Denote c3=f0, c4=f1, c5=f2 for convince self.fpn = nn.ModuleDict() self.fpn['c22_1'] = build_concat_cell(True, True) self.fpn['c22_2'] = build_concat_cell(True, True) self.fpn['c32'] = build_concat_cell(True, False) self.fpn['c02'] = build_concat_cell(True, False) self.fpn['c42'] = build_concat_cell(True, True) self.fpn['c36'] = build_concat_cell(True, True) self.fpn['c61'] = build_concat_cell(True, True) # f9 self.extra_downsamples = nn.ModuleList() for i in range(extra_levels): extra_act_cfg = None if i == 0 \ else dict(type='ReLU', inplace=False) self.extra_downsamples.append( ConvModule( out_channels, out_channels, 3, stride=2, padding=1, act_cfg=extra_act_cfg, order=('act', 'norm', 'conv'))) def forward(self, inputs): """Forward function.""" feats = [ adapt_conv(inputs[i + self.start_level]) for i, adapt_conv in enumerate(self.adapt_convs) ] for (i, module_name) in enumerate(self.fpn): idx_1, idx_2 = int(module_name[1]), int(module_name[2]) res = self.fpn[module_name](feats[idx_1], feats[idx_2]) feats.append(res) ret = [] for (idx, input_idx) in zip([9, 8, 7], [1, 2, 3]): # add P3, P4, P5 feats1, feats2 = feats[idx], feats[5] feats2_resize = F.interpolate( feats2, size=feats1.size()[2:], mode='bilinear', align_corners=False) feats_sum = feats1 + feats2_resize ret.append( F.interpolate( feats_sum, size=inputs[input_idx].size()[2:], mode='bilinear', align_corners=False)) for submodule in self.extra_downsamples: ret.append(submodule(ret[-1])) return tuple(ret) def init_weights(self): """Initialize the weights of module.""" super(NASFCOS_FPN, self).init_weights() for module in self.fpn.values(): if hasattr(module, 'conv_out'): caffe2_xavier_init(module.out_conv.conv) for modules in [ self.adapt_convs.modules(), self.extra_downsamples.modules() ]: for module in modules: if isinstance(module, nn.Conv2d): caffe2_xavier_init(module) ================================================ FILE: mmdet/models/necks/pafpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmdet.registry import MODELS from .fpn import FPN @MODELS.register_module() class PAFPN(FPN): """Path Aggregation Network for Instance Segmentation. This is an implementation of the `PAFPN in Path Aggregation Network `_. Args: in_channels (List[int]): Number of input channels per scale. out_channels (int): Number of output channels (used at each scale) num_outs (int): Number of output scales. start_level (int): Index of the start input backbone level used to build the feature pyramid. Default: 0. end_level (int): Index of the end input backbone level (exclusive) to build the feature pyramid. Default: -1, which means the last level. add_extra_convs (bool | str): If bool, it decides whether to add conv layers on top of the original feature maps. Default to False. If True, it is equivalent to `add_extra_convs='on_input'`. If str, it specifies the source feature map of the extra convs. Only the following options are allowed - 'on_input': Last feat map of neck inputs (i.e. backbone feature). - 'on_lateral': Last feature map after lateral convs. - 'on_output': The last output feature map after fpn convs. relu_before_extra_convs (bool): Whether to apply relu before the extra conv. Default: False. no_norm_on_lateral (bool): Whether to apply norm on lateral. Default: False. conv_cfg (dict): Config dict for convolution layer. Default: None. norm_cfg (dict): Config dict for normalization layer. Default: None. act_cfg (str): Config dict for activation layer in ConvModule. Default: None. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, in_channels, out_channels, num_outs, start_level=0, end_level=-1, add_extra_convs=False, relu_before_extra_convs=False, no_norm_on_lateral=False, conv_cfg=None, norm_cfg=None, act_cfg=None, init_cfg=dict( type='Xavier', layer='Conv2d', distribution='uniform')): super(PAFPN, self).__init__( in_channels, out_channels, num_outs, start_level, end_level, add_extra_convs, relu_before_extra_convs, no_norm_on_lateral, conv_cfg, norm_cfg, act_cfg, init_cfg=init_cfg) # add extra bottom up pathway self.downsample_convs = nn.ModuleList() self.pafpn_convs = nn.ModuleList() for i in range(self.start_level + 1, self.backbone_end_level): d_conv = ConvModule( out_channels, out_channels, 3, stride=2, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg, inplace=False) pafpn_conv = ConvModule( out_channels, out_channels, 3, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg, inplace=False) self.downsample_convs.append(d_conv) self.pafpn_convs.append(pafpn_conv) def forward(self, inputs): """Forward function.""" assert len(inputs) == len(self.in_channels) # build laterals laterals = [ lateral_conv(inputs[i + self.start_level]) for i, lateral_conv in enumerate(self.lateral_convs) ] # build top-down path used_backbone_levels = len(laterals) for i in range(used_backbone_levels - 1, 0, -1): prev_shape = laterals[i - 1].shape[2:] laterals[i - 1] = laterals[i - 1] + F.interpolate( laterals[i], size=prev_shape, mode='nearest') # build outputs # part 1: from original levels inter_outs = [ self.fpn_convs[i](laterals[i]) for i in range(used_backbone_levels) ] # part 2: add bottom-up path for i in range(0, used_backbone_levels - 1): inter_outs[i + 1] = inter_outs[i + 1] + \ self.downsample_convs[i](inter_outs[i]) outs = [] outs.append(inter_outs[0]) outs.extend([ self.pafpn_convs[i - 1](inter_outs[i]) for i in range(1, used_backbone_levels) ]) # part 3: add extra levels if self.num_outs > len(outs): # use max pool to get more levels on top of outputs # (e.g., Faster R-CNN, Mask R-CNN) if not self.add_extra_convs: for i in range(self.num_outs - used_backbone_levels): outs.append(F.max_pool2d(outs[-1], 1, stride=2)) # add conv layers on top of original feature maps (RetinaNet) else: if self.add_extra_convs == 'on_input': orig = inputs[self.backbone_end_level - 1] outs.append(self.fpn_convs[used_backbone_levels](orig)) elif self.add_extra_convs == 'on_lateral': outs.append(self.fpn_convs[used_backbone_levels]( laterals[-1])) elif self.add_extra_convs == 'on_output': outs.append(self.fpn_convs[used_backbone_levels](outs[-1])) else: raise NotImplementedError for i in range(used_backbone_levels + 1, self.num_outs): if self.relu_before_extra_convs: outs.append(self.fpn_convs[i](F.relu(outs[-1]))) else: outs.append(self.fpn_convs[i](outs[-1])) return tuple(outs) ================================================ FILE: mmdet/models/necks/rfp.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn import torch.nn.functional as F from mmengine.model import BaseModule, ModuleList, constant_init, xavier_init from mmdet.registry import MODELS from .fpn import FPN class ASPP(BaseModule): """ASPP (Atrous Spatial Pyramid Pooling) This is an implementation of the ASPP module used in DetectoRS (https://arxiv.org/pdf/2006.02334.pdf) Args: in_channels (int): Number of input channels. out_channels (int): Number of channels produced by this module dilations (tuple[int]): Dilations of the four branches. Default: (1, 3, 6, 1) init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, in_channels, out_channels, dilations=(1, 3, 6, 1), init_cfg=dict(type='Kaiming', layer='Conv2d')): super().__init__(init_cfg) assert dilations[-1] == 1 self.aspp = nn.ModuleList() for dilation in dilations: kernel_size = 3 if dilation > 1 else 1 padding = dilation if dilation > 1 else 0 conv = nn.Conv2d( in_channels, out_channels, kernel_size=kernel_size, stride=1, dilation=dilation, padding=padding, bias=True) self.aspp.append(conv) self.gap = nn.AdaptiveAvgPool2d(1) def forward(self, x): avg_x = self.gap(x) out = [] for aspp_idx in range(len(self.aspp)): inp = avg_x if (aspp_idx == len(self.aspp) - 1) else x out.append(F.relu_(self.aspp[aspp_idx](inp))) out[-1] = out[-1].expand_as(out[-2]) out = torch.cat(out, dim=1) return out @MODELS.register_module() class RFP(FPN): """RFP (Recursive Feature Pyramid) This is an implementation of RFP in `DetectoRS `_. Different from standard FPN, the input of RFP should be multi level features along with origin input image of backbone. Args: rfp_steps (int): Number of unrolled steps of RFP. rfp_backbone (dict): Configuration of the backbone for RFP. aspp_out_channels (int): Number of output channels of ASPP module. aspp_dilations (tuple[int]): Dilation rates of four branches. Default: (1, 3, 6, 1) init_cfg (dict or list[dict], optional): Initialization config dict. Default: None """ def __init__(self, rfp_steps, rfp_backbone, aspp_out_channels, aspp_dilations=(1, 3, 6, 1), init_cfg=None, **kwargs): assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' super().__init__(init_cfg=init_cfg, **kwargs) self.rfp_steps = rfp_steps # Be careful! Pretrained weights cannot be loaded when use # nn.ModuleList self.rfp_modules = ModuleList() for rfp_idx in range(1, rfp_steps): rfp_module = MODELS.build(rfp_backbone) self.rfp_modules.append(rfp_module) self.rfp_aspp = ASPP(self.out_channels, aspp_out_channels, aspp_dilations) self.rfp_weight = nn.Conv2d( self.out_channels, 1, kernel_size=1, stride=1, padding=0, bias=True) def init_weights(self): # Avoid using super().init_weights(), which may alter the default # initialization of the modules in self.rfp_modules that have missing # keys in the pretrained checkpoint. for convs in [self.lateral_convs, self.fpn_convs]: for m in convs.modules(): if isinstance(m, nn.Conv2d): xavier_init(m, distribution='uniform') for rfp_idx in range(self.rfp_steps - 1): self.rfp_modules[rfp_idx].init_weights() constant_init(self.rfp_weight, 0) def forward(self, inputs): inputs = list(inputs) assert len(inputs) == len(self.in_channels) + 1 # +1 for input image img = inputs.pop(0) # FPN forward x = super().forward(tuple(inputs)) for rfp_idx in range(self.rfp_steps - 1): rfp_feats = [x[0]] + list( self.rfp_aspp(x[i]) for i in range(1, len(x))) x_idx = self.rfp_modules[rfp_idx].rfp_forward(img, rfp_feats) # FPN forward x_idx = super().forward(x_idx) x_new = [] for ft_idx in range(len(x_idx)): add_weight = torch.sigmoid(self.rfp_weight(x_idx[ft_idx])) x_new.append(add_weight * x_idx[ft_idx] + (1 - add_weight) * x[ft_idx]) x = x_new return x ================================================ FILE: mmdet/models/necks/ssd_neck.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule from mmengine.model import BaseModule from mmdet.registry import MODELS @MODELS.register_module() class SSDNeck(BaseModule): """Extra layers of SSD backbone to generate multi-scale feature maps. Args: in_channels (Sequence[int]): Number of input channels per scale. out_channels (Sequence[int]): Number of output channels per scale. level_strides (Sequence[int]): Stride of 3x3 conv per level. level_paddings (Sequence[int]): Padding size of 3x3 conv per level. l2_norm_scale (float|None): L2 normalization layer init scale. If None, not use L2 normalization on the first input feature. last_kernel_size (int): Kernel size of the last conv layer. Default: 3. use_depthwise (bool): Whether to use DepthwiseSeparableConv. Default: False. conv_cfg (dict): Config dict for convolution layer. Default: None. norm_cfg (dict): Dictionary to construct and config norm layer. Default: None. act_cfg (dict): Config dict for activation layer. Default: dict(type='ReLU'). init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, in_channels, out_channels, level_strides, level_paddings, l2_norm_scale=20., last_kernel_size=3, use_depthwise=False, conv_cfg=None, norm_cfg=None, act_cfg=dict(type='ReLU'), init_cfg=[ dict( type='Xavier', distribution='uniform', layer='Conv2d'), dict(type='Constant', val=1, layer='BatchNorm2d'), ]): super(SSDNeck, self).__init__(init_cfg) assert len(out_channels) > len(in_channels) assert len(out_channels) - len(in_channels) == len(level_strides) assert len(level_strides) == len(level_paddings) assert in_channels == out_channels[:len(in_channels)] if l2_norm_scale: self.l2_norm = L2Norm(in_channels[0], l2_norm_scale) self.init_cfg += [ dict( type='Constant', val=self.l2_norm.scale, override=dict(name='l2_norm')) ] self.extra_layers = nn.ModuleList() extra_layer_channels = out_channels[len(in_channels):] second_conv = DepthwiseSeparableConvModule if \ use_depthwise else ConvModule for i, (out_channel, stride, padding) in enumerate( zip(extra_layer_channels, level_strides, level_paddings)): kernel_size = last_kernel_size \ if i == len(extra_layer_channels) - 1 else 3 per_lvl_convs = nn.Sequential( ConvModule( out_channels[len(in_channels) - 1 + i], out_channel // 2, 1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg), second_conv( out_channel // 2, out_channel, kernel_size, stride=stride, padding=padding, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) self.extra_layers.append(per_lvl_convs) def forward(self, inputs): """Forward function.""" outs = [feat for feat in inputs] if hasattr(self, 'l2_norm'): outs[0] = self.l2_norm(outs[0]) feat = outs[-1] for layer in self.extra_layers: feat = layer(feat) outs.append(feat) return tuple(outs) class L2Norm(nn.Module): def __init__(self, n_dims, scale=20., eps=1e-10): """L2 normalization layer. Args: n_dims (int): Number of dimensions to be normalized scale (float, optional): Defaults to 20.. eps (float, optional): Used to avoid division by zero. Defaults to 1e-10. """ super(L2Norm, self).__init__() self.n_dims = n_dims self.weight = nn.Parameter(torch.Tensor(self.n_dims)) self.eps = eps self.scale = scale def forward(self, x): """Forward function.""" # normalization layer convert to FP32 in FP16 training x_float = x.float() norm = x_float.pow(2).sum(1, keepdim=True).sqrt() + self.eps return (self.weight[None, :, None, None].float().expand_as(x_float) * x_float / norm).type_as(x) ================================================ FILE: mmdet/models/necks/ssh.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import torch import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.model import BaseModule from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig class SSHContextModule(BaseModule): """This is an implementation of `SSH context module` described in `SSH: Single Stage Headless Face Detector. `_. Args: in_channels (int): Number of input channels used at each scale. out_channels (int): Number of output channels used at each scale. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict): Config dict for normalization layer. Defaults to dict(type='BN'). init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, in_channels: int, out_channels: int, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN'), init_cfg: OptMultiConfig = None): super().__init__(init_cfg=init_cfg) assert out_channels % 4 == 0 self.in_channels = in_channels self.out_channels = out_channels self.conv5x5_1 = ConvModule( self.in_channels, self.out_channels // 4, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, ) self.conv5x5_2 = ConvModule( self.out_channels // 4, self.out_channels // 4, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=None) self.conv7x7_2 = ConvModule( self.out_channels // 4, self.out_channels // 4, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, ) self.conv7x7_3 = ConvModule( self.out_channels // 4, self.out_channels // 4, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=None, ) def forward(self, x: torch.Tensor) -> tuple: conv5x5_1 = self.conv5x5_1(x) conv5x5 = self.conv5x5_2(conv5x5_1) conv7x7_2 = self.conv7x7_2(conv5x5_1) conv7x7 = self.conv7x7_3(conv7x7_2) return (conv5x5, conv7x7) class SSHDetModule(BaseModule): """This is an implementation of `SSH detection module` described in `SSH: Single Stage Headless Face Detector. `_. Args: in_channels (int): Number of input channels used at each scale. out_channels (int): Number of output channels used at each scale. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict): Config dict for normalization layer. Defaults to dict(type='BN'). init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, in_channels: int, out_channels: int, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN'), init_cfg: OptMultiConfig = None): super().__init__(init_cfg=init_cfg) assert out_channels % 4 == 0 self.in_channels = in_channels self.out_channels = out_channels self.conv3x3 = ConvModule( self.in_channels, self.out_channels // 2, 3, stride=1, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=None) self.context_module = SSHContextModule( in_channels=self.in_channels, out_channels=self.out_channels, conv_cfg=conv_cfg, norm_cfg=norm_cfg) def forward(self, x: torch.Tensor) -> torch.Tensor: conv3x3 = self.conv3x3(x) conv5x5, conv7x7 = self.context_module(x) out = torch.cat([conv3x3, conv5x5, conv7x7], dim=1) out = F.relu(out) return out @MODELS.register_module() class SSH(BaseModule): """`SSH Neck` used in `SSH: Single Stage Headless Face Detector. `_. Args: num_scales (int): The number of scales / stages. in_channels (list[int]): The number of input channels per scale. out_channels (list[int]): The number of output channels per scale. conv_cfg (:obj:`ConfigDict` or dict, optional): Config dict for convolution layer. Defaults to None. norm_cfg (:obj:`ConfigDict` or dict): Config dict for normalization layer. Defaults to dict(type='BN'). init_cfg (:obj:`ConfigDict` or list[:obj:`ConfigDict`] or dict or list[dict], optional): Initialization config dict. Example: >>> import torch >>> in_channels = [8, 16, 32, 64] >>> out_channels = [16, 32, 64, 128] >>> scales = [340, 170, 84, 43] >>> inputs = [torch.rand(1, c, s, s) ... for c, s in zip(in_channels, scales)] >>> self = SSH(num_scales=4, in_channels=in_channels, ... out_channels=out_channels) >>> outputs = self.forward(inputs) >>> for i in range(len(outputs)): ... print(f'outputs[{i}].shape = {outputs[i].shape}') outputs[0].shape = torch.Size([1, 16, 340, 340]) outputs[1].shape = torch.Size([1, 32, 170, 170]) outputs[2].shape = torch.Size([1, 64, 84, 84]) outputs[3].shape = torch.Size([1, 128, 43, 43]) """ def __init__(self, num_scales: int, in_channels: List[int], out_channels: List[int], conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN'), init_cfg: OptMultiConfig = dict( type='Xavier', layer='Conv2d', distribution='uniform')): super().__init__(init_cfg=init_cfg) assert (num_scales == len(in_channels) == len(out_channels)) self.num_scales = num_scales self.in_channels = in_channels self.out_channels = out_channels for idx in range(self.num_scales): in_c, out_c = self.in_channels[idx], self.out_channels[idx] self.add_module( f'ssh_module{idx}', SSHDetModule( in_channels=in_c, out_channels=out_c, conv_cfg=conv_cfg, norm_cfg=norm_cfg)) def forward(self, inputs: Tuple[torch.Tensor]) -> tuple: assert len(inputs) == self.num_scales outs = [] for idx, x in enumerate(inputs): ssh_module = getattr(self, f'ssh_module{idx}') out = ssh_module(x) outs.append(out) return tuple(outs) ================================================ FILE: mmdet/models/necks/yolo_neck.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Copyright (c) 2019 Western Digital Corporation or its affiliates. from typing import List, Tuple import torch import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.model import BaseModule from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig class DetectionBlock(BaseModule): """Detection block in YOLO neck. Let out_channels = n, the DetectionBlock contains: Six ConvLayers, 1 Conv2D Layer and 1 YoloLayer. The first 6 ConvLayers are formed the following way: 1x1xn, 3x3x2n, 1x1xn, 3x3x2n, 1x1xn, 3x3x2n. The Conv2D layer is 1x1x255. Some block will have branch after the fifth ConvLayer. The input channel is arbitrary (in_channels) Args: in_channels (int): The number of input channels. out_channels (int): The number of output channels. conv_cfg (dict): Config dict for convolution layer. Default: None. norm_cfg (dict): Dictionary to construct and config norm layer. Default: dict(type='BN', requires_grad=True) act_cfg (dict): Config dict for activation layer. Default: dict(type='LeakyReLU', negative_slope=0.1). init_cfg (dict or list[dict], optional): Initialization config dict. Default: None """ def __init__(self, in_channels: int, out_channels: int, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN', requires_grad=True), act_cfg: ConfigType = dict( type='LeakyReLU', negative_slope=0.1), init_cfg: OptMultiConfig = None) -> None: super(DetectionBlock, self).__init__(init_cfg) double_out_channels = out_channels * 2 # shortcut cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.conv1 = ConvModule(in_channels, out_channels, 1, **cfg) self.conv2 = ConvModule( out_channels, double_out_channels, 3, padding=1, **cfg) self.conv3 = ConvModule(double_out_channels, out_channels, 1, **cfg) self.conv4 = ConvModule( out_channels, double_out_channels, 3, padding=1, **cfg) self.conv5 = ConvModule(double_out_channels, out_channels, 1, **cfg) def forward(self, x: Tensor) -> Tensor: tmp = self.conv1(x) tmp = self.conv2(tmp) tmp = self.conv3(tmp) tmp = self.conv4(tmp) out = self.conv5(tmp) return out @MODELS.register_module() class YOLOV3Neck(BaseModule): """The neck of YOLOV3. It can be treated as a simplified version of FPN. It will take the result from Darknet backbone and do some upsampling and concatenation. It will finally output the detection result. Note: The input feats should be from top to bottom. i.e., from high-lvl to low-lvl But YOLOV3Neck will process them in reversed order. i.e., from bottom (high-lvl) to top (low-lvl) Args: num_scales (int): The number of scales / stages. in_channels (List[int]): The number of input channels per scale. out_channels (List[int]): The number of output channels per scale. conv_cfg (dict, optional): Config dict for convolution layer. Default: None. norm_cfg (dict, optional): Dictionary to construct and config norm layer. Default: dict(type='BN', requires_grad=True) act_cfg (dict, optional): Config dict for activation layer. Default: dict(type='LeakyReLU', negative_slope=0.1). init_cfg (dict or list[dict], optional): Initialization config dict. Default: None """ def __init__(self, num_scales: int, in_channels: List[int], out_channels: List[int], conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN', requires_grad=True), act_cfg: ConfigType = dict( type='LeakyReLU', negative_slope=0.1), init_cfg: OptMultiConfig = None) -> None: super(YOLOV3Neck, self).__init__(init_cfg) assert (num_scales == len(in_channels) == len(out_channels)) self.num_scales = num_scales self.in_channels = in_channels self.out_channels = out_channels # shortcut cfg = dict(conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) # To support arbitrary scales, the code looks awful, but it works. # Better solution is welcomed. self.detect1 = DetectionBlock(in_channels[0], out_channels[0], **cfg) for i in range(1, self.num_scales): in_c, out_c = self.in_channels[i], self.out_channels[i] inter_c = out_channels[i - 1] self.add_module(f'conv{i}', ConvModule(inter_c, out_c, 1, **cfg)) # in_c + out_c : High-lvl feats will be cat with low-lvl feats self.add_module(f'detect{i+1}', DetectionBlock(in_c + out_c, out_c, **cfg)) def forward(self, feats=Tuple[Tensor]) -> Tuple[Tensor]: assert len(feats) == self.num_scales # processed from bottom (high-lvl) to top (low-lvl) outs = [] out = self.detect1(feats[-1]) outs.append(out) for i, x in enumerate(reversed(feats[:-1])): conv = getattr(self, f'conv{i+1}') tmp = conv(out) # Cat with low-lvl feats tmp = F.interpolate(tmp, scale_factor=2) tmp = torch.cat((tmp, x), 1) detect = getattr(self, f'detect{i+2}') out = detect(tmp) outs.append(out) return tuple(outs) ================================================ FILE: mmdet/models/necks/yolox_pafpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math import torch import torch.nn as nn from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule from mmengine.model import BaseModule from mmdet.registry import MODELS from ..layers import CSPLayer @MODELS.register_module() class YOLOXPAFPN(BaseModule): """Path Aggregation Network used in YOLOX. Args: in_channels (List[int]): Number of input channels per scale. out_channels (int): Number of output channels (used at each scale) num_csp_blocks (int): Number of bottlenecks in CSPLayer. Default: 3 use_depthwise (bool): Whether to depthwise separable convolution in blocks. Default: False upsample_cfg (dict): Config dict for interpolate layer. Default: `dict(scale_factor=2, mode='nearest')` conv_cfg (dict, optional): Config dict for convolution layer. Default: None, which means using conv2d. norm_cfg (dict): Config dict for normalization layer. Default: dict(type='BN') act_cfg (dict): Config dict for activation layer. Default: dict(type='Swish') init_cfg (dict or list[dict], optional): Initialization config dict. Default: None. """ def __init__(self, in_channels, out_channels, num_csp_blocks=3, use_depthwise=False, upsample_cfg=dict(scale_factor=2, mode='nearest'), conv_cfg=None, norm_cfg=dict(type='BN', momentum=0.03, eps=0.001), act_cfg=dict(type='Swish'), init_cfg=dict( type='Kaiming', layer='Conv2d', a=math.sqrt(5), distribution='uniform', mode='fan_in', nonlinearity='leaky_relu')): super(YOLOXPAFPN, self).__init__(init_cfg) self.in_channels = in_channels self.out_channels = out_channels conv = DepthwiseSeparableConvModule if use_depthwise else ConvModule # build top-down blocks self.upsample = nn.Upsample(**upsample_cfg) self.reduce_layers = nn.ModuleList() self.top_down_blocks = nn.ModuleList() for idx in range(len(in_channels) - 1, 0, -1): self.reduce_layers.append( ConvModule( in_channels[idx], in_channels[idx - 1], 1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) self.top_down_blocks.append( CSPLayer( in_channels[idx - 1] * 2, in_channels[idx - 1], num_blocks=num_csp_blocks, add_identity=False, use_depthwise=use_depthwise, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) # build bottom-up blocks self.downsamples = nn.ModuleList() self.bottom_up_blocks = nn.ModuleList() for idx in range(len(in_channels) - 1): self.downsamples.append( conv( in_channels[idx], in_channels[idx], 3, stride=2, padding=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) self.bottom_up_blocks.append( CSPLayer( in_channels[idx] * 2, in_channels[idx + 1], num_blocks=num_csp_blocks, add_identity=False, use_depthwise=use_depthwise, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) self.out_convs = nn.ModuleList() for i in range(len(in_channels)): self.out_convs.append( ConvModule( in_channels[i], out_channels, 1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg)) def forward(self, inputs): """ Args: inputs (tuple[Tensor]): input features. Returns: tuple[Tensor]: YOLOXPAFPN features. """ assert len(inputs) == len(self.in_channels) # top-down path inner_outs = [inputs[-1]] for idx in range(len(self.in_channels) - 1, 0, -1): feat_heigh = inner_outs[0] feat_low = inputs[idx - 1] feat_heigh = self.reduce_layers[len(self.in_channels) - 1 - idx]( feat_heigh) inner_outs[0] = feat_heigh upsample_feat = self.upsample(feat_heigh) inner_out = self.top_down_blocks[len(self.in_channels) - 1 - idx]( torch.cat([upsample_feat, feat_low], 1)) inner_outs.insert(0, inner_out) # bottom-up path outs = [inner_outs[0]] for idx in range(len(self.in_channels) - 1): feat_low = outs[-1] feat_height = inner_outs[idx + 1] downsample_feat = self.downsamples[idx](feat_low) out = self.bottom_up_blocks[idx]( torch.cat([downsample_feat, feat_height], 1)) outs.append(out) # out convs for idx, conv in enumerate(self.out_convs): outs[idx] = conv(outs[idx]) return tuple(outs) ================================================ FILE: mmdet/models/roi_heads/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .base_roi_head import BaseRoIHead from .bbox_heads import (BBoxHead, ConvFCBBoxHead, DIIHead, DoubleConvFCBBoxHead, SABLHead, SCNetBBoxHead, Shared2FCBBoxHead, Shared4Conv1FCBBoxHead) from .cascade_roi_head import CascadeRoIHead from .double_roi_head import DoubleHeadRoIHead from .dynamic_roi_head import DynamicRoIHead from .grid_roi_head import GridRoIHead from .htc_roi_head import HybridTaskCascadeRoIHead from .mask_heads import (CoarseMaskHead, FCNMaskHead, FeatureRelayHead, FusedSemanticHead, GlobalContextHead, GridHead, HTCMaskHead, MaskIoUHead, MaskPointHead, SCNetMaskHead, SCNetSemanticHead) from .mask_scoring_roi_head import MaskScoringRoIHead from .multi_instance_roi_head import MultiInstanceRoIHead from .pisa_roi_head import PISARoIHead from .point_rend_roi_head import PointRendRoIHead from .roi_extractors import (BaseRoIExtractor, GenericRoIExtractor, SingleRoIExtractor) from .scnet_roi_head import SCNetRoIHead from .shared_heads import ResLayer from .sparse_roi_head import SparseRoIHead from .standard_roi_head import StandardRoIHead from .trident_roi_head import TridentRoIHead __all__ = [ 'BaseRoIHead', 'CascadeRoIHead', 'DoubleHeadRoIHead', 'MaskScoringRoIHead', 'HybridTaskCascadeRoIHead', 'GridRoIHead', 'ResLayer', 'BBoxHead', 'ConvFCBBoxHead', 'DIIHead', 'SABLHead', 'Shared2FCBBoxHead', 'StandardRoIHead', 'Shared4Conv1FCBBoxHead', 'DoubleConvFCBBoxHead', 'FCNMaskHead', 'HTCMaskHead', 'FusedSemanticHead', 'GridHead', 'MaskIoUHead', 'BaseRoIExtractor', 'GenericRoIExtractor', 'SingleRoIExtractor', 'PISARoIHead', 'PointRendRoIHead', 'MaskPointHead', 'CoarseMaskHead', 'DynamicRoIHead', 'SparseRoIHead', 'TridentRoIHead', 'SCNetRoIHead', 'SCNetMaskHead', 'SCNetSemanticHead', 'SCNetBBoxHead', 'FeatureRelayHead', 'GlobalContextHead', 'MultiInstanceRoIHead' ] ================================================ FILE: mmdet/models/roi_heads/base_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from typing import Tuple from mmengine.model import BaseModule from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import InstanceList, OptConfigType, OptMultiConfig class BaseRoIHead(BaseModule, metaclass=ABCMeta): """Base class for RoIHeads.""" def __init__(self, bbox_roi_extractor: OptMultiConfig = None, bbox_head: OptMultiConfig = None, mask_roi_extractor: OptMultiConfig = None, mask_head: OptMultiConfig = None, shared_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) self.train_cfg = train_cfg self.test_cfg = test_cfg if shared_head is not None: self.shared_head = MODELS.build(shared_head) if bbox_head is not None: self.init_bbox_head(bbox_roi_extractor, bbox_head) if mask_head is not None: self.init_mask_head(mask_roi_extractor, mask_head) self.init_assigner_sampler() @property def with_bbox(self) -> bool: """bool: whether the RoI head contains a `bbox_head`""" return hasattr(self, 'bbox_head') and self.bbox_head is not None @property def with_mask(self) -> bool: """bool: whether the RoI head contains a `mask_head`""" return hasattr(self, 'mask_head') and self.mask_head is not None @property def with_shared_head(self) -> bool: """bool: whether the RoI head contains a `shared_head`""" return hasattr(self, 'shared_head') and self.shared_head is not None @abstractmethod def init_bbox_head(self, *args, **kwargs): """Initialize ``bbox_head``""" pass @abstractmethod def init_mask_head(self, *args, **kwargs): """Initialize ``mask_head``""" pass @abstractmethod def init_assigner_sampler(self, *args, **kwargs): """Initialize assigner and sampler.""" pass @abstractmethod def loss(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList): """Perform forward propagation and loss calculation of the roi head on the features of the upstream network.""" def predict(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the roi head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Features from upstream network. Each has shape (N, C, H, W). rpn_results_list (list[:obj:`InstanceData`]): list of region proposals. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results to the original image. Defaults to True. Returns: list[obj:`InstanceData`]: Detection results of each image. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ assert self.with_bbox, 'Bbox head must be implemented.' batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] # TODO: nms_op in mmcv need be enhanced, the bbox result may get # difference when not rescale in bbox_head # If it has the mask branch, the bbox branch does not need # to be scaled to the original image scale, because the mask # branch will scale both bbox and mask at the same time. bbox_rescale = rescale if not self.with_mask else False results_list = self.predict_bbox( x, batch_img_metas, rpn_results_list, rcnn_test_cfg=self.test_cfg, rescale=bbox_rescale) if self.with_mask: results_list = self.predict_mask( x, batch_img_metas, results_list, rescale=rescale) return results_list ================================================ FILE: mmdet/models/roi_heads/bbox_heads/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .bbox_head import BBoxHead from .convfc_bbox_head import (ConvFCBBoxHead, Shared2FCBBoxHead, Shared4Conv1FCBBoxHead) from .dii_head import DIIHead from .double_bbox_head import DoubleConvFCBBoxHead from .multi_instance_bbox_head import MultiInstanceBBoxHead from .sabl_head import SABLHead from .scnet_bbox_head import SCNetBBoxHead __all__ = [ 'BBoxHead', 'ConvFCBBoxHead', 'Shared2FCBBoxHead', 'Shared4Conv1FCBBoxHead', 'DoubleConvFCBBoxHead', 'SABLHead', 'DIIHead', 'SCNetBBoxHead', 'MultiInstanceBBoxHead' ] ================================================ FILE: mmdet/models/roi_heads/bbox_heads/bbox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F from mmengine.config import ConfigDict from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import Tensor from torch.nn.modules.utils import _pair from mmdet.models.layers import multiclass_nms from mmdet.models.losses import accuracy from mmdet.models.task_modules.samplers import SamplingResult from mmdet.models.utils import empty_instances, multi_apply from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures.bbox import get_box_tensor, scale_boxes from mmdet.utils import ConfigType, InstanceList, OptMultiConfig @MODELS.register_module() class BBoxHead(BaseModule): """Simplest RoI head, with only two fc layers for classification and regression respectively.""" def __init__(self, with_avg_pool: bool = False, with_cls: bool = True, with_reg: bool = True, roi_feat_size: int = 7, in_channels: int = 256, num_classes: int = 80, bbox_coder: ConfigType = dict( type='DeltaXYWHBBoxCoder', clip_border=True, target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), predict_box_type: str = 'hbox', reg_class_agnostic: bool = False, reg_decoded_bbox: bool = False, reg_predictor_cfg: ConfigType = dict(type='Linear'), cls_predictor_cfg: ConfigType = dict(type='Linear'), loss_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox: ConfigType = dict( type='SmoothL1Loss', beta=1.0, loss_weight=1.0), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) assert with_cls or with_reg self.with_avg_pool = with_avg_pool self.with_cls = with_cls self.with_reg = with_reg self.roi_feat_size = _pair(roi_feat_size) self.roi_feat_area = self.roi_feat_size[0] * self.roi_feat_size[1] self.in_channels = in_channels self.num_classes = num_classes self.predict_box_type = predict_box_type self.reg_class_agnostic = reg_class_agnostic self.reg_decoded_bbox = reg_decoded_bbox self.reg_predictor_cfg = reg_predictor_cfg self.cls_predictor_cfg = cls_predictor_cfg self.bbox_coder = TASK_UTILS.build(bbox_coder) self.loss_cls = MODELS.build(loss_cls) self.loss_bbox = MODELS.build(loss_bbox) in_channels = self.in_channels if self.with_avg_pool: self.avg_pool = nn.AvgPool2d(self.roi_feat_size) else: in_channels *= self.roi_feat_area if self.with_cls: # need to add background class if self.custom_cls_channels: cls_channels = self.loss_cls.get_cls_channels(self.num_classes) else: cls_channels = num_classes + 1 cls_predictor_cfg_ = self.cls_predictor_cfg.copy() cls_predictor_cfg_.update( in_features=in_channels, out_features=cls_channels) self.fc_cls = MODELS.build(cls_predictor_cfg_) if self.with_reg: box_dim = self.bbox_coder.encode_size out_dim_reg = box_dim if reg_class_agnostic else \ box_dim * num_classes reg_predictor_cfg_ = self.reg_predictor_cfg.copy() if isinstance(reg_predictor_cfg_, (dict, ConfigDict)): reg_predictor_cfg_.update( in_features=in_channels, out_features=out_dim_reg) self.fc_reg = MODELS.build(reg_predictor_cfg_) self.debug_imgs = None if init_cfg is None: self.init_cfg = [] if self.with_cls: self.init_cfg += [ dict( type='Normal', std=0.01, override=dict(name='fc_cls')) ] if self.with_reg: self.init_cfg += [ dict( type='Normal', std=0.001, override=dict(name='fc_reg')) ] # TODO: Create a SeasawBBoxHead to simplified logic in BBoxHead @property def custom_cls_channels(self) -> bool: """get custom_cls_channels from loss_cls.""" return getattr(self.loss_cls, 'custom_cls_channels', False) # TODO: Create a SeasawBBoxHead to simplified logic in BBoxHead @property def custom_activation(self) -> bool: """get custom_activation from loss_cls.""" return getattr(self.loss_cls, 'custom_activation', False) # TODO: Create a SeasawBBoxHead to simplified logic in BBoxHead @property def custom_accuracy(self) -> bool: """get custom_accuracy from loss_cls.""" return getattr(self.loss_cls, 'custom_accuracy', False) def forward(self, x: Tuple[Tensor]) -> tuple: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of classification scores and bbox prediction. - cls_score (Tensor): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * num_classes. - bbox_pred (Tensor): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * 4. """ if self.with_avg_pool: if x.numel() > 0: x = self.avg_pool(x) x = x.view(x.size(0), -1) else: # avg_pool does not support empty tensor, # so use torch.mean instead it x = torch.mean(x, dim=(-1, -2)) cls_score = self.fc_cls(x) if self.with_cls else None bbox_pred = self.fc_reg(x) if self.with_reg else None return cls_score, bbox_pred def _get_targets_single(self, pos_priors: Tensor, neg_priors: Tensor, pos_gt_bboxes: Tensor, pos_gt_labels: Tensor, cfg: ConfigDict) -> tuple: """Calculate the ground truth for proposals in the single image according to the sampling results. Args: pos_priors (Tensor): Contains all the positive boxes, has shape (num_pos, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. neg_priors (Tensor): Contains all the negative boxes, has shape (num_neg, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. pos_gt_bboxes (Tensor): Contains gt_boxes for all positive samples, has shape (num_pos, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. pos_gt_labels (Tensor): Contains gt_labels for all positive samples, has shape (num_pos, ). cfg (obj:`ConfigDict`): `train_cfg` of R-CNN. Returns: Tuple[Tensor]: Ground truth for proposals in a single image. Containing the following Tensors: - labels(Tensor): Gt_labels for all proposals, has shape (num_proposals,). - label_weights(Tensor): Labels_weights for all proposals, has shape (num_proposals,). - bbox_targets(Tensor):Regression target for all proposals, has shape (num_proposals, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. - bbox_weights(Tensor):Regression weights for all proposals, has shape (num_proposals, 4). """ num_pos = pos_priors.size(0) num_neg = neg_priors.size(0) num_samples = num_pos + num_neg # original implementation uses new_zeros since BG are set to be 0 # now use empty & fill because BG cat_id = num_classes, # FG cat_id = [0, num_classes-1] labels = pos_priors.new_full((num_samples, ), self.num_classes, dtype=torch.long) reg_dim = pos_gt_bboxes.size(-1) if self.reg_decoded_bbox \ else self.bbox_coder.encode_size label_weights = pos_priors.new_zeros(num_samples) bbox_targets = pos_priors.new_zeros(num_samples, reg_dim) bbox_weights = pos_priors.new_zeros(num_samples, reg_dim) if num_pos > 0: labels[:num_pos] = pos_gt_labels pos_weight = 1.0 if cfg.pos_weight <= 0 else cfg.pos_weight label_weights[:num_pos] = pos_weight if not self.reg_decoded_bbox: pos_bbox_targets = self.bbox_coder.encode( pos_priors, pos_gt_bboxes) else: # When the regression loss (e.g. `IouLoss`, `GIouLoss`) # is applied directly on the decoded bounding boxes, both # the predicted boxes and regression targets should be with # absolute coordinate format. pos_bbox_targets = get_box_tensor(pos_gt_bboxes) bbox_targets[:num_pos, :] = pos_bbox_targets bbox_weights[:num_pos, :] = 1 if num_neg > 0: label_weights[-num_neg:] = 1.0 return labels, label_weights, bbox_targets, bbox_weights def get_targets(self, sampling_results: List[SamplingResult], rcnn_train_cfg: ConfigDict, concat: bool = True) -> tuple: """Calculate the ground truth for all samples in a batch according to the sampling_results. Almost the same as the implementation in bbox_head, we passed additional parameters pos_inds_list and neg_inds_list to `_get_targets_single` function. Args: sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. rcnn_train_cfg (obj:ConfigDict): `train_cfg` of RCNN. concat (bool): Whether to concatenate the results of all the images in a single batch. Returns: Tuple[Tensor]: Ground truth for proposals in a single image. Containing the following list of Tensors: - labels (list[Tensor],Tensor): Gt_labels for all proposals in a batch, each tensor in list has shape (num_proposals,) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals,). - label_weights (list[Tensor]): Labels_weights for all proposals in a batch, each tensor in list has shape (num_proposals,) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals,). - bbox_targets (list[Tensor],Tensor): Regression target for all proposals in a batch, each tensor in list has shape (num_proposals, 4) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. - bbox_weights (list[tensor],Tensor): Regression weights for all proposals in a batch, each tensor in list has shape (num_proposals, 4) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals, 4). """ pos_priors_list = [res.pos_priors for res in sampling_results] neg_priors_list = [res.neg_priors for res in sampling_results] pos_gt_bboxes_list = [res.pos_gt_bboxes for res in sampling_results] pos_gt_labels_list = [res.pos_gt_labels for res in sampling_results] labels, label_weights, bbox_targets, bbox_weights = multi_apply( self._get_targets_single, pos_priors_list, neg_priors_list, pos_gt_bboxes_list, pos_gt_labels_list, cfg=rcnn_train_cfg) if concat: labels = torch.cat(labels, 0) label_weights = torch.cat(label_weights, 0) bbox_targets = torch.cat(bbox_targets, 0) bbox_weights = torch.cat(bbox_weights, 0) return labels, label_weights, bbox_targets, bbox_weights def loss_and_target(self, cls_score: Tensor, bbox_pred: Tensor, rois: Tensor, sampling_results: List[SamplingResult], rcnn_train_cfg: ConfigDict, concat: bool = True, reduction_override: Optional[str] = None) -> dict: """Calculate the loss based on the features extracted by the bbox head. Args: cls_score (Tensor): Classification prediction results of all class, has shape (batch_size * num_proposals_single_image, num_classes) bbox_pred (Tensor): Regression prediction results, has shape (batch_size * num_proposals_single_image, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. rois (Tensor): RoIs with the shape (batch_size * num_proposals_single_image, 5) where the first column indicates batch id of each RoI. sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. rcnn_train_cfg (obj:ConfigDict): `train_cfg` of RCNN. concat (bool): Whether to concatenate the results of all the images in a single batch. Defaults to True. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Options are "none", "mean" and "sum". Defaults to None, Returns: dict: A dictionary of loss and targets components. The targets are only used for cascade rcnn. """ cls_reg_targets = self.get_targets( sampling_results, rcnn_train_cfg, concat=concat) losses = self.loss( cls_score, bbox_pred, rois, *cls_reg_targets, reduction_override=reduction_override) # cls_reg_targets is only for cascade rcnn return dict(loss_bbox=losses, bbox_targets=cls_reg_targets) def loss(self, cls_score: Tensor, bbox_pred: Tensor, rois: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, bbox_weights: Tensor, reduction_override: Optional[str] = None) -> dict: """Calculate the loss based on the network predictions and targets. Args: cls_score (Tensor): Classification prediction results of all class, has shape (batch_size * num_proposals_single_image, num_classes) bbox_pred (Tensor): Regression prediction results, has shape (batch_size * num_proposals_single_image, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. rois (Tensor): RoIs with the shape (batch_size * num_proposals_single_image, 5) where the first column indicates batch id of each RoI. labels (Tensor): Gt_labels for all proposals in a batch, has shape (batch_size * num_proposals_single_image, ). label_weights (Tensor): Labels_weights for all proposals in a batch, has shape (batch_size * num_proposals_single_image, ). bbox_targets (Tensor): Regression target for all proposals in a batch, has shape (batch_size * num_proposals_single_image, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. bbox_weights (Tensor): Regression weights for all proposals in a batch, has shape (batch_size * num_proposals_single_image, 4). reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Options are "none", "mean" and "sum". Defaults to None, Returns: dict: A dictionary of loss. """ losses = dict() if cls_score is not None: avg_factor = max(torch.sum(label_weights > 0).float().item(), 1.) if cls_score.numel() > 0: loss_cls_ = self.loss_cls( cls_score, labels, label_weights, avg_factor=avg_factor, reduction_override=reduction_override) if isinstance(loss_cls_, dict): losses.update(loss_cls_) else: losses['loss_cls'] = loss_cls_ if self.custom_activation: acc_ = self.loss_cls.get_accuracy(cls_score, labels) losses.update(acc_) else: losses['acc'] = accuracy(cls_score, labels) if bbox_pred is not None: bg_class_ind = self.num_classes # 0~self.num_classes-1 are FG, self.num_classes is BG pos_inds = (labels >= 0) & (labels < bg_class_ind) # do not perform bounding box regression for BG anymore. if pos_inds.any(): if self.reg_decoded_bbox: # When the regression loss (e.g. `IouLoss`, # `GIouLoss`, `DIouLoss`) is applied directly on # the decoded bounding boxes, it decodes the # already encoded coordinates to absolute format. bbox_pred = self.bbox_coder.decode(rois[:, 1:], bbox_pred) bbox_pred = get_box_tensor(bbox_pred) if self.reg_class_agnostic: pos_bbox_pred = bbox_pred.view( bbox_pred.size(0), -1)[pos_inds.type(torch.bool)] else: pos_bbox_pred = bbox_pred.view( bbox_pred.size(0), self.num_classes, -1)[pos_inds.type(torch.bool), labels[pos_inds.type(torch.bool)]] losses['loss_bbox'] = self.loss_bbox( pos_bbox_pred, bbox_targets[pos_inds.type(torch.bool)], bbox_weights[pos_inds.type(torch.bool)], avg_factor=bbox_targets.size(0), reduction_override=reduction_override) else: losses['loss_bbox'] = bbox_pred[pos_inds].sum() return losses def predict_by_feat(self, rois: Tuple[Tensor], cls_scores: Tuple[Tensor], bbox_preds: Tuple[Tensor], batch_img_metas: List[dict], rcnn_test_cfg: Optional[ConfigDict] = None, rescale: bool = False) -> InstanceList: """Transform a batch of output features extracted from the head into bbox results. Args: rois (tuple[Tensor]): Tuple of boxes to be transformed. Each has shape (num_boxes, 5). last dimension 5 arrange as (batch_index, x1, y1, x2, y2). cls_scores (tuple[Tensor]): Tuple of box scores, each has shape (num_boxes, num_classes + 1). bbox_preds (tuple[Tensor]): Tuple of box energies / deltas, each has shape (num_boxes, num_classes * 4). batch_img_metas (list[dict]): List of image information. rcnn_test_cfg (obj:`ConfigDict`, optional): `test_cfg` of R-CNN. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Instance segmentation results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ assert len(cls_scores) == len(bbox_preds) result_list = [] for img_id in range(len(batch_img_metas)): img_meta = batch_img_metas[img_id] results = self._predict_by_feat_single( roi=rois[img_id], cls_score=cls_scores[img_id], bbox_pred=bbox_preds[img_id], img_meta=img_meta, rescale=rescale, rcnn_test_cfg=rcnn_test_cfg) result_list.append(results) return result_list def _predict_by_feat_single( self, roi: Tensor, cls_score: Tensor, bbox_pred: Tensor, img_meta: dict, rescale: bool = False, rcnn_test_cfg: Optional[ConfigDict] = None) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: roi (Tensor): Boxes to be transformed. Has shape (num_boxes, 5). last dimension 5 arrange as (batch_index, x1, y1, x2, y2). cls_score (Tensor): Box scores, has shape (num_boxes, num_classes + 1). bbox_pred (Tensor): Box energies / deltas. has shape (num_boxes, num_classes * 4). img_meta (dict): image information. rescale (bool): If True, return boxes in original image space. Defaults to False. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of Bbox Head. Defaults to None Returns: :obj:`InstanceData`: Detection results of each image\ Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ results = InstanceData() if roi.shape[0] == 0: return empty_instances([img_meta], roi.device, task_type='bbox', instance_results=[results], box_type=self.predict_box_type, use_box_type=False, num_classes=self.num_classes, score_per_cls=rcnn_test_cfg is None)[0] # some loss (Seesaw loss..) may have custom activation if self.custom_cls_channels: scores = self.loss_cls.get_activation(cls_score) else: scores = F.softmax( cls_score, dim=-1) if cls_score is not None else None img_shape = img_meta['img_shape'] num_rois = roi.size(0) # bbox_pred would be None in some detector when with_reg is False, # e.g. Grid R-CNN. if bbox_pred is not None: num_classes = 1 if self.reg_class_agnostic else self.num_classes roi = roi.repeat_interleave(num_classes, dim=0) bbox_pred = bbox_pred.view(-1, self.bbox_coder.encode_size) bboxes = self.bbox_coder.decode( roi[..., 1:], bbox_pred, max_shape=img_shape) else: bboxes = roi[:, 1:].clone() if img_shape is not None and bboxes.size(-1) == 4: bboxes[:, [0, 2]].clamp_(min=0, max=img_shape[1]) bboxes[:, [1, 3]].clamp_(min=0, max=img_shape[0]) if rescale and bboxes.size(0) > 0: assert img_meta.get('scale_factor') is not None scale_factor = [1 / s for s in img_meta['scale_factor']] bboxes = scale_boxes(bboxes, scale_factor) # Get the inside tensor when `bboxes` is a box type bboxes = get_box_tensor(bboxes) box_dim = bboxes.size(-1) bboxes = bboxes.view(num_rois, -1) if rcnn_test_cfg is None: # This means that it is aug test. # It needs to return the raw results without nms. results.bboxes = bboxes results.scores = scores else: det_bboxes, det_labels = multiclass_nms( bboxes, scores, rcnn_test_cfg.score_thr, rcnn_test_cfg.nms, rcnn_test_cfg.max_per_img, box_dim=box_dim) results.bboxes = det_bboxes[:, :-1] results.scores = det_bboxes[:, -1] results.labels = det_labels return results def refine_bboxes(self, sampling_results: Union[List[SamplingResult], InstanceList], bbox_results: dict, batch_img_metas: List[dict]) -> InstanceList: """Refine bboxes during training. Args: sampling_results (List[:obj:`SamplingResult`] or List[:obj:`InstanceData`]): Sampling results. :obj:`SamplingResult` is the real sampling results calculate from bbox_head, while :obj:`InstanceData` is fake sampling results, e.g., in Sparse R-CNN or QueryInst, etc. bbox_results (dict): Usually is a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `rois` (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. - `bbox_targets` (tuple): Ground truth for proposals in a single image. Containing the following list of Tensors: (labels, label_weights, bbox_targets, bbox_weights) batch_img_metas (List[dict]): List of image information. Returns: list[:obj:`InstanceData`]: Refined bboxes of each image. Example: >>> # xdoctest: +REQUIRES(module:kwarray) >>> import numpy as np >>> from mmdet.models.task_modules.samplers. ... sampling_result import random_boxes >>> from mmdet.models.task_modules.samplers import SamplingResult >>> self = BBoxHead(reg_class_agnostic=True) >>> n_roi = 2 >>> n_img = 4 >>> scale = 512 >>> rng = np.random.RandomState(0) ... batch_img_metas = [{'img_shape': (scale, scale)} >>> for _ in range(n_img)] >>> sampling_results = [SamplingResult.random(rng=10) ... for _ in range(n_img)] >>> # Create rois in the expected format >>> roi_boxes = random_boxes(n_roi, scale=scale, rng=rng) >>> img_ids = torch.randint(0, n_img, (n_roi,)) >>> img_ids = img_ids.float() >>> rois = torch.cat([img_ids[:, None], roi_boxes], dim=1) >>> # Create other args >>> labels = torch.randint(0, 81, (scale,)).long() >>> bbox_preds = random_boxes(n_roi, scale=scale, rng=rng) >>> cls_score = torch.randn((scale, 81)) ... # For each image, pretend random positive boxes are gts >>> bbox_targets = (labels, None, None, None) ... bbox_results = dict(rois=rois, bbox_pred=bbox_preds, ... cls_score=cls_score, ... bbox_targets=bbox_targets) >>> bboxes_list = self.refine_bboxes(sampling_results, ... bbox_results, ... batch_img_metas) >>> print(bboxes_list) """ pos_is_gts = [res.pos_is_gt for res in sampling_results] # bbox_targets is a tuple labels = bbox_results['bbox_targets'][0] cls_scores = bbox_results['cls_score'] rois = bbox_results['rois'] bbox_preds = bbox_results['bbox_pred'] if self.custom_activation: # TODO: Create a SeasawBBoxHead to simplified logic in BBoxHead cls_scores = self.loss_cls.get_activation(cls_scores) if cls_scores.numel() == 0: return None if cls_scores.shape[-1] == self.num_classes + 1: # remove background class cls_scores = cls_scores[:, :-1] elif cls_scores.shape[-1] != self.num_classes: raise ValueError('The last dim of `cls_scores` should equal to ' '`num_classes` or `num_classes + 1`,' f'but got {cls_scores.shape[-1]}.') labels = torch.where(labels == self.num_classes, cls_scores.argmax(1), labels) img_ids = rois[:, 0].long().unique(sorted=True) assert img_ids.numel() <= len(batch_img_metas) results_list = [] for i in range(len(batch_img_metas)): inds = torch.nonzero( rois[:, 0] == i, as_tuple=False).squeeze(dim=1) num_rois = inds.numel() bboxes_ = rois[inds, 1:] label_ = labels[inds] bbox_pred_ = bbox_preds[inds] img_meta_ = batch_img_metas[i] pos_is_gts_ = pos_is_gts[i] bboxes = self.regress_by_class(bboxes_, label_, bbox_pred_, img_meta_) # filter gt bboxes pos_keep = 1 - pos_is_gts_ keep_inds = pos_is_gts_.new_ones(num_rois) keep_inds[:len(pos_is_gts_)] = pos_keep results = InstanceData(bboxes=bboxes[keep_inds.type(torch.bool)]) results_list.append(results) return results_list def regress_by_class(self, priors: Tensor, label: Tensor, bbox_pred: Tensor, img_meta: dict) -> Tensor: """Regress the bbox for the predicted class. Used in Cascade R-CNN. Args: priors (Tensor): Priors from `rpn_head` or last stage `bbox_head`, has shape (num_proposals, 4). label (Tensor): Only used when `self.reg_class_agnostic` is False, has shape (num_proposals, ). bbox_pred (Tensor): Regression prediction of current stage `bbox_head`. When `self.reg_class_agnostic` is False, it has shape (n, num_classes * 4), otherwise it has shape (n, 4). img_meta (dict): Image meta info. Returns: Tensor: Regressed bboxes, the same shape as input rois. """ reg_dim = self.bbox_coder.encode_size if not self.reg_class_agnostic: label = label * reg_dim inds = torch.stack([label + i for i in range(reg_dim)], 1) bbox_pred = torch.gather(bbox_pred, 1, inds) assert bbox_pred.size()[1] == reg_dim max_shape = img_meta['img_shape'] regressed_bboxes = self.bbox_coder.decode( priors, bbox_pred, max_shape=max_shape) return regressed_bboxes ================================================ FILE: mmdet/models/roi_heads/bbox_heads/convfc_bbox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple, Union import torch.nn as nn from mmcv.cnn import ConvModule from mmengine.config import ConfigDict from torch import Tensor from mmdet.registry import MODELS from .bbox_head import BBoxHead @MODELS.register_module() class ConvFCBBoxHead(BBoxHead): r"""More general bbox head, with shared conv and fc layers and two optional separated branches. .. code-block:: none /-> cls convs -> cls fcs -> cls shared convs -> shared fcs \-> reg convs -> reg fcs -> reg """ # noqa: W605 def __init__(self, num_shared_convs: int = 0, num_shared_fcs: int = 0, num_cls_convs: int = 0, num_cls_fcs: int = 0, num_reg_convs: int = 0, num_reg_fcs: int = 0, conv_out_channels: int = 256, fc_out_channels: int = 1024, conv_cfg: Optional[Union[dict, ConfigDict]] = None, norm_cfg: Optional[Union[dict, ConfigDict]] = None, init_cfg: Optional[Union[dict, ConfigDict]] = None, *args, **kwargs) -> None: super().__init__(*args, init_cfg=init_cfg, **kwargs) assert (num_shared_convs + num_shared_fcs + num_cls_convs + num_cls_fcs + num_reg_convs + num_reg_fcs > 0) if num_cls_convs > 0 or num_reg_convs > 0: assert num_shared_fcs == 0 if not self.with_cls: assert num_cls_convs == 0 and num_cls_fcs == 0 if not self.with_reg: assert num_reg_convs == 0 and num_reg_fcs == 0 self.num_shared_convs = num_shared_convs self.num_shared_fcs = num_shared_fcs self.num_cls_convs = num_cls_convs self.num_cls_fcs = num_cls_fcs self.num_reg_convs = num_reg_convs self.num_reg_fcs = num_reg_fcs self.conv_out_channels = conv_out_channels self.fc_out_channels = fc_out_channels self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg # add shared convs and fcs self.shared_convs, self.shared_fcs, last_layer_dim = \ self._add_conv_fc_branch( self.num_shared_convs, self.num_shared_fcs, self.in_channels, True) self.shared_out_channels = last_layer_dim # add cls specific branch self.cls_convs, self.cls_fcs, self.cls_last_dim = \ self._add_conv_fc_branch( self.num_cls_convs, self.num_cls_fcs, self.shared_out_channels) # add reg specific branch self.reg_convs, self.reg_fcs, self.reg_last_dim = \ self._add_conv_fc_branch( self.num_reg_convs, self.num_reg_fcs, self.shared_out_channels) if self.num_shared_fcs == 0 and not self.with_avg_pool: if self.num_cls_fcs == 0: self.cls_last_dim *= self.roi_feat_area if self.num_reg_fcs == 0: self.reg_last_dim *= self.roi_feat_area self.relu = nn.ReLU(inplace=True) # reconstruct fc_cls and fc_reg since input channels are changed if self.with_cls: if self.custom_cls_channels: cls_channels = self.loss_cls.get_cls_channels(self.num_classes) else: cls_channels = self.num_classes + 1 cls_predictor_cfg_ = self.cls_predictor_cfg.copy() cls_predictor_cfg_.update( in_features=self.cls_last_dim, out_features=cls_channels) self.fc_cls = MODELS.build(cls_predictor_cfg_) if self.with_reg: box_dim = self.bbox_coder.encode_size out_dim_reg = box_dim if self.reg_class_agnostic else \ box_dim * self.num_classes reg_predictor_cfg_ = self.reg_predictor_cfg.copy() if isinstance(reg_predictor_cfg_, (dict, ConfigDict)): reg_predictor_cfg_.update( in_features=self.reg_last_dim, out_features=out_dim_reg) self.fc_reg = MODELS.build(reg_predictor_cfg_) if init_cfg is None: # when init_cfg is None, # It has been set to # [[dict(type='Normal', std=0.01, override=dict(name='fc_cls'))], # [dict(type='Normal', std=0.001, override=dict(name='fc_reg'))] # after `super(ConvFCBBoxHead, self).__init__()` # we only need to append additional configuration # for `shared_fcs`, `cls_fcs` and `reg_fcs` self.init_cfg += [ dict( type='Xavier', distribution='uniform', override=[ dict(name='shared_fcs'), dict(name='cls_fcs'), dict(name='reg_fcs') ]) ] def _add_conv_fc_branch(self, num_branch_convs: int, num_branch_fcs: int, in_channels: int, is_shared: bool = False) -> tuple: """Add shared or separable branch. convs -> avg pool (optional) -> fcs """ last_layer_dim = in_channels # add branch specific conv layers branch_convs = nn.ModuleList() if num_branch_convs > 0: for i in range(num_branch_convs): conv_in_channels = ( last_layer_dim if i == 0 else self.conv_out_channels) branch_convs.append( ConvModule( conv_in_channels, self.conv_out_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) last_layer_dim = self.conv_out_channels # add branch specific fc layers branch_fcs = nn.ModuleList() if num_branch_fcs > 0: # for shared branch, only consider self.with_avg_pool # for separated branches, also consider self.num_shared_fcs if (is_shared or self.num_shared_fcs == 0) and not self.with_avg_pool: last_layer_dim *= self.roi_feat_area for i in range(num_branch_fcs): fc_in_channels = ( last_layer_dim if i == 0 else self.fc_out_channels) branch_fcs.append( nn.Linear(fc_in_channels, self.fc_out_channels)) last_layer_dim = self.fc_out_channels return branch_convs, branch_fcs, last_layer_dim def forward(self, x: Tuple[Tensor]) -> tuple: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of classification scores and bbox prediction. - cls_score (Tensor): Classification scores for all \ scale levels, each is a 4D-tensor, the channels number \ is num_base_priors * num_classes. - bbox_pred (Tensor): Box energies / deltas for all \ scale levels, each is a 4D-tensor, the channels number \ is num_base_priors * 4. """ # shared part if self.num_shared_convs > 0: for conv in self.shared_convs: x = conv(x) if self.num_shared_fcs > 0: if self.with_avg_pool: x = self.avg_pool(x) x = x.flatten(1) for fc in self.shared_fcs: x = self.relu(fc(x)) # separate branches x_cls = x x_reg = x for conv in self.cls_convs: x_cls = conv(x_cls) if x_cls.dim() > 2: if self.with_avg_pool: x_cls = self.avg_pool(x_cls) x_cls = x_cls.flatten(1) for fc in self.cls_fcs: x_cls = self.relu(fc(x_cls)) for conv in self.reg_convs: x_reg = conv(x_reg) if x_reg.dim() > 2: if self.with_avg_pool: x_reg = self.avg_pool(x_reg) x_reg = x_reg.flatten(1) for fc in self.reg_fcs: x_reg = self.relu(fc(x_reg)) cls_score = self.fc_cls(x_cls) if self.with_cls else None bbox_pred = self.fc_reg(x_reg) if self.with_reg else None return cls_score, bbox_pred @MODELS.register_module() class Shared2FCBBoxHead(ConvFCBBoxHead): def __init__(self, fc_out_channels: int = 1024, *args, **kwargs) -> None: super().__init__( num_shared_convs=0, num_shared_fcs=2, num_cls_convs=0, num_cls_fcs=0, num_reg_convs=0, num_reg_fcs=0, fc_out_channels=fc_out_channels, *args, **kwargs) @MODELS.register_module() class Shared4Conv1FCBBoxHead(ConvFCBBoxHead): def __init__(self, fc_out_channels: int = 1024, *args, **kwargs) -> None: super().__init__( num_shared_convs=4, num_shared_fcs=1, num_cls_convs=0, num_cls_fcs=0, num_reg_convs=0, num_reg_fcs=0, fc_out_channels=fc_out_channels, *args, **kwargs) ================================================ FILE: mmdet/models/roi_heads/bbox_heads/dii_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List import torch import torch.nn as nn from mmcv.cnn import build_activation_layer, build_norm_layer from mmcv.cnn.bricks.transformer import FFN, MultiheadAttention from mmengine.config import ConfigDict from mmengine.model import bias_init_with_prob from torch import Tensor from mmdet.models.losses import accuracy from mmdet.models.task_modules import SamplingResult from mmdet.models.utils import multi_apply from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, reduce_mean from .bbox_head import BBoxHead @MODELS.register_module() class DIIHead(BBoxHead): r"""Dynamic Instance Interactive Head for `Sparse R-CNN: End-to-End Object Detection with Learnable Proposals `_ Args: num_classes (int): Number of class in dataset. Defaults to 80. num_ffn_fcs (int): The number of fully-connected layers in FFNs. Defaults to 2. num_heads (int): The hidden dimension of FFNs. Defaults to 8. num_cls_fcs (int): The number of fully-connected layers in classification subnet. Defaults to 1. num_reg_fcs (int): The number of fully-connected layers in regression subnet. Defaults to 3. feedforward_channels (int): The hidden dimension of FFNs. Defaults to 2048 in_channels (int): Hidden_channels of MultiheadAttention. Defaults to 256. dropout (float): Probability of drop the channel. Defaults to 0.0 ffn_act_cfg (:obj:`ConfigDict` or dict): The activation config for FFNs. dynamic_conv_cfg (:obj:`ConfigDict` or dict): The convolution config for DynamicConv. loss_iou (:obj:`ConfigDict` or dict): The config for iou or giou loss. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict]): Initialization config dict. Defaults to None. """ def __init__(self, num_classes: int = 80, num_ffn_fcs: int = 2, num_heads: int = 8, num_cls_fcs: int = 1, num_reg_fcs: int = 3, feedforward_channels: int = 2048, in_channels: int = 256, dropout: float = 0.0, ffn_act_cfg: ConfigType = dict(type='ReLU', inplace=True), dynamic_conv_cfg: ConfigType = dict( type='DynamicConv', in_channels=256, feat_channels=64, out_channels=256, input_feat_shape=7, act_cfg=dict(type='ReLU', inplace=True), norm_cfg=dict(type='LN')), loss_iou: ConfigType = dict(type='GIoULoss', loss_weight=2.0), init_cfg: OptConfigType = None, **kwargs) -> None: assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' super().__init__( num_classes=num_classes, reg_decoded_bbox=True, reg_class_agnostic=True, init_cfg=init_cfg, **kwargs) self.loss_iou = MODELS.build(loss_iou) self.in_channels = in_channels self.fp16_enabled = False self.attention = MultiheadAttention(in_channels, num_heads, dropout) self.attention_norm = build_norm_layer(dict(type='LN'), in_channels)[1] self.instance_interactive_conv = MODELS.build(dynamic_conv_cfg) self.instance_interactive_conv_dropout = nn.Dropout(dropout) self.instance_interactive_conv_norm = build_norm_layer( dict(type='LN'), in_channels)[1] self.ffn = FFN( in_channels, feedforward_channels, num_ffn_fcs, act_cfg=ffn_act_cfg, dropout=dropout) self.ffn_norm = build_norm_layer(dict(type='LN'), in_channels)[1] self.cls_fcs = nn.ModuleList() for _ in range(num_cls_fcs): self.cls_fcs.append( nn.Linear(in_channels, in_channels, bias=False)) self.cls_fcs.append( build_norm_layer(dict(type='LN'), in_channels)[1]) self.cls_fcs.append( build_activation_layer(dict(type='ReLU', inplace=True))) # over load the self.fc_cls in BBoxHead if self.loss_cls.use_sigmoid: self.fc_cls = nn.Linear(in_channels, self.num_classes) else: self.fc_cls = nn.Linear(in_channels, self.num_classes + 1) self.reg_fcs = nn.ModuleList() for _ in range(num_reg_fcs): self.reg_fcs.append( nn.Linear(in_channels, in_channels, bias=False)) self.reg_fcs.append( build_norm_layer(dict(type='LN'), in_channels)[1]) self.reg_fcs.append( build_activation_layer(dict(type='ReLU', inplace=True))) # over load the self.fc_cls in BBoxHead self.fc_reg = nn.Linear(in_channels, 4) assert self.reg_class_agnostic, 'DIIHead only ' \ 'suppport `reg_class_agnostic=True` ' assert self.reg_decoded_bbox, 'DIIHead only ' \ 'suppport `reg_decoded_bbox=True`' def init_weights(self) -> None: """Use xavier initialization for all weight parameter and set classification head bias as a specific value when use focal loss.""" super().init_weights() for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) else: # adopt the default initialization for # the weight and bias of the layer norm pass if self.loss_cls.use_sigmoid: bias_init = bias_init_with_prob(0.01) nn.init.constant_(self.fc_cls.bias, bias_init) def forward(self, roi_feat: Tensor, proposal_feat: Tensor) -> tuple: """Forward function of Dynamic Instance Interactive Head. Args: roi_feat (Tensor): Roi-pooling features with shape (batch_size*num_proposals, feature_dimensions, pooling_h , pooling_w). proposal_feat (Tensor): Intermediate feature get from diihead in last stage, has shape (batch_size, num_proposals, feature_dimensions) Returns: tuple[Tensor]: Usually a tuple of classification scores and bbox prediction and a intermediate feature. - cls_scores (Tensor): Classification scores for all proposals, has shape (batch_size, num_proposals, num_classes). - bbox_preds (Tensor): Box energies / deltas for all proposals, has shape (batch_size, num_proposals, 4). - obj_feat (Tensor): Object feature before classification and regression subnet, has shape (batch_size, num_proposal, feature_dimensions). - attn_feats (Tensor): Intermediate feature. """ N, num_proposals = proposal_feat.shape[:2] # Self attention proposal_feat = proposal_feat.permute(1, 0, 2) proposal_feat = self.attention_norm(self.attention(proposal_feat)) attn_feats = proposal_feat.permute(1, 0, 2) # instance interactive proposal_feat = attn_feats.reshape(-1, self.in_channels) proposal_feat_iic = self.instance_interactive_conv( proposal_feat, roi_feat) proposal_feat = proposal_feat + self.instance_interactive_conv_dropout( proposal_feat_iic) obj_feat = self.instance_interactive_conv_norm(proposal_feat) # FFN obj_feat = self.ffn_norm(self.ffn(obj_feat)) cls_feat = obj_feat reg_feat = obj_feat for cls_layer in self.cls_fcs: cls_feat = cls_layer(cls_feat) for reg_layer in self.reg_fcs: reg_feat = reg_layer(reg_feat) cls_score = self.fc_cls(cls_feat).view( N, num_proposals, self.num_classes if self.loss_cls.use_sigmoid else self.num_classes + 1) bbox_delta = self.fc_reg(reg_feat).view(N, num_proposals, 4) return cls_score, bbox_delta, obj_feat.view( N, num_proposals, self.in_channels), attn_feats def loss_and_target(self, cls_score: Tensor, bbox_pred: Tensor, sampling_results: List[SamplingResult], rcnn_train_cfg: ConfigType, imgs_whwh: Tensor, concat: bool = True, reduction_override: str = None) -> dict: """Calculate the loss based on the features extracted by the DIIHead. Args: cls_score (Tensor): Classification prediction results of all class, has shape (batch_size * num_proposals_single_image, num_classes) bbox_pred (Tensor): Regression prediction results, has shape (batch_size * num_proposals_single_image, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. rcnn_train_cfg (obj:ConfigDict): `train_cfg` of RCNN. imgs_whwh (Tensor): imgs_whwh (Tensor): Tensor with\ shape (batch_size, num_proposals, 4), the last dimension means [img_width,img_height, img_width, img_height]. concat (bool): Whether to concatenate the results of all the images in a single batch. Defaults to True. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Options are "none", "mean" and "sum". Defaults to None. Returns: dict: A dictionary of loss and targets components. The targets are only used for cascade rcnn. """ cls_reg_targets = self.get_targets( sampling_results=sampling_results, rcnn_train_cfg=rcnn_train_cfg, concat=concat) (labels, label_weights, bbox_targets, bbox_weights) = cls_reg_targets losses = dict() bg_class_ind = self.num_classes # note in spare rcnn num_gt == num_pos pos_inds = (labels >= 0) & (labels < bg_class_ind) num_pos = pos_inds.sum().float() avg_factor = reduce_mean(num_pos) if cls_score is not None: if cls_score.numel() > 0: losses['loss_cls'] = self.loss_cls( cls_score, labels, label_weights, avg_factor=avg_factor, reduction_override=reduction_override) losses['pos_acc'] = accuracy(cls_score[pos_inds], labels[pos_inds]) if bbox_pred is not None: # 0~self.num_classes-1 are FG, self.num_classes is BG # do not perform bounding box regression for BG anymore. if pos_inds.any(): pos_bbox_pred = bbox_pred.reshape(bbox_pred.size(0), 4)[pos_inds.type(torch.bool)] imgs_whwh = imgs_whwh.reshape(bbox_pred.size(0), 4)[pos_inds.type(torch.bool)] losses['loss_bbox'] = self.loss_bbox( pos_bbox_pred / imgs_whwh, bbox_targets[pos_inds.type(torch.bool)] / imgs_whwh, bbox_weights[pos_inds.type(torch.bool)], avg_factor=avg_factor) losses['loss_iou'] = self.loss_iou( pos_bbox_pred, bbox_targets[pos_inds.type(torch.bool)], bbox_weights[pos_inds.type(torch.bool)], avg_factor=avg_factor) else: losses['loss_bbox'] = bbox_pred.sum() * 0 losses['loss_iou'] = bbox_pred.sum() * 0 return dict(loss_bbox=losses, bbox_targets=cls_reg_targets) def _get_targets_single(self, pos_inds: Tensor, neg_inds: Tensor, pos_priors: Tensor, neg_priors: Tensor, pos_gt_bboxes: Tensor, pos_gt_labels: Tensor, cfg: ConfigDict) -> tuple: """Calculate the ground truth for proposals in the single image according to the sampling results. Almost the same as the implementation in `bbox_head`, we add pos_inds and neg_inds to select positive and negative samples instead of selecting the first num_pos as positive samples. Args: pos_inds (Tensor): The length is equal to the positive sample numbers contain all index of the positive sample in the origin proposal set. neg_inds (Tensor): The length is equal to the negative sample numbers contain all index of the negative sample in the origin proposal set. pos_priors (Tensor): Contains all the positive boxes, has shape (num_pos, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. neg_priors (Tensor): Contains all the negative boxes, has shape (num_neg, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. pos_gt_bboxes (Tensor): Contains gt_boxes for all positive samples, has shape (num_pos, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. pos_gt_labels (Tensor): Contains gt_labels for all positive samples, has shape (num_pos, ). cfg (obj:`ConfigDict`): `train_cfg` of R-CNN. Returns: Tuple[Tensor]: Ground truth for proposals in a single image. Containing the following Tensors: - labels(Tensor): Gt_labels for all proposals, has shape (num_proposals,). - label_weights(Tensor): Labels_weights for all proposals, has shape (num_proposals,). - bbox_targets(Tensor):Regression target for all proposals, has shape (num_proposals, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. - bbox_weights(Tensor):Regression weights for all proposals, has shape (num_proposals, 4). """ num_pos = pos_priors.size(0) num_neg = neg_priors.size(0) num_samples = num_pos + num_neg # original implementation uses new_zeros since BG are set to be 0 # now use empty & fill because BG cat_id = num_classes, # FG cat_id = [0, num_classes-1] labels = pos_priors.new_full((num_samples, ), self.num_classes, dtype=torch.long) label_weights = pos_priors.new_zeros(num_samples) bbox_targets = pos_priors.new_zeros(num_samples, 4) bbox_weights = pos_priors.new_zeros(num_samples, 4) if num_pos > 0: labels[pos_inds] = pos_gt_labels pos_weight = 1.0 if cfg.pos_weight <= 0 else cfg.pos_weight label_weights[pos_inds] = pos_weight if not self.reg_decoded_bbox: pos_bbox_targets = self.bbox_coder.encode( pos_priors, pos_gt_bboxes) else: pos_bbox_targets = pos_gt_bboxes bbox_targets[pos_inds, :] = pos_bbox_targets bbox_weights[pos_inds, :] = 1 if num_neg > 0: label_weights[neg_inds] = 1.0 return labels, label_weights, bbox_targets, bbox_weights def get_targets(self, sampling_results: List[SamplingResult], rcnn_train_cfg: ConfigDict, concat: bool = True) -> tuple: """Calculate the ground truth for all samples in a batch according to the sampling_results. Almost the same as the implementation in bbox_head, we passed additional parameters pos_inds_list and neg_inds_list to `_get_targets_single` function. Args: sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. rcnn_train_cfg (obj:ConfigDict): `train_cfg` of RCNN. concat (bool): Whether to concatenate the results of all the images in a single batch. Returns: Tuple[Tensor]: Ground truth for proposals in a single image. Containing the following list of Tensors: - labels (list[Tensor],Tensor): Gt_labels for all proposals in a batch, each tensor in list has shape (num_proposals,) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals,). - label_weights (list[Tensor]): Labels_weights for all proposals in a batch, each tensor in list has shape (num_proposals,) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals,). - bbox_targets (list[Tensor],Tensor): Regression target for all proposals in a batch, each tensor in list has shape (num_proposals, 4) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. - bbox_weights (list[tensor],Tensor): Regression weights for all proposals in a batch, each tensor in list has shape (num_proposals, 4) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals, 4). """ pos_inds_list = [res.pos_inds for res in sampling_results] neg_inds_list = [res.neg_inds for res in sampling_results] pos_priors_list = [res.pos_priors for res in sampling_results] neg_priors_list = [res.neg_priors for res in sampling_results] pos_gt_bboxes_list = [res.pos_gt_bboxes for res in sampling_results] pos_gt_labels_list = [res.pos_gt_labels for res in sampling_results] labels, label_weights, bbox_targets, bbox_weights = multi_apply( self._get_targets_single, pos_inds_list, neg_inds_list, pos_priors_list, neg_priors_list, pos_gt_bboxes_list, pos_gt_labels_list, cfg=rcnn_train_cfg) if concat: labels = torch.cat(labels, 0) label_weights = torch.cat(label_weights, 0) bbox_targets = torch.cat(bbox_targets, 0) bbox_weights = torch.cat(bbox_weights, 0) return labels, label_weights, bbox_targets, bbox_weights ================================================ FILE: mmdet/models/roi_heads/bbox_heads/double_bbox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple import torch.nn as nn from mmcv.cnn import ConvModule from mmengine.model import BaseModule, ModuleList from torch import Tensor from mmdet.models.backbones.resnet import Bottleneck from mmdet.registry import MODELS from mmdet.utils import ConfigType, MultiConfig, OptConfigType, OptMultiConfig from .bbox_head import BBoxHead class BasicResBlock(BaseModule): """Basic residual block. This block is a little different from the block in the ResNet backbone. The kernel size of conv1 is 1 in this block while 3 in ResNet BasicBlock. Args: in_channels (int): Channels of the input feature map. out_channels (int): Channels of the output feature map. conv_cfg (:obj:`ConfigDict` or dict, optional): The config dict for convolution layers. norm_cfg (:obj:`ConfigDict` or dict): The config dict for normalization layers. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. Defaults to None """ def __init__(self, in_channels: int, out_channels: int, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN'), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) # main path self.conv1 = ConvModule( in_channels, in_channels, kernel_size=3, padding=1, bias=False, conv_cfg=conv_cfg, norm_cfg=norm_cfg) self.conv2 = ConvModule( in_channels, out_channels, kernel_size=1, bias=False, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=None) # identity path self.conv_identity = ConvModule( in_channels, out_channels, kernel_size=1, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=None) self.relu = nn.ReLU(inplace=True) def forward(self, x: Tensor) -> Tensor: """Forward function.""" identity = x x = self.conv1(x) x = self.conv2(x) identity = self.conv_identity(identity) out = x + identity out = self.relu(out) return out @MODELS.register_module() class DoubleConvFCBBoxHead(BBoxHead): r"""Bbox head used in Double-Head R-CNN .. code-block:: none /-> cls /-> shared convs -> \-> reg roi features /-> cls \-> shared fc -> \-> reg """ # noqa: W605 def __init__(self, num_convs: int = 0, num_fcs: int = 0, conv_out_channels: int = 1024, fc_out_channels: int = 1024, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='BN'), init_cfg: MultiConfig = dict( type='Normal', override=[ dict(type='Normal', name='fc_cls', std=0.01), dict(type='Normal', name='fc_reg', std=0.001), dict( type='Xavier', name='fc_branch', distribution='uniform') ]), **kwargs) -> None: kwargs.setdefault('with_avg_pool', True) super().__init__(init_cfg=init_cfg, **kwargs) assert self.with_avg_pool assert num_convs > 0 assert num_fcs > 0 self.num_convs = num_convs self.num_fcs = num_fcs self.conv_out_channels = conv_out_channels self.fc_out_channels = fc_out_channels self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg # increase the channel of input features self.res_block = BasicResBlock(self.in_channels, self.conv_out_channels) # add conv heads self.conv_branch = self._add_conv_branch() # add fc heads self.fc_branch = self._add_fc_branch() out_dim_reg = 4 if self.reg_class_agnostic else 4 * self.num_classes self.fc_reg = nn.Linear(self.conv_out_channels, out_dim_reg) self.fc_cls = nn.Linear(self.fc_out_channels, self.num_classes + 1) self.relu = nn.ReLU() def _add_conv_branch(self) -> None: """Add the fc branch which consists of a sequential of conv layers.""" branch_convs = ModuleList() for i in range(self.num_convs): branch_convs.append( Bottleneck( inplanes=self.conv_out_channels, planes=self.conv_out_channels // 4, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) return branch_convs def _add_fc_branch(self) -> None: """Add the fc branch which consists of a sequential of fc layers.""" branch_fcs = ModuleList() for i in range(self.num_fcs): fc_in_channels = ( self.in_channels * self.roi_feat_area if i == 0 else self.fc_out_channels) branch_fcs.append(nn.Linear(fc_in_channels, self.fc_out_channels)) return branch_fcs def forward(self, x_cls: Tensor, x_reg: Tensor) -> Tuple[Tensor]: """Forward features from the upstream network. Args: x_cls (Tensor): Classification features of rois x_reg (Tensor): Regression features from the upstream network. Returns: tuple: A tuple of classification scores and bbox prediction. - cls_score (Tensor): Classification score predictions of rois. each roi predicts num_classes + 1 channels. - bbox_pred (Tensor): BBox deltas predictions of rois. each roi predicts 4 * num_classes channels. """ # conv head x_conv = self.res_block(x_reg) for conv in self.conv_branch: x_conv = conv(x_conv) if self.with_avg_pool: x_conv = self.avg_pool(x_conv) x_conv = x_conv.view(x_conv.size(0), -1) bbox_pred = self.fc_reg(x_conv) # fc head x_fc = x_cls.view(x_cls.size(0), -1) for fc in self.fc_branch: x_fc = self.relu(fc(x_fc)) cls_score = self.fc_cls(x_fc) return cls_score, bbox_pred ================================================ FILE: mmdet/models/roi_heads/bbox_heads/multi_instance_bbox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple, Union import numpy as np import torch import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.config import ConfigDict from mmengine.structures import InstanceData from torch import Tensor, nn from mmdet.models.roi_heads.bbox_heads.bbox_head import BBoxHead from mmdet.models.task_modules.samplers import SamplingResult from mmdet.models.utils import empty_instances from mmdet.registry import MODELS from mmdet.structures.bbox import bbox_overlaps @MODELS.register_module() class MultiInstanceBBoxHead(BBoxHead): r"""Bbox head used in CrowdDet. .. code-block:: none /-> cls convs_1 -> cls fcs_1 -> cls_1 |-- | \-> reg convs_1 -> reg fcs_1 -> reg_1 | | /-> cls convs_2 -> cls fcs_2 -> cls_2 shared convs -> shared fcs |-- | \-> reg convs_2 -> reg fcs_2 -> reg_2 | | ... | | /-> cls convs_k -> cls fcs_k -> cls_k |-- \-> reg convs_k -> reg fcs_k -> reg_k Args: num_instance (int): The number of branches after shared fcs. Defaults to 2. with_refine (bool): Whether to use refine module. Defaults to False. num_shared_convs (int): The number of shared convs. Defaults to 0. num_shared_fcs (int): The number of shared fcs. Defaults to 2. num_cls_convs (int): The number of cls convs. Defaults to 0. num_cls_fcs (int): The number of cls fcs. Defaults to 0. num_reg_convs (int): The number of reg convs. Defaults to 0. num_reg_fcs (int): The number of reg fcs. Defaults to 0. conv_out_channels (int): The number of conv out channels. Defaults to 256. fc_out_channels (int): The number of fc out channels. Defaults to 1024. init_cfg (dict or list[dict], optional): Initialization config dict. Defaults to None. """ # noqa: W605 def __init__(self, num_instance: int = 2, with_refine: bool = False, num_shared_convs: int = 0, num_shared_fcs: int = 2, num_cls_convs: int = 0, num_cls_fcs: int = 0, num_reg_convs: int = 0, num_reg_fcs: int = 0, conv_out_channels: int = 256, fc_out_channels: int = 1024, init_cfg: Optional[Union[dict, ConfigDict]] = None, *args, **kwargs) -> None: super().__init__(*args, init_cfg=init_cfg, **kwargs) assert (num_shared_convs + num_shared_fcs + num_cls_convs + num_cls_fcs + num_reg_convs + num_reg_fcs > 0) assert num_instance == 2, 'Currently only 2 instances are supported' if num_cls_convs > 0 or num_reg_convs > 0: assert num_shared_fcs == 0 if not self.with_cls: assert num_cls_convs == 0 and num_cls_fcs == 0 if not self.with_reg: assert num_reg_convs == 0 and num_reg_fcs == 0 self.num_instance = num_instance self.num_shared_convs = num_shared_convs self.num_shared_fcs = num_shared_fcs self.num_cls_convs = num_cls_convs self.num_cls_fcs = num_cls_fcs self.num_reg_convs = num_reg_convs self.num_reg_fcs = num_reg_fcs self.conv_out_channels = conv_out_channels self.fc_out_channels = fc_out_channels self.with_refine = with_refine # add shared convs and fcs self.shared_convs, self.shared_fcs, last_layer_dim = \ self._add_conv_fc_branch( self.num_shared_convs, self.num_shared_fcs, self.in_channels, True) self.shared_out_channels = last_layer_dim self.relu = nn.ReLU(inplace=True) if self.with_refine: refine_model_cfg = { 'type': 'Linear', 'in_features': self.shared_out_channels + 20, 'out_features': self.shared_out_channels } self.shared_fcs_ref = MODELS.build(refine_model_cfg) self.fc_cls_ref = nn.ModuleList() self.fc_reg_ref = nn.ModuleList() self.cls_convs = nn.ModuleList() self.cls_fcs = nn.ModuleList() self.reg_convs = nn.ModuleList() self.reg_fcs = nn.ModuleList() self.cls_last_dim = list() self.reg_last_dim = list() self.fc_cls = nn.ModuleList() self.fc_reg = nn.ModuleList() for k in range(self.num_instance): # add cls specific branch cls_convs, cls_fcs, cls_last_dim = self._add_conv_fc_branch( self.num_cls_convs, self.num_cls_fcs, self.shared_out_channels) self.cls_convs.append(cls_convs) self.cls_fcs.append(cls_fcs) self.cls_last_dim.append(cls_last_dim) # add reg specific branch reg_convs, reg_fcs, reg_last_dim = self._add_conv_fc_branch( self.num_reg_convs, self.num_reg_fcs, self.shared_out_channels) self.reg_convs.append(reg_convs) self.reg_fcs.append(reg_fcs) self.reg_last_dim.append(reg_last_dim) if self.num_shared_fcs == 0 and not self.with_avg_pool: if self.num_cls_fcs == 0: self.cls_last_dim *= self.roi_feat_area if self.num_reg_fcs == 0: self.reg_last_dim *= self.roi_feat_area if self.with_cls: if self.custom_cls_channels: cls_channels = self.loss_cls.get_cls_channels( self.num_classes) else: cls_channels = self.num_classes + 1 cls_predictor_cfg_ = self.cls_predictor_cfg.copy() # deepcopy cls_predictor_cfg_.update( in_features=self.cls_last_dim[k], out_features=cls_channels) self.fc_cls.append(MODELS.build(cls_predictor_cfg_)) if self.with_refine: self.fc_cls_ref.append(MODELS.build(cls_predictor_cfg_)) if self.with_reg: out_dim_reg = (4 if self.reg_class_agnostic else 4 * self.num_classes) reg_predictor_cfg_ = self.reg_predictor_cfg.copy() reg_predictor_cfg_.update( in_features=self.reg_last_dim[k], out_features=out_dim_reg) self.fc_reg.append(MODELS.build(reg_predictor_cfg_)) if self.with_refine: self.fc_reg_ref.append(MODELS.build(reg_predictor_cfg_)) if init_cfg is None: # when init_cfg is None, # It has been set to # [[dict(type='Normal', std=0.01, override=dict(name='fc_cls'))], # [dict(type='Normal', std=0.001, override=dict(name='fc_reg'))] # after `super(ConvFCBBoxHead, self).__init__()` # we only need to append additional configuration # for `shared_fcs`, `cls_fcs` and `reg_fcs` self.init_cfg += [ dict( type='Xavier', distribution='uniform', override=[ dict(name='shared_fcs'), dict(name='cls_fcs'), dict(name='reg_fcs') ]) ] def _add_conv_fc_branch(self, num_branch_convs: int, num_branch_fcs: int, in_channels: int, is_shared: bool = False) -> tuple: """Add shared or separable branch. convs -> avg pool (optional) -> fcs """ last_layer_dim = in_channels # add branch specific conv layers branch_convs = nn.ModuleList() if num_branch_convs > 0: for i in range(num_branch_convs): conv_in_channels = ( last_layer_dim if i == 0 else self.conv_out_channels) branch_convs.append( ConvModule( conv_in_channels, self.conv_out_channels, 3, padding=1)) last_layer_dim = self.conv_out_channels # add branch specific fc layers branch_fcs = nn.ModuleList() if num_branch_fcs > 0: # for shared branch, only consider self.with_avg_pool # for separated branches, also consider self.num_shared_fcs if (is_shared or self.num_shared_fcs == 0) and not self.with_avg_pool: last_layer_dim *= self.roi_feat_area for i in range(num_branch_fcs): fc_in_channels = ( last_layer_dim if i == 0 else self.fc_out_channels) branch_fcs.append( nn.Linear(fc_in_channels, self.fc_out_channels)) last_layer_dim = self.fc_out_channels return branch_convs, branch_fcs, last_layer_dim def forward(self, x: Tuple[Tensor]) -> tuple: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of classification scores and bbox prediction. - cls_score (Tensor): Classification scores for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * num_classes. - bbox_pred (Tensor): Box energies / deltas for all scale levels, each is a 4D-tensor, the channels number is num_base_priors * 4. - cls_score_ref (Tensor): The cls_score after refine model. - bbox_pred_ref (Tensor): The bbox_pred after refine model. """ # shared part if self.num_shared_convs > 0: for conv in self.shared_convs: x = conv(x) if self.num_shared_fcs > 0: if self.with_avg_pool: x = self.avg_pool(x) x = x.flatten(1) for fc in self.shared_fcs: x = self.relu(fc(x)) x_cls = x x_reg = x # separate branches cls_score = list() bbox_pred = list() for k in range(self.num_instance): for conv in self.cls_convs[k]: x_cls = conv(x_cls) if x_cls.dim() > 2: if self.with_avg_pool: x_cls = self.avg_pool(x_cls) x_cls = x_cls.flatten(1) for fc in self.cls_fcs[k]: x_cls = self.relu(fc(x_cls)) for conv in self.reg_convs[k]: x_reg = conv(x_reg) if x_reg.dim() > 2: if self.with_avg_pool: x_reg = self.avg_pool(x_reg) x_reg = x_reg.flatten(1) for fc in self.reg_fcs[k]: x_reg = self.relu(fc(x_reg)) cls_score.append(self.fc_cls[k](x_cls) if self.with_cls else None) bbox_pred.append(self.fc_reg[k](x_reg) if self.with_reg else None) if self.with_refine: x_ref = x cls_score_ref = list() bbox_pred_ref = list() for k in range(self.num_instance): feat_ref = cls_score[k].softmax(dim=-1) feat_ref = torch.cat((bbox_pred[k], feat_ref[:, 1][:, None]), dim=1).repeat(1, 4) feat_ref = torch.cat((x_ref, feat_ref), dim=1) feat_ref = F.relu_(self.shared_fcs_ref(feat_ref)) cls_score_ref.append(self.fc_cls_ref[k](feat_ref)) bbox_pred_ref.append(self.fc_reg_ref[k](feat_ref)) cls_score = torch.cat(cls_score, dim=1) bbox_pred = torch.cat(bbox_pred, dim=1) cls_score_ref = torch.cat(cls_score_ref, dim=1) bbox_pred_ref = torch.cat(bbox_pred_ref, dim=1) return cls_score, bbox_pred, cls_score_ref, bbox_pred_ref cls_score = torch.cat(cls_score, dim=1) bbox_pred = torch.cat(bbox_pred, dim=1) return cls_score, bbox_pred def get_targets(self, sampling_results: List[SamplingResult], rcnn_train_cfg: ConfigDict, concat: bool = True) -> tuple: """Calculate the ground truth for all samples in a batch according to the sampling_results. Almost the same as the implementation in bbox_head, we passed additional parameters pos_inds_list and neg_inds_list to `_get_targets_single` function. Args: sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. rcnn_train_cfg (obj:ConfigDict): `train_cfg` of RCNN. concat (bool): Whether to concatenate the results of all the images in a single batch. Returns: Tuple[Tensor]: Ground truth for proposals in a single image. Containing the following list of Tensors: - labels (list[Tensor],Tensor): Gt_labels for all proposals in a batch, each tensor in list has shape (num_proposals,) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals,). - label_weights (list[Tensor]): Labels_weights for all proposals in a batch, each tensor in list has shape (num_proposals,) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals,). - bbox_targets (list[Tensor],Tensor): Regression target for all proposals in a batch, each tensor in list has shape (num_proposals, 4) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. - bbox_weights (list[tensor],Tensor): Regression weights for all proposals in a batch, each tensor in list has shape (num_proposals, 4) when `concat=False`, otherwise just a single tensor has shape (num_all_proposals, 4). """ labels = [] bbox_targets = [] bbox_weights = [] label_weights = [] for i in range(len(sampling_results)): sample_bboxes = torch.cat([ sampling_results[i].pos_gt_bboxes, sampling_results[i].neg_gt_bboxes ]) sample_priors = sampling_results[i].priors sample_priors = sample_priors.repeat(1, self.num_instance).reshape( -1, 4) sample_bboxes = sample_bboxes.reshape(-1, 4) if not self.reg_decoded_bbox: _bbox_targets = self.bbox_coder.encode(sample_priors, sample_bboxes) else: _bbox_targets = sample_priors _bbox_targets = _bbox_targets.reshape(-1, self.num_instance * 4) _bbox_weights = torch.ones(_bbox_targets.shape) _labels = torch.cat([ sampling_results[i].pos_gt_labels, sampling_results[i].neg_gt_labels ]) _labels_weights = torch.ones(_labels.shape) bbox_targets.append(_bbox_targets) bbox_weights.append(_bbox_weights) labels.append(_labels) label_weights.append(_labels_weights) if concat: labels = torch.cat(labels, 0) label_weights = torch.cat(label_weights, 0) bbox_targets = torch.cat(bbox_targets, 0) bbox_weights = torch.cat(bbox_weights, 0) return labels, label_weights, bbox_targets, bbox_weights def loss(self, cls_score: Tensor, bbox_pred: Tensor, rois: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tensor, bbox_weights: Tensor, **kwargs) -> dict: """Calculate the loss based on the network predictions and targets. Args: cls_score (Tensor): Classification prediction results of all class, has shape (batch_size * num_proposals_single_image, (num_classes + 1) * k), k represents the number of prediction boxes generated by each proposal box. bbox_pred (Tensor): Regression prediction results, has shape (batch_size * num_proposals_single_image, 4 * k), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. rois (Tensor): RoIs with the shape (batch_size * num_proposals_single_image, 5) where the first column indicates batch id of each RoI. labels (Tensor): Gt_labels for all proposals in a batch, has shape (batch_size * num_proposals_single_image, k). label_weights (Tensor): Labels_weights for all proposals in a batch, has shape (batch_size * num_proposals_single_image, k). bbox_targets (Tensor): Regression target for all proposals in a batch, has shape (batch_size * num_proposals_single_image, 4 * k), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. bbox_weights (Tensor): Regression weights for all proposals in a batch, has shape (batch_size * num_proposals_single_image, 4 * k). Returns: dict: A dictionary of loss. """ losses = dict() if bbox_pred.numel(): loss_0 = self.emd_loss(bbox_pred[:, 0:4], cls_score[:, 0:2], bbox_pred[:, 4:8], cls_score[:, 2:4], bbox_targets, labels) loss_1 = self.emd_loss(bbox_pred[:, 4:8], cls_score[:, 2:4], bbox_pred[:, 0:4], cls_score[:, 0:2], bbox_targets, labels) loss = torch.cat([loss_0, loss_1], dim=1) _, min_indices = loss.min(dim=1) loss_emd = loss[torch.arange(loss.shape[0]), min_indices] loss_emd = loss_emd.mean() else: loss_emd = bbox_pred.sum() losses['loss_rcnn_emd'] = loss_emd return losses def emd_loss(self, bbox_pred_0: Tensor, cls_score_0: Tensor, bbox_pred_1: Tensor, cls_score_1: Tensor, targets: Tensor, labels: Tensor) -> Tensor: """Calculate the emd loss. Note: This implementation is modified from https://github.com/Purkialo/ CrowdDet/blob/master/lib/det_oprs/loss_opr.py Args: bbox_pred_0 (Tensor): Part of regression prediction results, has shape (batch_size * num_proposals_single_image, 4), the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. cls_score_0 (Tensor): Part of classification prediction results, has shape (batch_size * num_proposals_single_image, (num_classes + 1)), where 1 represents the background. bbox_pred_1 (Tensor): The other part of regression prediction results, has shape (batch_size*num_proposals_single_image, 4). cls_score_1 (Tensor):The other part of classification prediction results, has shape (batch_size * num_proposals_single_image, (num_classes + 1)). targets (Tensor):Regression target for all proposals in a batch, has shape (batch_size * num_proposals_single_image, 4 * k), the last dimension 4 represents [tl_x, tl_y, br_x, br_y], k represents the number of prediction boxes generated by each proposal box. labels (Tensor): Gt_labels for all proposals in a batch, has shape (batch_size * num_proposals_single_image, k). Returns: torch.Tensor: The calculated loss. """ bbox_pred = torch.cat([bbox_pred_0, bbox_pred_1], dim=1).reshape(-1, bbox_pred_0.shape[-1]) cls_score = torch.cat([cls_score_0, cls_score_1], dim=1).reshape(-1, cls_score_0.shape[-1]) targets = targets.reshape(-1, 4) labels = labels.long().flatten() # masks valid_masks = labels >= 0 fg_masks = labels > 0 # multiple class bbox_pred = bbox_pred.reshape(-1, self.num_classes, 4) fg_gt_classes = labels[fg_masks] bbox_pred = bbox_pred[fg_masks, fg_gt_classes - 1, :] # loss for regression loss_bbox = self.loss_bbox(bbox_pred, targets[fg_masks]) loss_bbox = loss_bbox.sum(dim=1) # loss for classification labels = labels * valid_masks loss_cls = self.loss_cls(cls_score, labels) loss_cls[fg_masks] = loss_cls[fg_masks] + loss_bbox loss = loss_cls.reshape(-1, 2).sum(dim=1) return loss.reshape(-1, 1) def _predict_by_feat_single( self, roi: Tensor, cls_score: Tensor, bbox_pred: Tensor, img_meta: dict, rescale: bool = False, rcnn_test_cfg: Optional[ConfigDict] = None) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: roi (Tensor): Boxes to be transformed. Has shape (num_boxes, 5). last dimension 5 arrange as (batch_index, x1, y1, x2, y2). cls_score (Tensor): Box scores, has shape (num_boxes, num_classes + 1). bbox_pred (Tensor): Box energies / deltas. has shape (num_boxes, num_classes * 4). img_meta (dict): image information. rescale (bool): If True, return boxes in original image space. Defaults to False. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of Bbox Head. Defaults to None Returns: :obj:`InstanceData`: Detection results of each image. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ cls_score = cls_score.reshape(-1, self.num_classes + 1) bbox_pred = bbox_pred.reshape(-1, 4) roi = roi.repeat_interleave(self.num_instance, dim=0) results = InstanceData() if roi.shape[0] == 0: return empty_instances([img_meta], roi.device, task_type='bbox', instance_results=[results])[0] scores = cls_score.softmax(dim=-1) if cls_score is not None else None img_shape = img_meta['img_shape'] bboxes = self.bbox_coder.decode( roi[..., 1:], bbox_pred, max_shape=img_shape) if rescale and bboxes.size(0) > 0: assert img_meta.get('scale_factor') is not None scale_factor = bboxes.new_tensor(img_meta['scale_factor']).repeat( (1, 2)) bboxes = (bboxes.view(bboxes.size(0), -1, 4) / scale_factor).view( bboxes.size()[0], -1) if rcnn_test_cfg is None: # This means that it is aug test. # It needs to return the raw results without nms. results.bboxes = bboxes results.scores = scores else: roi_idx = np.tile( np.arange(bboxes.shape[0] / self.num_instance)[:, None], (1, self.num_instance)).reshape(-1, 1)[:, 0] roi_idx = torch.from_numpy(roi_idx).to(bboxes.device).reshape( -1, 1) bboxes = torch.cat([bboxes, roi_idx], dim=1) det_bboxes, det_scores = self.set_nms( bboxes, scores[:, 1], rcnn_test_cfg.score_thr, rcnn_test_cfg.nms['iou_threshold'], rcnn_test_cfg.max_per_img) results.bboxes = det_bboxes[:, :-1] results.scores = det_scores results.labels = torch.zeros_like(det_scores) return results @staticmethod def set_nms(bboxes: Tensor, scores: Tensor, score_thr: float, iou_threshold: float, max_num: int = -1) -> Tuple[Tensor, Tensor]: """NMS for multi-instance prediction. Please refer to https://github.com/Purkialo/CrowdDet for more details. Args: bboxes (Tensor): predict bboxes. scores (Tensor): The score of each predict bbox. score_thr (float): bbox threshold, bboxes with scores lower than it will not be considered. iou_threshold (float): IoU threshold to be considered as conflicted. max_num (int, optional): if there are more than max_num bboxes after NMS, only top max_num will be kept. Default to -1. Returns: Tuple[Tensor, Tensor]: (bboxes, scores). """ bboxes = bboxes[scores > score_thr] scores = scores[scores > score_thr] ordered_scores, order = scores.sort(descending=True) ordered_bboxes = bboxes[order] roi_idx = ordered_bboxes[:, -1] keep = torch.ones(len(ordered_bboxes)) == 1 ruler = torch.arange(len(ordered_bboxes)) while ruler.shape[0] > 0: basement = ruler[0] ruler = ruler[1:] idx = roi_idx[basement] # calculate the body overlap basement_bbox = ordered_bboxes[:, :4][basement].reshape(-1, 4) ruler_bbox = ordered_bboxes[:, :4][ruler].reshape(-1, 4) overlap = bbox_overlaps(basement_bbox, ruler_bbox) indices = torch.where(overlap > iou_threshold)[1] loc = torch.where(roi_idx[ruler][indices] == idx) # the mask won't change in the step mask = keep[ruler[indices][loc]] keep[ruler[indices]] = False keep[ruler[indices][loc][mask]] = True ruler[~keep[ruler]] = -1 ruler = ruler[ruler > 0] keep = keep[order.sort()[1]] return bboxes[keep][:max_num, :], scores[keep][:max_num] ================================================ FILE: mmdet/models/roi_heads/bbox_heads/sabl_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Sequence, Tuple import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.config import ConfigDict from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.layers import multiclass_nms from mmdet.models.losses import accuracy from mmdet.models.task_modules import SamplingResult from mmdet.models.utils import multi_apply from mmdet.registry import MODELS, TASK_UTILS from mmdet.utils import ConfigType, InstanceList, OptConfigType, OptMultiConfig from .bbox_head import BBoxHead @MODELS.register_module() class SABLHead(BBoxHead): """Side-Aware Boundary Localization (SABL) for RoI-Head. Side-Aware features are extracted by conv layers with an attention mechanism. Boundary Localization with Bucketing and Bucketing Guided Rescoring are implemented in BucketingBBoxCoder. Please refer to https://arxiv.org/abs/1912.04260 for more details. Args: cls_in_channels (int): Input channels of cls RoI feature. \ Defaults to 256. reg_in_channels (int): Input channels of reg RoI feature. \ Defaults to 256. roi_feat_size (int): Size of RoI features. Defaults to 7. reg_feat_up_ratio (int): Upsample ratio of reg features. \ Defaults to 2. reg_pre_kernel (int): Kernel of 2D conv layers before \ attention pooling. Defaults to 3. reg_post_kernel (int): Kernel of 1D conv layers after \ attention pooling. Defaults to 3. reg_pre_num (int): Number of pre convs. Defaults to 2. reg_post_num (int): Number of post convs. Defaults to 1. num_classes (int): Number of classes in dataset. Defaults to 80. cls_out_channels (int): Hidden channels in cls fcs. Defaults to 1024. reg_offset_out_channels (int): Hidden and output channel \ of reg offset branch. Defaults to 256. reg_cls_out_channels (int): Hidden and output channel \ of reg cls branch. Defaults to 256. num_cls_fcs (int): Number of fcs for cls branch. Defaults to 1. num_reg_fcs (int): Number of fcs for reg branch.. Defaults to 0. reg_class_agnostic (bool): Class agnostic regression or not. \ Defaults to True. norm_cfg (dict): Config of norm layers. Defaults to None. bbox_coder (dict): Config of bbox coder. Defaults 'BucketingBBoxCoder'. loss_cls (dict): Config of classification loss. loss_bbox_cls (dict): Config of classification loss for bbox branch. loss_bbox_reg (dict): Config of regression loss for bbox branch. init_cfg (dict or list[dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, num_classes: int, cls_in_channels: int = 256, reg_in_channels: int = 256, roi_feat_size: int = 7, reg_feat_up_ratio: int = 2, reg_pre_kernel: int = 3, reg_post_kernel: int = 3, reg_pre_num: int = 2, reg_post_num: int = 1, cls_out_channels: int = 1024, reg_offset_out_channels: int = 256, reg_cls_out_channels: int = 256, num_cls_fcs: int = 1, num_reg_fcs: int = 0, reg_class_agnostic: bool = True, norm_cfg: OptConfigType = None, bbox_coder: ConfigType = dict( type='BucketingBBoxCoder', num_buckets=14, scale_factor=1.7), loss_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=False, loss_weight=1.0), loss_bbox_cls: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox_reg: ConfigType = dict( type='SmoothL1Loss', beta=0.1, loss_weight=1.0), init_cfg: OptMultiConfig = None) -> None: super(BBoxHead, self).__init__(init_cfg=init_cfg) self.cls_in_channels = cls_in_channels self.reg_in_channels = reg_in_channels self.roi_feat_size = roi_feat_size self.reg_feat_up_ratio = int(reg_feat_up_ratio) self.num_buckets = bbox_coder['num_buckets'] assert self.reg_feat_up_ratio // 2 >= 1 self.up_reg_feat_size = roi_feat_size * self.reg_feat_up_ratio assert self.up_reg_feat_size == bbox_coder['num_buckets'] self.reg_pre_kernel = reg_pre_kernel self.reg_post_kernel = reg_post_kernel self.reg_pre_num = reg_pre_num self.reg_post_num = reg_post_num self.num_classes = num_classes self.cls_out_channels = cls_out_channels self.reg_offset_out_channels = reg_offset_out_channels self.reg_cls_out_channels = reg_cls_out_channels self.num_cls_fcs = num_cls_fcs self.num_reg_fcs = num_reg_fcs self.reg_class_agnostic = reg_class_agnostic assert self.reg_class_agnostic self.norm_cfg = norm_cfg self.bbox_coder = TASK_UTILS.build(bbox_coder) self.loss_cls = MODELS.build(loss_cls) self.loss_bbox_cls = MODELS.build(loss_bbox_cls) self.loss_bbox_reg = MODELS.build(loss_bbox_reg) self.cls_fcs = self._add_fc_branch(self.num_cls_fcs, self.cls_in_channels, self.roi_feat_size, self.cls_out_channels) self.side_num = int(np.ceil(self.num_buckets / 2)) if self.reg_feat_up_ratio > 1: self.upsample_x = nn.ConvTranspose1d( reg_in_channels, reg_in_channels, self.reg_feat_up_ratio, stride=self.reg_feat_up_ratio) self.upsample_y = nn.ConvTranspose1d( reg_in_channels, reg_in_channels, self.reg_feat_up_ratio, stride=self.reg_feat_up_ratio) self.reg_pre_convs = nn.ModuleList() for i in range(self.reg_pre_num): reg_pre_conv = ConvModule( reg_in_channels, reg_in_channels, kernel_size=reg_pre_kernel, padding=reg_pre_kernel // 2, norm_cfg=norm_cfg, act_cfg=dict(type='ReLU')) self.reg_pre_convs.append(reg_pre_conv) self.reg_post_conv_xs = nn.ModuleList() for i in range(self.reg_post_num): reg_post_conv_x = ConvModule( reg_in_channels, reg_in_channels, kernel_size=(1, reg_post_kernel), padding=(0, reg_post_kernel // 2), norm_cfg=norm_cfg, act_cfg=dict(type='ReLU')) self.reg_post_conv_xs.append(reg_post_conv_x) self.reg_post_conv_ys = nn.ModuleList() for i in range(self.reg_post_num): reg_post_conv_y = ConvModule( reg_in_channels, reg_in_channels, kernel_size=(reg_post_kernel, 1), padding=(reg_post_kernel // 2, 0), norm_cfg=norm_cfg, act_cfg=dict(type='ReLU')) self.reg_post_conv_ys.append(reg_post_conv_y) self.reg_conv_att_x = nn.Conv2d(reg_in_channels, 1, 1) self.reg_conv_att_y = nn.Conv2d(reg_in_channels, 1, 1) self.fc_cls = nn.Linear(self.cls_out_channels, self.num_classes + 1) self.relu = nn.ReLU(inplace=True) self.reg_cls_fcs = self._add_fc_branch(self.num_reg_fcs, self.reg_in_channels, 1, self.reg_cls_out_channels) self.reg_offset_fcs = self._add_fc_branch(self.num_reg_fcs, self.reg_in_channels, 1, self.reg_offset_out_channels) self.fc_reg_cls = nn.Linear(self.reg_cls_out_channels, 1) self.fc_reg_offset = nn.Linear(self.reg_offset_out_channels, 1) if init_cfg is None: self.init_cfg = [ dict( type='Xavier', layer='Linear', distribution='uniform', override=[ dict(type='Normal', name='reg_conv_att_x', std=0.01), dict(type='Normal', name='reg_conv_att_y', std=0.01), dict(type='Normal', name='fc_reg_cls', std=0.01), dict(type='Normal', name='fc_cls', std=0.01), dict(type='Normal', name='fc_reg_offset', std=0.001) ]) ] if self.reg_feat_up_ratio > 1: self.init_cfg += [ dict( type='Kaiming', distribution='normal', override=[ dict(name='upsample_x'), dict(name='upsample_y') ]) ] def _add_fc_branch(self, num_branch_fcs: int, in_channels: int, roi_feat_size: int, fc_out_channels: int) -> nn.ModuleList: """build fc layers.""" in_channels = in_channels * roi_feat_size * roi_feat_size branch_fcs = nn.ModuleList() for i in range(num_branch_fcs): fc_in_channels = (in_channels if i == 0 else fc_out_channels) branch_fcs.append(nn.Linear(fc_in_channels, fc_out_channels)) return branch_fcs def cls_forward(self, cls_x: Tensor) -> Tensor: """forward of classification fc layers.""" cls_x = cls_x.view(cls_x.size(0), -1) for fc in self.cls_fcs: cls_x = self.relu(fc(cls_x)) cls_score = self.fc_cls(cls_x) return cls_score def attention_pool(self, reg_x: Tensor) -> tuple: """Extract direction-specific features fx and fy with attention methanism.""" reg_fx = reg_x reg_fy = reg_x reg_fx_att = self.reg_conv_att_x(reg_fx).sigmoid() reg_fy_att = self.reg_conv_att_y(reg_fy).sigmoid() reg_fx_att = reg_fx_att / reg_fx_att.sum(dim=2).unsqueeze(2) reg_fy_att = reg_fy_att / reg_fy_att.sum(dim=3).unsqueeze(3) reg_fx = (reg_fx * reg_fx_att).sum(dim=2) reg_fy = (reg_fy * reg_fy_att).sum(dim=3) return reg_fx, reg_fy def side_aware_feature_extractor(self, reg_x: Tensor) -> tuple: """Refine and extract side-aware features without split them.""" for reg_pre_conv in self.reg_pre_convs: reg_x = reg_pre_conv(reg_x) reg_fx, reg_fy = self.attention_pool(reg_x) if self.reg_post_num > 0: reg_fx = reg_fx.unsqueeze(2) reg_fy = reg_fy.unsqueeze(3) for i in range(self.reg_post_num): reg_fx = self.reg_post_conv_xs[i](reg_fx) reg_fy = self.reg_post_conv_ys[i](reg_fy) reg_fx = reg_fx.squeeze(2) reg_fy = reg_fy.squeeze(3) if self.reg_feat_up_ratio > 1: reg_fx = self.relu(self.upsample_x(reg_fx)) reg_fy = self.relu(self.upsample_y(reg_fy)) reg_fx = torch.transpose(reg_fx, 1, 2) reg_fy = torch.transpose(reg_fy, 1, 2) return reg_fx.contiguous(), reg_fy.contiguous() def reg_pred(self, x: Tensor, offset_fcs: nn.ModuleList, cls_fcs: nn.ModuleList) -> tuple: """Predict bucketing estimation (cls_pred) and fine regression (offset pred) with side-aware features.""" x_offset = x.view(-1, self.reg_in_channels) x_cls = x.view(-1, self.reg_in_channels) for fc in offset_fcs: x_offset = self.relu(fc(x_offset)) for fc in cls_fcs: x_cls = self.relu(fc(x_cls)) offset_pred = self.fc_reg_offset(x_offset) cls_pred = self.fc_reg_cls(x_cls) offset_pred = offset_pred.view(x.size(0), -1) cls_pred = cls_pred.view(x.size(0), -1) return offset_pred, cls_pred def side_aware_split(self, feat: Tensor) -> Tensor: """Split side-aware features aligned with orders of bucketing targets.""" l_end = int(np.ceil(self.up_reg_feat_size / 2)) r_start = int(np.floor(self.up_reg_feat_size / 2)) feat_fl = feat[:, :l_end] feat_fr = feat[:, r_start:].flip(dims=(1, )) feat_fl = feat_fl.contiguous() feat_fr = feat_fr.contiguous() feat = torch.cat([feat_fl, feat_fr], dim=-1) return feat def bbox_pred_split(self, bbox_pred: tuple, num_proposals_per_img: Sequence[int]) -> tuple: """Split batch bbox prediction back to each image.""" bucket_cls_preds, bucket_offset_preds = bbox_pred bucket_cls_preds = bucket_cls_preds.split(num_proposals_per_img, 0) bucket_offset_preds = bucket_offset_preds.split( num_proposals_per_img, 0) bbox_pred = tuple(zip(bucket_cls_preds, bucket_offset_preds)) return bbox_pred def reg_forward(self, reg_x: Tensor) -> tuple: """forward of regression branch.""" outs = self.side_aware_feature_extractor(reg_x) edge_offset_preds = [] edge_cls_preds = [] reg_fx = outs[0] reg_fy = outs[1] offset_pred_x, cls_pred_x = self.reg_pred(reg_fx, self.reg_offset_fcs, self.reg_cls_fcs) offset_pred_y, cls_pred_y = self.reg_pred(reg_fy, self.reg_offset_fcs, self.reg_cls_fcs) offset_pred_x = self.side_aware_split(offset_pred_x) offset_pred_y = self.side_aware_split(offset_pred_y) cls_pred_x = self.side_aware_split(cls_pred_x) cls_pred_y = self.side_aware_split(cls_pred_y) edge_offset_preds = torch.cat([offset_pred_x, offset_pred_y], dim=-1) edge_cls_preds = torch.cat([cls_pred_x, cls_pred_y], dim=-1) return edge_cls_preds, edge_offset_preds def forward(self, x: Tensor) -> tuple: """Forward features from the upstream network.""" bbox_pred = self.reg_forward(x) cls_score = self.cls_forward(x) return cls_score, bbox_pred def get_targets(self, sampling_results: List[SamplingResult], rcnn_train_cfg: ConfigDict, concat: bool = True) -> tuple: """Calculate the ground truth for all samples in a batch according to the sampling_results.""" pos_proposals = [res.pos_bboxes for res in sampling_results] neg_proposals = [res.neg_bboxes for res in sampling_results] pos_gt_bboxes = [res.pos_gt_bboxes for res in sampling_results] pos_gt_labels = [res.pos_gt_labels for res in sampling_results] cls_reg_targets = self.bucket_target( pos_proposals, neg_proposals, pos_gt_bboxes, pos_gt_labels, rcnn_train_cfg, concat=concat) (labels, label_weights, bucket_cls_targets, bucket_cls_weights, bucket_offset_targets, bucket_offset_weights) = cls_reg_targets return (labels, label_weights, (bucket_cls_targets, bucket_offset_targets), (bucket_cls_weights, bucket_offset_weights)) def bucket_target(self, pos_proposals_list: list, neg_proposals_list: list, pos_gt_bboxes_list: list, pos_gt_labels_list: list, rcnn_train_cfg: ConfigDict, concat: bool = True) -> tuple: """Compute bucketing estimation targets and fine regression targets for a batch of images.""" (labels, label_weights, bucket_cls_targets, bucket_cls_weights, bucket_offset_targets, bucket_offset_weights) = multi_apply( self._bucket_target_single, pos_proposals_list, neg_proposals_list, pos_gt_bboxes_list, pos_gt_labels_list, cfg=rcnn_train_cfg) if concat: labels = torch.cat(labels, 0) label_weights = torch.cat(label_weights, 0) bucket_cls_targets = torch.cat(bucket_cls_targets, 0) bucket_cls_weights = torch.cat(bucket_cls_weights, 0) bucket_offset_targets = torch.cat(bucket_offset_targets, 0) bucket_offset_weights = torch.cat(bucket_offset_weights, 0) return (labels, label_weights, bucket_cls_targets, bucket_cls_weights, bucket_offset_targets, bucket_offset_weights) def _bucket_target_single(self, pos_proposals: Tensor, neg_proposals: Tensor, pos_gt_bboxes: Tensor, pos_gt_labels: Tensor, cfg: ConfigDict) -> tuple: """Compute bucketing estimation targets and fine regression targets for a single image. Args: pos_proposals (Tensor): positive proposals of a single image, Shape (n_pos, 4) neg_proposals (Tensor): negative proposals of a single image, Shape (n_neg, 4). pos_gt_bboxes (Tensor): gt bboxes assigned to positive proposals of a single image, Shape (n_pos, 4). pos_gt_labels (Tensor): gt labels assigned to positive proposals of a single image, Shape (n_pos, ). cfg (dict): Config of calculating targets Returns: tuple: - labels (Tensor): Labels in a single image. Shape (n,). - label_weights (Tensor): Label weights in a single image. Shape (n,) - bucket_cls_targets (Tensor): Bucket cls targets in a single image. Shape (n, num_buckets*2). - bucket_cls_weights (Tensor): Bucket cls weights in a single image. Shape (n, num_buckets*2). - bucket_offset_targets (Tensor): Bucket offset targets in a single image. Shape (n, num_buckets*2). - bucket_offset_targets (Tensor): Bucket offset weights in a single image. Shape (n, num_buckets*2). """ num_pos = pos_proposals.size(0) num_neg = neg_proposals.size(0) num_samples = num_pos + num_neg labels = pos_gt_bboxes.new_full((num_samples, ), self.num_classes, dtype=torch.long) label_weights = pos_proposals.new_zeros(num_samples) bucket_cls_targets = pos_proposals.new_zeros(num_samples, 4 * self.side_num) bucket_cls_weights = pos_proposals.new_zeros(num_samples, 4 * self.side_num) bucket_offset_targets = pos_proposals.new_zeros( num_samples, 4 * self.side_num) bucket_offset_weights = pos_proposals.new_zeros( num_samples, 4 * self.side_num) if num_pos > 0: labels[:num_pos] = pos_gt_labels label_weights[:num_pos] = 1.0 (pos_bucket_offset_targets, pos_bucket_offset_weights, pos_bucket_cls_targets, pos_bucket_cls_weights) = self.bbox_coder.encode( pos_proposals, pos_gt_bboxes) bucket_cls_targets[:num_pos, :] = pos_bucket_cls_targets bucket_cls_weights[:num_pos, :] = pos_bucket_cls_weights bucket_offset_targets[:num_pos, :] = pos_bucket_offset_targets bucket_offset_weights[:num_pos, :] = pos_bucket_offset_weights if num_neg > 0: label_weights[-num_neg:] = 1.0 return (labels, label_weights, bucket_cls_targets, bucket_cls_weights, bucket_offset_targets, bucket_offset_weights) def loss(self, cls_score: Tensor, bbox_pred: Tuple[Tensor, Tensor], rois: Tensor, labels: Tensor, label_weights: Tensor, bbox_targets: Tuple[Tensor, Tensor], bbox_weights: Tuple[Tensor, Tensor], reduction_override: Optional[str] = None) -> dict: """Calculate the loss based on the network predictions and targets. Args: cls_score (Tensor): Classification prediction results of all class, has shape (batch_size * num_proposals_single_image, num_classes) bbox_pred (Tensor): A tuple of regression prediction results containing `bucket_cls_preds and` `bucket_offset_preds`. rois (Tensor): RoIs with the shape (batch_size * num_proposals_single_image, 5) where the first column indicates batch id of each RoI. labels (Tensor): Gt_labels for all proposals in a batch, has shape (batch_size * num_proposals_single_image, ). label_weights (Tensor): Labels_weights for all proposals in a batch, has shape (batch_size * num_proposals_single_image, ). bbox_targets (Tuple[Tensor, Tensor]): A tuple of regression target containing `bucket_cls_targets` and `bucket_offset_targets`. the last dimension 4 represents [tl_x, tl_y, br_x, br_y]. bbox_weights (Tuple[Tensor, Tensor]): A tuple of regression weights containing `bucket_cls_weights` and `bucket_offset_weights`. reduction_override (str, optional): The reduction method used to override the original reduction method of the loss. Options are "none", "mean" and "sum". Defaults to None, Returns: dict: A dictionary of loss. """ losses = dict() if cls_score is not None: avg_factor = max(torch.sum(label_weights > 0).float().item(), 1.) losses['loss_cls'] = self.loss_cls( cls_score, labels, label_weights, avg_factor=avg_factor, reduction_override=reduction_override) losses['acc'] = accuracy(cls_score, labels) if bbox_pred is not None: bucket_cls_preds, bucket_offset_preds = bbox_pred bucket_cls_targets, bucket_offset_targets = bbox_targets bucket_cls_weights, bucket_offset_weights = bbox_weights # edge cls bucket_cls_preds = bucket_cls_preds.view(-1, self.side_num) bucket_cls_targets = bucket_cls_targets.view(-1, self.side_num) bucket_cls_weights = bucket_cls_weights.view(-1, self.side_num) losses['loss_bbox_cls'] = self.loss_bbox_cls( bucket_cls_preds, bucket_cls_targets, bucket_cls_weights, avg_factor=bucket_cls_targets.size(0), reduction_override=reduction_override) losses['loss_bbox_reg'] = self.loss_bbox_reg( bucket_offset_preds, bucket_offset_targets, bucket_offset_weights, avg_factor=bucket_offset_targets.size(0), reduction_override=reduction_override) return losses def _predict_by_feat_single( self, roi: Tensor, cls_score: Tensor, bbox_pred: Tuple[Tensor, Tensor], img_meta: dict, rescale: bool = False, rcnn_test_cfg: Optional[ConfigDict] = None) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: roi (Tensor): Boxes to be transformed. Has shape (num_boxes, 5). last dimension 5 arrange as (batch_index, x1, y1, x2, y2). cls_score (Tensor): Box scores, has shape (num_boxes, num_classes + 1). bbox_pred (Tuple[Tensor, Tensor]): Box cls preds and offset preds. img_meta (dict): image information. rescale (bool): If True, return boxes in original image space. Defaults to False. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of Bbox Head. Defaults to None Returns: :obj:`InstanceData`: Detection results of each image Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ results = InstanceData() if isinstance(cls_score, list): cls_score = sum(cls_score) / float(len(cls_score)) scores = F.softmax(cls_score, dim=1) if cls_score is not None else None img_shape = img_meta['img_shape'] if bbox_pred is not None: bboxes, confidences = self.bbox_coder.decode( roi[:, 1:], bbox_pred, img_shape) else: bboxes = roi[:, 1:].clone() confidences = None if img_shape is not None: bboxes[:, [0, 2]].clamp_(min=0, max=img_shape[1] - 1) bboxes[:, [1, 3]].clamp_(min=0, max=img_shape[0] - 1) if rescale and bboxes.size(0) > 0: assert img_meta.get('scale_factor') is not None scale_factor = bboxes.new_tensor(img_meta['scale_factor']).repeat( (1, 2)) bboxes = (bboxes.view(bboxes.size(0), -1, 4) / scale_factor).view( bboxes.size()[0], -1) if rcnn_test_cfg is None: results.bboxes = bboxes results.scores = scores else: det_bboxes, det_labels = multiclass_nms( bboxes, scores, rcnn_test_cfg.score_thr, rcnn_test_cfg.nms, rcnn_test_cfg.max_per_img, score_factors=confidences) results.bboxes = det_bboxes[:, :4] results.scores = det_bboxes[:, -1] results.labels = det_labels return results def refine_bboxes(self, sampling_results: List[SamplingResult], bbox_results: dict, batch_img_metas: List[dict]) -> InstanceList: """Refine bboxes during training. Args: sampling_results (List[:obj:`SamplingResult`]): Sampling results. bbox_results (dict): Usually is a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `rois` (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. - `bbox_targets` (tuple): Ground truth for proposals in a single image. Containing the following list of Tensors: (labels, label_weights, bbox_targets, bbox_weights) batch_img_metas (List[dict]): List of image information. Returns: list[:obj:`InstanceData`]: Refined bboxes of each image. """ pos_is_gts = [res.pos_is_gt for res in sampling_results] # bbox_targets is a tuple labels = bbox_results['bbox_targets'][0] cls_scores = bbox_results['cls_score'] rois = bbox_results['rois'] bbox_preds = bbox_results['bbox_pred'] if cls_scores.numel() == 0: return None labels = torch.where(labels == self.num_classes, cls_scores[:, :-1].argmax(1), labels) img_ids = rois[:, 0].long().unique(sorted=True) assert img_ids.numel() <= len(batch_img_metas) results_list = [] for i in range(len(batch_img_metas)): inds = torch.nonzero( rois[:, 0] == i, as_tuple=False).squeeze(dim=1) num_rois = inds.numel() bboxes_ = rois[inds, 1:] label_ = labels[inds] edge_cls_preds, edge_offset_preds = bbox_preds edge_cls_preds_ = edge_cls_preds[inds] edge_offset_preds_ = edge_offset_preds[inds] bbox_pred_ = (edge_cls_preds_, edge_offset_preds_) img_meta_ = batch_img_metas[i] pos_is_gts_ = pos_is_gts[i] bboxes = self.regress_by_class(bboxes_, label_, bbox_pred_, img_meta_) # filter gt bboxes pos_keep = 1 - pos_is_gts_ keep_inds = pos_is_gts_.new_ones(num_rois) keep_inds[:len(pos_is_gts_)] = pos_keep results = InstanceData(bboxes=bboxes[keep_inds.type(torch.bool)]) results_list.append(results) return results_list def regress_by_class(self, rois: Tensor, label: Tensor, bbox_pred: tuple, img_meta: dict) -> Tensor: """Regress the bbox for the predicted class. Used in Cascade R-CNN. Args: rois (Tensor): shape (n, 4) or (n, 5) label (Tensor): shape (n, ) bbox_pred (Tuple[Tensor]): shape [(n, num_buckets *2), \ (n, num_buckets *2)] img_meta (dict): Image meta info. Returns: Tensor: Regressed bboxes, the same shape as input rois. """ assert rois.size(1) == 4 or rois.size(1) == 5 if rois.size(1) == 4: new_rois, _ = self.bbox_coder.decode(rois, bbox_pred, img_meta['img_shape']) else: bboxes, _ = self.bbox_coder.decode(rois[:, 1:], bbox_pred, img_meta['img_shape']) new_rois = torch.cat((rois[:, [0]], bboxes), dim=1) return new_rois ================================================ FILE: mmdet/models/roi_heads/bbox_heads/scnet_bbox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple, Union from torch import Tensor from mmdet.registry import MODELS from .convfc_bbox_head import ConvFCBBoxHead @MODELS.register_module() class SCNetBBoxHead(ConvFCBBoxHead): """BBox head for `SCNet `_. This inherits ``ConvFCBBoxHead`` with modified forward() function, allow us to get intermediate shared feature. """ def _forward_shared(self, x: Tensor) -> Tensor: """Forward function for shared part. Args: x (Tensor): Input feature. Returns: Tensor: Shared feature. """ if self.num_shared_convs > 0: for conv in self.shared_convs: x = conv(x) if self.num_shared_fcs > 0: if self.with_avg_pool: x = self.avg_pool(x) x = x.flatten(1) for fc in self.shared_fcs: x = self.relu(fc(x)) return x def _forward_cls_reg(self, x: Tensor) -> Tuple[Tensor]: """Forward function for classification and regression parts. Args: x (Tensor): Input feature. Returns: tuple[Tensor]: - cls_score (Tensor): classification prediction. - bbox_pred (Tensor): bbox prediction. """ x_cls = x x_reg = x for conv in self.cls_convs: x_cls = conv(x_cls) if x_cls.dim() > 2: if self.with_avg_pool: x_cls = self.avg_pool(x_cls) x_cls = x_cls.flatten(1) for fc in self.cls_fcs: x_cls = self.relu(fc(x_cls)) for conv in self.reg_convs: x_reg = conv(x_reg) if x_reg.dim() > 2: if self.with_avg_pool: x_reg = self.avg_pool(x_reg) x_reg = x_reg.flatten(1) for fc in self.reg_fcs: x_reg = self.relu(fc(x_reg)) cls_score = self.fc_cls(x_cls) if self.with_cls else None bbox_pred = self.fc_reg(x_reg) if self.with_reg else None return cls_score, bbox_pred def forward( self, x: Tensor, return_shared_feat: bool = False) -> Union[Tensor, Tuple[Tensor]]: """Forward function. Args: x (Tensor): input features return_shared_feat (bool): If True, return cls-reg-shared feature. Return: out (tuple[Tensor]): contain ``cls_score`` and ``bbox_pred``, if ``return_shared_feat`` is True, append ``x_shared`` to the returned tuple. """ x_shared = self._forward_shared(x) out = self._forward_cls_reg(x_shared) if return_shared_feat: out += (x_shared, ) return out ================================================ FILE: mmdet/models/roi_heads/cascade_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Sequence, Tuple, Union import torch import torch.nn as nn from mmengine.model import ModuleList from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.task_modules.samplers import SamplingResult from mmdet.models.test_time_augs import merge_aug_masks from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox2roi, get_box_tensor from mmdet.utils import (ConfigType, InstanceList, MultiConfig, OptConfigType, OptMultiConfig) from ..utils.misc import empty_instances, unpack_gt_instances from .base_roi_head import BaseRoIHead @MODELS.register_module() class CascadeRoIHead(BaseRoIHead): """Cascade roi head including one bbox head and one mask head. https://arxiv.org/abs/1712.00726 """ def __init__(self, num_stages: int, stage_loss_weights: Union[List[float], Tuple[float]], bbox_roi_extractor: OptMultiConfig = None, bbox_head: OptMultiConfig = None, mask_roi_extractor: OptMultiConfig = None, mask_head: OptMultiConfig = None, shared_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: assert bbox_roi_extractor is not None assert bbox_head is not None assert shared_head is None, \ 'Shared head is not supported in Cascade RCNN anymore' self.num_stages = num_stages self.stage_loss_weights = stage_loss_weights super().__init__( bbox_roi_extractor=bbox_roi_extractor, bbox_head=bbox_head, mask_roi_extractor=mask_roi_extractor, mask_head=mask_head, shared_head=shared_head, train_cfg=train_cfg, test_cfg=test_cfg, init_cfg=init_cfg) def init_bbox_head(self, bbox_roi_extractor: MultiConfig, bbox_head: MultiConfig) -> None: """Initialize box head and box roi extractor. Args: bbox_roi_extractor (:obj:`ConfigDict`, dict or list): Config of box roi extractor. bbox_head (:obj:`ConfigDict`, dict or list): Config of box in box head. """ self.bbox_roi_extractor = ModuleList() self.bbox_head = ModuleList() if not isinstance(bbox_roi_extractor, list): bbox_roi_extractor = [ bbox_roi_extractor for _ in range(self.num_stages) ] if not isinstance(bbox_head, list): bbox_head = [bbox_head for _ in range(self.num_stages)] assert len(bbox_roi_extractor) == len(bbox_head) == self.num_stages for roi_extractor, head in zip(bbox_roi_extractor, bbox_head): self.bbox_roi_extractor.append(MODELS.build(roi_extractor)) self.bbox_head.append(MODELS.build(head)) def init_mask_head(self, mask_roi_extractor: MultiConfig, mask_head: MultiConfig) -> None: """Initialize mask head and mask roi extractor. Args: mask_head (dict): Config of mask in mask head. mask_roi_extractor (:obj:`ConfigDict`, dict or list): Config of mask roi extractor. """ self.mask_head = nn.ModuleList() if not isinstance(mask_head, list): mask_head = [mask_head for _ in range(self.num_stages)] assert len(mask_head) == self.num_stages for head in mask_head: self.mask_head.append(MODELS.build(head)) if mask_roi_extractor is not None: self.share_roi_extractor = False self.mask_roi_extractor = ModuleList() if not isinstance(mask_roi_extractor, list): mask_roi_extractor = [ mask_roi_extractor for _ in range(self.num_stages) ] assert len(mask_roi_extractor) == self.num_stages for roi_extractor in mask_roi_extractor: self.mask_roi_extractor.append(MODELS.build(roi_extractor)) else: self.share_roi_extractor = True self.mask_roi_extractor = self.bbox_roi_extractor def init_assigner_sampler(self) -> None: """Initialize assigner and sampler for each stage.""" self.bbox_assigner = [] self.bbox_sampler = [] if self.train_cfg is not None: for idx, rcnn_train_cfg in enumerate(self.train_cfg): self.bbox_assigner.append( TASK_UTILS.build(rcnn_train_cfg.assigner)) self.current_stage = idx self.bbox_sampler.append( TASK_UTILS.build( rcnn_train_cfg.sampler, default_args=dict(context=self))) def _bbox_forward(self, stage: int, x: Tuple[Tensor], rois: Tensor) -> dict: """Box head forward function used in both training and testing. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): List of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. """ bbox_roi_extractor = self.bbox_roi_extractor[stage] bbox_head = self.bbox_head[stage] bbox_feats = bbox_roi_extractor(x[:bbox_roi_extractor.num_inputs], rois) # do not support caffe_c4 model anymore cls_score, bbox_pred = bbox_head(bbox_feats) bbox_results = dict( cls_score=cls_score, bbox_pred=bbox_pred, bbox_feats=bbox_feats) return bbox_results def bbox_loss(self, stage: int, x: Tuple[Tensor], sampling_results: List[SamplingResult]) -> dict: """Run forward function and calculate loss for box head in training. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): List of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. Returns: dict: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. - `loss_bbox` (dict): A dictionary of bbox loss components. - `rois` (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. - `bbox_targets` (tuple): Ground truth for proposals in a single image. Containing the following list of Tensors: (labels, label_weights, bbox_targets, bbox_weights) """ bbox_head = self.bbox_head[stage] rois = bbox2roi([res.priors for res in sampling_results]) bbox_results = self._bbox_forward(stage, x, rois) bbox_results.update(rois=rois) bbox_loss_and_target = bbox_head.loss_and_target( cls_score=bbox_results['cls_score'], bbox_pred=bbox_results['bbox_pred'], rois=rois, sampling_results=sampling_results, rcnn_train_cfg=self.train_cfg[stage]) bbox_results.update(bbox_loss_and_target) return bbox_results def _mask_forward(self, stage: int, x: Tuple[Tensor], rois: Tensor) -> dict: """Mask head forward function used in both training and testing. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): Tuple of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. """ mask_roi_extractor = self.mask_roi_extractor[stage] mask_head = self.mask_head[stage] mask_feats = mask_roi_extractor(x[:mask_roi_extractor.num_inputs], rois) # do not support caffe_c4 model anymore mask_preds = mask_head(mask_feats) mask_results = dict(mask_preds=mask_preds) return mask_results def mask_loss(self, stage: int, x: Tuple[Tensor], sampling_results: List[SamplingResult], batch_gt_instances: InstanceList) -> dict: """Run forward function and calculate loss for mask head in training. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): Tuple of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. - `loss_mask` (dict): A dictionary of mask loss components. """ pos_rois = bbox2roi([res.pos_priors for res in sampling_results]) mask_results = self._mask_forward(stage, x, pos_rois) mask_head = self.mask_head[stage] mask_loss_and_target = mask_head.loss_and_target( mask_preds=mask_results['mask_preds'], sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, rcnn_train_cfg=self.train_cfg[stage]) mask_results.update(mask_loss_and_target) return mask_results def loss(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection roi on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components """ # TODO: May add a new function in baseroihead assert len(rpn_results_list) == len(batch_data_samples) outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, batch_img_metas \ = outputs num_imgs = len(batch_data_samples) losses = dict() results_list = rpn_results_list for stage in range(self.num_stages): self.current_stage = stage stage_loss_weight = self.stage_loss_weights[stage] # assign gts and sample proposals sampling_results = [] if self.with_bbox or self.with_mask: bbox_assigner = self.bbox_assigner[stage] bbox_sampler = self.bbox_sampler[stage] for i in range(num_imgs): results = results_list[i] # rename rpn_results.bboxes to rpn_results.priors results.priors = results.pop('bboxes') assign_result = bbox_assigner.assign( results, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = bbox_sampler.sample( assign_result, results, batch_gt_instances[i], feats=[lvl_feat[i][None] for lvl_feat in x]) sampling_results.append(sampling_result) # bbox head forward and loss bbox_results = self.bbox_loss(stage, x, sampling_results) for name, value in bbox_results['loss_bbox'].items(): losses[f's{stage}.{name}'] = ( value * stage_loss_weight if 'loss' in name else value) # mask head forward and loss if self.with_mask: mask_results = self.mask_loss(stage, x, sampling_results, batch_gt_instances) for name, value in mask_results['loss_mask'].items(): losses[f's{stage}.{name}'] = ( value * stage_loss_weight if 'loss' in name else value) # refine bboxes if stage < self.num_stages - 1: bbox_head = self.bbox_head[stage] with torch.no_grad(): results_list = bbox_head.refine_bboxes( sampling_results, bbox_results, batch_img_metas) # Empty proposal if results_list is None: break return losses def predict_bbox(self, x: Tuple[Tensor], batch_img_metas: List[dict], rpn_results_list: InstanceList, rcnn_test_cfg: ConfigType, rescale: bool = False, **kwargs) -> InstanceList: """Perform forward propagation of the bbox head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of R-CNN. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ proposals = [res.bboxes for res in rpn_results_list] num_proposals_per_img = tuple(len(p) for p in proposals) rois = bbox2roi(proposals) if rois.shape[0] == 0: return empty_instances( batch_img_metas, rois.device, task_type='bbox', box_type=self.bbox_head[-1].predict_box_type, num_classes=self.bbox_head[-1].num_classes, score_per_cls=rcnn_test_cfg is None) rois, cls_scores, bbox_preds = self._refine_roi( x=x, rois=rois, batch_img_metas=batch_img_metas, num_proposals_per_img=num_proposals_per_img, **kwargs) results_list = self.bbox_head[-1].predict_by_feat( rois=rois, cls_scores=cls_scores, bbox_preds=bbox_preds, batch_img_metas=batch_img_metas, rescale=rescale, rcnn_test_cfg=rcnn_test_cfg) return results_list def predict_mask(self, x: Tuple[Tensor], batch_img_metas: List[dict], results_list: List[InstanceData], rescale: bool = False) -> List[InstanceData]: """Perform forward propagation of the mask head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. results_list (list[:obj:`InstanceData`]): Detection results of each image. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ bboxes = [res.bboxes for res in results_list] mask_rois = bbox2roi(bboxes) if mask_rois.shape[0] == 0: results_list = empty_instances( batch_img_metas, mask_rois.device, task_type='mask', instance_results=results_list, mask_thr_binary=self.test_cfg.mask_thr_binary) return results_list num_mask_rois_per_img = [len(res) for res in results_list] aug_masks = [] for stage in range(self.num_stages): mask_results = self._mask_forward(stage, x, mask_rois) mask_preds = mask_results['mask_preds'] # split batch mask prediction back to each image mask_preds = mask_preds.split(num_mask_rois_per_img, 0) aug_masks.append([m.sigmoid().detach() for m in mask_preds]) merged_masks = [] for i in range(len(batch_img_metas)): aug_mask = [mask[i] for mask in aug_masks] merged_mask = merge_aug_masks(aug_mask, batch_img_metas[i]) merged_masks.append(merged_mask) results_list = self.mask_head[-1].predict_by_feat( mask_preds=merged_masks, results_list=results_list, batch_img_metas=batch_img_metas, rcnn_test_cfg=self.test_cfg, rescale=rescale, activate_map=True) return results_list def _refine_roi(self, x: Tuple[Tensor], rois: Tensor, batch_img_metas: List[dict], num_proposals_per_img: Sequence[int], **kwargs) -> tuple: """Multi-stage refinement of RoI. Args: x (tuple[Tensor]): List of multi-level img features. rois (Tensor): shape (n, 5), [batch_ind, x1, y1, x2, y2] batch_img_metas (list[dict]): List of image information. num_proposals_per_img (sequence[int]): number of proposals in each image. Returns: tuple: - rois (Tensor): Refined RoI. - cls_scores (list[Tensor]): Average predicted cls score per image. - bbox_preds (list[Tensor]): Bbox branch predictions for the last stage of per image. """ # "ms" in variable names means multi-stage ms_scores = [] for stage in range(self.num_stages): bbox_results = self._bbox_forward( stage=stage, x=x, rois=rois, **kwargs) # split batch bbox prediction back to each image cls_scores = bbox_results['cls_score'] bbox_preds = bbox_results['bbox_pred'] rois = rois.split(num_proposals_per_img, 0) cls_scores = cls_scores.split(num_proposals_per_img, 0) ms_scores.append(cls_scores) # some detector with_reg is False, bbox_preds will be None if bbox_preds is not None: # TODO move this to a sabl_roi_head # the bbox prediction of some detectors like SABL is not Tensor if isinstance(bbox_preds, torch.Tensor): bbox_preds = bbox_preds.split(num_proposals_per_img, 0) else: bbox_preds = self.bbox_head[stage].bbox_pred_split( bbox_preds, num_proposals_per_img) else: bbox_preds = (None, ) * len(batch_img_metas) if stage < self.num_stages - 1: bbox_head = self.bbox_head[stage] if bbox_head.custom_activation: cls_scores = [ bbox_head.loss_cls.get_activation(s) for s in cls_scores ] refine_rois_list = [] for i in range(len(batch_img_metas)): if rois[i].shape[0] > 0: bbox_label = cls_scores[i][:, :-1].argmax(dim=1) # Refactor `bbox_head.regress_by_class` to only accept # box tensor without img_idx concatenated. refined_bboxes = bbox_head.regress_by_class( rois[i][:, 1:], bbox_label, bbox_preds[i], batch_img_metas[i]) refined_bboxes = get_box_tensor(refined_bboxes) refined_rois = torch.cat( [rois[i][:, [0]], refined_bboxes], dim=1) refine_rois_list.append(refined_rois) rois = torch.cat(refine_rois_list) # average scores of each image by stages cls_scores = [ sum([score[i] for score in ms_scores]) / float(len(ms_scores)) for i in range(len(batch_img_metas)) ] return rois, cls_scores, bbox_preds def forward(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList) -> tuple: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: x (List[Tensor]): Multi-level features that may have different resolutions. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. Returns tuple: A tuple of features from ``bbox_head`` and ``mask_head`` forward. """ results = () batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] proposals = [rpn_results.bboxes for rpn_results in rpn_results_list] num_proposals_per_img = tuple(len(p) for p in proposals) rois = bbox2roi(proposals) # bbox head if self.with_bbox: rois, cls_scores, bbox_preds = self._refine_roi( x, rois, batch_img_metas, num_proposals_per_img) results = results + (cls_scores, bbox_preds) # mask head if self.with_mask: aug_masks = [] rois = torch.cat(rois) for stage in range(self.num_stages): mask_results = self._mask_forward(stage, x, rois) mask_preds = mask_results['mask_preds'] mask_preds = mask_preds.split(num_proposals_per_img, 0) aug_masks.append([m.sigmoid().detach() for m in mask_preds]) merged_masks = [] for i in range(len(batch_img_metas)): aug_mask = [mask[i] for mask in aug_masks] merged_mask = merge_aug_masks(aug_mask, batch_img_metas[i]) merged_masks.append(merged_mask) results = results + (merged_masks, ) return results ================================================ FILE: mmdet/models/roi_heads/double_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple from torch import Tensor from mmdet.registry import MODELS from .standard_roi_head import StandardRoIHead @MODELS.register_module() class DoubleHeadRoIHead(StandardRoIHead): """RoI head for `Double Head RCNN `_. Args: reg_roi_scale_factor (float): The scale factor to extend the rois used to extract the regression features. """ def __init__(self, reg_roi_scale_factor: float, **kwargs): super().__init__(**kwargs) self.reg_roi_scale_factor = reg_roi_scale_factor def _bbox_forward(self, x: Tuple[Tensor], rois: Tensor) -> dict: """Box head forward function used in both training and testing. Args: x (tuple[Tensor]): List of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. """ bbox_cls_feats = self.bbox_roi_extractor( x[:self.bbox_roi_extractor.num_inputs], rois) bbox_reg_feats = self.bbox_roi_extractor( x[:self.bbox_roi_extractor.num_inputs], rois, roi_scale_factor=self.reg_roi_scale_factor) if self.with_shared_head: bbox_cls_feats = self.shared_head(bbox_cls_feats) bbox_reg_feats = self.shared_head(bbox_reg_feats) cls_score, bbox_pred = self.bbox_head(bbox_cls_feats, bbox_reg_feats) bbox_results = dict( cls_score=cls_score, bbox_pred=bbox_pred, bbox_feats=bbox_cls_feats) return bbox_results ================================================ FILE: mmdet/models/roi_heads/dynamic_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import numpy as np import torch from torch import Tensor from mmdet.models.losses import SmoothL1Loss from mmdet.models.task_modules.samplers import SamplingResult from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox2roi from mmdet.utils import InstanceList from ..utils.misc import unpack_gt_instances from .standard_roi_head import StandardRoIHead EPS = 1e-15 @MODELS.register_module() class DynamicRoIHead(StandardRoIHead): """RoI head for `Dynamic R-CNN `_.""" def __init__(self, **kwargs) -> None: super().__init__(**kwargs) assert isinstance(self.bbox_head.loss_bbox, SmoothL1Loss) # the IoU history of the past `update_iter_interval` iterations self.iou_history = [] # the beta history of the past `update_iter_interval` iterations self.beta_history = [] def loss(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList) -> dict: """Forward function for training. Args: x (tuple[Tensor]): List of multi-level img features. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: a dictionary of loss components """ assert len(rpn_results_list) == len(batch_data_samples) outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, _ = outputs # assign gts and sample proposals num_imgs = len(batch_data_samples) sampling_results = [] cur_iou = [] for i in range(num_imgs): # rename rpn_results.bboxes to rpn_results.priors rpn_results = rpn_results_list[i] rpn_results.priors = rpn_results.pop('bboxes') assign_result = self.bbox_assigner.assign( rpn_results, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = self.bbox_sampler.sample( assign_result, rpn_results, batch_gt_instances[i], feats=[lvl_feat[i][None] for lvl_feat in x]) # record the `iou_topk`-th largest IoU in an image iou_topk = min(self.train_cfg.dynamic_rcnn.iou_topk, len(assign_result.max_overlaps)) ious, _ = torch.topk(assign_result.max_overlaps, iou_topk) cur_iou.append(ious[-1].item()) sampling_results.append(sampling_result) # average the current IoUs over images cur_iou = np.mean(cur_iou) self.iou_history.append(cur_iou) losses = dict() # bbox head forward and loss if self.with_bbox: bbox_results = self.bbox_loss(x, sampling_results) losses.update(bbox_results['loss_bbox']) # mask head forward and loss if self.with_mask: mask_results = self.mask_loss(x, sampling_results, bbox_results['bbox_feats'], batch_gt_instances) losses.update(mask_results['loss_mask']) # update IoU threshold and SmoothL1 beta update_iter_interval = self.train_cfg.dynamic_rcnn.update_iter_interval if len(self.iou_history) % update_iter_interval == 0: new_iou_thr, new_beta = self.update_hyperparameters() return losses def bbox_loss(self, x: Tuple[Tensor], sampling_results: List[SamplingResult]) -> dict: """Perform forward propagation and loss calculation of the bbox head on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. - `loss_bbox` (dict): A dictionary of bbox loss components. """ rois = bbox2roi([res.priors for res in sampling_results]) bbox_results = self._bbox_forward(x, rois) bbox_loss_and_target = self.bbox_head.loss_and_target( cls_score=bbox_results['cls_score'], bbox_pred=bbox_results['bbox_pred'], rois=rois, sampling_results=sampling_results, rcnn_train_cfg=self.train_cfg) bbox_results.update(loss_bbox=bbox_loss_and_target['loss_bbox']) # record the `beta_topk`-th smallest target # `bbox_targets[2]` and `bbox_targets[3]` stand for bbox_targets # and bbox_weights, respectively bbox_targets = bbox_loss_and_target['bbox_targets'] pos_inds = bbox_targets[3][:, 0].nonzero().squeeze(1) num_pos = len(pos_inds) num_imgs = len(sampling_results) if num_pos > 0: cur_target = bbox_targets[2][pos_inds, :2].abs().mean(dim=1) beta_topk = min(self.train_cfg.dynamic_rcnn.beta_topk * num_imgs, num_pos) cur_target = torch.kthvalue(cur_target, beta_topk)[0].item() self.beta_history.append(cur_target) return bbox_results def update_hyperparameters(self): """Update hyperparameters like IoU thresholds for assigner and beta for SmoothL1 loss based on the training statistics. Returns: tuple[float]: the updated ``iou_thr`` and ``beta``. """ new_iou_thr = max(self.train_cfg.dynamic_rcnn.initial_iou, np.mean(self.iou_history)) self.iou_history = [] self.bbox_assigner.pos_iou_thr = new_iou_thr self.bbox_assigner.neg_iou_thr = new_iou_thr self.bbox_assigner.min_pos_iou = new_iou_thr if (not self.beta_history) or (np.median(self.beta_history) < EPS): # avoid 0 or too small value for new_beta new_beta = self.bbox_head.loss_bbox.beta else: new_beta = min(self.train_cfg.dynamic_rcnn.initial_beta, np.median(self.beta_history)) self.beta_history = [] self.bbox_head.loss_bbox.beta = new_beta return new_iou_thr, new_beta ================================================ FILE: mmdet/models/roi_heads/grid_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox2roi from mmdet.utils import ConfigType, InstanceList from ..task_modules.samplers import SamplingResult from ..utils.misc import unpack_gt_instances from .standard_roi_head import StandardRoIHead @MODELS.register_module() class GridRoIHead(StandardRoIHead): """Implementation of `Grid RoI Head `_ Args: grid_roi_extractor (:obj:`ConfigDict` or dict): Config of roi extractor. grid_head (:obj:`ConfigDict` or dict): Config of grid head """ def __init__(self, grid_roi_extractor: ConfigType, grid_head: ConfigType, **kwargs) -> None: assert grid_head is not None super().__init__(**kwargs) if grid_roi_extractor is not None: self.grid_roi_extractor = MODELS.build(grid_roi_extractor) self.share_roi_extractor = False else: self.share_roi_extractor = True self.grid_roi_extractor = self.bbox_roi_extractor self.grid_head = MODELS.build(grid_head) def _random_jitter(self, sampling_results: List[SamplingResult], batch_img_metas: List[dict], amplitude: float = 0.15) -> List[SamplingResult]: """Ramdom jitter positive proposals for training. Args: sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. batch_img_metas (list[dict]): List of image information. amplitude (float): Amplitude of random offset. Defaults to 0.15. Returns: list[obj:SamplingResult]: SamplingResults after random jittering. """ for sampling_result, img_meta in zip(sampling_results, batch_img_metas): bboxes = sampling_result.pos_priors random_offsets = bboxes.new_empty(bboxes.shape[0], 4).uniform_( -amplitude, amplitude) # before jittering cxcy = (bboxes[:, 2:4] + bboxes[:, :2]) / 2 wh = (bboxes[:, 2:4] - bboxes[:, :2]).abs() # after jittering new_cxcy = cxcy + wh * random_offsets[:, :2] new_wh = wh * (1 + random_offsets[:, 2:]) # xywh to xyxy new_x1y1 = (new_cxcy - new_wh / 2) new_x2y2 = (new_cxcy + new_wh / 2) new_bboxes = torch.cat([new_x1y1, new_x2y2], dim=1) # clip bboxes max_shape = img_meta['img_shape'] if max_shape is not None: new_bboxes[:, 0::2].clamp_(min=0, max=max_shape[1] - 1) new_bboxes[:, 1::2].clamp_(min=0, max=max_shape[0] - 1) sampling_result.pos_priors = new_bboxes return sampling_results # TODO: Forward is incorrect and need to refactor. def forward(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList = None) -> tuple: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: x (Tuple[Tensor]): Multi-level features that may have different resolutions. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. Returns tuple: A tuple of features from ``bbox_head`` and ``mask_head`` forward. """ results = () proposals = [rpn_results.bboxes for rpn_results in rpn_results_list] rois = bbox2roi(proposals) # bbox head if self.with_bbox: bbox_results = self._bbox_forward(x, rois) results = results + (bbox_results['cls_score'], ) if self.bbox_head.with_reg: results = results + (bbox_results['bbox_pred'], ) # grid head grid_rois = rois[:100] grid_feats = self.grid_roi_extractor( x[:len(self.grid_roi_extractor.featmap_strides)], grid_rois) if self.with_shared_head: grid_feats = self.shared_head(grid_feats) self.grid_head.test_mode = True grid_preds = self.grid_head(grid_feats) results = results + (grid_preds, ) # mask head if self.with_mask: mask_rois = rois[:100] mask_results = self._mask_forward(x, mask_rois) results = results + (mask_results['mask_preds'], ) return results def loss(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList, **kwargs) -> dict: """Perform forward propagation and loss calculation of the detection roi on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components """ assert len(rpn_results_list) == len(batch_data_samples) outputs = unpack_gt_instances(batch_data_samples) (batch_gt_instances, batch_gt_instances_ignore, batch_img_metas) = outputs # assign gts and sample proposals num_imgs = len(batch_data_samples) sampling_results = [] for i in range(num_imgs): # rename rpn_results.bboxes to rpn_results.priors rpn_results = rpn_results_list[i] rpn_results.priors = rpn_results.pop('bboxes') assign_result = self.bbox_assigner.assign( rpn_results, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = self.bbox_sampler.sample( assign_result, rpn_results, batch_gt_instances[i], feats=[lvl_feat[i][None] for lvl_feat in x]) sampling_results.append(sampling_result) losses = dict() # bbox head loss if self.with_bbox: bbox_results = self.bbox_loss(x, sampling_results, batch_img_metas) losses.update(bbox_results['loss_bbox']) # mask head forward and loss if self.with_mask: mask_results = self.mask_loss(x, sampling_results, bbox_results['bbox_feats'], batch_gt_instances) losses.update(mask_results['loss_mask']) return losses def bbox_loss(self, x: Tuple[Tensor], sampling_results: List[SamplingResult], batch_img_metas: Optional[List[dict]] = None) -> dict: """Perform forward propagation and loss calculation of the bbox head on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. sampling_results (list[:obj:`SamplingResult`]): Sampling results. batch_img_metas (list[dict], optional): Meta information of each image, e.g., image size, scaling factor, etc. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. - `loss_bbox` (dict): A dictionary of bbox loss components. """ assert batch_img_metas is not None bbox_results = super().bbox_loss(x, sampling_results) # Grid head forward and loss sampling_results = self._random_jitter(sampling_results, batch_img_metas) pos_rois = bbox2roi([res.pos_bboxes for res in sampling_results]) # GN in head does not support zero shape input if pos_rois.shape[0] == 0: return bbox_results grid_feats = self.grid_roi_extractor( x[:self.grid_roi_extractor.num_inputs], pos_rois) if self.with_shared_head: grid_feats = self.shared_head(grid_feats) # Accelerate training max_sample_num_grid = self.train_cfg.get('max_num_grid', 192) sample_idx = torch.randperm( grid_feats.shape[0])[:min(grid_feats.shape[0], max_sample_num_grid )] grid_feats = grid_feats[sample_idx] grid_pred = self.grid_head(grid_feats) loss_grid = self.grid_head.loss(grid_pred, sample_idx, sampling_results, self.train_cfg) bbox_results['loss_bbox'].update(loss_grid) return bbox_results def predict_bbox(self, x: Tuple[Tensor], batch_img_metas: List[dict], rpn_results_list: InstanceList, rcnn_test_cfg: ConfigType, rescale: bool = False) -> InstanceList: """Perform forward propagation of the bbox head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. rcnn_test_cfg (:obj:`ConfigDict`): `test_cfg` of R-CNN. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape \ (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last \ dimension 4 arrange as (x1, y1, x2, y2). """ results_list = super().predict_bbox( x, batch_img_metas=batch_img_metas, rpn_results_list=rpn_results_list, rcnn_test_cfg=rcnn_test_cfg, rescale=False) grid_rois = bbox2roi([res.bboxes for res in results_list]) if grid_rois.shape[0] != 0: grid_feats = self.grid_roi_extractor( x[:len(self.grid_roi_extractor.featmap_strides)], grid_rois) if self.with_shared_head: grid_feats = self.shared_head(grid_feats) self.grid_head.test_mode = True grid_preds = self.grid_head(grid_feats) results_list = self.grid_head.predict_by_feat( grid_preds=grid_preds, results_list=results_list, batch_img_metas=batch_img_metas, rescale=rescale) return results_list ================================================ FILE: mmdet/models/roi_heads/htc_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Optional, Tuple import torch import torch.nn.functional as F from torch import Tensor from mmdet.models.test_time_augs import merge_aug_masks from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox2roi from mmdet.utils import InstanceList, OptConfigType from ..layers import adaptive_avg_pool2d from ..task_modules.samplers import SamplingResult from ..utils import empty_instances, unpack_gt_instances from .cascade_roi_head import CascadeRoIHead @MODELS.register_module() class HybridTaskCascadeRoIHead(CascadeRoIHead): """Hybrid task cascade roi head including one bbox head and one mask head. https://arxiv.org/abs/1901.07518 Args: num_stages (int): Number of cascade stages. stage_loss_weights (list[float]): Loss weight for every stage. semantic_roi_extractor (:obj:`ConfigDict` or dict, optional): Config of semantic roi extractor. Defaults to None. Semantic_head (:obj:`ConfigDict` or dict, optional): Config of semantic head. Defaults to None. interleaved (bool): Whether to interleaves the box branch and mask branch. If True, the mask branch can take the refined bounding box predictions. Defaults to True. mask_info_flow (bool): Whether to turn on the mask information flow, which means that feeding the mask features of the preceding stage to the current stage. Defaults to True. """ def __init__(self, num_stages: int, stage_loss_weights: List[float], semantic_roi_extractor: OptConfigType = None, semantic_head: OptConfigType = None, semantic_fusion: Tuple[str] = ('bbox', 'mask'), interleaved: bool = True, mask_info_flow: bool = True, **kwargs) -> None: super().__init__( num_stages=num_stages, stage_loss_weights=stage_loss_weights, **kwargs) assert self.with_bbox assert not self.with_shared_head # shared head is not supported if semantic_head is not None: self.semantic_roi_extractor = MODELS.build(semantic_roi_extractor) self.semantic_head = MODELS.build(semantic_head) self.semantic_fusion = semantic_fusion self.interleaved = interleaved self.mask_info_flow = mask_info_flow # TODO move to base_roi_head later @property def with_semantic(self) -> bool: """bool: whether the head has semantic head""" return hasattr(self, 'semantic_head') and self.semantic_head is not None def _bbox_forward( self, stage: int, x: Tuple[Tensor], rois: Tensor, semantic_feat: Optional[Tensor] = None) -> Dict[str, Tensor]: """Box head forward function used in both training and testing. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): List of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. semantic_feat (Tensor, optional): Semantic feature. Defaults to None. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. """ bbox_roi_extractor = self.bbox_roi_extractor[stage] bbox_head = self.bbox_head[stage] bbox_feats = bbox_roi_extractor(x[:bbox_roi_extractor.num_inputs], rois) if self.with_semantic and 'bbox' in self.semantic_fusion: bbox_semantic_feat = self.semantic_roi_extractor([semantic_feat], rois) if bbox_semantic_feat.shape[-2:] != bbox_feats.shape[-2:]: bbox_semantic_feat = adaptive_avg_pool2d( bbox_semantic_feat, bbox_feats.shape[-2:]) bbox_feats += bbox_semantic_feat cls_score, bbox_pred = bbox_head(bbox_feats) bbox_results = dict(cls_score=cls_score, bbox_pred=bbox_pred) return bbox_results def bbox_loss(self, stage: int, x: Tuple[Tensor], sampling_results: List[SamplingResult], semantic_feat: Optional[Tensor] = None) -> dict: """Run forward function and calculate loss for box head in training. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): List of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. semantic_feat (Tensor, optional): Semantic feature. Defaults to None. Returns: dict: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. - `loss_bbox` (dict): A dictionary of bbox loss components. - `rois` (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. - `bbox_targets` (tuple): Ground truth for proposals in a single image. Containing the following list of Tensors: (labels, label_weights, bbox_targets, bbox_weights) """ bbox_head = self.bbox_head[stage] rois = bbox2roi([res.priors for res in sampling_results]) bbox_results = self._bbox_forward( stage, x, rois, semantic_feat=semantic_feat) bbox_results.update(rois=rois) bbox_loss_and_target = bbox_head.loss_and_target( cls_score=bbox_results['cls_score'], bbox_pred=bbox_results['bbox_pred'], rois=rois, sampling_results=sampling_results, rcnn_train_cfg=self.train_cfg[stage]) bbox_results.update(bbox_loss_and_target) return bbox_results def _mask_forward(self, stage: int, x: Tuple[Tensor], rois: Tensor, semantic_feat: Optional[Tensor] = None, training: bool = True) -> Dict[str, Tensor]: """Mask head forward function used only in training. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): Tuple of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. semantic_feat (Tensor, optional): Semantic feature. Defaults to None. training (bool): Mask Forward is different between training and testing. If True, use the mask forward in training. Defaults to True. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. """ mask_roi_extractor = self.mask_roi_extractor[stage] mask_head = self.mask_head[stage] mask_feats = mask_roi_extractor(x[:mask_roi_extractor.num_inputs], rois) # semantic feature fusion # element-wise sum for original features and pooled semantic features if self.with_semantic and 'mask' in self.semantic_fusion: mask_semantic_feat = self.semantic_roi_extractor([semantic_feat], rois) if mask_semantic_feat.shape[-2:] != mask_feats.shape[-2:]: mask_semantic_feat = F.adaptive_avg_pool2d( mask_semantic_feat, mask_feats.shape[-2:]) mask_feats = mask_feats + mask_semantic_feat # mask information flow # forward all previous mask heads to obtain last_feat, and fuse it # with the normal mask feature if training: if self.mask_info_flow: last_feat = None for i in range(stage): last_feat = self.mask_head[i]( mask_feats, last_feat, return_logits=False) mask_preds = mask_head( mask_feats, last_feat, return_feat=False) else: mask_preds = mask_head(mask_feats, return_feat=False) mask_results = dict(mask_preds=mask_preds) else: aug_masks = [] last_feat = None for i in range(self.num_stages): mask_head = self.mask_head[i] if self.mask_info_flow: mask_preds, last_feat = mask_head(mask_feats, last_feat) else: mask_preds = mask_head(mask_feats) aug_masks.append(mask_preds) mask_results = dict(mask_preds=aug_masks) return mask_results def mask_loss(self, stage: int, x: Tuple[Tensor], sampling_results: List[SamplingResult], batch_gt_instances: InstanceList, semantic_feat: Optional[Tensor] = None) -> dict: """Run forward function and calculate loss for mask head in training. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): Tuple of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. semantic_feat (Tensor, optional): Semantic feature. Defaults to None. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. - `loss_mask` (dict): A dictionary of mask loss components. """ pos_rois = bbox2roi([res.pos_priors for res in sampling_results]) mask_results = self._mask_forward( stage=stage, x=x, rois=pos_rois, semantic_feat=semantic_feat, training=True) mask_head = self.mask_head[stage] mask_loss_and_target = mask_head.loss_and_target( mask_preds=mask_results['mask_preds'], sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, rcnn_train_cfg=self.train_cfg[stage]) mask_results.update(mask_loss_and_target) return mask_results def loss(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection roi on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components """ assert len(rpn_results_list) == len(batch_data_samples) outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, batch_img_metas \ = outputs # semantic segmentation part # 2 outputs: segmentation prediction and embedded features losses = dict() if self.with_semantic: gt_semantic_segs = [ data_sample.gt_sem_seg.sem_seg for data_sample in batch_data_samples ] gt_semantic_segs = torch.stack(gt_semantic_segs) semantic_pred, semantic_feat = self.semantic_head(x) loss_seg = self.semantic_head.loss(semantic_pred, gt_semantic_segs) losses['loss_semantic_seg'] = loss_seg else: semantic_feat = None results_list = rpn_results_list num_imgs = len(batch_img_metas) for stage in range(self.num_stages): self.current_stage = stage stage_loss_weight = self.stage_loss_weights[stage] # assign gts and sample proposals sampling_results = [] bbox_assigner = self.bbox_assigner[stage] bbox_sampler = self.bbox_sampler[stage] for i in range(num_imgs): results = results_list[i] # rename rpn_results.bboxes to rpn_results.priors if 'bboxes' in results: results.priors = results.pop('bboxes') assign_result = bbox_assigner.assign( results, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = bbox_sampler.sample( assign_result, results, batch_gt_instances[i], feats=[lvl_feat[i][None] for lvl_feat in x]) sampling_results.append(sampling_result) # bbox head forward and loss bbox_results = self.bbox_loss( stage=stage, x=x, sampling_results=sampling_results, semantic_feat=semantic_feat) for name, value in bbox_results['loss_bbox'].items(): losses[f's{stage}.{name}'] = ( value * stage_loss_weight if 'loss' in name else value) # mask head forward and loss if self.with_mask: # interleaved execution: use regressed bboxes by the box branch # to train the mask branch if self.interleaved: bbox_head = self.bbox_head[stage] with torch.no_grad(): results_list = bbox_head.refine_bboxes( sampling_results, bbox_results, batch_img_metas) # re-assign and sample 512 RoIs from 512 RoIs sampling_results = [] for i in range(num_imgs): results = results_list[i] # rename rpn_results.bboxes to rpn_results.priors results.priors = results.pop('bboxes') assign_result = bbox_assigner.assign( results, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = bbox_sampler.sample( assign_result, results, batch_gt_instances[i], feats=[lvl_feat[i][None] for lvl_feat in x]) sampling_results.append(sampling_result) mask_results = self.mask_loss( stage=stage, x=x, sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, semantic_feat=semantic_feat) for name, value in mask_results['loss_mask'].items(): losses[f's{stage}.{name}'] = ( value * stage_loss_weight if 'loss' in name else value) # refine bboxes (same as Cascade R-CNN) if stage < self.num_stages - 1 and not self.interleaved: bbox_head = self.bbox_head[stage] with torch.no_grad(): results_list = bbox_head.refine_bboxes( sampling_results=sampling_results, bbox_results=bbox_results, batch_img_metas=batch_img_metas) return losses def predict(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the roi head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Features from upstream network. Each has shape (N, C, H, W). rpn_results_list (list[:obj:`InstanceData`]): list of region proposals. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results to the original image. Defaults to False. Returns: list[obj:`InstanceData`]: Detection results of each image. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ assert self.with_bbox, 'Bbox head must be implemented.' batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] if self.with_semantic: _, semantic_feat = self.semantic_head(x) else: semantic_feat = None # TODO: nms_op in mmcv need be enhanced, the bbox result may get # difference when not rescale in bbox_head # If it has the mask branch, the bbox branch does not need # to be scaled to the original image scale, because the mask # branch will scale both bbox and mask at the same time. bbox_rescale = rescale if not self.with_mask else False results_list = self.predict_bbox( x=x, semantic_feat=semantic_feat, batch_img_metas=batch_img_metas, rpn_results_list=rpn_results_list, rcnn_test_cfg=self.test_cfg, rescale=bbox_rescale) if self.with_mask: results_list = self.predict_mask( x=x, semantic_heat=semantic_feat, batch_img_metas=batch_img_metas, results_list=results_list, rescale=rescale) return results_list def predict_mask(self, x: Tuple[Tensor], semantic_heat: Tensor, batch_img_metas: List[dict], results_list: InstanceList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the mask head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. semantic_feat (Tensor): Semantic feature. batch_img_metas (list[dict]): List of image information. results_list (list[:obj:`InstanceData`]): Detection results of each image. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ num_imgs = len(batch_img_metas) bboxes = [res.bboxes for res in results_list] mask_rois = bbox2roi(bboxes) if mask_rois.shape[0] == 0: results_list = empty_instances( batch_img_metas=batch_img_metas, device=mask_rois.device, task_type='mask', instance_results=results_list, mask_thr_binary=self.test_cfg.mask_thr_binary) return results_list num_mask_rois_per_img = [len(res) for res in results_list] mask_results = self._mask_forward( stage=-1, x=x, rois=mask_rois, semantic_feat=semantic_heat, training=False) # split batch mask prediction back to each image aug_masks = [[ mask.sigmoid().detach() for mask in mask_preds.split(num_mask_rois_per_img, 0) ] for mask_preds in mask_results['mask_preds']] merged_masks = [] for i in range(num_imgs): aug_mask = [mask[i] for mask in aug_masks] merged_mask = merge_aug_masks(aug_mask, batch_img_metas[i]) merged_masks.append(merged_mask) results_list = self.mask_head[-1].predict_by_feat( mask_preds=merged_masks, results_list=results_list, batch_img_metas=batch_img_metas, rcnn_test_cfg=self.test_cfg, rescale=rescale, activate_map=True) return results_list def forward(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList) -> tuple: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: x (List[Tensor]): Multi-level features that may have different resolutions. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. Returns tuple: A tuple of features from ``bbox_head`` and ``mask_head`` forward. """ results = () batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] num_imgs = len(batch_img_metas) if self.with_semantic: _, semantic_feat = self.semantic_head(x) else: semantic_feat = None proposals = [rpn_results.bboxes for rpn_results in rpn_results_list] num_proposals_per_img = tuple(len(p) for p in proposals) rois = bbox2roi(proposals) # bbox head if self.with_bbox: rois, cls_scores, bbox_preds = self._refine_roi( x=x, rois=rois, semantic_feat=semantic_feat, batch_img_metas=batch_img_metas, num_proposals_per_img=num_proposals_per_img) results = results + (cls_scores, bbox_preds) # mask head if self.with_mask: rois = torch.cat(rois) mask_results = self._mask_forward( stage=-1, x=x, rois=rois, semantic_feat=semantic_feat, training=False) aug_masks = [[ mask.sigmoid().detach() for mask in mask_preds.split(num_proposals_per_img, 0) ] for mask_preds in mask_results['mask_preds']] merged_masks = [] for i in range(num_imgs): aug_mask = [mask[i] for mask in aug_masks] merged_mask = merge_aug_masks(aug_mask, batch_img_metas[i]) merged_masks.append(merged_mask) results = results + (merged_masks, ) return results ================================================ FILE: mmdet/models/roi_heads/mask_heads/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .coarse_mask_head import CoarseMaskHead from .dynamic_mask_head import DynamicMaskHead from .fcn_mask_head import FCNMaskHead from .feature_relay_head import FeatureRelayHead from .fused_semantic_head import FusedSemanticHead from .global_context_head import GlobalContextHead from .grid_head import GridHead from .htc_mask_head import HTCMaskHead from .mask_point_head import MaskPointHead from .maskiou_head import MaskIoUHead from .scnet_mask_head import SCNetMaskHead from .scnet_semantic_head import SCNetSemanticHead __all__ = [ 'FCNMaskHead', 'HTCMaskHead', 'FusedSemanticHead', 'GridHead', 'MaskIoUHead', 'CoarseMaskHead', 'MaskPointHead', 'SCNetMaskHead', 'SCNetSemanticHead', 'GlobalContextHead', 'FeatureRelayHead', 'DynamicMaskHead' ] ================================================ FILE: mmdet/models/roi_heads/mask_heads/coarse_mask_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmcv.cnn import ConvModule, Linear from mmengine.model import ModuleList from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import MultiConfig from .fcn_mask_head import FCNMaskHead @MODELS.register_module() class CoarseMaskHead(FCNMaskHead): """Coarse mask head used in PointRend. Compared with standard ``FCNMaskHead``, ``CoarseMaskHead`` will downsample the input feature map instead of upsample it. Args: num_convs (int): Number of conv layers in the head. Defaults to 0. num_fcs (int): Number of fc layers in the head. Defaults to 2. fc_out_channels (int): Number of output channels of fc layer. Defaults to 1024. downsample_factor (int): The factor that feature map is downsampled by. Defaults to 2. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, num_convs: int = 0, num_fcs: int = 2, fc_out_channels: int = 1024, downsample_factor: int = 2, init_cfg: MultiConfig = dict( type='Xavier', override=[ dict(name='fcs'), dict(type='Constant', val=0.001, name='fc_logits') ]), *arg, **kwarg) -> None: super().__init__( *arg, num_convs=num_convs, upsample_cfg=dict(type=None), init_cfg=None, **kwarg) self.init_cfg = init_cfg self.num_fcs = num_fcs assert self.num_fcs > 0 self.fc_out_channels = fc_out_channels self.downsample_factor = downsample_factor assert self.downsample_factor >= 1 # remove conv_logit delattr(self, 'conv_logits') if downsample_factor > 1: downsample_in_channels = ( self.conv_out_channels if self.num_convs > 0 else self.in_channels) self.downsample_conv = ConvModule( downsample_in_channels, self.conv_out_channels, kernel_size=downsample_factor, stride=downsample_factor, padding=0, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg) else: self.downsample_conv = None self.output_size = (self.roi_feat_size[0] // downsample_factor, self.roi_feat_size[1] // downsample_factor) self.output_area = self.output_size[0] * self.output_size[1] last_layer_dim = self.conv_out_channels * self.output_area self.fcs = ModuleList() for i in range(num_fcs): fc_in_channels = ( last_layer_dim if i == 0 else self.fc_out_channels) self.fcs.append(Linear(fc_in_channels, self.fc_out_channels)) last_layer_dim = self.fc_out_channels output_channels = self.num_classes * self.output_area self.fc_logits = Linear(last_layer_dim, output_channels) def init_weights(self) -> None: """Initialize weights.""" super(FCNMaskHead, self).init_weights() def forward(self, x: Tensor) -> Tensor: """Forward features from the upstream network. Args: x (Tensor): Extract mask RoI features. Returns: Tensor: Predicted foreground masks. """ for conv in self.convs: x = conv(x) if self.downsample_conv is not None: x = self.downsample_conv(x) x = x.flatten(1) for fc in self.fcs: x = self.relu(fc(x)) mask_preds = self.fc_logits(x).view( x.size(0), self.num_classes, *self.output_size) return mask_preds ================================================ FILE: mmdet/models/roi_heads/mask_heads/dynamic_mask_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List import torch import torch.nn as nn from mmengine.config import ConfigDict from torch import Tensor from mmdet.models.task_modules import SamplingResult from mmdet.registry import MODELS from mmdet.utils import ConfigType, InstanceList, OptConfigType, reduce_mean from .fcn_mask_head import FCNMaskHead @MODELS.register_module() class DynamicMaskHead(FCNMaskHead): r"""Dynamic Mask Head for `Instances as Queries `_ Args: num_convs (int): Number of convolution layer. Defaults to 4. roi_feat_size (int): The output size of RoI extractor, Defaults to 14. in_channels (int): Input feature channels. Defaults to 256. conv_kernel_size (int): Kernel size of convolution layers. Defaults to 3. conv_out_channels (int): Output channels of convolution layers. Defaults to 256. num_classes (int): Number of classes. Defaults to 80 class_agnostic (int): Whether generate class agnostic prediction. Defaults to False. dropout (float): Probability of drop the channel. Defaults to 0.0 upsample_cfg (:obj:`ConfigDict` or dict): The config for upsample layer. conv_cfg (:obj:`ConfigDict` or dict, optional): The convolution layer config. norm_cfg (:obj:`ConfigDict` or dict, optional): The norm layer config. dynamic_conv_cfg (:obj:`ConfigDict` or dict): The dynamic convolution layer config. loss_mask (:obj:`ConfigDict` or dict): The config for mask loss. """ def __init__(self, num_convs: int = 4, roi_feat_size: int = 14, in_channels: int = 256, conv_kernel_size: int = 3, conv_out_channels: int = 256, num_classes: int = 80, class_agnostic: bool = False, upsample_cfg: ConfigType = dict( type='deconv', scale_factor=2), conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, dynamic_conv_cfg: ConfigType = dict( type='DynamicConv', in_channels=256, feat_channels=64, out_channels=256, input_feat_shape=14, with_proj=False, act_cfg=dict(type='ReLU', inplace=True), norm_cfg=dict(type='LN')), loss_mask: ConfigType = dict( type='DiceLoss', loss_weight=8.0), **kwargs) -> None: super().__init__( num_convs=num_convs, roi_feat_size=roi_feat_size, in_channels=in_channels, conv_kernel_size=conv_kernel_size, conv_out_channels=conv_out_channels, num_classes=num_classes, class_agnostic=class_agnostic, upsample_cfg=upsample_cfg, conv_cfg=conv_cfg, norm_cfg=norm_cfg, loss_mask=loss_mask, **kwargs) assert class_agnostic is False, \ 'DynamicMaskHead only support class_agnostic=False' self.fp16_enabled = False self.instance_interactive_conv = MODELS.build(dynamic_conv_cfg) def init_weights(self) -> None: """Use xavier initialization for all weight parameter and set classification head bias as a specific value when use focal loss.""" for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) nn.init.constant_(self.conv_logits.bias, 0.) def forward(self, roi_feat: Tensor, proposal_feat: Tensor) -> Tensor: """Forward function of DynamicMaskHead. Args: roi_feat (Tensor): Roi-pooling features with shape (batch_size*num_proposals, feature_dimensions, pooling_h , pooling_w). proposal_feat (Tensor): Intermediate feature get from diihead in last stage, has shape (batch_size*num_proposals, feature_dimensions) Returns: mask_preds (Tensor): Predicted foreground masks with shape (batch_size*num_proposals, num_classes, pooling_h*2, pooling_w*2). """ proposal_feat = proposal_feat.reshape(-1, self.in_channels) proposal_feat_iic = self.instance_interactive_conv( proposal_feat, roi_feat) x = proposal_feat_iic.permute(0, 2, 1).reshape(roi_feat.size()) for conv in self.convs: x = conv(x) if self.upsample is not None: x = self.upsample(x) if self.upsample_method == 'deconv': x = self.relu(x) mask_preds = self.conv_logits(x) return mask_preds def loss_and_target(self, mask_preds: Tensor, sampling_results: List[SamplingResult], batch_gt_instances: InstanceList, rcnn_train_cfg: ConfigDict) -> dict: """Calculate the loss based on the features extracted by the mask head. Args: mask_preds (Tensor): Predicted foreground masks, has shape (num_pos, num_classes, h, w). sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. rcnn_train_cfg (obj:ConfigDict): `train_cfg` of RCNN. Returns: dict: A dictionary of loss and targets components. """ mask_targets = self.get_targets( sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, rcnn_train_cfg=rcnn_train_cfg) pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) num_pos = pos_labels.new_ones(pos_labels.size()).float().sum() avg_factor = torch.clamp(reduce_mean(num_pos), min=1.).item() loss = dict() if mask_preds.size(0) == 0: loss_mask = mask_preds.sum() else: loss_mask = self.loss_mask( mask_preds[torch.arange(num_pos).long(), pos_labels, ...].sigmoid(), mask_targets, avg_factor=avg_factor) loss['loss_mask'] = loss_mask return dict(loss_mask=loss, mask_targets=mask_targets) ================================================ FILE: mmdet/models/roi_heads/mask_heads/fcn_mask_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule, build_conv_layer, build_upsample_layer from mmcv.ops.carafe import CARAFEPack from mmengine.config import ConfigDict from mmengine.model import BaseModule, ModuleList from mmengine.structures import InstanceData from torch import Tensor from torch.nn.modules.utils import _pair from mmdet.models.task_modules.samplers import SamplingResult from mmdet.models.utils import empty_instances from mmdet.registry import MODELS from mmdet.structures.mask import mask_target from mmdet.utils import ConfigType, InstanceList, OptConfigType, OptMultiConfig BYTES_PER_FLOAT = 4 # TODO: This memory limit may be too much or too little. It would be better to # determine it based on available resources. GPU_MEM_LIMIT = 1024**3 # 1 GB memory limit @MODELS.register_module() class FCNMaskHead(BaseModule): def __init__(self, num_convs: int = 4, roi_feat_size: int = 14, in_channels: int = 256, conv_kernel_size: int = 3, conv_out_channels: int = 256, num_classes: int = 80, class_agnostic: int = False, upsample_cfg: ConfigType = dict( type='deconv', scale_factor=2), conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, predictor_cfg: ConfigType = dict(type='Conv'), loss_mask: ConfigType = dict( type='CrossEntropyLoss', use_mask=True, loss_weight=1.0), init_cfg: OptMultiConfig = None) -> None: assert init_cfg is None, 'To prevent abnormal initialization ' \ 'behavior, init_cfg is not allowed to be set' super().__init__(init_cfg=init_cfg) self.upsample_cfg = upsample_cfg.copy() if self.upsample_cfg['type'] not in [ None, 'deconv', 'nearest', 'bilinear', 'carafe' ]: raise ValueError( f'Invalid upsample method {self.upsample_cfg["type"]}, ' 'accepted methods are "deconv", "nearest", "bilinear", ' '"carafe"') self.num_convs = num_convs # WARN: roi_feat_size is reserved and not used self.roi_feat_size = _pair(roi_feat_size) self.in_channels = in_channels self.conv_kernel_size = conv_kernel_size self.conv_out_channels = conv_out_channels self.upsample_method = self.upsample_cfg.get('type') self.scale_factor = self.upsample_cfg.pop('scale_factor', None) self.num_classes = num_classes self.class_agnostic = class_agnostic self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.predictor_cfg = predictor_cfg self.loss_mask = MODELS.build(loss_mask) self.convs = ModuleList() for i in range(self.num_convs): in_channels = ( self.in_channels if i == 0 else self.conv_out_channels) padding = (self.conv_kernel_size - 1) // 2 self.convs.append( ConvModule( in_channels, self.conv_out_channels, self.conv_kernel_size, padding=padding, conv_cfg=conv_cfg, norm_cfg=norm_cfg)) upsample_in_channels = ( self.conv_out_channels if self.num_convs > 0 else in_channels) upsample_cfg_ = self.upsample_cfg.copy() if self.upsample_method is None: self.upsample = None elif self.upsample_method == 'deconv': upsample_cfg_.update( in_channels=upsample_in_channels, out_channels=self.conv_out_channels, kernel_size=self.scale_factor, stride=self.scale_factor) self.upsample = build_upsample_layer(upsample_cfg_) elif self.upsample_method == 'carafe': upsample_cfg_.update( channels=upsample_in_channels, scale_factor=self.scale_factor) self.upsample = build_upsample_layer(upsample_cfg_) else: # suppress warnings align_corners = (None if self.upsample_method == 'nearest' else False) upsample_cfg_.update( scale_factor=self.scale_factor, mode=self.upsample_method, align_corners=align_corners) self.upsample = build_upsample_layer(upsample_cfg_) out_channels = 1 if self.class_agnostic else self.num_classes logits_in_channel = ( self.conv_out_channels if self.upsample_method == 'deconv' else upsample_in_channels) self.conv_logits = build_conv_layer(self.predictor_cfg, logits_in_channel, out_channels, 1) self.relu = nn.ReLU(inplace=True) self.debug_imgs = None def init_weights(self) -> None: """Initialize the weights.""" super().init_weights() for m in [self.upsample, self.conv_logits]: if m is None: continue elif isinstance(m, CARAFEPack): m.init_weights() elif hasattr(m, 'weight') and hasattr(m, 'bias'): nn.init.kaiming_normal_( m.weight, mode='fan_out', nonlinearity='relu') nn.init.constant_(m.bias, 0) def forward(self, x: Tensor) -> Tensor: """Forward features from the upstream network. Args: x (Tensor): Extract mask RoI features. Returns: Tensor: Predicted foreground masks. """ for conv in self.convs: x = conv(x) if self.upsample is not None: x = self.upsample(x) if self.upsample_method == 'deconv': x = self.relu(x) mask_preds = self.conv_logits(x) return mask_preds def get_targets(self, sampling_results: List[SamplingResult], batch_gt_instances: InstanceList, rcnn_train_cfg: ConfigDict) -> Tensor: """Calculate the ground truth for all samples in a batch according to the sampling_results. Args: sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. rcnn_train_cfg (obj:ConfigDict): `train_cfg` of RCNN. Returns: Tensor: Mask target of each positive proposals in the image. """ pos_proposals = [res.pos_priors for res in sampling_results] pos_assigned_gt_inds = [ res.pos_assigned_gt_inds for res in sampling_results ] gt_masks = [res.masks for res in batch_gt_instances] mask_targets = mask_target(pos_proposals, pos_assigned_gt_inds, gt_masks, rcnn_train_cfg) return mask_targets def loss_and_target(self, mask_preds: Tensor, sampling_results: List[SamplingResult], batch_gt_instances: InstanceList, rcnn_train_cfg: ConfigDict) -> dict: """Calculate the loss based on the features extracted by the mask head. Args: mask_preds (Tensor): Predicted foreground masks, has shape (num_pos, num_classes, h, w). sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. rcnn_train_cfg (obj:ConfigDict): `train_cfg` of RCNN. Returns: dict: A dictionary of loss and targets components. """ mask_targets = self.get_targets( sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, rcnn_train_cfg=rcnn_train_cfg) pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) loss = dict() if mask_preds.size(0) == 0: loss_mask = mask_preds.sum() else: if self.class_agnostic: loss_mask = self.loss_mask(mask_preds, mask_targets, torch.zeros_like(pos_labels)) else: loss_mask = self.loss_mask(mask_preds, mask_targets, pos_labels) loss['loss_mask'] = loss_mask # TODO: which algorithm requires mask_targets? return dict(loss_mask=loss, mask_targets=mask_targets) def predict_by_feat(self, mask_preds: Tuple[Tensor], results_list: List[InstanceData], batch_img_metas: List[dict], rcnn_test_cfg: ConfigDict, rescale: bool = False, activate_map: bool = False) -> InstanceList: """Transform a batch of output features extracted from the head into mask results. Args: mask_preds (tuple[Tensor]): Tuple of predicted foreground masks, each has shape (n, num_classes, h, w). results_list (list[:obj:`InstanceData`]): Detection results of each image. batch_img_metas (list[dict]): List of image information. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of Bbox Head. rescale (bool): If True, return boxes in original image space. Defaults to False. activate_map (book): Whether get results with augmentations test. If True, the `mask_preds` will not process with sigmoid. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ assert len(mask_preds) == len(results_list) == len(batch_img_metas) for img_id in range(len(batch_img_metas)): img_meta = batch_img_metas[img_id] results = results_list[img_id] bboxes = results.bboxes if bboxes.shape[0] == 0: results_list[img_id] = empty_instances( [img_meta], bboxes.device, task_type='mask', instance_results=[results], mask_thr_binary=rcnn_test_cfg.mask_thr_binary)[0] else: im_mask = self._predict_by_feat_single( mask_preds=mask_preds[img_id], bboxes=bboxes, labels=results.labels, img_meta=img_meta, rcnn_test_cfg=rcnn_test_cfg, rescale=rescale, activate_map=activate_map) results.masks = im_mask return results_list def _predict_by_feat_single(self, mask_preds: Tensor, bboxes: Tensor, labels: Tensor, img_meta: dict, rcnn_test_cfg: ConfigDict, rescale: bool = False, activate_map: bool = False) -> Tensor: """Get segmentation masks from mask_preds and bboxes. Args: mask_preds (Tensor): Predicted foreground masks, has shape (n, num_classes, h, w). bboxes (Tensor): Predicted bboxes, has shape (n, 4) labels (Tensor): Labels of bboxes, has shape (n, ) img_meta (dict): image information. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of Bbox Head. Defaults to None. rescale (bool): If True, return boxes in original image space. Defaults to False. activate_map (book): Whether get results with augmentations test. If True, the `mask_preds` will not process with sigmoid. Defaults to False. Returns: Tensor: Encoded masks, has shape (n, img_w, img_h) Example: >>> from mmengine.config import Config >>> from mmdet.models.roi_heads.mask_heads.fcn_mask_head import * # NOQA >>> N = 7 # N = number of extracted ROIs >>> C, H, W = 11, 32, 32 >>> # Create example instance of FCN Mask Head. >>> self = FCNMaskHead(num_classes=C, num_convs=0) >>> inputs = torch.rand(N, self.in_channels, H, W) >>> mask_preds = self.forward(inputs) >>> # Each input is associated with some bounding box >>> bboxes = torch.Tensor([[1, 1, 42, 42 ]] * N) >>> labels = torch.randint(0, C, size=(N,)) >>> rcnn_test_cfg = Config({'mask_thr_binary': 0, }) >>> ori_shape = (H * 4, W * 4) >>> scale_factor = (1, 1) >>> rescale = False >>> img_meta = {'scale_factor': scale_factor, ... 'ori_shape': ori_shape} >>> # Encoded masks are a list for each category. >>> encoded_masks = self._get_seg_masks_single( ... mask_preds, bboxes, labels, ... img_meta, rcnn_test_cfg, rescale) >>> assert encoded_masks.size()[0] == N >>> assert encoded_masks.size()[1:] == ori_shape """ scale_factor = bboxes.new_tensor(img_meta['scale_factor']).repeat( (1, 2)) img_h, img_w = img_meta['ori_shape'][:2] device = bboxes.device if not activate_map: mask_preds = mask_preds.sigmoid() else: # In AugTest, has been activated before mask_preds = bboxes.new_tensor(mask_preds) if rescale: # in-placed rescale the bboxes bboxes /= scale_factor else: w_scale, h_scale = scale_factor[0, 0], scale_factor[0, 1] img_h = np.round(img_h * h_scale.item()).astype(np.int32) img_w = np.round(img_w * w_scale.item()).astype(np.int32) N = len(mask_preds) # The actual implementation split the input into chunks, # and paste them chunk by chunk. if device.type == 'cpu': # CPU is most efficient when they are pasted one by one with # skip_empty=True, so that it performs minimal number of # operations. num_chunks = N else: # GPU benefits from parallelism for larger chunks, # but may have memory issue # the types of img_w and img_h are np.int32, # when the image resolution is large, # the calculation of num_chunks will overflow. # so we need to change the types of img_w and img_h to int. # See https://github.com/open-mmlab/mmdetection/pull/5191 num_chunks = int( np.ceil(N * int(img_h) * int(img_w) * BYTES_PER_FLOAT / GPU_MEM_LIMIT)) assert (num_chunks <= N), 'Default GPU_MEM_LIMIT is too small; try increasing it' chunks = torch.chunk(torch.arange(N, device=device), num_chunks) threshold = rcnn_test_cfg.mask_thr_binary im_mask = torch.zeros( N, img_h, img_w, device=device, dtype=torch.bool if threshold >= 0 else torch.uint8) if not self.class_agnostic: mask_preds = mask_preds[range(N), labels][:, None] for inds in chunks: masks_chunk, spatial_inds = _do_paste_mask( mask_preds[inds], bboxes[inds], img_h, img_w, skip_empty=device.type == 'cpu') if threshold >= 0: masks_chunk = (masks_chunk >= threshold).to(dtype=torch.bool) else: # for visualization and debugging masks_chunk = (masks_chunk * 255).to(dtype=torch.uint8) im_mask[(inds, ) + spatial_inds] = masks_chunk return im_mask def _do_paste_mask(masks: Tensor, boxes: Tensor, img_h: int, img_w: int, skip_empty: bool = True) -> tuple: """Paste instance masks according to boxes. This implementation is modified from https://github.com/facebookresearch/detectron2/ Args: masks (Tensor): N, 1, H, W boxes (Tensor): N, 4 img_h (int): Height of the image to be pasted. img_w (int): Width of the image to be pasted. skip_empty (bool): Only paste masks within the region that tightly bound all boxes, and returns the results this region only. An important optimization for CPU. Returns: tuple: (Tensor, tuple). The first item is mask tensor, the second one is the slice object. If skip_empty == False, the whole image will be pasted. It will return a mask of shape (N, img_h, img_w) and an empty tuple. If skip_empty == True, only area around the mask will be pasted. A mask of shape (N, h', w') and its start and end coordinates in the original image will be returned. """ # On GPU, paste all masks together (up to chunk size) # by using the entire image to sample the masks # Compared to pasting them one by one, # this has more operations but is faster on COCO-scale dataset. device = masks.device if skip_empty: x0_int, y0_int = torch.clamp( boxes.min(dim=0).values.floor()[:2] - 1, min=0).to(dtype=torch.int32) x1_int = torch.clamp( boxes[:, 2].max().ceil() + 1, max=img_w).to(dtype=torch.int32) y1_int = torch.clamp( boxes[:, 3].max().ceil() + 1, max=img_h).to(dtype=torch.int32) else: x0_int, y0_int = 0, 0 x1_int, y1_int = img_w, img_h x0, y0, x1, y1 = torch.split(boxes, 1, dim=1) # each is Nx1 N = masks.shape[0] img_y = torch.arange(y0_int, y1_int, device=device).to(torch.float32) + 0.5 img_x = torch.arange(x0_int, x1_int, device=device).to(torch.float32) + 0.5 img_y = (img_y - y0) / (y1 - y0) * 2 - 1 img_x = (img_x - x0) / (x1 - x0) * 2 - 1 # img_x, img_y have shapes (N, w), (N, h) # IsInf op is not supported with ONNX<=1.7.0 if not torch.onnx.is_in_onnx_export(): if torch.isinf(img_x).any(): inds = torch.where(torch.isinf(img_x)) img_x[inds] = 0 if torch.isinf(img_y).any(): inds = torch.where(torch.isinf(img_y)) img_y[inds] = 0 gx = img_x[:, None, :].expand(N, img_y.size(1), img_x.size(1)) gy = img_y[:, :, None].expand(N, img_y.size(1), img_x.size(1)) grid = torch.stack([gx, gy], dim=3) img_masks = F.grid_sample( masks.to(dtype=torch.float32), grid, align_corners=False) if skip_empty: return img_masks[:, 0], (slice(y0_int, y1_int), slice(x0_int, x1_int)) else: return img_masks[:, 0], () ================================================ FILE: mmdet/models/roi_heads/mask_heads/feature_relay_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch.nn as nn from mmengine.model import BaseModule from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import MultiConfig @MODELS.register_module() class FeatureRelayHead(BaseModule): """Feature Relay Head used in `SCNet `_. Args: in_channels (int): number of input channels. Defaults to 256. conv_out_channels (int): number of output channels before classification layer. Defaults to 256. roi_feat_size (int): roi feat size at box head. Default: 7. scale_factor (int): scale factor to match roi feat size at mask head. Defaults to 2. init_cfg (:obj:`ConfigDict` or dict or list[dict] or list[:obj:`ConfigDict`]): Initialization config dict. Defaults to dict(type='Kaiming', layer='Linear'). """ def __init__( self, in_channels: int = 1024, out_conv_channels: int = 256, roi_feat_size: int = 7, scale_factor: int = 2, init_cfg: MultiConfig = dict(type='Kaiming', layer='Linear') ) -> None: super().__init__(init_cfg=init_cfg) assert isinstance(roi_feat_size, int) self.in_channels = in_channels self.out_conv_channels = out_conv_channels self.roi_feat_size = roi_feat_size self.out_channels = (roi_feat_size**2) * out_conv_channels self.scale_factor = scale_factor self.fp16_enabled = False self.fc = nn.Linear(self.in_channels, self.out_channels) self.upsample = nn.Upsample( scale_factor=scale_factor, mode='bilinear', align_corners=True) def forward(self, x: Tensor) -> Optional[Tensor]: """Forward function. Args: x (Tensor): Input feature. Returns: Optional[Tensor]: Output feature. When the first dim of input is 0, None is returned. """ N, _ = x.shape if N > 0: out_C = self.out_conv_channels out_HW = self.roi_feat_size x = self.fc(x) x = x.reshape(N, out_C, out_HW, out_HW) x = self.upsample(x) return x return None ================================================ FILE: mmdet/models/roi_heads/mask_heads/fused_semantic_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings from typing import Tuple import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.config import ConfigDict from mmengine.model import BaseModule from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import MultiConfig, OptConfigType @MODELS.register_module() class FusedSemanticHead(BaseModule): r"""Multi-level fused semantic segmentation head. .. code-block:: none in_1 -> 1x1 conv --- | in_2 -> 1x1 conv -- | || in_3 -> 1x1 conv - || ||| /-> 1x1 conv (mask prediction) in_4 -> 1x1 conv -----> 3x3 convs (*4) | \-> 1x1 conv (feature) in_5 -> 1x1 conv --- """ # noqa: W605 def __init__( self, num_ins: int, fusion_level: int, seg_scale_factor=1 / 8, num_convs: int = 4, in_channels: int = 256, conv_out_channels: int = 256, num_classes: int = 183, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, ignore_label: int = None, loss_weight: float = None, loss_seg: ConfigDict = dict( type='CrossEntropyLoss', ignore_index=255, loss_weight=0.2), init_cfg: MultiConfig = dict( type='Kaiming', override=dict(name='conv_logits')) ) -> None: super().__init__(init_cfg=init_cfg) self.num_ins = num_ins self.fusion_level = fusion_level self.seg_scale_factor = seg_scale_factor self.num_convs = num_convs self.in_channels = in_channels self.conv_out_channels = conv_out_channels self.num_classes = num_classes self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.fp16_enabled = False self.lateral_convs = nn.ModuleList() for i in range(self.num_ins): self.lateral_convs.append( ConvModule( self.in_channels, self.in_channels, 1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, inplace=False)) self.convs = nn.ModuleList() for i in range(self.num_convs): in_channels = self.in_channels if i == 0 else conv_out_channels self.convs.append( ConvModule( in_channels, conv_out_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.conv_embedding = ConvModule( conv_out_channels, conv_out_channels, 1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg) self.conv_logits = nn.Conv2d(conv_out_channels, self.num_classes, 1) if ignore_label: loss_seg['ignore_index'] = ignore_label if loss_weight: loss_seg['loss_weight'] = loss_weight if ignore_label or loss_weight: warnings.warn('``ignore_label`` and ``loss_weight`` would be ' 'deprecated soon. Please set ``ingore_index`` and ' '``loss_weight`` in ``loss_seg`` instead.') self.criterion = MODELS.build(loss_seg) def forward(self, feats: Tuple[Tensor]) -> Tuple[Tensor]: """Forward function. Args: feats (tuple[Tensor]): Multi scale feature maps. Returns: tuple[Tensor]: - mask_preds (Tensor): Predicted mask logits. - x (Tensor): Fused feature. """ x = self.lateral_convs[self.fusion_level](feats[self.fusion_level]) fused_size = tuple(x.shape[-2:]) for i, feat in enumerate(feats): if i != self.fusion_level: feat = F.interpolate( feat, size=fused_size, mode='bilinear', align_corners=True) # fix runtime error of "+=" inplace operation in PyTorch 1.10 x = x + self.lateral_convs[i](feat) for i in range(self.num_convs): x = self.convs[i](x) mask_preds = self.conv_logits(x) x = self.conv_embedding(x) return mask_preds, x def loss(self, mask_preds: Tensor, labels: Tensor) -> Tensor: """Loss function. Args: mask_preds (Tensor): Predicted mask logits. labels (Tensor): Ground truth. Returns: Tensor: Semantic segmentation loss. """ labels = F.interpolate( labels.float(), scale_factor=self.seg_scale_factor, mode='nearest') labels = labels.squeeze(1).long() loss_semantic_seg = self.criterion(mask_preds, labels) return loss_semantic_seg ================================================ FILE: mmdet/models/roi_heads/mask_heads/global_context_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import torch.nn as nn from mmcv.cnn import ConvModule from mmengine.model import BaseModule from torch import Tensor from mmdet.models.layers import ResLayer, SimplifiedBasicBlock from mmdet.registry import MODELS from mmdet.utils import MultiConfig, OptConfigType @MODELS.register_module() class GlobalContextHead(BaseModule): """Global context head used in `SCNet `_. Args: num_convs (int, optional): number of convolutional layer in GlbCtxHead. Defaults to 4. in_channels (int, optional): number of input channels. Defaults to 256. conv_out_channels (int, optional): number of output channels before classification layer. Defaults to 256. num_classes (int, optional): number of classes. Defaults to 80. loss_weight (float, optional): global context loss weight. Defaults to 1. conv_cfg (dict, optional): config to init conv layer. Defaults to None. norm_cfg (dict, optional): config to init norm layer. Defaults to None. conv_to_res (bool, optional): if True, 2 convs will be grouped into 1 `SimplifiedBasicBlock` using a skip connection. Defaults to False. init_cfg (:obj:`ConfigDict` or dict or list[dict] or list[:obj:`ConfigDict`]): Initialization config dict. Defaults to dict(type='Normal', std=0.01, override=dict(name='fc')). """ def __init__( self, num_convs: int = 4, in_channels: int = 256, conv_out_channels: int = 256, num_classes: int = 80, loss_weight: float = 1.0, conv_cfg: OptConfigType = None, norm_cfg: OptConfigType = None, conv_to_res: bool = False, init_cfg: MultiConfig = dict( type='Normal', std=0.01, override=dict(name='fc')) ) -> None: super().__init__(init_cfg=init_cfg) self.num_convs = num_convs self.in_channels = in_channels self.conv_out_channels = conv_out_channels self.num_classes = num_classes self.loss_weight = loss_weight self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.conv_to_res = conv_to_res self.fp16_enabled = False if self.conv_to_res: num_res_blocks = num_convs // 2 self.convs = ResLayer( SimplifiedBasicBlock, in_channels, self.conv_out_channels, num_res_blocks, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg) self.num_convs = num_res_blocks else: self.convs = nn.ModuleList() for i in range(self.num_convs): in_channels = self.in_channels if i == 0 else conv_out_channels self.convs.append( ConvModule( in_channels, conv_out_channels, 3, padding=1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg)) self.pool = nn.AdaptiveAvgPool2d(1) self.fc = nn.Linear(conv_out_channels, num_classes) self.criterion = nn.BCEWithLogitsLoss() def forward(self, feats: Tuple[Tensor]) -> Tuple[Tensor]: """Forward function. Args: feats (Tuple[Tensor]): Multi-scale feature maps. Returns: Tuple[Tensor]: - mc_pred (Tensor): Multi-class prediction. - x (Tensor): Global context feature. """ x = feats[-1] for i in range(self.num_convs): x = self.convs[i](x) x = self.pool(x) # multi-class prediction mc_pred = x.reshape(x.size(0), -1) mc_pred = self.fc(mc_pred) return mc_pred, x def loss(self, pred: Tensor, labels: List[Tensor]) -> Tensor: """Loss function. Args: pred (Tensor): Logits. labels (list[Tensor]): Grouth truths. Returns: Tensor: Loss. """ labels = [lbl.unique() for lbl in labels] targets = pred.new_zeros(pred.size()) for i, label in enumerate(labels): targets[i, label] = 1.0 loss = self.loss_weight * self.criterion(pred, targets) return loss ================================================ FILE: mmdet/models/roi_heads/mask_heads/grid_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Tuple import numpy as np import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import ConvModule from mmengine.config import ConfigDict from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.task_modules.samplers import SamplingResult from mmdet.registry import MODELS from mmdet.utils import ConfigType, InstanceList, MultiConfig, OptConfigType @MODELS.register_module() class GridHead(BaseModule): """Implementation of `Grid Head `_ Args: grid_points (int): The number of grid points. Defaults to 9. num_convs (int): The number of convolution layers. Defaults to 8. roi_feat_size (int): RoI feature size. Default to 14. in_channels (int): The channel number of inputs features. Defaults to 256. conv_kernel_size (int): The kernel size of convolution layers. Defaults to 3. point_feat_channels (int): The number of channels of each point features. Defaults to 64. class_agnostic (bool): Whether use class agnostic classification. If so, the output channels of logits will be 1. Defaults to False. loss_grid (:obj:`ConfigDict` or dict): Config of grid loss. conv_cfg (:obj:`ConfigDict` or dict, optional) dictionary to construct and config conv layer. norm_cfg (:obj:`ConfigDict` or dict): dictionary to construct and config norm layer. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict]): Initialization config dict. """ def __init__( self, grid_points: int = 9, num_convs: int = 8, roi_feat_size: int = 14, in_channels: int = 256, conv_kernel_size: int = 3, point_feat_channels: int = 64, deconv_kernel_size: int = 4, class_agnostic: bool = False, loss_grid: ConfigType = dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=15), conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict(type='GN', num_groups=36), init_cfg: MultiConfig = [ dict(type='Kaiming', layer=['Conv2d', 'Linear']), dict( type='Normal', layer='ConvTranspose2d', std=0.001, override=dict( type='Normal', name='deconv2', std=0.001, bias=-np.log(0.99 / 0.01))) ] ) -> None: super().__init__(init_cfg=init_cfg) self.grid_points = grid_points self.num_convs = num_convs self.roi_feat_size = roi_feat_size self.in_channels = in_channels self.conv_kernel_size = conv_kernel_size self.point_feat_channels = point_feat_channels self.conv_out_channels = self.point_feat_channels * self.grid_points self.class_agnostic = class_agnostic self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg if isinstance(norm_cfg, dict) and norm_cfg['type'] == 'GN': assert self.conv_out_channels % norm_cfg['num_groups'] == 0 assert self.grid_points >= 4 self.grid_size = int(np.sqrt(self.grid_points)) if self.grid_size * self.grid_size != self.grid_points: raise ValueError('grid_points must be a square number') # the predicted heatmap is half of whole_map_size if not isinstance(self.roi_feat_size, int): raise ValueError('Only square RoIs are supporeted in Grid R-CNN') self.whole_map_size = self.roi_feat_size * 4 # compute point-wise sub-regions self.sub_regions = self.calc_sub_regions() self.convs = [] for i in range(self.num_convs): in_channels = ( self.in_channels if i == 0 else self.conv_out_channels) stride = 2 if i == 0 else 1 padding = (self.conv_kernel_size - 1) // 2 self.convs.append( ConvModule( in_channels, self.conv_out_channels, self.conv_kernel_size, stride=stride, padding=padding, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg, bias=True)) self.convs = nn.Sequential(*self.convs) self.deconv1 = nn.ConvTranspose2d( self.conv_out_channels, self.conv_out_channels, kernel_size=deconv_kernel_size, stride=2, padding=(deconv_kernel_size - 2) // 2, groups=grid_points) self.norm1 = nn.GroupNorm(grid_points, self.conv_out_channels) self.deconv2 = nn.ConvTranspose2d( self.conv_out_channels, grid_points, kernel_size=deconv_kernel_size, stride=2, padding=(deconv_kernel_size - 2) // 2, groups=grid_points) # find the 4-neighbor of each grid point self.neighbor_points = [] grid_size = self.grid_size for i in range(grid_size): # i-th column for j in range(grid_size): # j-th row neighbors = [] if i > 0: # left: (i - 1, j) neighbors.append((i - 1) * grid_size + j) if j > 0: # up: (i, j - 1) neighbors.append(i * grid_size + j - 1) if j < grid_size - 1: # down: (i, j + 1) neighbors.append(i * grid_size + j + 1) if i < grid_size - 1: # right: (i + 1, j) neighbors.append((i + 1) * grid_size + j) self.neighbor_points.append(tuple(neighbors)) # total edges in the grid self.num_edges = sum([len(p) for p in self.neighbor_points]) self.forder_trans = nn.ModuleList() # first-order feature transition self.sorder_trans = nn.ModuleList() # second-order feature transition for neighbors in self.neighbor_points: fo_trans = nn.ModuleList() so_trans = nn.ModuleList() for _ in range(len(neighbors)): # each transition module consists of a 5x5 depth-wise conv and # 1x1 conv. fo_trans.append( nn.Sequential( nn.Conv2d( self.point_feat_channels, self.point_feat_channels, 5, stride=1, padding=2, groups=self.point_feat_channels), nn.Conv2d(self.point_feat_channels, self.point_feat_channels, 1))) so_trans.append( nn.Sequential( nn.Conv2d( self.point_feat_channels, self.point_feat_channels, 5, 1, 2, groups=self.point_feat_channels), nn.Conv2d(self.point_feat_channels, self.point_feat_channels, 1))) self.forder_trans.append(fo_trans) self.sorder_trans.append(so_trans) self.loss_grid = MODELS.build(loss_grid) def forward(self, x: Tensor) -> Dict[str, Tensor]: """forward function of ``GridHead``. Args: x (Tensor): RoI features, has shape (num_rois, num_channels, roi_feat_size, roi_feat_size). Returns: Dict[str, Tensor]: Return a dict including fused and unfused heatmap. """ assert x.shape[-1] == x.shape[-2] == self.roi_feat_size # RoI feature transformation, downsample 2x x = self.convs(x) c = self.point_feat_channels # first-order fusion x_fo = [None for _ in range(self.grid_points)] for i, points in enumerate(self.neighbor_points): x_fo[i] = x[:, i * c:(i + 1) * c] for j, point_idx in enumerate(points): x_fo[i] = x_fo[i] + self.forder_trans[i][j]( x[:, point_idx * c:(point_idx + 1) * c]) # second-order fusion x_so = [None for _ in range(self.grid_points)] for i, points in enumerate(self.neighbor_points): x_so[i] = x[:, i * c:(i + 1) * c] for j, point_idx in enumerate(points): x_so[i] = x_so[i] + self.sorder_trans[i][j](x_fo[point_idx]) # predicted heatmap with fused features x2 = torch.cat(x_so, dim=1) x2 = self.deconv1(x2) x2 = F.relu(self.norm1(x2), inplace=True) heatmap = self.deconv2(x2) # predicted heatmap with original features (applicable during training) if self.training: x1 = x x1 = self.deconv1(x1) x1 = F.relu(self.norm1(x1), inplace=True) heatmap_unfused = self.deconv2(x1) else: heatmap_unfused = heatmap return dict(fused=heatmap, unfused=heatmap_unfused) def calc_sub_regions(self) -> List[Tuple[float]]: """Compute point specific representation regions. See `Grid R-CNN Plus `_ for details. """ # to make it consistent with the original implementation, half_size # is computed as 2 * quarter_size, which is smaller half_size = self.whole_map_size // 4 * 2 sub_regions = [] for i in range(self.grid_points): x_idx = i // self.grid_size y_idx = i % self.grid_size if x_idx == 0: sub_x1 = 0 elif x_idx == self.grid_size - 1: sub_x1 = half_size else: ratio = x_idx / (self.grid_size - 1) - 0.25 sub_x1 = max(int(ratio * self.whole_map_size), 0) if y_idx == 0: sub_y1 = 0 elif y_idx == self.grid_size - 1: sub_y1 = half_size else: ratio = y_idx / (self.grid_size - 1) - 0.25 sub_y1 = max(int(ratio * self.whole_map_size), 0) sub_regions.append( (sub_x1, sub_y1, sub_x1 + half_size, sub_y1 + half_size)) return sub_regions def get_targets(self, sampling_results: List[SamplingResult], rcnn_train_cfg: ConfigDict) -> Tensor: """Calculate the ground truth for all samples in a batch according to the sampling_results.". Args: sampling_results (List[:obj:`SamplingResult`]): Assign results of all images in a batch after sampling. rcnn_train_cfg (:obj:`ConfigDict`): `train_cfg` of RCNN. Returns: Tensor: Grid heatmap targets. """ # mix all samples (across images) together. pos_bboxes = torch.cat([res.pos_bboxes for res in sampling_results], dim=0).cpu() pos_gt_bboxes = torch.cat( [res.pos_gt_bboxes for res in sampling_results], dim=0).cpu() assert pos_bboxes.shape == pos_gt_bboxes.shape # expand pos_bboxes to 2x of original size x1 = pos_bboxes[:, 0] - (pos_bboxes[:, 2] - pos_bboxes[:, 0]) / 2 y1 = pos_bboxes[:, 1] - (pos_bboxes[:, 3] - pos_bboxes[:, 1]) / 2 x2 = pos_bboxes[:, 2] + (pos_bboxes[:, 2] - pos_bboxes[:, 0]) / 2 y2 = pos_bboxes[:, 3] + (pos_bboxes[:, 3] - pos_bboxes[:, 1]) / 2 pos_bboxes = torch.stack([x1, y1, x2, y2], dim=-1) pos_bbox_ws = (pos_bboxes[:, 2] - pos_bboxes[:, 0]).unsqueeze(-1) pos_bbox_hs = (pos_bboxes[:, 3] - pos_bboxes[:, 1]).unsqueeze(-1) num_rois = pos_bboxes.shape[0] map_size = self.whole_map_size # this is not the final target shape targets = torch.zeros((num_rois, self.grid_points, map_size, map_size), dtype=torch.float) # pre-compute interpolation factors for all grid points. # the first item is the factor of x-dim, and the second is y-dim. # for a 9-point grid, factors are like (1, 0), (0.5, 0.5), (0, 1) factors = [] for j in range(self.grid_points): x_idx = j // self.grid_size y_idx = j % self.grid_size factors.append((1 - x_idx / (self.grid_size - 1), 1 - y_idx / (self.grid_size - 1))) radius = rcnn_train_cfg.pos_radius radius2 = radius**2 for i in range(num_rois): # ignore small bboxes if (pos_bbox_ws[i] <= self.grid_size or pos_bbox_hs[i] <= self.grid_size): continue # for each grid point, mark a small circle as positive for j in range(self.grid_points): factor_x, factor_y = factors[j] gridpoint_x = factor_x * pos_gt_bboxes[i, 0] + ( 1 - factor_x) * pos_gt_bboxes[i, 2] gridpoint_y = factor_y * pos_gt_bboxes[i, 1] + ( 1 - factor_y) * pos_gt_bboxes[i, 3] cx = int((gridpoint_x - pos_bboxes[i, 0]) / pos_bbox_ws[i] * map_size) cy = int((gridpoint_y - pos_bboxes[i, 1]) / pos_bbox_hs[i] * map_size) for x in range(cx - radius, cx + radius + 1): for y in range(cy - radius, cy + radius + 1): if x >= 0 and x < map_size and y >= 0 and y < map_size: if (x - cx)**2 + (y - cy)**2 <= radius2: targets[i, j, y, x] = 1 # reduce the target heatmap size by a half # proposed in Grid R-CNN Plus (https://arxiv.org/abs/1906.05688). sub_targets = [] for i in range(self.grid_points): sub_x1, sub_y1, sub_x2, sub_y2 = self.sub_regions[i] sub_targets.append(targets[:, [i], sub_y1:sub_y2, sub_x1:sub_x2]) sub_targets = torch.cat(sub_targets, dim=1) sub_targets = sub_targets.to(sampling_results[0].pos_bboxes.device) return sub_targets def loss(self, grid_pred: Tensor, sample_idx: Tensor, sampling_results: List[SamplingResult], rcnn_train_cfg: ConfigDict) -> dict: """Calculate the loss based on the features extracted by the grid head. Args: grid_pred (dict[str, Tensor]): Outputs of grid_head forward. sample_idx (Tensor): The sampling index of ``grid_pred``. sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. rcnn_train_cfg (obj:`ConfigDict`): `train_cfg` of RCNN. Returns: dict: A dictionary of loss and targets components. """ grid_targets = self.get_targets(sampling_results, rcnn_train_cfg) grid_targets = grid_targets[sample_idx] loss_fused = self.loss_grid(grid_pred['fused'], grid_targets) loss_unfused = self.loss_grid(grid_pred['unfused'], grid_targets) loss_grid = loss_fused + loss_unfused return dict(loss_grid=loss_grid) def predict_by_feat(self, grid_preds: Dict[str, Tensor], results_list: List[InstanceData], batch_img_metas: List[dict], rescale: bool = False) -> InstanceList: """Adjust the predicted bboxes from bbox head. Args: grid_preds (dict[str, Tensor]): dictionary outputted by forward function. results_list (list[:obj:`InstanceData`]): Detection results of each image. batch_img_metas (list[dict]): List of image information. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape \ (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last \ dimension 4 arrange as (x1, y1, x2, y2). """ num_roi_per_img = tuple(res.bboxes.size(0) for res in results_list) grid_preds = { k: v.split(num_roi_per_img, 0) for k, v in grid_preds.items() } for i, results in enumerate(results_list): if len(results) != 0: bboxes = self._predict_by_feat_single( grid_pred=grid_preds['fused'][i], bboxes=results.bboxes, img_meta=batch_img_metas[i], rescale=rescale) results.bboxes = bboxes return results_list def _predict_by_feat_single(self, grid_pred: Tensor, bboxes: Tensor, img_meta: dict, rescale: bool = False) -> Tensor: """Adjust ``bboxes`` according to ``grid_pred``. Args: grid_pred (Tensor): Grid fused heatmap. bboxes (Tensor): Predicted bboxes, has shape (n, 4) img_meta (dict): image information. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: Tensor: adjusted bboxes. """ assert bboxes.size(0) == grid_pred.size(0) grid_pred = grid_pred.sigmoid() R, c, h, w = grid_pred.shape half_size = self.whole_map_size // 4 * 2 assert h == w == half_size assert c == self.grid_points # find the point with max scores in the half-sized heatmap grid_pred = grid_pred.view(R * c, h * w) pred_scores, pred_position = grid_pred.max(dim=1) xs = pred_position % w ys = pred_position // w # get the position in the whole heatmap instead of half-sized heatmap for i in range(self.grid_points): xs[i::self.grid_points] += self.sub_regions[i][0] ys[i::self.grid_points] += self.sub_regions[i][1] # reshape to (num_rois, grid_points) pred_scores, xs, ys = tuple( map(lambda x: x.view(R, c), [pred_scores, xs, ys])) # get expanded pos_bboxes widths = (bboxes[:, 2] - bboxes[:, 0]).unsqueeze(-1) heights = (bboxes[:, 3] - bboxes[:, 1]).unsqueeze(-1) x1 = (bboxes[:, 0, None] - widths / 2) y1 = (bboxes[:, 1, None] - heights / 2) # map the grid point to the absolute coordinates abs_xs = (xs.float() + 0.5) / w * widths + x1 abs_ys = (ys.float() + 0.5) / h * heights + y1 # get the grid points indices that fall on the bbox boundaries x1_inds = [i for i in range(self.grid_size)] y1_inds = [i * self.grid_size for i in range(self.grid_size)] x2_inds = [ self.grid_points - self.grid_size + i for i in range(self.grid_size) ] y2_inds = [(i + 1) * self.grid_size - 1 for i in range(self.grid_size)] # voting of all grid points on some boundary bboxes_x1 = (abs_xs[:, x1_inds] * pred_scores[:, x1_inds]).sum( dim=1, keepdim=True) / ( pred_scores[:, x1_inds].sum(dim=1, keepdim=True)) bboxes_y1 = (abs_ys[:, y1_inds] * pred_scores[:, y1_inds]).sum( dim=1, keepdim=True) / ( pred_scores[:, y1_inds].sum(dim=1, keepdim=True)) bboxes_x2 = (abs_xs[:, x2_inds] * pred_scores[:, x2_inds]).sum( dim=1, keepdim=True) / ( pred_scores[:, x2_inds].sum(dim=1, keepdim=True)) bboxes_y2 = (abs_ys[:, y2_inds] * pred_scores[:, y2_inds]).sum( dim=1, keepdim=True) / ( pred_scores[:, y2_inds].sum(dim=1, keepdim=True)) bboxes = torch.cat([bboxes_x1, bboxes_y1, bboxes_x2, bboxes_y2], dim=1) bboxes[:, [0, 2]].clamp_(min=0, max=img_meta['img_shape'][1]) bboxes[:, [1, 3]].clamp_(min=0, max=img_meta['img_shape'][0]) if rescale: assert img_meta.get('scale_factor') is not None bboxes /= bboxes.new_tensor(img_meta['scale_factor']).repeat( (1, 2)) return bboxes ================================================ FILE: mmdet/models/roi_heads/mask_heads/htc_mask_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Union from mmcv.cnn import ConvModule from torch import Tensor from mmdet.registry import MODELS from .fcn_mask_head import FCNMaskHead @MODELS.register_module() class HTCMaskHead(FCNMaskHead): """Mask head for HTC. Args: with_conv_res (bool): Whether add conv layer for ``res_feat``. Defaults to True. """ def __init__(self, with_conv_res: bool = True, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.with_conv_res = with_conv_res if self.with_conv_res: self.conv_res = ConvModule( self.conv_out_channels, self.conv_out_channels, 1, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg) def forward(self, x: Tensor, res_feat: Optional[Tensor] = None, return_logits: bool = True, return_feat: bool = True) -> Union[Tensor, List[Tensor]]: """ Args: x (Tensor): Feature map. res_feat (Tensor, optional): Feature for residual connection. Defaults to None. return_logits (bool): Whether return mask logits. Defaults to True. return_feat (bool): Whether return feature map. Defaults to True. Returns: Union[Tensor, List[Tensor]]: The return result is one of three results: res_feat, logits, or [logits, res_feat]. """ assert not (not return_logits and not return_feat) if res_feat is not None: assert self.with_conv_res res_feat = self.conv_res(res_feat) x = x + res_feat for conv in self.convs: x = conv(x) res_feat = x outs = [] if return_logits: x = self.upsample(x) if self.upsample_method == 'deconv': x = self.relu(x) mask_preds = self.conv_logits(x) outs.append(mask_preds) if return_feat: outs.append(res_feat) return outs if len(outs) > 1 else outs[0] ================================================ FILE: mmdet/models/roi_heads/mask_heads/mask_point_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Modified from https://github.com/facebookresearch/detectron2/tree/master/projects/PointRend/point_head/point_head.py # noqa from typing import List, Tuple import torch import torch.nn as nn from mmcv.cnn import ConvModule from mmcv.ops import point_sample, rel_roi_point_to_rel_img_point from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.task_modules.samplers import SamplingResult from mmdet.models.utils import (get_uncertain_point_coords_with_randomness, get_uncertainty) from mmdet.registry import MODELS from mmdet.structures.bbox import bbox2roi from mmdet.utils import ConfigType, InstanceList, MultiConfig, OptConfigType @MODELS.register_module() class MaskPointHead(BaseModule): """A mask point head use in PointRend. ``MaskPointHead`` use shared multi-layer perceptron (equivalent to nn.Conv1d) to predict the logit of input points. The fine-grained feature and coarse feature will be concatenate together for predication. Args: num_fcs (int): Number of fc layers in the head. Defaults to 3. in_channels (int): Number of input channels. Defaults to 256. fc_channels (int): Number of fc channels. Defaults to 256. num_classes (int): Number of classes for logits. Defaults to 80. class_agnostic (bool): Whether use class agnostic classification. If so, the output channels of logits will be 1. Defaults to False. coarse_pred_each_layer (bool): Whether concatenate coarse feature with the output of each fc layer. Defaults to True. conv_cfg (:obj:`ConfigDict` or dict): Dictionary to construct and config conv layer. Defaults to dict(type='Conv1d')). norm_cfg (:obj:`ConfigDict` or dict, optional): Dictionary to construct and config norm layer. Defaults to None. loss_point (:obj:`ConfigDict` or dict): Dictionary to construct and config loss layer of point head. Defaults to dict(type='CrossEntropyLoss', use_mask=True, loss_weight=1.0). init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. """ def __init__( self, num_classes: int, num_fcs: int = 3, in_channels: int = 256, fc_channels: int = 256, class_agnostic: bool = False, coarse_pred_each_layer: bool = True, conv_cfg: ConfigType = dict(type='Conv1d'), norm_cfg: OptConfigType = None, act_cfg: ConfigType = dict(type='ReLU'), loss_point: ConfigType = dict( type='CrossEntropyLoss', use_mask=True, loss_weight=1.0), init_cfg: MultiConfig = dict( type='Normal', std=0.001, override=dict(name='fc_logits')) ) -> None: super().__init__(init_cfg=init_cfg) self.num_fcs = num_fcs self.in_channels = in_channels self.fc_channels = fc_channels self.num_classes = num_classes self.class_agnostic = class_agnostic self.coarse_pred_each_layer = coarse_pred_each_layer self.conv_cfg = conv_cfg self.norm_cfg = norm_cfg self.loss_point = MODELS.build(loss_point) fc_in_channels = in_channels + num_classes self.fcs = nn.ModuleList() for _ in range(num_fcs): fc = ConvModule( fc_in_channels, fc_channels, kernel_size=1, stride=1, padding=0, conv_cfg=conv_cfg, norm_cfg=norm_cfg, act_cfg=act_cfg) self.fcs.append(fc) fc_in_channels = fc_channels fc_in_channels += num_classes if self.coarse_pred_each_layer else 0 out_channels = 1 if self.class_agnostic else self.num_classes self.fc_logits = nn.Conv1d( fc_in_channels, out_channels, kernel_size=1, stride=1, padding=0) def forward(self, fine_grained_feats: Tensor, coarse_feats: Tensor) -> Tensor: """Classify each point base on fine grained and coarse feats. Args: fine_grained_feats (Tensor): Fine grained feature sampled from FPN, shape (num_rois, in_channels, num_points). coarse_feats (Tensor): Coarse feature sampled from CoarseMaskHead, shape (num_rois, num_classes, num_points). Returns: Tensor: Point classification results, shape (num_rois, num_class, num_points). """ x = torch.cat([fine_grained_feats, coarse_feats], dim=1) for fc in self.fcs: x = fc(x) if self.coarse_pred_each_layer: x = torch.cat((x, coarse_feats), dim=1) return self.fc_logits(x) def get_targets(self, rois: Tensor, rel_roi_points: Tensor, sampling_results: List[SamplingResult], batch_gt_instances: InstanceList, cfg: ConfigType) -> Tensor: """Get training targets of MaskPointHead for all images. Args: rois (Tensor): Region of Interest, shape (num_rois, 5). rel_roi_points (Tensor): Points coordinates relative to RoI, shape (num_rois, num_points, 2). sampling_results (:obj:`SamplingResult`): Sampling result after sampling and assignment. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. cfg (obj:`ConfigDict` or dict): Training cfg. Returns: Tensor: Point target, shape (num_rois, num_points). """ num_imgs = len(sampling_results) rois_list = [] rel_roi_points_list = [] for batch_ind in range(num_imgs): inds = (rois[:, 0] == batch_ind) rois_list.append(rois[inds]) rel_roi_points_list.append(rel_roi_points[inds]) pos_assigned_gt_inds_list = [ res.pos_assigned_gt_inds for res in sampling_results ] cfg_list = [cfg for _ in range(num_imgs)] point_targets = map(self._get_targets_single, rois_list, rel_roi_points_list, pos_assigned_gt_inds_list, batch_gt_instances, cfg_list) point_targets = list(point_targets) if len(point_targets) > 0: point_targets = torch.cat(point_targets) return point_targets def _get_targets_single(self, rois: Tensor, rel_roi_points: Tensor, pos_assigned_gt_inds: Tensor, gt_instances: InstanceData, cfg: ConfigType) -> Tensor: """Get training target of MaskPointHead for each image.""" num_pos = rois.size(0) num_points = cfg.num_points if num_pos > 0: gt_masks_th = ( gt_instances.masks.to_tensor(rois.dtype, rois.device).index_select( 0, pos_assigned_gt_inds)) gt_masks_th = gt_masks_th.unsqueeze(1) rel_img_points = rel_roi_point_to_rel_img_point( rois, rel_roi_points, gt_masks_th) point_targets = point_sample(gt_masks_th, rel_img_points).squeeze(1) else: point_targets = rois.new_zeros((0, num_points)) return point_targets def loss_and_target(self, point_pred: Tensor, rel_roi_points: Tensor, sampling_results: List[SamplingResult], batch_gt_instances: InstanceList, cfg: ConfigType) -> dict: """Calculate loss for MaskPointHead. Args: point_pred (Tensor): Point predication result, shape (num_rois, num_classes, num_points). rel_roi_points (Tensor): Points coordinates relative to RoI, shape (num_rois, num_points, 2). sampling_results (:obj:`SamplingResult`): Sampling result after sampling and assignment. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. cfg (obj:`ConfigDict` or dict): Training cfg. Returns: dict: a dictionary of point loss and point target. """ rois = bbox2roi([res.pos_bboxes for res in sampling_results]) pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) point_target = self.get_targets(rois, rel_roi_points, sampling_results, batch_gt_instances, cfg) if self.class_agnostic: loss_point = self.loss_point(point_pred, point_target, torch.zeros_like(pos_labels)) else: loss_point = self.loss_point(point_pred, point_target, pos_labels) return dict(loss_point=loss_point, point_target=point_target) def get_roi_rel_points_train(self, mask_preds: Tensor, labels: Tensor, cfg: ConfigType) -> Tensor: """Get ``num_points`` most uncertain points with random points during train. Sample points in [0, 1] x [0, 1] coordinate space based on their uncertainty. The uncertainties are calculated for each point using '_get_uncertainty()' function that takes point's logit prediction as input. Args: mask_preds (Tensor): A tensor of shape (num_rois, num_classes, mask_height, mask_width) for class-specific or class-agnostic prediction. labels (Tensor): The ground truth class for each instance. cfg (:obj:`ConfigDict` or dict): Training config of point head. Returns: point_coords (Tensor): A tensor of shape (num_rois, num_points, 2) that contains the coordinates sampled points. """ point_coords = get_uncertain_point_coords_with_randomness( mask_preds, labels, cfg.num_points, cfg.oversample_ratio, cfg.importance_sample_ratio) return point_coords def get_roi_rel_points_test(self, mask_preds: Tensor, label_preds: Tensor, cfg: ConfigType) -> Tuple[Tensor, Tensor]: """Get ``num_points`` most uncertain points during test. Args: mask_preds (Tensor): A tensor of shape (num_rois, num_classes, mask_height, mask_width) for class-specific or class-agnostic prediction. label_preds (Tensor): The predication class for each instance. cfg (:obj:`ConfigDict` or dict): Testing config of point head. Returns: tuple: - point_indices (Tensor): A tensor of shape (num_rois, num_points) that contains indices from [0, mask_height x mask_width) of the most uncertain points. - point_coords (Tensor): A tensor of shape (num_rois, num_points, 2) that contains [0, 1] x [0, 1] normalized coordinates of the most uncertain points from the [mask_height, mask_width] grid. """ num_points = cfg.subdivision_num_points uncertainty_map = get_uncertainty(mask_preds, label_preds) num_rois, _, mask_height, mask_width = uncertainty_map.shape # During ONNX exporting, the type of each elements of 'shape' is # `Tensor(float)`, while it is `float` during PyTorch inference. if isinstance(mask_height, torch.Tensor): h_step = 1.0 / mask_height.float() w_step = 1.0 / mask_width.float() else: h_step = 1.0 / mask_height w_step = 1.0 / mask_width # cast to int to avoid dynamic K for TopK op in ONNX mask_size = int(mask_height * mask_width) uncertainty_map = uncertainty_map.view(num_rois, mask_size) num_points = min(mask_size, num_points) point_indices = uncertainty_map.topk(num_points, dim=1)[1] xs = w_step / 2.0 + (point_indices % mask_width).float() * w_step ys = h_step / 2.0 + (point_indices // mask_width).float() * h_step point_coords = torch.stack([xs, ys], dim=2) return point_indices, point_coords ================================================ FILE: mmdet/models/roi_heads/mask_heads/maskiou_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import numpy as np import torch import torch.nn as nn from mmcv.cnn import Conv2d, Linear, MaxPool2d from mmengine.config import ConfigDict from mmengine.model import BaseModule from mmengine.structures import InstanceData from torch import Tensor from torch.nn.modules.utils import _pair from mmdet.models.task_modules.samplers import SamplingResult from mmdet.registry import MODELS from mmdet.utils import ConfigType, InstanceList, OptMultiConfig @MODELS.register_module() class MaskIoUHead(BaseModule): """Mask IoU Head. This head predicts the IoU of predicted masks and corresponding gt masks. Args: num_convs (int): The number of convolution layers. Defaults to 4. num_fcs (int): The number of fully connected layers. Defaults to 2. roi_feat_size (int): RoI feature size. Default to 14. in_channels (int): The channel number of inputs features. Defaults to 256. conv_out_channels (int): The feature channels of convolution layers. Defaults to 256. fc_out_channels (int): The feature channels of fully connected layers. Defaults to 1024. num_classes (int): Number of categories excluding the background category. Defaults to 80. loss_iou (:obj:`ConfigDict` or dict): IoU loss. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. """ def __init__( self, num_convs: int = 4, num_fcs: int = 2, roi_feat_size: int = 14, in_channels: int = 256, conv_out_channels: int = 256, fc_out_channels: int = 1024, num_classes: int = 80, loss_iou: ConfigType = dict(type='MSELoss', loss_weight=0.5), init_cfg: OptMultiConfig = [ dict(type='Kaiming', override=dict(name='convs')), dict(type='Caffe2Xavier', override=dict(name='fcs')), dict(type='Normal', std=0.01, override=dict(name='fc_mask_iou')) ] ) -> None: super().__init__(init_cfg=init_cfg) self.in_channels = in_channels self.conv_out_channels = conv_out_channels self.fc_out_channels = fc_out_channels self.num_classes = num_classes self.convs = nn.ModuleList() for i in range(num_convs): if i == 0: # concatenation of mask feature and mask prediction in_channels = self.in_channels + 1 else: in_channels = self.conv_out_channels stride = 2 if i == num_convs - 1 else 1 self.convs.append( Conv2d( in_channels, self.conv_out_channels, 3, stride=stride, padding=1)) roi_feat_size = _pair(roi_feat_size) pooled_area = (roi_feat_size[0] // 2) * (roi_feat_size[1] // 2) self.fcs = nn.ModuleList() for i in range(num_fcs): in_channels = ( self.conv_out_channels * pooled_area if i == 0 else self.fc_out_channels) self.fcs.append(Linear(in_channels, self.fc_out_channels)) self.fc_mask_iou = Linear(self.fc_out_channels, self.num_classes) self.relu = nn.ReLU() self.max_pool = MaxPool2d(2, 2) self.loss_iou = MODELS.build(loss_iou) def forward(self, mask_feat: Tensor, mask_preds: Tensor) -> Tensor: """Forward function. Args: mask_feat (Tensor): Mask features from upstream models. mask_preds (Tensor): Mask predictions from mask head. Returns: Tensor: Mask IoU predictions. """ mask_preds = mask_preds.sigmoid() mask_pred_pooled = self.max_pool(mask_preds.unsqueeze(1)) x = torch.cat((mask_feat, mask_pred_pooled), 1) for conv in self.convs: x = self.relu(conv(x)) x = x.flatten(1) for fc in self.fcs: x = self.relu(fc(x)) mask_iou = self.fc_mask_iou(x) return mask_iou def loss_and_target(self, mask_iou_pred: Tensor, mask_preds: Tensor, mask_targets: Tensor, sampling_results: List[SamplingResult], batch_gt_instances: InstanceList, rcnn_train_cfg: ConfigDict) -> dict: """Calculate the loss and targets of MaskIoUHead. Args: mask_iou_pred (Tensor): Mask IoU predictions results, has shape (num_pos, num_classes) mask_preds (Tensor): Mask predictions from mask head, has shape (num_pos, mask_size, mask_size). mask_targets (Tensor): The ground truth masks assigned with predictions, has shape (num_pos, mask_size, mask_size). sampling_results (List[obj:SamplingResult]): Assign results of all images in a batch after sampling. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It includes ``masks`` inside. rcnn_train_cfg (obj:`ConfigDict`): `train_cfg` of RCNN. Returns: dict: A dictionary of loss and targets components. The targets are only used for cascade rcnn. """ mask_iou_targets = self.get_targets( sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, mask_preds=mask_preds, mask_targets=mask_targets, rcnn_train_cfg=rcnn_train_cfg) pos_inds = mask_iou_targets > 0 if pos_inds.sum() > 0: loss_mask_iou = self.loss_iou(mask_iou_pred[pos_inds], mask_iou_targets[pos_inds]) else: loss_mask_iou = mask_iou_pred.sum() * 0 return dict(loss_mask_iou=loss_mask_iou) def get_targets(self, sampling_results: List[SamplingResult], batch_gt_instances: InstanceList, mask_preds: Tensor, mask_targets: Tensor, rcnn_train_cfg: ConfigDict) -> Tensor: """Compute target of mask IoU. Mask IoU target is the IoU of the predicted mask (inside a bbox) and the gt mask of corresponding gt mask (the whole instance). The intersection area is computed inside the bbox, and the gt mask area is computed with two steps, firstly we compute the gt area inside the bbox, then divide it by the area ratio of gt area inside the bbox and the gt area of the whole instance. Args: sampling_results (list[:obj:`SamplingResult`]): sampling results. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It includes ``masks`` inside. mask_preds (Tensor): Predicted masks of each positive proposal, shape (num_pos, h, w). mask_targets (Tensor): Gt mask of each positive proposal, binary map of the shape (num_pos, h, w). rcnn_train_cfg (obj:`ConfigDict`): Training config for R-CNN part. Returns: Tensor: mask iou target (length == num positive). """ pos_proposals = [res.pos_priors for res in sampling_results] pos_assigned_gt_inds = [ res.pos_assigned_gt_inds for res in sampling_results ] gt_masks = [res.masks for res in batch_gt_instances] # compute the area ratio of gt areas inside the proposals and # the whole instance area_ratios = map(self._get_area_ratio, pos_proposals, pos_assigned_gt_inds, gt_masks) area_ratios = torch.cat(list(area_ratios)) assert mask_targets.size(0) == area_ratios.size(0) mask_preds = (mask_preds > rcnn_train_cfg.mask_thr_binary).float() mask_pred_areas = mask_preds.sum((-1, -2)) # mask_preds and mask_targets are binary maps overlap_areas = (mask_preds * mask_targets).sum((-1, -2)) # compute the mask area of the whole instance gt_full_areas = mask_targets.sum((-1, -2)) / (area_ratios + 1e-7) mask_iou_targets = overlap_areas / ( mask_pred_areas + gt_full_areas - overlap_areas) return mask_iou_targets def _get_area_ratio(self, pos_proposals: Tensor, pos_assigned_gt_inds: Tensor, gt_masks: InstanceData) -> Tensor: """Compute area ratio of the gt mask inside the proposal and the gt mask of the corresponding instance. Args: pos_proposals (Tensor): Positive proposals, has shape (num_pos, 4). pos_assigned_gt_inds (Tensor): positive proposals assigned ground truth index. gt_masks (BitmapMask or PolygonMask): Gt masks (the whole instance) of each image, with the same shape of the input image. Returns: Tensor: The area ratio of the gt mask inside the proposal and the gt mask of the corresponding instance. """ num_pos = pos_proposals.size(0) if num_pos > 0: area_ratios = [] proposals_np = pos_proposals.cpu().numpy() pos_assigned_gt_inds = pos_assigned_gt_inds.cpu().numpy() # compute mask areas of gt instances (batch processing for speedup) gt_instance_mask_area = gt_masks.areas for i in range(num_pos): gt_mask = gt_masks[pos_assigned_gt_inds[i]] # crop the gt mask inside the proposal bbox = proposals_np[i, :].astype(np.int32) gt_mask_in_proposal = gt_mask.crop(bbox) ratio = gt_mask_in_proposal.areas[0] / ( gt_instance_mask_area[pos_assigned_gt_inds[i]] + 1e-7) area_ratios.append(ratio) area_ratios = torch.from_numpy(np.stack(area_ratios)).float().to( pos_proposals.device) else: area_ratios = pos_proposals.new_zeros((0, )) return area_ratios def predict_by_feat(self, mask_iou_preds: Tuple[Tensor], results_list: InstanceList) -> InstanceList: """Predict the mask iou and calculate it into ``results.scores``. Args: mask_iou_preds (Tensor): Mask IoU predictions results, has shape (num_proposals, num_classes) results_list (list[:obj:`InstanceData`]): Detection results of each image. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ assert len(mask_iou_preds) == len(results_list) for results, mask_iou_pred in zip(results_list, mask_iou_preds): labels = results.labels scores = results.scores results.scores = scores * mask_iou_pred[range(labels.size(0)), labels] return results_list ================================================ FILE: mmdet/models/roi_heads/mask_heads/scnet_mask_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.layers import ResLayer, SimplifiedBasicBlock from mmdet.registry import MODELS from .fcn_mask_head import FCNMaskHead @MODELS.register_module() class SCNetMaskHead(FCNMaskHead): """Mask head for `SCNet `_. Args: conv_to_res (bool, optional): if True, change the conv layers to ``SimplifiedBasicBlock``. """ def __init__(self, conv_to_res: bool = True, **kwargs) -> None: super().__init__(**kwargs) self.conv_to_res = conv_to_res if conv_to_res: assert self.conv_kernel_size == 3 self.num_res_blocks = self.num_convs // 2 self.convs = ResLayer( SimplifiedBasicBlock, self.in_channels, self.conv_out_channels, self.num_res_blocks, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg) ================================================ FILE: mmdet/models/roi_heads/mask_heads/scnet_semantic_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.layers import ResLayer, SimplifiedBasicBlock from mmdet.registry import MODELS from .fused_semantic_head import FusedSemanticHead @MODELS.register_module() class SCNetSemanticHead(FusedSemanticHead): """Mask head for `SCNet `_. Args: conv_to_res (bool, optional): if True, change the conv layers to ``SimplifiedBasicBlock``. """ def __init__(self, conv_to_res: bool = True, **kwargs) -> None: super().__init__(**kwargs) self.conv_to_res = conv_to_res if self.conv_to_res: num_res_blocks = self.num_convs // 2 self.convs = ResLayer( SimplifiedBasicBlock, self.in_channels, self.conv_out_channels, num_res_blocks, conv_cfg=self.conv_cfg, norm_cfg=self.norm_cfg) self.num_convs = num_res_blocks ================================================ FILE: mmdet/models/roi_heads/mask_scoring_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import torch from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox2roi from mmdet.utils import ConfigType, InstanceList from ..task_modules.samplers import SamplingResult from ..utils.misc import empty_instances from .standard_roi_head import StandardRoIHead @MODELS.register_module() class MaskScoringRoIHead(StandardRoIHead): """Mask Scoring RoIHead for `Mask Scoring RCNN. `_. Args: mask_iou_head (:obj`ConfigDict`, dict): The config of mask_iou_head. """ def __init__(self, mask_iou_head: ConfigType, **kwargs): assert mask_iou_head is not None super().__init__(**kwargs) self.mask_iou_head = MODELS.build(mask_iou_head) def forward(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList = None) -> tuple: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: x (List[Tensor]): Multi-level features that may have different resolutions. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. Returns tuple: A tuple of features from ``bbox_head`` and ``mask_head`` forward. """ results = () proposals = [rpn_results.bboxes for rpn_results in rpn_results_list] rois = bbox2roi(proposals) # bbox head if self.with_bbox: bbox_results = self._bbox_forward(x, rois) results = results + (bbox_results['cls_score'], bbox_results['bbox_pred']) # mask head if self.with_mask: mask_rois = rois[:100] mask_results = self._mask_forward(x, mask_rois) results = results + (mask_results['mask_preds'], ) # mask iou head cls_score = bbox_results['cls_score'][:100] mask_preds = mask_results['mask_preds'] mask_feats = mask_results['mask_feats'] _, labels = cls_score[:, :self.bbox_head.num_classes].max(dim=1) mask_iou_preds = self.mask_iou_head( mask_feats, mask_preds[range(labels.size(0)), labels]) results = results + (mask_iou_preds, ) return results def mask_loss(self, x: Tuple[Tensor], sampling_results: List[SamplingResult], bbox_feats, batch_gt_instances: InstanceList) -> dict: """Perform forward propagation and loss calculation of the mask head on the features of the upstream network. Args: x (tuple[Tensor]): Tuple of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. bbox_feats (Tensor): Extract bbox RoI features. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. - `mask_feats` (Tensor): Extract mask RoI features. - `mask_targets` (Tensor): Mask target of each positive\ proposals in the image. - `loss_mask` (dict): A dictionary of mask loss components. - `loss_mask_iou` (Tensor): mask iou loss. """ if not self.share_roi_extractor: pos_rois = bbox2roi([res.pos_priors for res in sampling_results]) mask_results = self._mask_forward(x, pos_rois) else: pos_inds = [] device = bbox_feats.device for res in sampling_results: pos_inds.append( torch.ones( res.pos_priors.shape[0], device=device, dtype=torch.uint8)) pos_inds.append( torch.zeros( res.neg_priors.shape[0], device=device, dtype=torch.uint8)) pos_inds = torch.cat(pos_inds) mask_results = self._mask_forward( x, pos_inds=pos_inds, bbox_feats=bbox_feats) mask_loss_and_target = self.mask_head.loss_and_target( mask_preds=mask_results['mask_preds'], sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, rcnn_train_cfg=self.train_cfg) mask_targets = mask_loss_and_target['mask_targets'] mask_results.update(loss_mask=mask_loss_and_target['loss_mask']) if mask_results['loss_mask'] is None: return mask_results # mask iou head forward and loss pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) pos_mask_pred = mask_results['mask_preds'][ range(mask_results['mask_preds'].size(0)), pos_labels] mask_iou_pred = self.mask_iou_head(mask_results['mask_feats'], pos_mask_pred) pos_mask_iou_pred = mask_iou_pred[range(mask_iou_pred.size(0)), pos_labels] loss_mask_iou = self.mask_iou_head.loss_and_target( pos_mask_iou_pred, pos_mask_pred, mask_targets, sampling_results, batch_gt_instances, self.train_cfg) mask_results['loss_mask'].update(loss_mask_iou) return mask_results def predict_mask(self, x: Tensor, batch_img_metas: List[dict], results_list: InstanceList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the mask head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. results_list (list[:obj:`InstanceData`]): Detection results of each image. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ bboxes = [res.bboxes for res in results_list] mask_rois = bbox2roi(bboxes) if mask_rois.shape[0] == 0: results_list = empty_instances( batch_img_metas, mask_rois.device, task_type='mask', instance_results=results_list, mask_thr_binary=self.test_cfg.mask_thr_binary) return results_list mask_results = self._mask_forward(x, mask_rois) mask_preds = mask_results['mask_preds'] mask_feats = mask_results['mask_feats'] # get mask scores with mask iou head labels = torch.cat([res.labels for res in results_list]) mask_iou_preds = self.mask_iou_head( mask_feats, mask_preds[range(labels.size(0)), labels]) # split batch mask prediction back to each image num_mask_rois_per_img = [len(res) for res in results_list] mask_preds = mask_preds.split(num_mask_rois_per_img, 0) mask_iou_preds = mask_iou_preds.split(num_mask_rois_per_img, 0) # TODO: Handle the case where rescale is false results_list = self.mask_head.predict_by_feat( mask_preds=mask_preds, results_list=results_list, batch_img_metas=batch_img_metas, rcnn_test_cfg=self.test_cfg, rescale=rescale) results_list = self.mask_iou_head.predict_by_feat( mask_iou_preds=mask_iou_preds, results_list=results_list) return results_list ================================================ FILE: mmdet/models/roi_heads/multi_instance_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.structures.bbox import bbox2roi from mmdet.utils import ConfigType, InstanceList from ..task_modules.samplers import SamplingResult from ..utils import empty_instances, unpack_gt_instances from .standard_roi_head import StandardRoIHead @MODELS.register_module() class MultiInstanceRoIHead(StandardRoIHead): """The roi head for Multi-instance prediction.""" def __init__(self, num_instance: int = 2, *args, **kwargs) -> None: self.num_instance = num_instance super().__init__(*args, **kwargs) def init_bbox_head(self, bbox_roi_extractor: ConfigType, bbox_head: ConfigType) -> None: """Initialize box head and box roi extractor. Args: bbox_roi_extractor (dict or ConfigDict): Config of box roi extractor. bbox_head (dict or ConfigDict): Config of box in box head. """ self.bbox_roi_extractor = MODELS.build(bbox_roi_extractor) self.bbox_head = MODELS.build(bbox_head) def _bbox_forward(self, x: Tuple[Tensor], rois: Tensor) -> dict: """Box head forward function used in both training and testing. Args: x (tuple[Tensor]): List of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `cls_score_ref` (Tensor): The cls_score after refine model. - `bbox_pred_ref` (Tensor): The bbox_pred after refine model. - `bbox_feats` (Tensor): Extract bbox RoI features. """ # TODO: a more flexible way to decide which feature maps to use bbox_feats = self.bbox_roi_extractor( x[:self.bbox_roi_extractor.num_inputs], rois) bbox_results = self.bbox_head(bbox_feats) if self.bbox_head.with_refine: bbox_results = dict( cls_score=bbox_results[0], bbox_pred=bbox_results[1], cls_score_ref=bbox_results[2], bbox_pred_ref=bbox_results[3], bbox_feats=bbox_feats) else: bbox_results = dict( cls_score=bbox_results[0], bbox_pred=bbox_results[1], bbox_feats=bbox_feats) return bbox_results def bbox_loss(self, x: Tuple[Tensor], sampling_results: List[SamplingResult]) -> dict: """Perform forward propagation and loss calculation of the bbox head on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. - `loss_bbox` (dict): A dictionary of bbox loss components. """ rois = bbox2roi([res.priors for res in sampling_results]) bbox_results = self._bbox_forward(x, rois) # If there is a refining process, add refine loss. if 'cls_score_ref' in bbox_results: bbox_loss_and_target = self.bbox_head.loss_and_target( cls_score=bbox_results['cls_score'], bbox_pred=bbox_results['bbox_pred'], rois=rois, sampling_results=sampling_results, rcnn_train_cfg=self.train_cfg) bbox_results.update(loss_bbox=bbox_loss_and_target['loss_bbox']) bbox_loss_and_target_ref = self.bbox_head.loss_and_target( cls_score=bbox_results['cls_score_ref'], bbox_pred=bbox_results['bbox_pred_ref'], rois=rois, sampling_results=sampling_results, rcnn_train_cfg=self.train_cfg) bbox_results['loss_bbox']['loss_rcnn_emd_ref'] = \ bbox_loss_and_target_ref['loss_bbox']['loss_rcnn_emd'] else: bbox_loss_and_target = self.bbox_head.loss_and_target( cls_score=bbox_results['cls_score'], bbox_pred=bbox_results['bbox_pred'], rois=rois, sampling_results=sampling_results, rcnn_train_cfg=self.train_cfg) bbox_results.update(loss_bbox=bbox_loss_and_target['loss_bbox']) return bbox_results def loss(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: List[DetDataSample]) -> dict: """Perform forward propagation and loss calculation of the detection roi on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components """ assert len(rpn_results_list) == len(batch_data_samples) outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, _ = outputs sampling_results = [] for i in range(len(batch_data_samples)): # rename rpn_results.bboxes to rpn_results.priors rpn_results = rpn_results_list[i] rpn_results.priors = rpn_results.pop('bboxes') assign_result = self.bbox_assigner.assign( rpn_results, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = self.bbox_sampler.sample( assign_result, rpn_results, batch_gt_instances[i], batch_gt_instances_ignore=batch_gt_instances_ignore[i]) sampling_results.append(sampling_result) losses = dict() # bbox head loss if self.with_bbox: bbox_results = self.bbox_loss(x, sampling_results) losses.update(bbox_results['loss_bbox']) return losses def predict_bbox(self, x: Tuple[Tensor], batch_img_metas: List[dict], rpn_results_list: InstanceList, rcnn_test_cfg: ConfigType, rescale: bool = False) -> InstanceList: """Perform forward propagation of the bbox head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of R-CNN. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ proposals = [res.bboxes for res in rpn_results_list] rois = bbox2roi(proposals) if rois.shape[0] == 0: return empty_instances( batch_img_metas, rois.device, task_type='bbox') bbox_results = self._bbox_forward(x, rois) # split batch bbox prediction back to each image if 'cls_score_ref' in bbox_results: cls_scores = bbox_results['cls_score_ref'] bbox_preds = bbox_results['bbox_pred_ref'] else: cls_scores = bbox_results['cls_score'] bbox_preds = bbox_results['bbox_pred'] num_proposals_per_img = tuple(len(p) for p in proposals) rois = rois.split(num_proposals_per_img, 0) cls_scores = cls_scores.split(num_proposals_per_img, 0) if bbox_preds is not None: bbox_preds = bbox_preds.split(num_proposals_per_img, 0) else: bbox_preds = (None, ) * len(proposals) result_list = self.bbox_head.predict_by_feat( rois=rois, cls_scores=cls_scores, bbox_preds=bbox_preds, batch_img_metas=batch_img_metas, rcnn_test_cfg=rcnn_test_cfg, rescale=rescale) return result_list ================================================ FILE: mmdet/models/roi_heads/pisa_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple from torch import Tensor from mmdet.models.task_modules import SamplingResult from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.structures.bbox import bbox2roi from mmdet.utils import InstanceList from ..losses.pisa_loss import carl_loss, isr_p from ..utils import unpack_gt_instances from .standard_roi_head import StandardRoIHead @MODELS.register_module() class PISARoIHead(StandardRoIHead): r"""The RoI head for `Prime Sample Attention in Object Detection `_.""" def loss(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: List[DetDataSample]) -> dict: """Perform forward propagation and loss calculation of the detection roi on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components """ assert len(rpn_results_list) == len(batch_data_samples) outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, _ = outputs # assign gts and sample proposals num_imgs = len(batch_data_samples) sampling_results = [] neg_label_weights = [] for i in range(num_imgs): # rename rpn_results.bboxes to rpn_results.priors rpn_results = rpn_results_list[i] rpn_results.priors = rpn_results.pop('bboxes') assign_result = self.bbox_assigner.assign( rpn_results, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = self.bbox_sampler.sample( assign_result, rpn_results, batch_gt_instances[i], feats=[lvl_feat[i][None] for lvl_feat in x]) if isinstance(sampling_result, tuple): sampling_result, neg_label_weight = sampling_result sampling_results.append(sampling_result) neg_label_weights.append(neg_label_weight) losses = dict() # bbox head forward and loss if self.with_bbox: bbox_results = self.bbox_loss( x, sampling_results, neg_label_weights=neg_label_weights) losses.update(bbox_results['loss_bbox']) # mask head forward and loss if self.with_mask: mask_results = self.mask_loss(x, sampling_results, bbox_results['bbox_feats'], batch_gt_instances) losses.update(mask_results['loss_mask']) return losses def bbox_loss(self, x: Tuple[Tensor], sampling_results: List[SamplingResult], neg_label_weights: List[Tensor] = None) -> dict: """Perform forward propagation and loss calculation of the bbox head on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. - `loss_bbox` (dict): A dictionary of bbox loss components. """ rois = bbox2roi([res.priors for res in sampling_results]) bbox_results = self._bbox_forward(x, rois) bbox_targets = self.bbox_head.get_targets(sampling_results, self.train_cfg) # neg_label_weights obtained by sampler is image-wise, mapping back to # the corresponding location in label weights if neg_label_weights[0] is not None: label_weights = bbox_targets[1] cur_num_rois = 0 for i in range(len(sampling_results)): num_pos = sampling_results[i].pos_inds.size(0) num_neg = sampling_results[i].neg_inds.size(0) label_weights[cur_num_rois + num_pos:cur_num_rois + num_pos + num_neg] = neg_label_weights[i] cur_num_rois += num_pos + num_neg cls_score = bbox_results['cls_score'] bbox_pred = bbox_results['bbox_pred'] # Apply ISR-P isr_cfg = self.train_cfg.get('isr', None) if isr_cfg is not None: bbox_targets = isr_p( cls_score, bbox_pred, bbox_targets, rois, sampling_results, self.bbox_head.loss_cls, self.bbox_head.bbox_coder, **isr_cfg, num_class=self.bbox_head.num_classes) loss_bbox = self.bbox_head.loss(cls_score, bbox_pred, rois, *bbox_targets) # Add CARL Loss carl_cfg = self.train_cfg.get('carl', None) if carl_cfg is not None: loss_carl = carl_loss( cls_score, bbox_targets[0], bbox_pred, bbox_targets[2], self.bbox_head.loss_bbox, **carl_cfg, num_class=self.bbox_head.num_classes) loss_bbox.update(loss_carl) bbox_results.update(loss_bbox=loss_bbox) return bbox_results ================================================ FILE: mmdet/models/roi_heads/point_rend_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Modified from https://github.com/facebookresearch/detectron2/tree/master/projects/PointRend # noqa from typing import List, Tuple import torch import torch.nn.functional as F from mmcv.ops import point_sample, rel_roi_point_to_rel_img_point from torch import Tensor from mmdet.registry import MODELS from mmdet.structures.bbox import bbox2roi from mmdet.utils import ConfigType, InstanceList from ..task_modules.samplers import SamplingResult from ..utils import empty_instances from .standard_roi_head import StandardRoIHead @MODELS.register_module() class PointRendRoIHead(StandardRoIHead): """`PointRend `_.""" def __init__(self, point_head: ConfigType, *args, **kwargs) -> None: super().__init__(*args, **kwargs) assert self.with_bbox and self.with_mask self.init_point_head(point_head) def init_point_head(self, point_head: ConfigType) -> None: """Initialize ``point_head``""" self.point_head = MODELS.build(point_head) def mask_loss(self, x: Tuple[Tensor], sampling_results: List[SamplingResult], bbox_feats: Tensor, batch_gt_instances: InstanceList) -> dict: """Run forward function and calculate loss for mask head and point head in training.""" mask_results = super().mask_loss( x=x, sampling_results=sampling_results, bbox_feats=bbox_feats, batch_gt_instances=batch_gt_instances) mask_point_results = self._mask_point_loss( x=x, sampling_results=sampling_results, mask_preds=mask_results['mask_preds'], batch_gt_instances=batch_gt_instances) mask_results['loss_mask'].update( loss_point=mask_point_results['loss_point']) return mask_results def _mask_point_loss(self, x: Tuple[Tensor], sampling_results: List[SamplingResult], mask_preds: Tensor, batch_gt_instances: InstanceList) -> dict: """Run forward function and calculate loss for point head in training.""" pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) rel_roi_points = self.point_head.get_roi_rel_points_train( mask_preds, pos_labels, cfg=self.train_cfg) rois = bbox2roi([res.pos_bboxes for res in sampling_results]) fine_grained_point_feats = self._get_fine_grained_point_feats( x, rois, rel_roi_points) coarse_point_feats = point_sample(mask_preds, rel_roi_points) mask_point_pred = self.point_head(fine_grained_point_feats, coarse_point_feats) loss_and_target = self.point_head.loss_and_target( point_pred=mask_point_pred, rel_roi_points=rel_roi_points, sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, cfg=self.train_cfg) return loss_and_target def _mask_point_forward_test(self, x: Tuple[Tensor], rois: Tensor, label_preds: Tensor, mask_preds: Tensor) -> Tensor: """Mask refining process with point head in testing. Args: x (tuple[Tensor]): Feature maps of all scale level. rois (Tensor): shape (num_rois, 5). label_preds (Tensor): The predication class for each rois. mask_preds (Tensor): The predication coarse masks of shape (num_rois, num_classes, small_size, small_size). Returns: Tensor: The refined masks of shape (num_rois, num_classes, large_size, large_size). """ refined_mask_pred = mask_preds.clone() for subdivision_step in range(self.test_cfg.subdivision_steps): refined_mask_pred = F.interpolate( refined_mask_pred, scale_factor=self.test_cfg.scale_factor, mode='bilinear', align_corners=False) # If `subdivision_num_points` is larger or equal to the # resolution of the next step, then we can skip this step num_rois, channels, mask_height, mask_width = \ refined_mask_pred.shape if (self.test_cfg.subdivision_num_points >= self.test_cfg.scale_factor**2 * mask_height * mask_width and subdivision_step < self.test_cfg.subdivision_steps - 1): continue point_indices, rel_roi_points = \ self.point_head.get_roi_rel_points_test( refined_mask_pred, label_preds, cfg=self.test_cfg) fine_grained_point_feats = self._get_fine_grained_point_feats( x=x, rois=rois, rel_roi_points=rel_roi_points) coarse_point_feats = point_sample(mask_preds, rel_roi_points) mask_point_pred = self.point_head(fine_grained_point_feats, coarse_point_feats) point_indices = point_indices.unsqueeze(1).expand(-1, channels, -1) refined_mask_pred = refined_mask_pred.reshape( num_rois, channels, mask_height * mask_width) refined_mask_pred = refined_mask_pred.scatter_( 2, point_indices, mask_point_pred) refined_mask_pred = refined_mask_pred.view(num_rois, channels, mask_height, mask_width) return refined_mask_pred def _get_fine_grained_point_feats(self, x: Tuple[Tensor], rois: Tensor, rel_roi_points: Tensor) -> Tensor: """Sample fine grained feats from each level feature map and concatenate them together. Args: x (tuple[Tensor]): Feature maps of all scale level. rois (Tensor): shape (num_rois, 5). rel_roi_points (Tensor): A tensor of shape (num_rois, num_points, 2) that contains [0, 1] x [0, 1] normalized coordinates of the most uncertain points from the [mask_height, mask_width] grid. Returns: Tensor: The fine grained features for each points, has shape (num_rois, feats_channels, num_points). """ assert rois.shape[0] > 0, 'RoI is a empty tensor.' num_imgs = x[0].shape[0] fine_grained_feats = [] for idx in range(self.mask_roi_extractor.num_inputs): feats = x[idx] spatial_scale = 1. / float( self.mask_roi_extractor.featmap_strides[idx]) point_feats = [] for batch_ind in range(num_imgs): # unravel batch dim feat = feats[batch_ind].unsqueeze(0) inds = (rois[:, 0].long() == batch_ind) if inds.any(): rel_img_points = rel_roi_point_to_rel_img_point( rois=rois[inds], rel_roi_points=rel_roi_points[inds], img=feat.shape[2:], spatial_scale=spatial_scale).unsqueeze(0) point_feat = point_sample(feat, rel_img_points) point_feat = point_feat.squeeze(0).transpose(0, 1) point_feats.append(point_feat) fine_grained_feats.append(torch.cat(point_feats, dim=0)) return torch.cat(fine_grained_feats, dim=1) def predict_mask(self, x: Tuple[Tensor], batch_img_metas: List[dict], results_list: InstanceList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the mask head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. results_list (list[:obj:`InstanceData`]): Detection results of each image. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ # don't need to consider aug_test. bboxes = [res.bboxes for res in results_list] mask_rois = bbox2roi(bboxes) if mask_rois.shape[0] == 0: results_list = empty_instances( batch_img_metas, mask_rois.device, task_type='mask', instance_results=results_list, mask_thr_binary=self.test_cfg.mask_thr_binary) return results_list mask_results = self._mask_forward(x, mask_rois) mask_preds = mask_results['mask_preds'] # split batch mask prediction back to each image num_mask_rois_per_img = [len(res) for res in results_list] mask_preds = mask_preds.split(num_mask_rois_per_img, 0) # refine mask_preds mask_rois = mask_rois.split(num_mask_rois_per_img, 0) mask_preds_refined = [] for i in range(len(batch_img_metas)): labels = results_list[i].labels x_i = [xx[[i]] for xx in x] mask_rois_i = mask_rois[i] mask_rois_i[:, 0] = 0 mask_pred_i = self._mask_point_forward_test( x_i, mask_rois_i, labels, mask_preds[i]) mask_preds_refined.append(mask_pred_i) # TODO: Handle the case where rescale is false results_list = self.mask_head.predict_by_feat( mask_preds=mask_preds_refined, results_list=results_list, batch_img_metas=batch_img_metas, rcnn_test_cfg=self.test_cfg, rescale=rescale) return results_list ================================================ FILE: mmdet/models/roi_heads/roi_extractors/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .base_roi_extractor import BaseRoIExtractor from .generic_roi_extractor import GenericRoIExtractor from .single_level_roi_extractor import SingleRoIExtractor __all__ = ['BaseRoIExtractor', 'SingleRoIExtractor', 'GenericRoIExtractor'] ================================================ FILE: mmdet/models/roi_heads/roi_extractors/base_roi_extractor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from typing import List, Optional, Tuple import torch import torch.nn as nn from mmcv import ops from mmengine.model import BaseModule from torch import Tensor from mmdet.utils import ConfigType, OptMultiConfig class BaseRoIExtractor(BaseModule, metaclass=ABCMeta): """Base class for RoI extractor. Args: roi_layer (:obj:`ConfigDict` or dict): Specify RoI layer type and arguments. out_channels (int): Output channels of RoI layers. featmap_strides (list[int]): Strides of input feature maps. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, roi_layer: ConfigType, out_channels: int, featmap_strides: List[int], init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) self.roi_layers = self.build_roi_layers(roi_layer, featmap_strides) self.out_channels = out_channels self.featmap_strides = featmap_strides @property def num_inputs(self) -> int: """int: Number of input feature maps.""" return len(self.featmap_strides) def build_roi_layers(self, layer_cfg: ConfigType, featmap_strides: List[int]) -> nn.ModuleList: """Build RoI operator to extract feature from each level feature map. Args: layer_cfg (:obj:`ConfigDict` or dict): Dictionary to construct and config RoI layer operation. Options are modules under ``mmcv/ops`` such as ``RoIAlign``. featmap_strides (list[int]): The stride of input feature map w.r.t to the original image size, which would be used to scale RoI coordinate (original image coordinate system) to feature coordinate system. Returns: :obj:`nn.ModuleList`: The RoI extractor modules for each level feature map. """ cfg = layer_cfg.copy() layer_type = cfg.pop('type') assert hasattr(ops, layer_type) layer_cls = getattr(ops, layer_type) roi_layers = nn.ModuleList( [layer_cls(spatial_scale=1 / s, **cfg) for s in featmap_strides]) return roi_layers def roi_rescale(self, rois: Tensor, scale_factor: float) -> Tensor: """Scale RoI coordinates by scale factor. Args: rois (Tensor): RoI (Region of Interest), shape (n, 5) scale_factor (float): Scale factor that RoI will be multiplied by. Returns: Tensor: Scaled RoI. """ cx = (rois[:, 1] + rois[:, 3]) * 0.5 cy = (rois[:, 2] + rois[:, 4]) * 0.5 w = rois[:, 3] - rois[:, 1] h = rois[:, 4] - rois[:, 2] new_w = w * scale_factor new_h = h * scale_factor x1 = cx - new_w * 0.5 x2 = cx + new_w * 0.5 y1 = cy - new_h * 0.5 y2 = cy + new_h * 0.5 new_rois = torch.stack((rois[:, 0], x1, y1, x2, y2), dim=-1) return new_rois @abstractmethod def forward(self, feats: Tuple[Tensor], rois: Tensor, roi_scale_factor: Optional[float] = None) -> Tensor: """Extractor ROI feats. Args: feats (Tuple[Tensor]): Multi-scale features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. roi_scale_factor (Optional[float]): RoI scale factor. Defaults to None. Returns: Tensor: RoI feature. """ pass ================================================ FILE: mmdet/models/roi_heads/roi_extractors/generic_roi_extractor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple from mmcv.cnn.bricks import build_plugin_layer from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import OptConfigType from .base_roi_extractor import BaseRoIExtractor @MODELS.register_module() class GenericRoIExtractor(BaseRoIExtractor): """Extract RoI features from all level feature maps levels. This is the implementation of `A novel Region of Interest Extraction Layer for Instance Segmentation `_. Args: aggregation (str): The method to aggregate multiple feature maps. Options are 'sum', 'concat'. Defaults to 'sum'. pre_cfg (:obj:`ConfigDict` or dict): Specify pre-processing modules. Defaults to None. post_cfg (:obj:`ConfigDict` or dict): Specify post-processing modules. Defaults to None. kwargs (keyword arguments): Arguments that are the same as :class:`BaseRoIExtractor`. """ def __init__(self, aggregation: str = 'sum', pre_cfg: OptConfigType = None, post_cfg: OptConfigType = None, **kwargs) -> None: super().__init__(**kwargs) assert aggregation in ['sum', 'concat'] self.aggregation = aggregation self.with_post = post_cfg is not None self.with_pre = pre_cfg is not None # build pre/post processing modules if self.with_post: self.post_module = build_plugin_layer(post_cfg, '_post_module')[1] if self.with_pre: self.pre_module = build_plugin_layer(pre_cfg, '_pre_module')[1] def forward(self, feats: Tuple[Tensor], rois: Tensor, roi_scale_factor: Optional[float] = None) -> Tensor: """Extractor ROI feats. Args: feats (Tuple[Tensor]): Multi-scale features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. roi_scale_factor (Optional[float]): RoI scale factor. Defaults to None. Returns: Tensor: RoI feature. """ out_size = self.roi_layers[0].output_size num_levels = len(feats) roi_feats = feats[0].new_zeros( rois.size(0), self.out_channels, *out_size) # some times rois is an empty tensor if roi_feats.shape[0] == 0: return roi_feats if num_levels == 1: return self.roi_layers[0](feats[0], rois) if roi_scale_factor is not None: rois = self.roi_rescale(rois, roi_scale_factor) # mark the starting channels for concat mode start_channels = 0 for i in range(num_levels): roi_feats_t = self.roi_layers[i](feats[i], rois) end_channels = start_channels + roi_feats_t.size(1) if self.with_pre: # apply pre-processing to a RoI extracted from each layer roi_feats_t = self.pre_module(roi_feats_t) if self.aggregation == 'sum': # and sum them all roi_feats += roi_feats_t else: # and concat them along channel dimension roi_feats[:, start_channels:end_channels] = roi_feats_t # update channels starting position start_channels = end_channels # check if concat channels match at the end if self.aggregation == 'concat': assert start_channels == self.out_channels if self.with_post: # apply post-processing before return the result roi_feats = self.post_module(roi_feats) return roi_feats ================================================ FILE: mmdet/models/roi_heads/roi_extractors/single_level_roi_extractor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch from torch import Tensor from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptMultiConfig from .base_roi_extractor import BaseRoIExtractor @MODELS.register_module() class SingleRoIExtractor(BaseRoIExtractor): """Extract RoI features from a single level feature map. If there are multiple input feature levels, each RoI is mapped to a level according to its scale. The mapping rule is proposed in `FPN `_. Args: roi_layer (:obj:`ConfigDict` or dict): Specify RoI layer type and arguments. out_channels (int): Output channels of RoI layers. featmap_strides (List[int]): Strides of input feature maps. finest_scale (int): Scale threshold of mapping to level 0. Defaults to 56. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict], optional): Initialization config dict. Defaults to None. """ def __init__(self, roi_layer: ConfigType, out_channels: int, featmap_strides: List[int], finest_scale: int = 56, init_cfg: OptMultiConfig = None) -> None: super().__init__( roi_layer=roi_layer, out_channels=out_channels, featmap_strides=featmap_strides, init_cfg=init_cfg) self.finest_scale = finest_scale def map_roi_levels(self, rois: Tensor, num_levels: int) -> Tensor: """Map rois to corresponding feature levels by scales. - scale < finest_scale * 2: level 0 - finest_scale * 2 <= scale < finest_scale * 4: level 1 - finest_scale * 4 <= scale < finest_scale * 8: level 2 - scale >= finest_scale * 8: level 3 Args: rois (Tensor): Input RoIs, shape (k, 5). num_levels (int): Total level number. Returns: Tensor: Level index (0-based) of each RoI, shape (k, ) """ scale = torch.sqrt( (rois[:, 3] - rois[:, 1]) * (rois[:, 4] - rois[:, 2])) target_lvls = torch.floor(torch.log2(scale / self.finest_scale + 1e-6)) target_lvls = target_lvls.clamp(min=0, max=num_levels - 1).long() return target_lvls def forward(self, feats: Tuple[Tensor], rois: Tensor, roi_scale_factor: Optional[float] = None): """Extractor ROI feats. Args: feats (Tuple[Tensor]): Multi-scale features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. roi_scale_factor (Optional[float]): RoI scale factor. Defaults to None. Returns: Tensor: RoI feature. """ # convert fp32 to fp16 when amp is on rois = rois.type_as(feats[0]) out_size = self.roi_layers[0].output_size num_levels = len(feats) roi_feats = feats[0].new_zeros( rois.size(0), self.out_channels, *out_size) # TODO: remove this when parrots supports if torch.__version__ == 'parrots': roi_feats.requires_grad = True if num_levels == 1: if len(rois) == 0: return roi_feats return self.roi_layers[0](feats[0], rois) target_lvls = self.map_roi_levels(rois, num_levels) if roi_scale_factor is not None: rois = self.roi_rescale(rois, roi_scale_factor) for i in range(num_levels): mask = target_lvls == i inds = mask.nonzero(as_tuple=False).squeeze(1) if inds.numel() > 0: rois_ = rois[inds] roi_feats_t = self.roi_layers[i](feats[i], rois_) roi_feats[inds] = roi_feats_t else: # Sometimes some pyramid levels will not be used for RoI # feature extraction and this will cause an incomplete # computation graph in one GPU, which is different from those # in other GPUs and will cause a hanging error. # Therefore, we add it to ensure each feature pyramid is # included in the computation graph to avoid runtime bugs. roi_feats += sum( x.view(-1)[0] for x in self.parameters()) * 0. + feats[i].sum() * 0. return roi_feats ================================================ FILE: mmdet/models/roi_heads/scnet_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch import torch.nn.functional as F from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox2roi from mmdet.utils import ConfigType, InstanceList, OptConfigType from ..layers import adaptive_avg_pool2d from ..task_modules.samplers import SamplingResult from ..utils import empty_instances, unpack_gt_instances from .cascade_roi_head import CascadeRoIHead @MODELS.register_module() class SCNetRoIHead(CascadeRoIHead): """RoIHead for `SCNet `_. Args: num_stages (int): number of cascade stages. stage_loss_weights (list): loss weight of cascade stages. semantic_roi_extractor (dict): config to init semantic roi extractor. semantic_head (dict): config to init semantic head. feat_relay_head (dict): config to init feature_relay_head. glbctx_head (dict): config to init global context head. """ def __init__(self, num_stages: int, stage_loss_weights: List[float], semantic_roi_extractor: OptConfigType = None, semantic_head: OptConfigType = None, feat_relay_head: OptConfigType = None, glbctx_head: OptConfigType = None, **kwargs) -> None: super().__init__( num_stages=num_stages, stage_loss_weights=stage_loss_weights, **kwargs) assert self.with_bbox and self.with_mask assert not self.with_shared_head # shared head is not supported if semantic_head is not None: self.semantic_roi_extractor = MODELS.build(semantic_roi_extractor) self.semantic_head = MODELS.build(semantic_head) if feat_relay_head is not None: self.feat_relay_head = MODELS.build(feat_relay_head) if glbctx_head is not None: self.glbctx_head = MODELS.build(glbctx_head) def init_mask_head(self, mask_roi_extractor: ConfigType, mask_head: ConfigType) -> None: """Initialize ``mask_head``""" if mask_roi_extractor is not None: self.mask_roi_extractor = MODELS.build(mask_roi_extractor) self.mask_head = MODELS.build(mask_head) # TODO move to base_roi_head later @property def with_semantic(self) -> bool: """bool: whether the head has semantic head""" return hasattr(self, 'semantic_head') and self.semantic_head is not None @property def with_feat_relay(self) -> bool: """bool: whether the head has feature relay head""" return (hasattr(self, 'feat_relay_head') and self.feat_relay_head is not None) @property def with_glbctx(self) -> bool: """bool: whether the head has global context head""" return hasattr(self, 'glbctx_head') and self.glbctx_head is not None def _fuse_glbctx(self, roi_feats: Tensor, glbctx_feat: Tensor, rois: Tensor) -> Tensor: """Fuse global context feats with roi feats. Args: roi_feats (Tensor): RoI features. glbctx_feat (Tensor): Global context feature.. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. Returns: Tensor: Fused feature. """ assert roi_feats.size(0) == rois.size(0) # RuntimeError: isDifferentiableType(variable.scalar_type()) # INTERNAL ASSERT FAILED if detach() is not used when calling # roi_head.predict(). img_inds = torch.unique(rois[:, 0].detach().cpu(), sorted=True).long() fused_feats = torch.zeros_like(roi_feats) for img_id in img_inds: inds = (rois[:, 0] == img_id.item()) fused_feats[inds] = roi_feats[inds] + glbctx_feat[img_id] return fused_feats def _slice_pos_feats(self, feats: Tensor, sampling_results: List[SamplingResult]) -> Tensor: """Get features from pos rois. Args: feats (Tensor): Input features. sampling_results (list["obj:`SamplingResult`]): Sampling results. Returns: Tensor: Sliced features. """ num_rois = [res.priors.size(0) for res in sampling_results] num_pos_rois = [res.pos_priors.size(0) for res in sampling_results] inds = torch.zeros(sum(num_rois), dtype=torch.bool) start = 0 for i in range(len(num_rois)): start = 0 if i == 0 else start + num_rois[i - 1] stop = start + num_pos_rois[i] inds[start:stop] = 1 sliced_feats = feats[inds] return sliced_feats def _bbox_forward(self, stage: int, x: Tuple[Tensor], rois: Tensor, semantic_feat: Optional[Tensor] = None, glbctx_feat: Optional[Tensor] = None) -> dict: """Box head forward function used in both training and testing. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): List of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. semantic_feat (Tensor): Semantic feature. Defaults to None. glbctx_feat (Tensor): Global context feature. Defaults to None. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. """ bbox_roi_extractor = self.bbox_roi_extractor[stage] bbox_head = self.bbox_head[stage] bbox_feats = bbox_roi_extractor(x[:bbox_roi_extractor.num_inputs], rois) if self.with_semantic and semantic_feat is not None: bbox_semantic_feat = self.semantic_roi_extractor([semantic_feat], rois) if bbox_semantic_feat.shape[-2:] != bbox_feats.shape[-2:]: bbox_semantic_feat = adaptive_avg_pool2d( bbox_semantic_feat, bbox_feats.shape[-2:]) bbox_feats += bbox_semantic_feat if self.with_glbctx and glbctx_feat is not None: bbox_feats = self._fuse_glbctx(bbox_feats, glbctx_feat, rois) cls_score, bbox_pred, relayed_feat = bbox_head( bbox_feats, return_shared_feat=True) bbox_results = dict( cls_score=cls_score, bbox_pred=bbox_pred, relayed_feat=relayed_feat) return bbox_results def _mask_forward(self, x: Tuple[Tensor], rois: Tensor, semantic_feat: Optional[Tensor] = None, glbctx_feat: Optional[Tensor] = None, relayed_feat: Optional[Tensor] = None) -> dict: """Mask head forward function used in both training and testing. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): Tuple of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. semantic_feat (Tensor): Semantic feature. Defaults to None. glbctx_feat (Tensor): Global context feature. Defaults to None. relayed_feat (Tensor): Relayed feature. Defaults to None. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. """ mask_feats = self.mask_roi_extractor( x[:self.mask_roi_extractor.num_inputs], rois) if self.with_semantic and semantic_feat is not None: mask_semantic_feat = self.semantic_roi_extractor([semantic_feat], rois) if mask_semantic_feat.shape[-2:] != mask_feats.shape[-2:]: mask_semantic_feat = F.adaptive_avg_pool2d( mask_semantic_feat, mask_feats.shape[-2:]) mask_feats += mask_semantic_feat if self.with_glbctx and glbctx_feat is not None: mask_feats = self._fuse_glbctx(mask_feats, glbctx_feat, rois) if self.with_feat_relay and relayed_feat is not None: mask_feats = mask_feats + relayed_feat mask_preds = self.mask_head(mask_feats) mask_results = dict(mask_preds=mask_preds) return mask_results def bbox_loss(self, stage: int, x: Tuple[Tensor], sampling_results: List[SamplingResult], semantic_feat: Optional[Tensor] = None, glbctx_feat: Optional[Tensor] = None) -> dict: """Run forward function and calculate loss for box head in training. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): List of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. semantic_feat (Tensor): Semantic feature. Defaults to None. glbctx_feat (Tensor): Global context feature. Defaults to None. Returns: dict: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. - `loss_bbox` (dict): A dictionary of bbox loss components. - `rois` (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. - `bbox_targets` (tuple): Ground truth for proposals in a single image. Containing the following list of Tensors: (labels, label_weights, bbox_targets, bbox_weights) """ bbox_head = self.bbox_head[stage] rois = bbox2roi([res.priors for res in sampling_results]) bbox_results = self._bbox_forward( stage, x, rois, semantic_feat=semantic_feat, glbctx_feat=glbctx_feat) bbox_results.update(rois=rois) bbox_loss_and_target = bbox_head.loss_and_target( cls_score=bbox_results['cls_score'], bbox_pred=bbox_results['bbox_pred'], rois=rois, sampling_results=sampling_results, rcnn_train_cfg=self.train_cfg[stage]) bbox_results.update(bbox_loss_and_target) return bbox_results def mask_loss(self, x: Tuple[Tensor], sampling_results: List[SamplingResult], batch_gt_instances: InstanceList, semantic_feat: Optional[Tensor] = None, glbctx_feat: Optional[Tensor] = None, relayed_feat: Optional[Tensor] = None) -> dict: """Run forward function and calculate loss for mask head in training. Args: x (tuple[Tensor]): Tuple of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. semantic_feat (Tensor): Semantic feature. Defaults to None. glbctx_feat (Tensor): Global context feature. Defaults to None. relayed_feat (Tensor): Relayed feature. Defaults to None. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. - `loss_mask` (dict): A dictionary of mask loss components. """ pos_rois = bbox2roi([res.pos_priors for res in sampling_results]) mask_results = self._mask_forward( x, pos_rois, semantic_feat=semantic_feat, glbctx_feat=glbctx_feat, relayed_feat=relayed_feat) mask_loss_and_target = self.mask_head.loss_and_target( mask_preds=mask_results['mask_preds'], sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, rcnn_train_cfg=self.train_cfg[-1]) mask_results.update(mask_loss_and_target) return mask_results def semantic_loss(self, x: Tuple[Tensor], batch_data_samples: SampleList) -> dict: """Semantic segmentation loss. Args: x (Tuple[Tensor]): Tuple of multi-level img features. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: Usually returns a dictionary with keys: - `semantic_feat` (Tensor): Semantic feature. - `loss_seg` (dict): Semantic segmentation loss. """ gt_semantic_segs = [ data_sample.gt_sem_seg.sem_seg for data_sample in batch_data_samples ] gt_semantic_segs = torch.stack(gt_semantic_segs) semantic_pred, semantic_feat = self.semantic_head(x) loss_seg = self.semantic_head.loss(semantic_pred, gt_semantic_segs) semantic_results = dict(loss_seg=loss_seg, semantic_feat=semantic_feat) return semantic_results def global_context_loss(self, x: Tuple[Tensor], batch_gt_instances: InstanceList) -> dict: """Global context loss. Args: x (Tuple[Tensor]): Tuple of multi-level img features. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. Returns: dict: Usually returns a dictionary with keys: - `glbctx_feat` (Tensor): Global context feature. - `loss_glbctx` (dict): Global context loss. """ gt_labels = [ gt_instances.labels for gt_instances in batch_gt_instances ] mc_pred, glbctx_feat = self.glbctx_head(x) loss_glbctx = self.glbctx_head.loss(mc_pred, gt_labels) global_context_results = dict( loss_glbctx=loss_glbctx, glbctx_feat=glbctx_feat) return global_context_results def loss(self, x: Tensor, rpn_results_list: InstanceList, batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection roi on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components """ assert len(rpn_results_list) == len(batch_data_samples) outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, batch_img_metas \ = outputs losses = dict() # semantic segmentation branch if self.with_semantic: semantic_results = self.semantic_loss( x=x, batch_data_samples=batch_data_samples) losses['loss_semantic_seg'] = semantic_results['loss_seg'] semantic_feat = semantic_results['semantic_feat'] else: semantic_feat = None # global context branch if self.with_glbctx: global_context_results = self.global_context_loss( x=x, batch_gt_instances=batch_gt_instances) losses['loss_glbctx'] = global_context_results['loss_glbctx'] glbctx_feat = global_context_results['glbctx_feat'] else: glbctx_feat = None results_list = rpn_results_list num_imgs = len(batch_img_metas) for stage in range(self.num_stages): stage_loss_weight = self.stage_loss_weights[stage] # assign gts and sample proposals sampling_results = [] bbox_assigner = self.bbox_assigner[stage] bbox_sampler = self.bbox_sampler[stage] for i in range(num_imgs): results = results_list[i] # rename rpn_results.bboxes to rpn_results.priors results.priors = results.pop('bboxes') assign_result = bbox_assigner.assign( results, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = bbox_sampler.sample( assign_result, results, batch_gt_instances[i], feats=[lvl_feat[i][None] for lvl_feat in x]) sampling_results.append(sampling_result) # bbox head forward and loss bbox_results = self.bbox_loss( stage=stage, x=x, sampling_results=sampling_results, semantic_feat=semantic_feat, glbctx_feat=glbctx_feat) for name, value in bbox_results['loss_bbox'].items(): losses[f's{stage}.{name}'] = ( value * stage_loss_weight if 'loss' in name else value) # refine bboxes if stage < self.num_stages - 1: bbox_head = self.bbox_head[stage] with torch.no_grad(): results_list = bbox_head.refine_bboxes( sampling_results=sampling_results, bbox_results=bbox_results, batch_img_metas=batch_img_metas) if self.with_feat_relay: relayed_feat = self._slice_pos_feats(bbox_results['relayed_feat'], sampling_results) relayed_feat = self.feat_relay_head(relayed_feat) else: relayed_feat = None # mask head forward and loss mask_results = self.mask_loss( x=x, sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, semantic_feat=semantic_feat, glbctx_feat=glbctx_feat, relayed_feat=relayed_feat) mask_stage_loss_weight = sum(self.stage_loss_weights) losses['loss_mask'] = mask_stage_loss_weight * mask_results[ 'loss_mask']['loss_mask'] return losses def predict(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the roi head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Features from upstream network. Each has shape (N, C, H, W). rpn_results_list (list[:obj:`InstanceData`]): list of region proposals. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results to the original image. Defaults to False. Returns: list[obj:`InstanceData`]: Detection results of each image. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ assert self.with_bbox, 'Bbox head must be implemented.' batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] if self.with_semantic: _, semantic_feat = self.semantic_head(x) else: semantic_feat = None if self.with_glbctx: _, glbctx_feat = self.glbctx_head(x) else: glbctx_feat = None # TODO: nms_op in mmcv need be enhanced, the bbox result may get # difference when not rescale in bbox_head # If it has the mask branch, the bbox branch does not need # to be scaled to the original image scale, because the mask # branch will scale both bbox and mask at the same time. bbox_rescale = rescale if not self.with_mask else False results_list = self.predict_bbox( x=x, semantic_feat=semantic_feat, glbctx_feat=glbctx_feat, batch_img_metas=batch_img_metas, rpn_results_list=rpn_results_list, rcnn_test_cfg=self.test_cfg, rescale=bbox_rescale) if self.with_mask: results_list = self.predict_mask( x=x, semantic_heat=semantic_feat, glbctx_feat=glbctx_feat, batch_img_metas=batch_img_metas, results_list=results_list, rescale=rescale) return results_list def predict_mask(self, x: Tuple[Tensor], semantic_heat: Tensor, glbctx_feat: Tensor, batch_img_metas: List[dict], results_list: List[InstanceData], rescale: bool = False) -> List[InstanceData]: """Perform forward propagation of the mask head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. semantic_feat (Tensor): Semantic feature. glbctx_feat (Tensor): Global context feature. batch_img_metas (list[dict]): List of image information. results_list (list[:obj:`InstanceData`]): Detection results of each image. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ bboxes = [res.bboxes for res in results_list] mask_rois = bbox2roi(bboxes) if mask_rois.shape[0] == 0: results_list = empty_instances( batch_img_metas=batch_img_metas, device=mask_rois.device, task_type='mask', instance_results=results_list, mask_thr_binary=self.test_cfg.mask_thr_binary) return results_list bboxes_results = self._bbox_forward( stage=-1, x=x, rois=mask_rois, semantic_feat=semantic_heat, glbctx_feat=glbctx_feat) relayed_feat = bboxes_results['relayed_feat'] relayed_feat = self.feat_relay_head(relayed_feat) mask_results = self._mask_forward( x=x, rois=mask_rois, semantic_feat=semantic_heat, glbctx_feat=glbctx_feat, relayed_feat=relayed_feat) mask_preds = mask_results['mask_preds'] # split batch mask prediction back to each image num_bbox_per_img = tuple(len(_bbox) for _bbox in bboxes) mask_preds = mask_preds.split(num_bbox_per_img, 0) results_list = self.mask_head.predict_by_feat( mask_preds=mask_preds, results_list=results_list, batch_img_metas=batch_img_metas, rcnn_test_cfg=self.test_cfg, rescale=rescale) return results_list def forward(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList) -> tuple: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: x (List[Tensor]): Multi-level features that may have different resolutions. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. Returns tuple: A tuple of features from ``bbox_head`` and ``mask_head`` forward. """ results = () batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] if self.with_semantic: _, semantic_feat = self.semantic_head(x) else: semantic_feat = None if self.with_glbctx: _, glbctx_feat = self.glbctx_head(x) else: glbctx_feat = None proposals = [rpn_results.bboxes for rpn_results in rpn_results_list] num_proposals_per_img = tuple(len(p) for p in proposals) rois = bbox2roi(proposals) # bbox head if self.with_bbox: rois, cls_scores, bbox_preds = self._refine_roi( x=x, rois=rois, semantic_feat=semantic_feat, glbctx_feat=glbctx_feat, batch_img_metas=batch_img_metas, num_proposals_per_img=num_proposals_per_img) results = results + (cls_scores, bbox_preds) # mask head if self.with_mask: rois = torch.cat(rois) bboxes_results = self._bbox_forward( stage=-1, x=x, rois=rois, semantic_feat=semantic_feat, glbctx_feat=glbctx_feat) relayed_feat = bboxes_results['relayed_feat'] relayed_feat = self.feat_relay_head(relayed_feat) mask_results = self._mask_forward( x=x, rois=rois, semantic_feat=semantic_feat, glbctx_feat=glbctx_feat, relayed_feat=relayed_feat) mask_preds = mask_results['mask_preds'] mask_preds = mask_preds.split(num_proposals_per_img, 0) results = results + (mask_preds, ) return results ================================================ FILE: mmdet/models/roi_heads/shared_heads/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .res_layer import ResLayer __all__ = ['ResLayer'] ================================================ FILE: mmdet/models/roi_heads/shared_heads/res_layer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings import torch.nn as nn from mmengine.model import BaseModule from mmdet.models.backbones import ResNet from mmdet.models.layers import ResLayer as _ResLayer from mmdet.registry import MODELS @MODELS.register_module() class ResLayer(BaseModule): def __init__(self, depth, stage=3, stride=2, dilation=1, style='pytorch', norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, with_cp=False, dcn=None, pretrained=None, init_cfg=None): super(ResLayer, self).__init__(init_cfg) self.norm_eval = norm_eval self.norm_cfg = norm_cfg self.stage = stage self.fp16_enabled = False block, stage_blocks = ResNet.arch_settings[depth] stage_block = stage_blocks[stage] planes = 64 * 2**stage inplanes = 64 * 2**(stage - 1) * block.expansion res_layer = _ResLayer( block, inplanes, planes, stage_block, stride=stride, dilation=dilation, style=style, with_cp=with_cp, norm_cfg=self.norm_cfg, dcn=dcn) self.add_module(f'layer{stage + 1}', res_layer) assert not (init_cfg and pretrained), \ 'init_cfg and pretrained cannot be specified at the same time' if isinstance(pretrained, str): warnings.warn('DeprecationWarning: pretrained is a deprecated, ' 'please use "init_cfg" instead') self.init_cfg = dict(type='Pretrained', checkpoint=pretrained) elif pretrained is None: if init_cfg is None: self.init_cfg = [ dict(type='Kaiming', layer='Conv2d'), dict( type='Constant', val=1, layer=['_BatchNorm', 'GroupNorm']) ] else: raise TypeError('pretrained must be a str or None') def forward(self, x): res_layer = getattr(self, f'layer{self.stage + 1}') out = res_layer(x) return out def train(self, mode=True): super(ResLayer, self).train(mode) if self.norm_eval: for m in self.modules(): if isinstance(m, nn.BatchNorm2d): m.eval() ================================================ FILE: mmdet/models/roi_heads/sparse_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.task_modules.samplers import PseudoSampler from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox2roi from mmdet.utils import ConfigType, InstanceList, OptConfigType from ..utils.misc import empty_instances, unpack_gt_instances from .cascade_roi_head import CascadeRoIHead @MODELS.register_module() class SparseRoIHead(CascadeRoIHead): r"""The RoIHead for `Sparse R-CNN: End-to-End Object Detection with Learnable Proposals `_ and `Instances as Queries `_ Args: num_stages (int): Number of stage whole iterative process. Defaults to 6. stage_loss_weights (Tuple[float]): The loss weight of each stage. By default all stages have the same weight 1. bbox_roi_extractor (:obj:`ConfigDict` or dict): Config of box roi extractor. mask_roi_extractor (:obj:`ConfigDict` or dict): Config of mask roi extractor. bbox_head (:obj:`ConfigDict` or dict): Config of box head. mask_head (:obj:`ConfigDict` or dict): Config of mask head. train_cfg (:obj:`ConfigDict` or dict, Optional): Configuration information in train stage. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, Optional): Configuration information in test stage. Defaults to None. init_cfg (:obj:`ConfigDict` or dict or list[:obj:`ConfigDict` or \ dict]): Initialization config dict. Defaults to None. """ def __init__(self, num_stages: int = 6, stage_loss_weights: Tuple[float] = (1, 1, 1, 1, 1, 1), proposal_feature_channel: int = 256, bbox_roi_extractor: ConfigType = dict( type='SingleRoIExtractor', roi_layer=dict( type='RoIAlign', output_size=7, sampling_ratio=2), out_channels=256, featmap_strides=[4, 8, 16, 32]), mask_roi_extractor: OptConfigType = None, bbox_head: ConfigType = dict( type='DIIHead', num_classes=80, num_fcs=2, num_heads=8, num_cls_fcs=1, num_reg_fcs=3, feedforward_channels=2048, hidden_channels=256, dropout=0.0, roi_feat_size=7, ffn_act_cfg=dict(type='ReLU', inplace=True)), mask_head: OptConfigType = None, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: OptConfigType = None) -> None: assert bbox_roi_extractor is not None assert bbox_head is not None assert len(stage_loss_weights) == num_stages self.num_stages = num_stages self.stage_loss_weights = stage_loss_weights self.proposal_feature_channel = proposal_feature_channel super().__init__( num_stages=num_stages, stage_loss_weights=stage_loss_weights, bbox_roi_extractor=bbox_roi_extractor, mask_roi_extractor=mask_roi_extractor, bbox_head=bbox_head, mask_head=mask_head, train_cfg=train_cfg, test_cfg=test_cfg, init_cfg=init_cfg) # train_cfg would be None when run the test.py if train_cfg is not None: for stage in range(num_stages): assert isinstance(self.bbox_sampler[stage], PseudoSampler), \ 'Sparse R-CNN and QueryInst only support `PseudoSampler`' def bbox_loss(self, stage: int, x: Tuple[Tensor], results_list: InstanceList, object_feats: Tensor, batch_img_metas: List[dict], batch_gt_instances: InstanceList) -> dict: """Perform forward propagation and loss calculation of the bbox head on the features of the upstream network. Args: stage (int): The current stage in iterative process. x (tuple[Tensor]): List of multi-level img features. results_list (List[:obj:`InstanceData`]) : List of region proposals. object_feats (Tensor): The object feature extracted from the previous stage. batch_img_metas (list[dict]): Meta information of each image. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. - `loss_bbox` (dict): A dictionary of bbox loss components. """ proposal_list = [res.bboxes for res in results_list] rois = bbox2roi(proposal_list) bbox_results = self._bbox_forward(stage, x, rois, object_feats, batch_img_metas) imgs_whwh = torch.cat( [res.imgs_whwh[None, ...] for res in results_list]) cls_pred_list = bbox_results['detached_cls_scores'] proposal_list = bbox_results['detached_proposals'] sampling_results = [] bbox_head = self.bbox_head[stage] for i in range(len(batch_img_metas)): pred_instances = InstanceData() # TODO: Enhance the logic pred_instances.bboxes = proposal_list[i] # for assinger pred_instances.scores = cls_pred_list[i] pred_instances.priors = proposal_list[i] # for sampler assign_result = self.bbox_assigner[stage].assign( pred_instances=pred_instances, gt_instances=batch_gt_instances[i], gt_instances_ignore=None, img_meta=batch_img_metas[i]) sampling_result = self.bbox_sampler[stage].sample( assign_result, pred_instances, batch_gt_instances[i]) sampling_results.append(sampling_result) bbox_results.update(sampling_results=sampling_results) cls_score = bbox_results['cls_score'] decoded_bboxes = bbox_results['decoded_bboxes'] cls_score = cls_score.view(-1, cls_score.size(-1)) decoded_bboxes = decoded_bboxes.view(-1, 4) bbox_loss_and_target = bbox_head.loss_and_target( cls_score, decoded_bboxes, sampling_results, self.train_cfg[stage], imgs_whwh=imgs_whwh, concat=True) bbox_results.update(bbox_loss_and_target) # propose for the new proposal_list proposal_list = [] for idx in range(len(batch_img_metas)): results = InstanceData() results.imgs_whwh = results_list[idx].imgs_whwh results.bboxes = bbox_results['detached_proposals'][idx] proposal_list.append(results) bbox_results.update(results_list=proposal_list) return bbox_results def _bbox_forward(self, stage: int, x: Tuple[Tensor], rois: Tensor, object_feats: Tensor, batch_img_metas: List[dict]) -> dict: """Box head forward function used in both training and testing. Returns all regression, classification results and a intermediate feature. Args: stage (int): The current stage in iterative process. x (tuple[Tensor]): List of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. Each dimension means (img_index, x1, y1, x2, y2). object_feats (Tensor): The object feature extracted from the previous stage. batch_img_metas (list[dict]): Meta information of each image. Returns: dict[str, Tensor]: a dictionary of bbox head outputs, Containing the following results: - cls_score (Tensor): The score of each class, has shape (batch_size, num_proposals, num_classes) when use focal loss or (batch_size, num_proposals, num_classes+1) otherwise. - decoded_bboxes (Tensor): The regression results with shape (batch_size, num_proposal, 4). The last dimension 4 represents [tl_x, tl_y, br_x, br_y]. - object_feats (Tensor): The object feature extracted from current stage - detached_cls_scores (list[Tensor]): The detached classification results, length is batch_size, and each tensor has shape (num_proposal, num_classes). - detached_proposals (list[tensor]): The detached regression results, length is batch_size, and each tensor has shape (num_proposal, 4). The last dimension 4 represents [tl_x, tl_y, br_x, br_y]. """ num_imgs = len(batch_img_metas) bbox_roi_extractor = self.bbox_roi_extractor[stage] bbox_head = self.bbox_head[stage] bbox_feats = bbox_roi_extractor(x[:bbox_roi_extractor.num_inputs], rois) cls_score, bbox_pred, object_feats, attn_feats = bbox_head( bbox_feats, object_feats) fake_bbox_results = dict( rois=rois, bbox_targets=(rois.new_zeros(len(rois), dtype=torch.long), None), bbox_pred=bbox_pred.view(-1, bbox_pred.size(-1)), cls_score=cls_score.view(-1, cls_score.size(-1))) fake_sampling_results = [ InstanceData(pos_is_gt=rois.new_zeros(object_feats.size(1))) for _ in range(len(batch_img_metas)) ] results_list = bbox_head.refine_bboxes( sampling_results=fake_sampling_results, bbox_results=fake_bbox_results, batch_img_metas=batch_img_metas) proposal_list = [res.bboxes for res in results_list] bbox_results = dict( cls_score=cls_score, decoded_bboxes=torch.cat(proposal_list), object_feats=object_feats, attn_feats=attn_feats, # detach then use it in label assign detached_cls_scores=[ cls_score[i].detach() for i in range(num_imgs) ], detached_proposals=[item.detach() for item in proposal_list]) return bbox_results def _mask_forward(self, stage: int, x: Tuple[Tensor], rois: Tensor, attn_feats) -> dict: """Mask head forward function used in both training and testing. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): Tuple of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. attn_feats (Tensot): Intermediate feature get from the last diihead, has shape (batch_size*num_proposals, feature_dimensions) Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. """ mask_roi_extractor = self.mask_roi_extractor[stage] mask_head = self.mask_head[stage] mask_feats = mask_roi_extractor(x[:mask_roi_extractor.num_inputs], rois) # do not support caffe_c4 model anymore mask_preds = mask_head(mask_feats, attn_feats) mask_results = dict(mask_preds=mask_preds) return mask_results def mask_loss(self, stage: int, x: Tuple[Tensor], bbox_results: dict, batch_gt_instances: InstanceList, rcnn_train_cfg: ConfigDict) -> dict: """Run forward function and calculate loss for mask head in training. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): Tuple of multi-level img features. bbox_results (dict): Results obtained from `bbox_loss`. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. rcnn_train_cfg (obj:ConfigDict): `train_cfg` of RCNN. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. - `loss_mask` (dict): A dictionary of mask loss components. """ attn_feats = bbox_results['attn_feats'] sampling_results = bbox_results['sampling_results'] pos_rois = bbox2roi([res.pos_priors for res in sampling_results]) attn_feats = torch.cat([ feats[res.pos_inds] for (feats, res) in zip(attn_feats, sampling_results) ]) mask_results = self._mask_forward(stage, x, pos_rois, attn_feats) mask_loss_and_target = self.mask_head[stage].loss_and_target( mask_preds=mask_results['mask_preds'], sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, rcnn_train_cfg=rcnn_train_cfg) mask_results.update(mask_loss_and_target) return mask_results def loss(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection roi on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. rpn_results_list (List[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: a dictionary of loss components of all stage. """ outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, batch_img_metas \ = outputs object_feats = torch.cat( [res.pop('features')[None, ...] for res in rpn_results_list]) results_list = rpn_results_list losses = {} for stage in range(self.num_stages): stage_loss_weight = self.stage_loss_weights[stage] # bbox head forward and loss bbox_results = self.bbox_loss( stage=stage, x=x, object_feats=object_feats, results_list=results_list, batch_img_metas=batch_img_metas, batch_gt_instances=batch_gt_instances) for name, value in bbox_results['loss_bbox'].items(): losses[f's{stage}.{name}'] = ( value * stage_loss_weight if 'loss' in name else value) if self.with_mask: mask_results = self.mask_loss( stage=stage, x=x, bbox_results=bbox_results, batch_gt_instances=batch_gt_instances, rcnn_train_cfg=self.train_cfg[stage]) for name, value in mask_results['loss_mask'].items(): losses[f's{stage}.{name}'] = ( value * stage_loss_weight if 'loss' in name else value) object_feats = bbox_results['object_feats'] results_list = bbox_results['results_list'] return losses def predict_bbox(self, x: Tuple[Tensor], batch_img_metas: List[dict], rpn_results_list: InstanceList, rcnn_test_cfg: ConfigType, rescale: bool = False) -> InstanceList: """Perform forward propagation of the bbox head and predict detection results on the features of the upstream network. Args: x(tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of R-CNN. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ proposal_list = [res.bboxes for res in rpn_results_list] object_feats = torch.cat( [res.pop('features')[None, ...] for res in rpn_results_list]) if all([proposal.shape[0] == 0 for proposal in proposal_list]): # There is no proposal in the whole batch return empty_instances( batch_img_metas, x[0].device, task_type='bbox') for stage in range(self.num_stages): rois = bbox2roi(proposal_list) bbox_results = self._bbox_forward(stage, x, rois, object_feats, batch_img_metas) object_feats = bbox_results['object_feats'] cls_score = bbox_results['cls_score'] proposal_list = bbox_results['detached_proposals'] num_classes = self.bbox_head[-1].num_classes if self.bbox_head[-1].loss_cls.use_sigmoid: cls_score = cls_score.sigmoid() else: cls_score = cls_score.softmax(-1)[..., :-1] topk_inds_list = [] results_list = [] for img_id in range(len(batch_img_metas)): cls_score_per_img = cls_score[img_id] scores_per_img, topk_inds = cls_score_per_img.flatten(0, 1).topk( self.test_cfg.max_per_img, sorted=False) labels_per_img = topk_inds % num_classes bboxes_per_img = proposal_list[img_id][topk_inds // num_classes] topk_inds_list.append(topk_inds) if rescale and bboxes_per_img.size(0) > 0: assert batch_img_metas[img_id].get('scale_factor') is not None scale_factor = bboxes_per_img.new_tensor( batch_img_metas[img_id]['scale_factor']).repeat((1, 2)) bboxes_per_img = ( bboxes_per_img.view(bboxes_per_img.size(0), -1, 4) / scale_factor).view(bboxes_per_img.size()[0], -1) results = InstanceData() results.bboxes = bboxes_per_img results.scores = scores_per_img results.labels = labels_per_img results_list.append(results) if self.with_mask: for img_id in range(len(batch_img_metas)): # add positive information in InstanceData to predict # mask results in `mask_head`. proposals = bbox_results['detached_proposals'][img_id] topk_inds = topk_inds_list[img_id] attn_feats = bbox_results['attn_feats'][img_id] results_list[img_id].proposals = proposals results_list[img_id].topk_inds = topk_inds results_list[img_id].attn_feats = attn_feats return results_list def predict_mask(self, x: Tuple[Tensor], batch_img_metas: List[dict], results_list: InstanceList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the mask head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. results_list (list[:obj:`InstanceData`]): Detection results of each image. Each item usually contains following keys: - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - proposal (Tensor): Bboxes predicted from bbox_head, has a shape (num_instances, 4). - topk_inds (Tensor): Topk indices of each image, has shape (num_instances, ) - attn_feats (Tensor): Intermediate feature get from the last diihead, has shape (num_instances, feature_dimensions) rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ proposal_list = [res.pop('proposals') for res in results_list] topk_inds_list = [res.pop('topk_inds') for res in results_list] attn_feats = torch.cat( [res.pop('attn_feats')[None, ...] for res in results_list]) rois = bbox2roi(proposal_list) if rois.shape[0] == 0: results_list = empty_instances( batch_img_metas, rois.device, task_type='mask', instance_results=results_list, mask_thr_binary=self.test_cfg.mask_thr_binary) return results_list last_stage = self.num_stages - 1 mask_results = self._mask_forward(last_stage, x, rois, attn_feats) num_imgs = len(batch_img_metas) mask_results['mask_preds'] = mask_results['mask_preds'].reshape( num_imgs, -1, *mask_results['mask_preds'].size()[1:]) num_classes = self.bbox_head[-1].num_classes mask_preds = [] for img_id in range(num_imgs): topk_inds = topk_inds_list[img_id] masks_per_img = mask_results['mask_preds'][img_id].flatten( 0, 1)[topk_inds] masks_per_img = masks_per_img[:, None, ...].repeat(1, num_classes, 1, 1) mask_preds.append(masks_per_img) results_list = self.mask_head[-1].predict_by_feat( mask_preds, results_list, batch_img_metas, rcnn_test_cfg=self.test_cfg, rescale=rescale) return results_list # TODO: Need to refactor later def forward(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList) -> tuple: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: x (List[Tensor]): Multi-level features that may have different resolutions. rpn_results_list (List[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns tuple: A tuple of features from ``bbox_head`` and ``mask_head`` forward. """ outputs = unpack_gt_instances(batch_data_samples) (batch_gt_instances, batch_gt_instances_ignore, batch_img_metas) = outputs all_stage_bbox_results = [] object_feats = torch.cat( [res.pop('features')[None, ...] for res in rpn_results_list]) results_list = rpn_results_list if self.with_bbox: for stage in range(self.num_stages): bbox_results = self.bbox_loss( stage=stage, x=x, results_list=results_list, object_feats=object_feats, batch_img_metas=batch_img_metas, batch_gt_instances=batch_gt_instances) bbox_results.pop('loss_bbox') # torch.jit does not support obj:SamplingResult bbox_results.pop('results_list') bbox_res = bbox_results.copy() bbox_res.pop('sampling_results') all_stage_bbox_results.append((bbox_res, )) if self.with_mask: attn_feats = bbox_results['attn_feats'] sampling_results = bbox_results['sampling_results'] pos_rois = bbox2roi( [res.pos_priors for res in sampling_results]) attn_feats = torch.cat([ feats[res.pos_inds] for (feats, res) in zip(attn_feats, sampling_results) ]) mask_results = self._mask_forward(stage, x, pos_rois, attn_feats) all_stage_bbox_results[-1] += (mask_results, ) return tuple(all_stage_bbox_results) ================================================ FILE: mmdet/models/roi_heads/standard_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures import DetDataSample, SampleList from mmdet.structures.bbox import bbox2roi from mmdet.utils import ConfigType, InstanceList from ..task_modules.samplers import SamplingResult from ..utils import empty_instances, unpack_gt_instances from .base_roi_head import BaseRoIHead @MODELS.register_module() class StandardRoIHead(BaseRoIHead): """Simplest base roi head including one bbox head and one mask head.""" def init_assigner_sampler(self) -> None: """Initialize assigner and sampler.""" self.bbox_assigner = None self.bbox_sampler = None if self.train_cfg: self.bbox_assigner = TASK_UTILS.build(self.train_cfg.assigner) self.bbox_sampler = TASK_UTILS.build( self.train_cfg.sampler, default_args=dict(context=self)) def init_bbox_head(self, bbox_roi_extractor: ConfigType, bbox_head: ConfigType) -> None: """Initialize box head and box roi extractor. Args: bbox_roi_extractor (dict or ConfigDict): Config of box roi extractor. bbox_head (dict or ConfigDict): Config of box in box head. """ self.bbox_roi_extractor = MODELS.build(bbox_roi_extractor) self.bbox_head = MODELS.build(bbox_head) def init_mask_head(self, mask_roi_extractor: ConfigType, mask_head: ConfigType) -> None: """Initialize mask head and mask roi extractor. Args: mask_roi_extractor (dict or ConfigDict): Config of mask roi extractor. mask_head (dict or ConfigDict): Config of mask in mask head. """ if mask_roi_extractor is not None: self.mask_roi_extractor = MODELS.build(mask_roi_extractor) self.share_roi_extractor = False else: self.share_roi_extractor = True self.mask_roi_extractor = self.bbox_roi_extractor self.mask_head = MODELS.build(mask_head) # TODO: Need to refactor later def forward(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList = None) -> tuple: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: x (List[Tensor]): Multi-level features that may have different resolutions. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): Each item contains the meta information of each image and corresponding annotations. Returns tuple: A tuple of features from ``bbox_head`` and ``mask_head`` forward. """ results = () proposals = [rpn_results.bboxes for rpn_results in rpn_results_list] rois = bbox2roi(proposals) # bbox head if self.with_bbox: bbox_results = self._bbox_forward(x, rois) results = results + (bbox_results['cls_score'], bbox_results['bbox_pred']) # mask head if self.with_mask: mask_rois = rois[:100] mask_results = self._mask_forward(x, mask_rois) results = results + (mask_results['mask_preds'], ) return results def loss(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: List[DetDataSample]) -> dict: """Perform forward propagation and loss calculation of the detection roi on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components """ assert len(rpn_results_list) == len(batch_data_samples) outputs = unpack_gt_instances(batch_data_samples) batch_gt_instances, batch_gt_instances_ignore, _ = outputs # assign gts and sample proposals num_imgs = len(batch_data_samples) sampling_results = [] for i in range(num_imgs): # rename rpn_results.bboxes to rpn_results.priors rpn_results = rpn_results_list[i] rpn_results.priors = rpn_results.pop('bboxes') assign_result = self.bbox_assigner.assign( rpn_results, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = self.bbox_sampler.sample( assign_result, rpn_results, batch_gt_instances[i], feats=[lvl_feat[i][None] for lvl_feat in x]) sampling_results.append(sampling_result) losses = dict() # bbox head loss if self.with_bbox: bbox_results = self.bbox_loss(x, sampling_results) losses.update(bbox_results['loss_bbox']) # mask head forward and loss if self.with_mask: mask_results = self.mask_loss(x, sampling_results, bbox_results['bbox_feats'], batch_gt_instances) losses.update(mask_results['loss_mask']) return losses def _bbox_forward(self, x: Tuple[Tensor], rois: Tensor) -> dict: """Box head forward function used in both training and testing. Args: x (tuple[Tensor]): List of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. """ # TODO: a more flexible way to decide which feature maps to use bbox_feats = self.bbox_roi_extractor( x[:self.bbox_roi_extractor.num_inputs], rois) if self.with_shared_head: bbox_feats = self.shared_head(bbox_feats) cls_score, bbox_pred = self.bbox_head(bbox_feats) bbox_results = dict( cls_score=cls_score, bbox_pred=bbox_pred, bbox_feats=bbox_feats) return bbox_results def bbox_loss(self, x: Tuple[Tensor], sampling_results: List[SamplingResult]) -> dict: """Perform forward propagation and loss calculation of the bbox head on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. - `loss_bbox` (dict): A dictionary of bbox loss components. """ rois = bbox2roi([res.priors for res in sampling_results]) bbox_results = self._bbox_forward(x, rois) bbox_loss_and_target = self.bbox_head.loss_and_target( cls_score=bbox_results['cls_score'], bbox_pred=bbox_results['bbox_pred'], rois=rois, sampling_results=sampling_results, rcnn_train_cfg=self.train_cfg) bbox_results.update(loss_bbox=bbox_loss_and_target['loss_bbox']) return bbox_results def mask_loss(self, x: Tuple[Tensor], sampling_results: List[SamplingResult], bbox_feats: Tensor, batch_gt_instances: InstanceList) -> dict: """Perform forward propagation and loss calculation of the mask head on the features of the upstream network. Args: x (tuple[Tensor]): Tuple of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. bbox_feats (Tensor): Extract bbox RoI features. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. - `mask_feats` (Tensor): Extract mask RoI features. - `mask_targets` (Tensor): Mask target of each positive\ proposals in the image. - `loss_mask` (dict): A dictionary of mask loss components. """ if not self.share_roi_extractor: pos_rois = bbox2roi([res.pos_priors for res in sampling_results]) mask_results = self._mask_forward(x, pos_rois) else: pos_inds = [] device = bbox_feats.device for res in sampling_results: pos_inds.append( torch.ones( res.pos_priors.shape[0], device=device, dtype=torch.uint8)) pos_inds.append( torch.zeros( res.neg_priors.shape[0], device=device, dtype=torch.uint8)) pos_inds = torch.cat(pos_inds) mask_results = self._mask_forward( x, pos_inds=pos_inds, bbox_feats=bbox_feats) mask_loss_and_target = self.mask_head.loss_and_target( mask_preds=mask_results['mask_preds'], sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, rcnn_train_cfg=self.train_cfg) mask_results.update(loss_mask=mask_loss_and_target['loss_mask']) return mask_results def _mask_forward(self, x: Tuple[Tensor], rois: Tensor = None, pos_inds: Optional[Tensor] = None, bbox_feats: Optional[Tensor] = None) -> dict: """Mask head forward function used in both training and testing. Args: x (tuple[Tensor]): Tuple of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. pos_inds (Tensor, optional): Indices of positive samples. Defaults to None. bbox_feats (Tensor): Extract bbox RoI features. Defaults to None. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. - `mask_feats` (Tensor): Extract mask RoI features. """ assert ((rois is not None) ^ (pos_inds is not None and bbox_feats is not None)) if rois is not None: mask_feats = self.mask_roi_extractor( x[:self.mask_roi_extractor.num_inputs], rois) if self.with_shared_head: mask_feats = self.shared_head(mask_feats) else: assert bbox_feats is not None mask_feats = bbox_feats[pos_inds] mask_preds = self.mask_head(mask_feats) mask_results = dict(mask_preds=mask_preds, mask_feats=mask_feats) return mask_results def predict_bbox(self, x: Tuple[Tensor], batch_img_metas: List[dict], rpn_results_list: InstanceList, rcnn_test_cfg: ConfigType, rescale: bool = False) -> InstanceList: """Perform forward propagation of the bbox head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of R-CNN. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ proposals = [res.bboxes for res in rpn_results_list] rois = bbox2roi(proposals) if rois.shape[0] == 0: return empty_instances( batch_img_metas, rois.device, task_type='bbox', box_type=self.bbox_head.predict_box_type, num_classes=self.bbox_head.num_classes, score_per_cls=rcnn_test_cfg is None) bbox_results = self._bbox_forward(x, rois) # split batch bbox prediction back to each image cls_scores = bbox_results['cls_score'] bbox_preds = bbox_results['bbox_pred'] num_proposals_per_img = tuple(len(p) for p in proposals) rois = rois.split(num_proposals_per_img, 0) cls_scores = cls_scores.split(num_proposals_per_img, 0) # some detector with_reg is False, bbox_preds will be None if bbox_preds is not None: # TODO move this to a sabl_roi_head # the bbox prediction of some detectors like SABL is not Tensor if isinstance(bbox_preds, torch.Tensor): bbox_preds = bbox_preds.split(num_proposals_per_img, 0) else: bbox_preds = self.bbox_head.bbox_pred_split( bbox_preds, num_proposals_per_img) else: bbox_preds = (None, ) * len(proposals) result_list = self.bbox_head.predict_by_feat( rois=rois, cls_scores=cls_scores, bbox_preds=bbox_preds, batch_img_metas=batch_img_metas, rcnn_test_cfg=rcnn_test_cfg, rescale=rescale) return result_list def predict_mask(self, x: Tuple[Tensor], batch_img_metas: List[dict], results_list: InstanceList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the mask head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. results_list (list[:obj:`InstanceData`]): Detection results of each image. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ # don't need to consider aug_test. bboxes = [res.bboxes for res in results_list] mask_rois = bbox2roi(bboxes) if mask_rois.shape[0] == 0: results_list = empty_instances( batch_img_metas, mask_rois.device, task_type='mask', instance_results=results_list, mask_thr_binary=self.test_cfg.mask_thr_binary) return results_list mask_results = self._mask_forward(x, mask_rois) mask_preds = mask_results['mask_preds'] # split batch mask prediction back to each image num_mask_rois_per_img = [len(res) for res in results_list] mask_preds = mask_preds.split(num_mask_rois_per_img, 0) # TODO: Handle the case where rescale is false results_list = self.mask_head.predict_by_feat( mask_preds=mask_preds, results_list=results_list, batch_img_metas=batch_img_metas, rcnn_test_cfg=self.test_cfg, rescale=rescale) return results_list ================================================ FILE: mmdet/models/roi_heads/test_mixins.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # TODO: delete this file after refactor import sys import torch from mmdet.models.layers import multiclass_nms from mmdet.models.test_time_augs import merge_aug_bboxes, merge_aug_masks from mmdet.structures.bbox import bbox2roi, bbox_mapping if sys.version_info >= (3, 7): from mmdet.utils.contextmanagers import completed class BBoxTestMixin: if sys.version_info >= (3, 7): # TODO: Currently not supported async def async_test_bboxes(self, x, img_metas, proposals, rcnn_test_cfg, rescale=False, **kwargs): """Asynchronized test for box head without augmentation.""" rois = bbox2roi(proposals) roi_feats = self.bbox_roi_extractor( x[:len(self.bbox_roi_extractor.featmap_strides)], rois) if self.with_shared_head: roi_feats = self.shared_head(roi_feats) sleep_interval = rcnn_test_cfg.get('async_sleep_interval', 0.017) async with completed( __name__, 'bbox_head_forward', sleep_interval=sleep_interval): cls_score, bbox_pred = self.bbox_head(roi_feats) img_shape = img_metas[0]['img_shape'] scale_factor = img_metas[0]['scale_factor'] det_bboxes, det_labels = self.bbox_head.get_bboxes( rois, cls_score, bbox_pred, img_shape, scale_factor, rescale=rescale, cfg=rcnn_test_cfg) return det_bboxes, det_labels # TODO: Currently not supported def aug_test_bboxes(self, feats, img_metas, rpn_results_list, rcnn_test_cfg): """Test det bboxes with test time augmentation.""" aug_bboxes = [] aug_scores = [] for x, img_meta in zip(feats, img_metas): # only one image in the batch img_shape = img_meta[0]['img_shape'] scale_factor = img_meta[0]['scale_factor'] flip = img_meta[0]['flip'] flip_direction = img_meta[0]['flip_direction'] # TODO more flexible proposals = bbox_mapping(rpn_results_list[0][:, :4], img_shape, scale_factor, flip, flip_direction) rois = bbox2roi([proposals]) bbox_results = self.bbox_forward(x, rois) bboxes, scores = self.bbox_head.get_bboxes( rois, bbox_results['cls_score'], bbox_results['bbox_pred'], img_shape, scale_factor, rescale=False, cfg=None) aug_bboxes.append(bboxes) aug_scores.append(scores) # after merging, bboxes will be rescaled to the original image size merged_bboxes, merged_scores = merge_aug_bboxes( aug_bboxes, aug_scores, img_metas, rcnn_test_cfg) if merged_bboxes.shape[0] == 0: # There is no proposal in the single image det_bboxes = merged_bboxes.new_zeros(0, 5) det_labels = merged_bboxes.new_zeros((0, ), dtype=torch.long) else: det_bboxes, det_labels = multiclass_nms(merged_bboxes, merged_scores, rcnn_test_cfg.score_thr, rcnn_test_cfg.nms, rcnn_test_cfg.max_per_img) return det_bboxes, det_labels class MaskTestMixin: if sys.version_info >= (3, 7): # TODO: Currently not supported async def async_test_mask(self, x, img_metas, det_bboxes, det_labels, rescale=False, mask_test_cfg=None): """Asynchronized test for mask head without augmentation.""" # image shape of the first image in the batch (only one) ori_shape = img_metas[0]['ori_shape'] scale_factor = img_metas[0]['scale_factor'] if det_bboxes.shape[0] == 0: segm_result = [[] for _ in range(self.mask_head.num_classes)] else: if rescale and not isinstance(scale_factor, (float, torch.Tensor)): scale_factor = det_bboxes.new_tensor(scale_factor) _bboxes = ( det_bboxes[:, :4] * scale_factor if rescale else det_bboxes) mask_rois = bbox2roi([_bboxes]) mask_feats = self.mask_roi_extractor( x[:len(self.mask_roi_extractor.featmap_strides)], mask_rois) if self.with_shared_head: mask_feats = self.shared_head(mask_feats) if mask_test_cfg and \ mask_test_cfg.get('async_sleep_interval'): sleep_interval = mask_test_cfg['async_sleep_interval'] else: sleep_interval = 0.035 async with completed( __name__, 'mask_head_forward', sleep_interval=sleep_interval): mask_pred = self.mask_head(mask_feats) segm_result = self.mask_head.get_results( mask_pred, _bboxes, det_labels, self.test_cfg, ori_shape, scale_factor, rescale) return segm_result # TODO: Currently not supported def aug_test_mask(self, feats, img_metas, det_bboxes, det_labels): """Test for mask head with test time augmentation.""" if det_bboxes.shape[0] == 0: segm_result = [[] for _ in range(self.mask_head.num_classes)] else: aug_masks = [] for x, img_meta in zip(feats, img_metas): img_shape = img_meta[0]['img_shape'] scale_factor = img_meta[0]['scale_factor'] flip = img_meta[0]['flip'] flip_direction = img_meta[0]['flip_direction'] _bboxes = bbox_mapping(det_bboxes[:, :4], img_shape, scale_factor, flip, flip_direction) mask_rois = bbox2roi([_bboxes]) mask_results = self._mask_forward(x, mask_rois) # convert to numpy array to save memory aug_masks.append( mask_results['mask_pred'].sigmoid().cpu().numpy()) merged_masks = merge_aug_masks(aug_masks, img_metas, self.test_cfg) ori_shape = img_metas[0][0]['ori_shape'] scale_factor = det_bboxes.new_ones(4) segm_result = self.mask_head.get_results( merged_masks, det_bboxes, det_labels, self.test_cfg, ori_shape, scale_factor=scale_factor, rescale=False) return segm_result ================================================ FILE: mmdet/models/roi_heads/trident_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple import torch from mmcv.ops import batched_nms from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import InstanceList from .standard_roi_head import StandardRoIHead @MODELS.register_module() class TridentRoIHead(StandardRoIHead): """Trident roi head. Args: num_branch (int): Number of branches in TridentNet. test_branch_idx (int): In inference, all 3 branches will be used if `test_branch_idx==-1`, otherwise only branch with index `test_branch_idx` will be used. """ def __init__(self, num_branch: int, test_branch_idx: int, **kwargs) -> None: self.num_branch = num_branch self.test_branch_idx = test_branch_idx super().__init__(**kwargs) def merge_trident_bboxes(self, trident_results: InstanceList) -> InstanceData: """Merge bbox predictions of each branch. Args: trident_results (List[:obj:`InstanceData`]): A list of InstanceData predicted from every branch. Returns: :obj:`InstanceData`: merged InstanceData. """ bboxes = torch.cat([res.bboxes for res in trident_results]) scores = torch.cat([res.scores for res in trident_results]) labels = torch.cat([res.labels for res in trident_results]) nms_cfg = self.test_cfg['nms'] results = InstanceData() if bboxes.numel() == 0: results.bboxes = bboxes results.scores = scores results.labels = labels else: det_bboxes, keep = batched_nms(bboxes, scores, labels, nms_cfg) results.bboxes = det_bboxes[:, :-1] results.scores = det_bboxes[:, -1] results.labels = labels[keep] if self.test_cfg['max_per_img'] > 0: results = results[:self.test_cfg['max_per_img']] return results def predict(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the roi head and predict detection results on the features of the upstream network. - Compute prediction bbox and label per branch. - Merge predictions of each branch according to scores of bboxes, i.e., bboxes with higher score are kept to give top-k prediction. Args: x (tuple[Tensor]): Features from upstream network. Each has shape (N, C, H, W). rpn_results_list (list[:obj:`InstanceData`]): list of region proposals. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results to the original image. Defaults to True. Returns: list[obj:`InstanceData`]: Detection results of each image. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ results_list = super().predict( x=x, rpn_results_list=rpn_results_list, batch_data_samples=batch_data_samples, rescale=rescale) num_branch = self.num_branch \ if self.training or self.test_branch_idx == -1 else 1 merged_results_list = [] for i in range(len(batch_data_samples) // num_branch): merged_results_list.append( self.merge_trident_bboxes(results_list[i * num_branch:(i + 1) * num_branch])) return merged_results_list ================================================ FILE: mmdet/models/seg_heads/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .panoptic_fpn_head import PanopticFPNHead # noqa: F401,F403 from .panoptic_fusion_heads import * # noqa: F401,F403 ================================================ FILE: mmdet/models/seg_heads/base_semantic_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from typing import Dict, List, Tuple, Union import torch.nn.functional as F from mmengine.model import BaseModule from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import ConfigType, OptMultiConfig @MODELS.register_module() class BaseSemanticHead(BaseModule, metaclass=ABCMeta): """Base module of Semantic Head. Args: num_classes (int): the number of classes. seg_rescale_factor (float): the rescale factor for ``gt_sem_seg``, which equals to ``1 / output_strides``. The output_strides is for ``seg_preds``. Defaults to 1 / 4. init_cfg (Optional[Union[:obj:`ConfigDict`, dict]]): the initialization config. loss_seg (Union[:obj:`ConfigDict`, dict]): the loss of the semantic head. """ def __init__(self, num_classes: int, seg_rescale_factor: float = 1 / 4., loss_seg: ConfigType = dict( type='CrossEntropyLoss', ignore_index=255, loss_weight=1.0), init_cfg: OptMultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) self.loss_seg = MODELS.build(loss_seg) self.num_classes = num_classes self.seg_rescale_factor = seg_rescale_factor @abstractmethod def forward(self, x: Union[Tensor, Tuple[Tensor]]) -> Dict[str, Tensor]: """Placeholder of forward function. Args: x (Tensor): Feature maps. Returns: Dict[str, Tensor]: A dictionary, including features and predicted scores. Required keys: 'seg_preds' and 'feats'. """ pass @abstractmethod def loss(self, x: Union[Tensor, Tuple[Tensor]], batch_data_samples: SampleList) -> Dict[str, Tensor]: """ Args: x (Union[Tensor, Tuple[Tensor]]): Feature maps. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Args: x (Tensor): Feature maps. Returns: Dict[str, Tensor]: The loss of semantic head. """ pass def predict(self, x: Union[Tensor, Tuple[Tensor]], batch_img_metas: List[dict], rescale: bool = False) -> List[Tensor]: """Test without Augmentation. Args: x (Union[Tensor, Tuple[Tensor]]): Feature maps. batch_img_metas (List[dict]): List of image information. rescale (bool): Whether to rescale the results. Defaults to False. Returns: list[Tensor]: semantic segmentation logits. """ seg_preds = self.forward(x)['seg_preds'] seg_preds = F.interpolate( seg_preds, size=batch_img_metas[0]['batch_input_shape'], mode='bilinear', align_corners=False) seg_preds = [seg_preds[i] for i in range(len(batch_img_metas))] if rescale: seg_pred_list = [] for i in range(len(batch_img_metas)): h, w = batch_img_metas[i]['img_shape'] seg_pred = seg_preds[i][:, :h, :w] h, w = batch_img_metas[i]['ori_shape'] seg_pred = F.interpolate( seg_pred[None], size=(h, w), mode='bilinear', align_corners=False)[0] seg_pred_list.append(seg_pred) else: seg_pred_list = seg_preds return seg_pred_list ================================================ FILE: mmdet/models/seg_heads/panoptic_fpn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, Tuple, Union import torch import torch.nn as nn import torch.nn.functional as F from mmengine.model import ModuleList from torch import Tensor from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig from ..layers import ConvUpsample from ..utils import interpolate_as from .base_semantic_head import BaseSemanticHead @MODELS.register_module() class PanopticFPNHead(BaseSemanticHead): """PanopticFPNHead used in Panoptic FPN. In this head, the number of output channels is ``num_stuff_classes + 1``, including all stuff classes and one thing class. The stuff classes will be reset from ``0`` to ``num_stuff_classes - 1``, the thing classes will be merged to ``num_stuff_classes``-th channel. Arg: num_things_classes (int): Number of thing classes. Default: 80. num_stuff_classes (int): Number of stuff classes. Default: 53. in_channels (int): Number of channels in the input feature map. inner_channels (int): Number of channels in inner features. start_level (int): The start level of the input features used in PanopticFPN. end_level (int): The end level of the used features, the ``end_level``-th layer will not be used. conv_cfg (Optional[Union[ConfigDict, dict]]): Dictionary to construct and config conv layer. norm_cfg (Union[ConfigDict, dict]): Dictionary to construct and config norm layer. Use ``GN`` by default. init_cfg (Optional[Union[ConfigDict, dict]]): Initialization config dict. loss_seg (Union[ConfigDict, dict]): the loss of the semantic head. """ def __init__(self, num_things_classes: int = 80, num_stuff_classes: int = 53, in_channels: int = 256, inner_channels: int = 128, start_level: int = 0, end_level: int = 4, conv_cfg: OptConfigType = None, norm_cfg: ConfigType = dict( type='GN', num_groups=32, requires_grad=True), loss_seg: ConfigType = dict( type='CrossEntropyLoss', ignore_index=-1, loss_weight=1.0), init_cfg: OptMultiConfig = None) -> None: seg_rescale_factor = 1 / 2**(start_level + 2) super().__init__( num_classes=num_stuff_classes + 1, seg_rescale_factor=seg_rescale_factor, loss_seg=loss_seg, init_cfg=init_cfg) self.num_things_classes = num_things_classes self.num_stuff_classes = num_stuff_classes # Used feature layers are [start_level, end_level) self.start_level = start_level self.end_level = end_level self.num_stages = end_level - start_level self.inner_channels = inner_channels self.conv_upsample_layers = ModuleList() for i in range(start_level, end_level): self.conv_upsample_layers.append( ConvUpsample( in_channels, inner_channels, num_layers=i if i > 0 else 1, num_upsample=i if i > 0 else 0, conv_cfg=conv_cfg, norm_cfg=norm_cfg, )) self.conv_logits = nn.Conv2d(inner_channels, self.num_classes, 1) def _set_things_to_void(self, gt_semantic_seg: Tensor) -> Tensor: """Merge thing classes to one class. In PanopticFPN, the background labels will be reset from `0` to `self.num_stuff_classes-1`, the foreground labels will be merged to `self.num_stuff_classes`-th channel. """ gt_semantic_seg = gt_semantic_seg.int() fg_mask = gt_semantic_seg < self.num_things_classes bg_mask = (gt_semantic_seg >= self.num_things_classes) * ( gt_semantic_seg < self.num_things_classes + self.num_stuff_classes) new_gt_seg = torch.clone(gt_semantic_seg) new_gt_seg = torch.where(bg_mask, gt_semantic_seg - self.num_things_classes, new_gt_seg) new_gt_seg = torch.where(fg_mask, fg_mask.int() * self.num_stuff_classes, new_gt_seg) return new_gt_seg def loss(self, x: Union[Tensor, Tuple[Tensor]], batch_data_samples: SampleList) -> Dict[str, Tensor]: """ Args: x (Union[Tensor, Tuple[Tensor]]): Feature maps. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: Dict[str, Tensor]: The loss of semantic head. """ seg_preds = self(x)['seg_preds'] gt_semantic_segs = [ data_sample.gt_sem_seg.sem_seg for data_sample in batch_data_samples ] gt_semantic_segs = torch.stack(gt_semantic_segs) if self.seg_rescale_factor != 1.0: gt_semantic_segs = F.interpolate( gt_semantic_segs.float(), scale_factor=self.seg_rescale_factor, mode='nearest').squeeze(1) # Things classes will be merged to one class in PanopticFPN. gt_semantic_segs = self._set_things_to_void(gt_semantic_segs) if seg_preds.shape[-2:] != gt_semantic_segs.shape[-2:]: seg_preds = interpolate_as(seg_preds, gt_semantic_segs) seg_preds = seg_preds.permute((0, 2, 3, 1)) loss_seg = self.loss_seg( seg_preds.reshape(-1, self.num_classes), # => [NxHxW, C] gt_semantic_segs.reshape(-1).long()) return dict(loss_seg=loss_seg) def init_weights(self) -> None: """Initialize weights.""" super().init_weights() nn.init.normal_(self.conv_logits.weight.data, 0, 0.01) self.conv_logits.bias.data.zero_() def forward(self, x: Tuple[Tensor]) -> Dict[str, Tensor]: """Forward. Args: x (Tuple[Tensor]): Multi scale Feature maps. Returns: dict[str, Tensor]: semantic segmentation predictions and feature maps. """ # the number of subnets must be not more than # the length of features. assert self.num_stages <= len(x) feats = [] for i, layer in enumerate(self.conv_upsample_layers): f = layer(x[self.start_level + i]) feats.append(f) seg_feats = torch.sum(torch.stack(feats, dim=0), dim=0) seg_preds = self.conv_logits(seg_feats) out = dict(seg_preds=seg_preds, seg_feats=seg_feats) return out ================================================ FILE: mmdet/models/seg_heads/panoptic_fusion_heads/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .base_panoptic_fusion_head import \ BasePanopticFusionHead # noqa: F401,F403 from .heuristic_fusion_head import HeuristicFusionHead # noqa: F401,F403 from .maskformer_fusion_head import MaskFormerFusionHead # noqa: F401,F403 ================================================ FILE: mmdet/models/seg_heads/panoptic_fusion_heads/base_panoptic_fusion_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from mmengine.model import BaseModule from mmdet.registry import MODELS from mmdet.utils import OptConfigType, OptMultiConfig @MODELS.register_module() class BasePanopticFusionHead(BaseModule, metaclass=ABCMeta): """Base class for panoptic heads.""" def __init__(self, num_things_classes: int = 80, num_stuff_classes: int = 53, test_cfg: OptConfigType = None, loss_panoptic: OptConfigType = None, init_cfg: OptMultiConfig = None, **kwargs) -> None: super().__init__(init_cfg=init_cfg) self.num_things_classes = num_things_classes self.num_stuff_classes = num_stuff_classes self.num_classes = num_things_classes + num_stuff_classes self.test_cfg = test_cfg if loss_panoptic: self.loss_panoptic = MODELS.build(loss_panoptic) else: self.loss_panoptic = None @property def with_loss(self) -> bool: """bool: whether the panoptic head contains loss function.""" return self.loss_panoptic is not None @abstractmethod def loss(self, **kwargs): """Loss function.""" @abstractmethod def predict(self, **kwargs): """Predict function.""" ================================================ FILE: mmdet/models/seg_heads/panoptic_fusion_heads/heuristic_fusion_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List import torch from mmengine.structures import InstanceData, PixelData from torch import Tensor from mmdet.evaluation.functional import INSTANCE_OFFSET from mmdet.registry import MODELS from mmdet.utils import InstanceList, OptConfigType, OptMultiConfig, PixelList from .base_panoptic_fusion_head import BasePanopticFusionHead @MODELS.register_module() class HeuristicFusionHead(BasePanopticFusionHead): """Fusion Head with Heuristic method.""" def __init__(self, num_things_classes: int = 80, num_stuff_classes: int = 53, test_cfg: OptConfigType = None, init_cfg: OptMultiConfig = None, **kwargs) -> None: super().__init__( num_things_classes=num_things_classes, num_stuff_classes=num_stuff_classes, test_cfg=test_cfg, loss_panoptic=None, init_cfg=init_cfg, **kwargs) def loss(self, **kwargs) -> dict: """HeuristicFusionHead has no training loss.""" return dict() def _lay_masks(self, mask_results: InstanceData, overlap_thr: float = 0.5) -> Tensor: """Lay instance masks to a result map. Args: mask_results (:obj:`InstanceData`): Instance segmentation results, each contains ``bboxes``, ``labels``, ``scores`` and ``masks``. overlap_thr (float): Threshold to determine whether two masks overlap. default: 0.5. Returns: Tensor: The result map, (H, W). """ bboxes = mask_results.bboxes scores = mask_results.scores labels = mask_results.labels masks = mask_results.masks num_insts = bboxes.shape[0] id_map = torch.zeros( masks.shape[-2:], device=bboxes.device, dtype=torch.long) if num_insts == 0: return id_map, labels # Sort by score to use heuristic fusion order = torch.argsort(-scores) bboxes = bboxes[order] labels = labels[order] segm_masks = masks[order] instance_id = 1 left_labels = [] for idx in range(bboxes.shape[0]): _cls = labels[idx] _mask = segm_masks[idx] instance_id_map = torch.ones_like( _mask, dtype=torch.long) * instance_id area = _mask.sum() if area == 0: continue pasted = id_map > 0 intersect = (_mask * pasted).sum() if (intersect / (area + 1e-5)) > overlap_thr: continue _part = _mask * (~pasted) id_map = torch.where(_part, instance_id_map, id_map) left_labels.append(_cls) instance_id += 1 if len(left_labels) > 0: instance_labels = torch.stack(left_labels) else: instance_labels = bboxes.new_zeros((0, ), dtype=torch.long) assert instance_id == (len(instance_labels) + 1) return id_map, instance_labels def _predict_single(self, mask_results: InstanceData, seg_preds: Tensor, **kwargs) -> PixelData: """Fuse the results of instance and semantic segmentations. Args: mask_results (:obj:`InstanceData`): Instance segmentation results, each contains ``bboxes``, ``labels``, ``scores`` and ``masks``. seg_preds (Tensor): The semantic segmentation results, (num_stuff + 1, H, W). Returns: Tensor: The panoptic segmentation result, (H, W). """ id_map, labels = self._lay_masks(mask_results, self.test_cfg.mask_overlap) seg_results = seg_preds.argmax(dim=0) seg_results = seg_results + self.num_things_classes pan_results = seg_results instance_id = 1 for idx in range(len(mask_results)): _mask = id_map == (idx + 1) if _mask.sum() == 0: continue _cls = labels[idx] # simply trust detection segment_id = _cls + instance_id * INSTANCE_OFFSET pan_results[_mask] = segment_id instance_id += 1 ids, counts = torch.unique( pan_results % INSTANCE_OFFSET, return_counts=True) stuff_ids = ids[ids >= self.num_things_classes] stuff_counts = counts[ids >= self.num_things_classes] ignore_stuff_ids = stuff_ids[ stuff_counts < self.test_cfg.stuff_area_limit] assert pan_results.ndim == 2 pan_results[(pan_results.unsqueeze(2) == ignore_stuff_ids.reshape( 1, 1, -1)).any(dim=2)] = self.num_classes pan_results = PixelData(sem_seg=pan_results[None].int()) return pan_results def predict(self, mask_results_list: InstanceList, seg_preds_list: List[Tensor], **kwargs) -> PixelList: """Predict results by fusing the results of instance and semantic segmentations. Args: mask_results_list (list[:obj:`InstanceData`]): Instance segmentation results, each contains ``bboxes``, ``labels``, ``scores`` and ``masks``. seg_preds_list (Tensor): List of semantic segmentation results. Returns: List[PixelData]: Panoptic segmentation result. """ results_list = [ self._predict_single(mask_results_list[i], seg_preds_list[i]) for i in range(len(mask_results_list)) ] return results_list ================================================ FILE: mmdet/models/seg_heads/panoptic_fusion_heads/maskformer_fusion_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List import torch import torch.nn.functional as F from mmengine.structures import InstanceData, PixelData from torch import Tensor from mmdet.evaluation.functional import INSTANCE_OFFSET from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.mask import mask2bbox from mmdet.utils import OptConfigType, OptMultiConfig from .base_panoptic_fusion_head import BasePanopticFusionHead @MODELS.register_module() class MaskFormerFusionHead(BasePanopticFusionHead): """MaskFormer fusion head which postprocesses results for panoptic segmentation, instance segmentation and semantic segmentation.""" def __init__(self, num_things_classes: int = 80, num_stuff_classes: int = 53, test_cfg: OptConfigType = None, loss_panoptic: OptConfigType = None, init_cfg: OptMultiConfig = None, **kwargs): super().__init__( num_things_classes=num_things_classes, num_stuff_classes=num_stuff_classes, test_cfg=test_cfg, loss_panoptic=loss_panoptic, init_cfg=init_cfg, **kwargs) def loss(self, **kwargs): """MaskFormerFusionHead has no training loss.""" return dict() def panoptic_postprocess(self, mask_cls: Tensor, mask_pred: Tensor) -> PixelData: """Panoptic segmengation inference. Args: mask_cls (Tensor): Classfication outputs of shape (num_queries, cls_out_channels) for a image. Note `cls_out_channels` should includes background. mask_pred (Tensor): Mask outputs of shape (num_queries, h, w) for a image. Returns: :obj:`PixelData`: Panoptic segment result of shape \ (h, w), each element in Tensor means: \ ``segment_id = _cls + instance_id * INSTANCE_OFFSET``. """ object_mask_thr = self.test_cfg.get('object_mask_thr', 0.8) iou_thr = self.test_cfg.get('iou_thr', 0.8) filter_low_score = self.test_cfg.get('filter_low_score', False) scores, labels = F.softmax(mask_cls, dim=-1).max(-1) mask_pred = mask_pred.sigmoid() keep = labels.ne(self.num_classes) & (scores > object_mask_thr) cur_scores = scores[keep] cur_classes = labels[keep] cur_masks = mask_pred[keep] cur_prob_masks = cur_scores.view(-1, 1, 1) * cur_masks h, w = cur_masks.shape[-2:] panoptic_seg = torch.full((h, w), self.num_classes, dtype=torch.int32, device=cur_masks.device) if cur_masks.shape[0] == 0: # We didn't detect any mask :( pass else: cur_mask_ids = cur_prob_masks.argmax(0) instance_id = 1 for k in range(cur_classes.shape[0]): pred_class = int(cur_classes[k].item()) isthing = pred_class < self.num_things_classes mask = cur_mask_ids == k mask_area = mask.sum().item() original_area = (cur_masks[k] >= 0.5).sum().item() if filter_low_score: mask = mask & (cur_masks[k] >= 0.5) if mask_area > 0 and original_area > 0: if mask_area / original_area < iou_thr: continue if not isthing: # different stuff regions of same class will be # merged here, and stuff share the instance_id 0. panoptic_seg[mask] = pred_class else: panoptic_seg[mask] = ( pred_class + instance_id * INSTANCE_OFFSET) instance_id += 1 return PixelData(sem_seg=panoptic_seg[None]) def semantic_postprocess(self, mask_cls: Tensor, mask_pred: Tensor) -> PixelData: """Semantic segmengation postprocess. Args: mask_cls (Tensor): Classfication outputs of shape (num_queries, cls_out_channels) for a image. Note `cls_out_channels` should includes background. mask_pred (Tensor): Mask outputs of shape (num_queries, h, w) for a image. Returns: :obj:`PixelData`: Semantic segment result. """ # TODO add semantic segmentation result raise NotImplementedError def instance_postprocess(self, mask_cls: Tensor, mask_pred: Tensor) -> InstanceData: """Instance segmengation postprocess. Args: mask_cls (Tensor): Classfication outputs of shape (num_queries, cls_out_channels) for a image. Note `cls_out_channels` should includes background. mask_pred (Tensor): Mask outputs of shape (num_queries, h, w) for a image. Returns: :obj:`InstanceData`: Instance segmentation results. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ max_per_image = self.test_cfg.get('max_per_image', 100) num_queries = mask_cls.shape[0] # shape (num_queries, num_class) scores = F.softmax(mask_cls, dim=-1)[:, :-1] # shape (num_queries * num_class, ) labels = torch.arange(self.num_classes, device=mask_cls.device).\ unsqueeze(0).repeat(num_queries, 1).flatten(0, 1) scores_per_image, top_indices = scores.flatten(0, 1).topk( max_per_image, sorted=False) labels_per_image = labels[top_indices] query_indices = top_indices // self.num_classes mask_pred = mask_pred[query_indices] # extract things is_thing = labels_per_image < self.num_things_classes scores_per_image = scores_per_image[is_thing] labels_per_image = labels_per_image[is_thing] mask_pred = mask_pred[is_thing] mask_pred_binary = (mask_pred > 0).float() mask_scores_per_image = (mask_pred.sigmoid() * mask_pred_binary).flatten(1).sum(1) / ( mask_pred_binary.flatten(1).sum(1) + 1e-6) det_scores = scores_per_image * mask_scores_per_image mask_pred_binary = mask_pred_binary.bool() bboxes = mask2bbox(mask_pred_binary) results = InstanceData() results.bboxes = bboxes results.labels = labels_per_image results.scores = det_scores results.masks = mask_pred_binary return results def predict(self, mask_cls_results: Tensor, mask_pred_results: Tensor, batch_data_samples: SampleList, rescale: bool = False, **kwargs) -> List[dict]: """Test segment without test-time aumengtation. Only the output of last decoder layers was used. Args: mask_cls_results (Tensor): Mask classification logits, shape (batch_size, num_queries, cls_out_channels). Note `cls_out_channels` should includes background. mask_pred_results (Tensor): Mask logits, shape (batch_size, num_queries, h, w). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): If True, return boxes in original image space. Default False. Returns: list[dict]: Instance segmentation \ results and panoptic segmentation results for each \ image. .. code-block:: none [ { 'pan_results': PixelData, 'ins_results': InstanceData, # semantic segmentation results are not supported yet 'sem_results': PixelData }, ... ] """ batch_img_metas = [ data_sample.metainfo for data_sample in batch_data_samples ] panoptic_on = self.test_cfg.get('panoptic_on', True) semantic_on = self.test_cfg.get('semantic_on', False) instance_on = self.test_cfg.get('instance_on', False) assert not semantic_on, 'segmantic segmentation '\ 'results are not supported yet.' results = [] for mask_cls_result, mask_pred_result, meta in zip( mask_cls_results, mask_pred_results, batch_img_metas): # remove padding img_height, img_width = meta['img_shape'][:2] mask_pred_result = mask_pred_result[:, :img_height, :img_width] if rescale: # return result in original resolution ori_height, ori_width = meta['ori_shape'][:2] mask_pred_result = F.interpolate( mask_pred_result[:, None], size=(ori_height, ori_width), mode='bilinear', align_corners=False)[:, 0] result = dict() if panoptic_on: pan_results = self.panoptic_postprocess( mask_cls_result, mask_pred_result) result['pan_results'] = pan_results if instance_on: ins_results = self.instance_postprocess( mask_cls_result, mask_pred_result) result['ins_results'] = ins_results if semantic_on: sem_results = self.semantic_postprocess( mask_cls_result, mask_pred_result) result['sem_results'] = sem_results results.append(result) return results ================================================ FILE: mmdet/models/task_modules/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .assigners import * # noqa: F401,F403 from .builder import (ANCHOR_GENERATORS, BBOX_ASSIGNERS, BBOX_CODERS, BBOX_SAMPLERS, IOU_CALCULATORS, MATCH_COSTS, PRIOR_GENERATORS, build_anchor_generator, build_assigner, build_bbox_coder, build_iou_calculator, build_match_cost, build_prior_generator, build_sampler) from .coders import * # noqa: F401,F403 from .prior_generators import * # noqa: F401,F403 from .samplers import * # noqa: F401,F403 __all__ = [ 'ANCHOR_GENERATORS', 'PRIOR_GENERATORS', 'BBOX_ASSIGNERS', 'BBOX_SAMPLERS', 'MATCH_COSTS', 'BBOX_CODERS', 'IOU_CALCULATORS', 'build_anchor_generator', 'build_prior_generator', 'build_assigner', 'build_sampler', 'build_iou_calculator', 'build_match_cost', 'build_bbox_coder' ] ================================================ FILE: mmdet/models/task_modules/assigners/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .approx_max_iou_assigner import ApproxMaxIoUAssigner from .assign_result import AssignResult from .atss_assigner import ATSSAssigner from .base_assigner import BaseAssigner from .center_region_assigner import CenterRegionAssigner from .dynamic_soft_label_assigner import DynamicSoftLabelAssigner from .grid_assigner import GridAssigner from .hungarian_assigner import HungarianAssigner from .iou2d_calculator import BboxOverlaps2D from .match_cost import (BBoxL1Cost, ClassificationCost, CrossEntropyLossCost, DiceCost, FocalLossCost, IoUCost) from .max_iou_assigner import MaxIoUAssigner from .multi_instance_assigner import MultiInstanceAssigner from .point_assigner import PointAssigner from .region_assigner import RegionAssigner from .sim_ota_assigner import SimOTAAssigner from .task_aligned_assigner import TaskAlignedAssigner from .uniform_assigner import UniformAssigner __all__ = [ 'BaseAssigner', 'MaxIoUAssigner', 'ApproxMaxIoUAssigner', 'AssignResult', 'PointAssigner', 'ATSSAssigner', 'CenterRegionAssigner', 'GridAssigner', 'HungarianAssigner', 'RegionAssigner', 'UniformAssigner', 'SimOTAAssigner', 'TaskAlignedAssigner', 'BBoxL1Cost', 'ClassificationCost', 'CrossEntropyLossCost', 'DiceCost', 'FocalLossCost', 'IoUCost', 'BboxOverlaps2D', 'DynamicSoftLabelAssigner', 'MultiInstanceAssigner' ] ================================================ FILE: mmdet/models/task_modules/assigners/approx_max_iou_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Union import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.registry import TASK_UTILS from .assign_result import AssignResult from .max_iou_assigner import MaxIoUAssigner @TASK_UTILS.register_module() class ApproxMaxIoUAssigner(MaxIoUAssigner): """Assign a corresponding gt bbox or background to each bbox. Each proposals will be assigned with an integer indicating the ground truth index. (semi-positive index: gt label (0-based), -1: background) - -1: negative sample, no assigned gt - semi-positive integer: positive sample, index (0-based) of assigned gt Args: pos_iou_thr (float): IoU threshold for positive bboxes. neg_iou_thr (float or tuple): IoU threshold for negative bboxes. min_pos_iou (float): Minimum iou for a bbox to be considered as a positive bbox. Positive samples can have smaller IoU than pos_iou_thr due to the 4th step (assign max IoU sample to each gt). gt_max_assign_all (bool): Whether to assign all bboxes with the same highest overlap with some gt to that gt. ignore_iof_thr (float): IoF threshold for ignoring bboxes (if `gt_bboxes_ignore` is specified). Negative values mean not ignoring any bboxes. ignore_wrt_candidates (bool): Whether to compute the iof between `bboxes` and `gt_bboxes_ignore`, or the contrary. match_low_quality (bool): Whether to allow quality matches. This is usually allowed for RPN and single stage detectors, but not allowed in the second stage. gpu_assign_thr (int): The upper bound of the number of GT for GPU assign. When the number of gt is above this threshold, will assign on CPU device. Negative values mean not assign on CPU. iou_calculator (:obj:`ConfigDict` or dict): Config of overlaps Calculator. """ def __init__( self, pos_iou_thr: float, neg_iou_thr: Union[float, tuple], min_pos_iou: float = .0, gt_max_assign_all: bool = True, ignore_iof_thr: float = -1, ignore_wrt_candidates: bool = True, match_low_quality: bool = True, gpu_assign_thr: int = -1, iou_calculator: Union[ConfigDict, dict] = dict(type='BboxOverlaps2D') ) -> None: self.pos_iou_thr = pos_iou_thr self.neg_iou_thr = neg_iou_thr self.min_pos_iou = min_pos_iou self.gt_max_assign_all = gt_max_assign_all self.ignore_iof_thr = ignore_iof_thr self.ignore_wrt_candidates = ignore_wrt_candidates self.gpu_assign_thr = gpu_assign_thr self.match_low_quality = match_low_quality self.iou_calculator = TASK_UTILS.build(iou_calculator) def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, **kwargs) -> AssignResult: """Assign gt to approxs. This method assign a gt bbox to each group of approxs (bboxes), each group of approxs is represent by a base approx (bbox) and will be assigned with -1, or a semi-positive number. background_label (-1) means negative sample, semi-positive number is the index (0-based) of assigned gt. The assignment is done in following steps, the order matters. 1. assign every bbox to background_label (-1) 2. use the max IoU of each group of approxs to assign 2. assign proposals whose iou with all gts < neg_iou_thr to background 3. for each bbox, if the iou with its nearest gt >= pos_iou_thr, assign it to that bbox 4. for each gt bbox, assign its nearest proposals (may be more than one) to itself Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). ``approxs`` means the group of approxs aligned with ``priors``, has shape (n, num_approxs, 4). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: :obj:`AssignResult`: The assign result. """ squares = pred_instances.priors approxs = pred_instances.approxs gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels gt_bboxes_ignore = None if gt_instances_ignore is None else \ gt_instances_ignore.get('bboxes', None) approxs_per_octave = approxs.size(1) num_squares = squares.size(0) num_gts = gt_bboxes.size(0) if num_squares == 0 or num_gts == 0: # No predictions and/or truth, return empty assignment overlaps = approxs.new(num_gts, num_squares) assign_result = self.assign_wrt_overlaps(overlaps, gt_labels) return assign_result # re-organize anchors by approxs_per_octave x num_squares approxs = torch.transpose(approxs, 0, 1).contiguous().view(-1, 4) assign_on_cpu = True if (self.gpu_assign_thr > 0) and ( num_gts > self.gpu_assign_thr) else False # compute overlap and assign gt on CPU when number of GT is large if assign_on_cpu: device = approxs.device approxs = approxs.cpu() gt_bboxes = gt_bboxes.cpu() if gt_bboxes_ignore is not None: gt_bboxes_ignore = gt_bboxes_ignore.cpu() if gt_labels is not None: gt_labels = gt_labels.cpu() all_overlaps = self.iou_calculator(approxs, gt_bboxes) overlaps, _ = all_overlaps.view(approxs_per_octave, num_squares, num_gts).max(dim=0) overlaps = torch.transpose(overlaps, 0, 1) if (self.ignore_iof_thr > 0 and gt_bboxes_ignore is not None and gt_bboxes_ignore.numel() > 0 and squares.numel() > 0): if self.ignore_wrt_candidates: ignore_overlaps = self.iou_calculator( squares, gt_bboxes_ignore, mode='iof') ignore_max_overlaps, _ = ignore_overlaps.max(dim=1) else: ignore_overlaps = self.iou_calculator( gt_bboxes_ignore, squares, mode='iof') ignore_max_overlaps, _ = ignore_overlaps.max(dim=0) overlaps[:, ignore_max_overlaps > self.ignore_iof_thr] = -1 assign_result = self.assign_wrt_overlaps(overlaps, gt_labels) if assign_on_cpu: assign_result.gt_inds = assign_result.gt_inds.to(device) assign_result.max_overlaps = assign_result.max_overlaps.to(device) if assign_result.labels is not None: assign_result.labels = assign_result.labels.to(device) return assign_result ================================================ FILE: mmdet/models/task_modules/assigners/assign_result.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch from torch import Tensor from mmdet.utils import util_mixins class AssignResult(util_mixins.NiceRepr): """Stores assignments between predicted and truth boxes. Attributes: num_gts (int): the number of truth boxes considered when computing this assignment gt_inds (Tensor): for each predicted box indicates the 1-based index of the assigned truth box. 0 means unassigned and -1 means ignore. max_overlaps (Tensor): the iou between the predicted box and its assigned truth box. labels (Tensor): If specified, for each predicted box indicates the category label of the assigned truth box. Example: >>> # An assign result between 4 predicted boxes and 9 true boxes >>> # where only two boxes were assigned. >>> num_gts = 9 >>> max_overlaps = torch.LongTensor([0, .5, .9, 0]) >>> gt_inds = torch.LongTensor([-1, 1, 2, 0]) >>> labels = torch.LongTensor([0, 3, 4, 0]) >>> self = AssignResult(num_gts, gt_inds, max_overlaps, labels) >>> print(str(self)) # xdoctest: +IGNORE_WANT >>> # Force addition of gt labels (when adding gt as proposals) >>> new_labels = torch.LongTensor([3, 4, 5]) >>> self.add_gt_(new_labels) >>> print(str(self)) # xdoctest: +IGNORE_WANT """ def __init__(self, num_gts: int, gt_inds: Tensor, max_overlaps: Tensor, labels: Tensor) -> None: self.num_gts = num_gts self.gt_inds = gt_inds self.max_overlaps = max_overlaps self.labels = labels # Interface for possible user-defined properties self._extra_properties = {} @property def num_preds(self): """int: the number of predictions in this assignment""" return len(self.gt_inds) def set_extra_property(self, key, value): """Set user-defined new property.""" assert key not in self.info self._extra_properties[key] = value def get_extra_property(self, key): """Get user-defined property.""" return self._extra_properties.get(key, None) @property def info(self): """dict: a dictionary of info about the object""" basic_info = { 'num_gts': self.num_gts, 'num_preds': self.num_preds, 'gt_inds': self.gt_inds, 'max_overlaps': self.max_overlaps, 'labels': self.labels, } basic_info.update(self._extra_properties) return basic_info def __nice__(self): """str: a "nice" summary string describing this assign result""" parts = [] parts.append(f'num_gts={self.num_gts!r}') if self.gt_inds is None: parts.append(f'gt_inds={self.gt_inds!r}') else: parts.append(f'gt_inds.shape={tuple(self.gt_inds.shape)!r}') if self.max_overlaps is None: parts.append(f'max_overlaps={self.max_overlaps!r}') else: parts.append('max_overlaps.shape=' f'{tuple(self.max_overlaps.shape)!r}') if self.labels is None: parts.append(f'labels={self.labels!r}') else: parts.append(f'labels.shape={tuple(self.labels.shape)!r}') return ', '.join(parts) @classmethod def random(cls, **kwargs): """Create random AssignResult for tests or debugging. Args: num_preds: number of predicted boxes num_gts: number of true boxes p_ignore (float): probability of a predicted box assigned to an ignored truth p_assigned (float): probability of a predicted box not being assigned p_use_label (float | bool): with labels or not rng (None | int | numpy.random.RandomState): seed or state Returns: :obj:`AssignResult`: Randomly generated assign results. Example: >>> from mmdet.models.task_modules.assigners.assign_result import * # NOQA >>> self = AssignResult.random() >>> print(self.info) """ from ..samplers.sampling_result import ensure_rng rng = ensure_rng(kwargs.get('rng', None)) num_gts = kwargs.get('num_gts', None) num_preds = kwargs.get('num_preds', None) p_ignore = kwargs.get('p_ignore', 0.3) p_assigned = kwargs.get('p_assigned', 0.7) num_classes = kwargs.get('num_classes', 3) if num_gts is None: num_gts = rng.randint(0, 8) if num_preds is None: num_preds = rng.randint(0, 16) if num_gts == 0: max_overlaps = torch.zeros(num_preds, dtype=torch.float32) gt_inds = torch.zeros(num_preds, dtype=torch.int64) labels = torch.zeros(num_preds, dtype=torch.int64) else: import numpy as np # Create an overlap for each predicted box max_overlaps = torch.from_numpy(rng.rand(num_preds)) # Construct gt_inds for each predicted box is_assigned = torch.from_numpy(rng.rand(num_preds) < p_assigned) # maximum number of assignments constraints n_assigned = min(num_preds, min(num_gts, is_assigned.sum())) assigned_idxs = np.where(is_assigned)[0] rng.shuffle(assigned_idxs) assigned_idxs = assigned_idxs[0:n_assigned] assigned_idxs.sort() is_assigned[:] = 0 is_assigned[assigned_idxs] = True is_ignore = torch.from_numpy( rng.rand(num_preds) < p_ignore) & is_assigned gt_inds = torch.zeros(num_preds, dtype=torch.int64) true_idxs = np.arange(num_gts) rng.shuffle(true_idxs) true_idxs = torch.from_numpy(true_idxs) gt_inds[is_assigned] = true_idxs[:n_assigned].long() gt_inds = torch.from_numpy( rng.randint(1, num_gts + 1, size=num_preds)) gt_inds[is_ignore] = -1 gt_inds[~is_assigned] = 0 max_overlaps[~is_assigned] = 0 if num_classes == 0: labels = torch.zeros(num_preds, dtype=torch.int64) else: labels = torch.from_numpy( # remind that we set FG labels to [0, num_class-1] # since mmdet v2.0 # BG cat_id: num_class rng.randint(0, num_classes, size=num_preds)) labels[~is_assigned] = 0 self = cls(num_gts, gt_inds, max_overlaps, labels) return self def add_gt_(self, gt_labels): """Add ground truth as assigned results. Args: gt_labels (torch.Tensor): Labels of gt boxes """ self_inds = torch.arange( 1, len(gt_labels) + 1, dtype=torch.long, device=gt_labels.device) self.gt_inds = torch.cat([self_inds, self.gt_inds]) self.max_overlaps = torch.cat( [self.max_overlaps.new_ones(len(gt_labels)), self.max_overlaps]) self.labels = torch.cat([gt_labels, self.labels]) ================================================ FILE: mmdet/models/task_modules/assigners/atss_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings from typing import List, Optional import torch from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import TASK_UTILS from mmdet.utils import ConfigType from .assign_result import AssignResult from .base_assigner import BaseAssigner def bbox_center_distance(bboxes: Tensor, priors: Tensor) -> Tensor: """Compute the center distance between bboxes and priors. Args: bboxes (Tensor): Shape (n, 4) for , "xyxy" format. priors (Tensor): Shape (n, 4) for priors, "xyxy" format. Returns: Tensor: Center distances between bboxes and priors. """ bbox_cx = (bboxes[:, 0] + bboxes[:, 2]) / 2.0 bbox_cy = (bboxes[:, 1] + bboxes[:, 3]) / 2.0 bbox_points = torch.stack((bbox_cx, bbox_cy), dim=1) priors_cx = (priors[:, 0] + priors[:, 2]) / 2.0 priors_cy = (priors[:, 1] + priors[:, 3]) / 2.0 priors_points = torch.stack((priors_cx, priors_cy), dim=1) distances = (priors_points[:, None, :] - bbox_points[None, :, :]).pow(2).sum(-1).sqrt() return distances @TASK_UTILS.register_module() class ATSSAssigner(BaseAssigner): """Assign a corresponding gt bbox or background to each prior. Each proposals will be assigned with `0` or a positive integer indicating the ground truth index. - 0: negative sample, no assigned gt - positive integer: positive sample, index (1-based) of assigned gt If ``alpha`` is not None, it means that the dynamic cost ATSSAssigner is adopted, which is currently only used in the DDOD. Args: topk (int): number of priors selected in each level alpha (float, optional): param of cost rate for each proposal only in DDOD. Defaults to None. iou_calculator (:obj:`ConfigDict` or dict): Config dict for iou calculator. Defaults to ``dict(type='BboxOverlaps2D')`` ignore_iof_thr (float): IoF threshold for ignoring bboxes (if `gt_bboxes_ignore` is specified). Negative values mean not ignoring any bboxes. Defaults to -1. """ def __init__(self, topk: int, alpha: Optional[float] = None, iou_calculator: ConfigType = dict(type='BboxOverlaps2D'), ignore_iof_thr: float = -1) -> None: self.topk = topk self.alpha = alpha self.iou_calculator = TASK_UTILS.build(iou_calculator) self.ignore_iof_thr = ignore_iof_thr # https://github.com/sfzhang15/ATSS/blob/master/atss_core/modeling/rpn/atss/loss.py def assign( self, pred_instances: InstanceData, num_level_priors: List[int], gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None ) -> AssignResult: """Assign gt to priors. The assignment is done in following steps 1. compute iou between all prior (prior of all pyramid levels) and gt 2. compute center distance between all prior and gt 3. on each pyramid level, for each gt, select k prior whose center are closest to the gt center, so we total select k*l prior as candidates for each gt 4. get corresponding iou for the these candidates, and compute the mean and std, set mean + std as the iou threshold 5. select these candidates whose iou are greater than or equal to the threshold as positive 6. limit the positive sample's center in gt If ``alpha`` is not None, and ``cls_scores`` and `bbox_preds` are not None, the overlaps calculation in the first step will also include dynamic cost, which is currently only used in the DDOD. Args: pred_instances (:obj:`InstaceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors, points, or bboxes predicted by the model, shape(n, 4). num_level_priors (List): Number of bboxes in each level gt_instances (:obj:`InstaceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. gt_instances_ignore (:obj:`InstaceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: :obj:`AssignResult`: The assign result. """ gt_bboxes = gt_instances.bboxes priors = pred_instances.priors gt_labels = gt_instances.labels if gt_instances_ignore is not None: gt_bboxes_ignore = gt_instances_ignore.bboxes else: gt_bboxes_ignore = None INF = 100000000 priors = priors[:, :4] num_gt, num_priors = gt_bboxes.size(0), priors.size(0) message = 'Invalid alpha parameter because cls_scores or ' \ 'bbox_preds are None. If you want to use the ' \ 'cost-based ATSSAssigner, please set cls_scores, ' \ 'bbox_preds and self.alpha at the same time. ' # compute iou between all bbox and gt if self.alpha is None: # ATSSAssigner overlaps = self.iou_calculator(priors, gt_bboxes) if ('scores' in pred_instances or 'bboxes' in pred_instances): warnings.warn(message) else: # Dynamic cost ATSSAssigner in DDOD assert ('scores' in pred_instances and 'bboxes' in pred_instances), message cls_scores = pred_instances.scores bbox_preds = pred_instances.bboxes # compute cls cost for bbox and GT cls_cost = torch.sigmoid(cls_scores[:, gt_labels]) # compute iou between all bbox and gt overlaps = self.iou_calculator(bbox_preds, gt_bboxes) # make sure that we are in element-wise multiplication assert cls_cost.shape == overlaps.shape # overlaps is actually a cost matrix overlaps = cls_cost**(1 - self.alpha) * overlaps**self.alpha # assign 0 by default assigned_gt_inds = overlaps.new_full((num_priors, ), 0, dtype=torch.long) if num_gt == 0 or num_priors == 0: # No ground truth or boxes, return empty assignment max_overlaps = overlaps.new_zeros((num_priors, )) if num_gt == 0: # No truth, assign everything to background assigned_gt_inds[:] = 0 assigned_labels = overlaps.new_full((num_priors, ), -1, dtype=torch.long) return AssignResult( num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) # compute center distance between all bbox and gt distances = bbox_center_distance(gt_bboxes, priors) if (self.ignore_iof_thr > 0 and gt_bboxes_ignore is not None and gt_bboxes_ignore.numel() > 0 and priors.numel() > 0): ignore_overlaps = self.iou_calculator( priors, gt_bboxes_ignore, mode='iof') ignore_max_overlaps, _ = ignore_overlaps.max(dim=1) ignore_idxs = ignore_max_overlaps > self.ignore_iof_thr distances[ignore_idxs, :] = INF assigned_gt_inds[ignore_idxs] = -1 # Selecting candidates based on the center distance candidate_idxs = [] start_idx = 0 for level, priors_per_level in enumerate(num_level_priors): # on each pyramid level, for each gt, # select k bbox whose center are closest to the gt center end_idx = start_idx + priors_per_level distances_per_level = distances[start_idx:end_idx, :] selectable_k = min(self.topk, priors_per_level) _, topk_idxs_per_level = distances_per_level.topk( selectable_k, dim=0, largest=False) candidate_idxs.append(topk_idxs_per_level + start_idx) start_idx = end_idx candidate_idxs = torch.cat(candidate_idxs, dim=0) # get corresponding iou for the these candidates, and compute the # mean and std, set mean + std as the iou threshold candidate_overlaps = overlaps[candidate_idxs, torch.arange(num_gt)] overlaps_mean_per_gt = candidate_overlaps.mean(0) overlaps_std_per_gt = candidate_overlaps.std(0) overlaps_thr_per_gt = overlaps_mean_per_gt + overlaps_std_per_gt is_pos = candidate_overlaps >= overlaps_thr_per_gt[None, :] # limit the positive sample's center in gt for gt_idx in range(num_gt): candidate_idxs[:, gt_idx] += gt_idx * num_priors priors_cx = (priors[:, 0] + priors[:, 2]) / 2.0 priors_cy = (priors[:, 1] + priors[:, 3]) / 2.0 ep_priors_cx = priors_cx.view(1, -1).expand( num_gt, num_priors).contiguous().view(-1) ep_priors_cy = priors_cy.view(1, -1).expand( num_gt, num_priors).contiguous().view(-1) candidate_idxs = candidate_idxs.view(-1) # calculate the left, top, right, bottom distance between positive # prior center and gt side l_ = ep_priors_cx[candidate_idxs].view(-1, num_gt) - gt_bboxes[:, 0] t_ = ep_priors_cy[candidate_idxs].view(-1, num_gt) - gt_bboxes[:, 1] r_ = gt_bboxes[:, 2] - ep_priors_cx[candidate_idxs].view(-1, num_gt) b_ = gt_bboxes[:, 3] - ep_priors_cy[candidate_idxs].view(-1, num_gt) is_in_gts = torch.stack([l_, t_, r_, b_], dim=1).min(dim=1)[0] > 0.01 is_pos = is_pos & is_in_gts # if an anchor box is assigned to multiple gts, # the one with the highest IoU will be selected. overlaps_inf = torch.full_like(overlaps, -INF).t().contiguous().view(-1) index = candidate_idxs.view(-1)[is_pos.view(-1)] overlaps_inf[index] = overlaps.t().contiguous().view(-1)[index] overlaps_inf = overlaps_inf.view(num_gt, -1).t() max_overlaps, argmax_overlaps = overlaps_inf.max(dim=1) assigned_gt_inds[ max_overlaps != -INF] = argmax_overlaps[max_overlaps != -INF] + 1 assigned_labels = assigned_gt_inds.new_full((num_priors, ), -1) pos_inds = torch.nonzero( assigned_gt_inds > 0, as_tuple=False).squeeze() if pos_inds.numel() > 0: assigned_labels[pos_inds] = gt_labels[assigned_gt_inds[pos_inds] - 1] return AssignResult( num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) ================================================ FILE: mmdet/models/task_modules/assigners/base_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod from typing import Optional from mmengine.structures import InstanceData class BaseAssigner(metaclass=ABCMeta): """Base assigner that assigns boxes to ground truth boxes.""" @abstractmethod def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, **kwargs): """Assign boxes to either a ground truth boxes or a negative boxes.""" ================================================ FILE: mmdet/models/task_modules/assigners/center_region_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple import torch from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import TASK_UTILS from mmdet.utils import ConfigType from .assign_result import AssignResult from .base_assigner import BaseAssigner def scale_boxes(bboxes: Tensor, scale: float) -> Tensor: """Expand an array of boxes by a given scale. Args: bboxes (Tensor): Shape (m, 4) scale (float): The scale factor of bboxes Returns: Tensor: Shape (m, 4). Scaled bboxes """ assert bboxes.size(1) == 4 w_half = (bboxes[:, 2] - bboxes[:, 0]) * .5 h_half = (bboxes[:, 3] - bboxes[:, 1]) * .5 x_c = (bboxes[:, 2] + bboxes[:, 0]) * .5 y_c = (bboxes[:, 3] + bboxes[:, 1]) * .5 w_half *= scale h_half *= scale boxes_scaled = torch.zeros_like(bboxes) boxes_scaled[:, 0] = x_c - w_half boxes_scaled[:, 2] = x_c + w_half boxes_scaled[:, 1] = y_c - h_half boxes_scaled[:, 3] = y_c + h_half return boxes_scaled def is_located_in(points: Tensor, bboxes: Tensor) -> Tensor: """Are points located in bboxes. Args: points (Tensor): Points, shape: (m, 2). bboxes (Tensor): Bounding boxes, shape: (n, 4). Return: Tensor: Flags indicating if points are located in bboxes, shape: (m, n). """ assert points.size(1) == 2 assert bboxes.size(1) == 4 return (points[:, 0].unsqueeze(1) > bboxes[:, 0].unsqueeze(0)) & \ (points[:, 0].unsqueeze(1) < bboxes[:, 2].unsqueeze(0)) & \ (points[:, 1].unsqueeze(1) > bboxes[:, 1].unsqueeze(0)) & \ (points[:, 1].unsqueeze(1) < bboxes[:, 3].unsqueeze(0)) def bboxes_area(bboxes: Tensor) -> Tensor: """Compute the area of an array of bboxes. Args: bboxes (Tensor): The coordinates ox bboxes. Shape: (m, 4) Returns: Tensor: Area of the bboxes. Shape: (m, ) """ assert bboxes.size(1) == 4 w = (bboxes[:, 2] - bboxes[:, 0]) h = (bboxes[:, 3] - bboxes[:, 1]) areas = w * h return areas @TASK_UTILS.register_module() class CenterRegionAssigner(BaseAssigner): """Assign pixels at the center region of a bbox as positive. Each proposals will be assigned with `-1`, `0`, or a positive integer indicating the ground truth index. - -1: negative samples - semi-positive numbers: positive sample, index (0-based) of assigned gt Args: pos_scale (float): Threshold within which pixels are labelled as positive. neg_scale (float): Threshold above which pixels are labelled as positive. min_pos_iof (float): Minimum iof of a pixel with a gt to be labelled as positive. Default: 1e-2 ignore_gt_scale (float): Threshold within which the pixels are ignored when the gt is labelled as shadowed. Default: 0.5 foreground_dominate (bool): If True, the bbox will be assigned as positive when a gt's kernel region overlaps with another's shadowed (ignored) region, otherwise it is set as ignored. Default to False. iou_calculator (:obj:`ConfigDict` or dict): Config of overlaps Calculator. """ def __init__( self, pos_scale: float, neg_scale: float, min_pos_iof: float = 1e-2, ignore_gt_scale: float = 0.5, foreground_dominate: bool = False, iou_calculator: ConfigType = dict(type='BboxOverlaps2D') ) -> None: self.pos_scale = pos_scale self.neg_scale = neg_scale self.min_pos_iof = min_pos_iof self.ignore_gt_scale = ignore_gt_scale self.foreground_dominate = foreground_dominate self.iou_calculator = TASK_UTILS.build(iou_calculator) def get_gt_priorities(self, gt_bboxes: Tensor) -> Tensor: """Get gt priorities according to their areas. Smaller gt has higher priority. Args: gt_bboxes (Tensor): Ground truth boxes, shape (k, 4). Returns: Tensor: The priority of gts so that gts with larger priority is more likely to be assigned. Shape (k, ) """ gt_areas = bboxes_area(gt_bboxes) # Rank all gt bbox areas. Smaller objects has larger priority _, sort_idx = gt_areas.sort(descending=True) sort_idx = sort_idx.argsort() return sort_idx def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, **kwargs) -> AssignResult: """Assign gt to bboxes. This method assigns gts to every prior (proposal/anchor), each prior will be assigned with -1, or a semi-positive number. -1 means negative sample, semi-positive number is the index (0-based) of assigned gt. Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: :obj:`AssignResult`: The assigned result. Note that shadowed_labels of shape (N, 2) is also added as an `assign_result` attribute. `shadowed_labels` is a tensor composed of N pairs of anchor_ind, class_label], where N is the number of anchors that lie in the outer region of a gt, anchor_ind is the shadowed anchor index and class_label is the shadowed class label. Example: >>> from mmengine.structures import InstanceData >>> self = CenterRegionAssigner(0.2, 0.2) >>> pred_instances.priors = torch.Tensor([[0, 0, 10, 10], ... [10, 10, 20, 20]]) >>> gt_instances = InstanceData() >>> gt_instances.bboxes = torch.Tensor([[0, 0, 10, 10]]) >>> gt_instances.labels = torch.Tensor([0]) >>> assign_result = self.assign(pred_instances, gt_instances) >>> expected_gt_inds = torch.LongTensor([1, 0]) >>> assert torch.all(assign_result.gt_inds == expected_gt_inds) """ # There are in total 5 steps in the pixel assignment # 1. Find core (the center region, say inner 0.2) # and shadow (the relatively ourter part, say inner 0.2-0.5) # regions of every gt. # 2. Find all prior bboxes that lie in gt_core and gt_shadow regions # 3. Assign prior bboxes in gt_core with a one-hot id of the gt in # the image. # 3.1. For overlapping objects, the prior bboxes in gt_core is # assigned with the object with smallest area # 4. Assign prior bboxes with class label according to its gt id. # 4.1. Assign -1 to prior bboxes lying in shadowed gts # 4.2. Assign positive prior boxes with the corresponding label # 5. Find pixels lying in the shadow of an object and assign them with # background label, but set the loss weight of its corresponding # gt to zero. # TODO not extract bboxes in assign. gt_bboxes = gt_instances.bboxes priors = pred_instances.priors gt_labels = gt_instances.labels assert priors.size(1) == 4, 'priors must have size of 4' # 1. Find core positive and shadow region of every gt gt_core = scale_boxes(gt_bboxes, self.pos_scale) gt_shadow = scale_boxes(gt_bboxes, self.neg_scale) # 2. Find prior bboxes that lie in gt_core and gt_shadow regions prior_centers = (priors[:, 2:4] + priors[:, 0:2]) / 2 # The center points lie within the gt boxes is_prior_in_gt = is_located_in(prior_centers, gt_bboxes) # Only calculate prior and gt_core IoF. This enables small prior bboxes # to match large gts prior_and_gt_core_overlaps = self.iou_calculator( priors, gt_core, mode='iof') # The center point of effective priors should be within the gt box is_prior_in_gt_core = is_prior_in_gt & ( prior_and_gt_core_overlaps > self.min_pos_iof) # shape (n, k) is_prior_in_gt_shadow = ( self.iou_calculator(priors, gt_shadow, mode='iof') > self.min_pos_iof) # Rule out center effective positive pixels is_prior_in_gt_shadow &= (~is_prior_in_gt_core) num_gts, num_priors = gt_bboxes.size(0), priors.size(0) if num_gts == 0 or num_priors == 0: # If no gts exist, assign all pixels to negative assigned_gt_ids = \ is_prior_in_gt_core.new_zeros((num_priors,), dtype=torch.long) pixels_in_gt_shadow = assigned_gt_ids.new_empty((0, 2)) else: # Step 3: assign a one-hot gt id to each pixel, and smaller objects # have high priority to assign the pixel. sort_idx = self.get_gt_priorities(gt_bboxes) assigned_gt_ids, pixels_in_gt_shadow = \ self.assign_one_hot_gt_indices(is_prior_in_gt_core, is_prior_in_gt_shadow, gt_priority=sort_idx) if (gt_instances_ignore is not None and gt_instances_ignore.bboxes.numel() > 0): # No ground truth or boxes, return empty assignment gt_bboxes_ignore = gt_instances_ignore.bboxes gt_bboxes_ignore = scale_boxes( gt_bboxes_ignore, scale=self.ignore_gt_scale) is_prior_in_ignored_gts = is_located_in(prior_centers, gt_bboxes_ignore) is_prior_in_ignored_gts = is_prior_in_ignored_gts.any(dim=1) assigned_gt_ids[is_prior_in_ignored_gts] = -1 # 4. Assign prior bboxes with class label according to its gt id. # Default assigned label is the background (-1) assigned_labels = assigned_gt_ids.new_full((num_priors, ), -1) pos_inds = torch.nonzero(assigned_gt_ids > 0, as_tuple=False).squeeze() if pos_inds.numel() > 0: assigned_labels[pos_inds] = gt_labels[assigned_gt_ids[pos_inds] - 1] # 5. Find pixels lying in the shadow of an object shadowed_pixel_labels = pixels_in_gt_shadow.clone() if pixels_in_gt_shadow.numel() > 0: pixel_idx, gt_idx =\ pixels_in_gt_shadow[:, 0], pixels_in_gt_shadow[:, 1] assert (assigned_gt_ids[pixel_idx] != gt_idx).all(), \ 'Some pixels are dually assigned to ignore and gt!' shadowed_pixel_labels[:, 1] = gt_labels[gt_idx - 1] override = ( assigned_labels[pixel_idx] == shadowed_pixel_labels[:, 1]) if self.foreground_dominate: # When a pixel is both positive and shadowed, set it as pos shadowed_pixel_labels = shadowed_pixel_labels[~override] else: # When a pixel is both pos and shadowed, set it as shadowed assigned_labels[pixel_idx[override]] = -1 assigned_gt_ids[pixel_idx[override]] = 0 assign_result = AssignResult( num_gts, assigned_gt_ids, None, labels=assigned_labels) # Add shadowed_labels as assign_result property. Shape: (num_shadow, 2) assign_result.set_extra_property('shadowed_labels', shadowed_pixel_labels) return assign_result def assign_one_hot_gt_indices( self, is_prior_in_gt_core: Tensor, is_prior_in_gt_shadow: Tensor, gt_priority: Optional[Tensor] = None) -> Tuple[Tensor, Tensor]: """Assign only one gt index to each prior box. Gts with large gt_priority are more likely to be assigned. Args: is_prior_in_gt_core (Tensor): Bool tensor indicating the prior center is in the core area of a gt (e.g. 0-0.2). Shape: (num_prior, num_gt). is_prior_in_gt_shadow (Tensor): Bool tensor indicating the prior center is in the shadowed area of a gt (e.g. 0.2-0.5). Shape: (num_prior, num_gt). gt_priority (Tensor): Priorities of gts. The gt with a higher priority is more likely to be assigned to the bbox when the bbox match with multiple gts. Shape: (num_gt, ). Returns: tuple: Returns (assigned_gt_inds, shadowed_gt_inds). - assigned_gt_inds: The assigned gt index of each prior bbox \ (i.e. index from 1 to num_gts). Shape: (num_prior, ). - shadowed_gt_inds: shadowed gt indices. It is a tensor of \ shape (num_ignore, 2) with first column being the shadowed prior \ bbox indices and the second column the shadowed gt \ indices (1-based). """ num_bboxes, num_gts = is_prior_in_gt_core.shape if gt_priority is None: gt_priority = torch.arange( num_gts, device=is_prior_in_gt_core.device) assert gt_priority.size(0) == num_gts # The bigger gt_priority, the more preferable to be assigned # The assigned inds are by default 0 (background) assigned_gt_inds = is_prior_in_gt_core.new_zeros((num_bboxes, ), dtype=torch.long) # Shadowed bboxes are assigned to be background. But the corresponding # label is ignored during loss calculation, which is done through # shadowed_gt_inds shadowed_gt_inds = torch.nonzero(is_prior_in_gt_shadow, as_tuple=False) if is_prior_in_gt_core.sum() == 0: # No gt match shadowed_gt_inds[:, 1] += 1 # 1-based. For consistency issue return assigned_gt_inds, shadowed_gt_inds # The priority of each prior box and gt pair. If one prior box is # matched bo multiple gts. Only the pair with the highest priority # is saved pair_priority = is_prior_in_gt_core.new_full((num_bboxes, num_gts), -1, dtype=torch.long) # Each bbox could match with multiple gts. # The following codes deal with this situation # Matched bboxes (to any gt). Shape: (num_pos_anchor, ) inds_of_match = torch.any(is_prior_in_gt_core, dim=1) # The matched gt index of each positive bbox. Length >= num_pos_anchor # , since one bbox could match multiple gts matched_bbox_gt_inds = torch.nonzero( is_prior_in_gt_core, as_tuple=False)[:, 1] # Assign priority to each bbox-gt pair. pair_priority[is_prior_in_gt_core] = gt_priority[matched_bbox_gt_inds] _, argmax_priority = pair_priority[inds_of_match].max(dim=1) assigned_gt_inds[inds_of_match] = argmax_priority + 1 # 1-based # Zero-out the assigned anchor box to filter the shadowed gt indices is_prior_in_gt_core[inds_of_match, argmax_priority] = 0 # Concat the shadowed indices due to overlapping with that out side of # effective scale. shape: (total_num_ignore, 2) shadowed_gt_inds = torch.cat( (shadowed_gt_inds, torch.nonzero(is_prior_in_gt_core, as_tuple=False)), dim=0) # Change `is_prior_in_gt_core` back to keep arguments intact. is_prior_in_gt_core[inds_of_match, argmax_priority] = 1 # 1-based shadowed gt indices, to be consistent with `assigned_gt_inds` if shadowed_gt_inds.numel() > 0: shadowed_gt_inds[:, 1] += 1 return assigned_gt_inds, shadowed_gt_inds ================================================ FILE: mmdet/models/task_modules/assigners/dynamic_soft_label_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple import torch import torch.nn.functional as F from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import BaseBoxes from mmdet.utils import ConfigType from .assign_result import AssignResult from .base_assigner import BaseAssigner INF = 100000000 EPS = 1.0e-7 def center_of_mass(masks: Tensor, eps: float = 1e-7) -> Tensor: """Compute the masks center of mass. Args: masks: Mask tensor, has shape (num_masks, H, W). eps: a small number to avoid normalizer to be zero. Defaults to 1e-7. Returns: Tensor: The masks center of mass. Has shape (num_masks, 2). """ n, h, w = masks.shape grid_h = torch.arange(h, device=masks.device)[:, None] grid_w = torch.arange(w, device=masks.device) normalizer = masks.sum(dim=(1, 2)).float().clamp(min=eps) center_y = (masks * grid_h).sum(dim=(1, 2)) / normalizer center_x = (masks * grid_w).sum(dim=(1, 2)) / normalizer center = torch.cat([center_x[:, None], center_y[:, None]], dim=1) return center @TASK_UTILS.register_module() class DynamicSoftLabelAssigner(BaseAssigner): """Computes matching between predictions and ground truth with dynamic soft label assignment. Args: soft_center_radius (float): Radius of the soft center prior. Defaults to 3.0. topk (int): Select top-k predictions to calculate dynamic k best matches for each gt. Defaults to 13. iou_weight (float): The scale factor of iou cost. Defaults to 3.0. iou_calculator (ConfigType): Config of overlaps Calculator. Defaults to dict(type='BboxOverlaps2D'). """ def __init__( self, soft_center_radius: float = 3.0, topk: int = 13, iou_weight: float = 3.0, iou_calculator: ConfigType = dict(type='BboxOverlaps2D') ) -> None: self.soft_center_radius = soft_center_radius self.topk = topk self.iou_weight = iou_weight self.iou_calculator = TASK_UTILS.build(iou_calculator) def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, **kwargs) -> AssignResult: """Assign gt to priors. Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: obj:`AssignResult`: The assigned result. """ gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels num_gt = gt_bboxes.size(0) decoded_bboxes = pred_instances.bboxes pred_scores = pred_instances.scores priors = pred_instances.priors num_bboxes = decoded_bboxes.size(0) # assign 0 by default assigned_gt_inds = decoded_bboxes.new_full((num_bboxes, ), 0, dtype=torch.long) if num_gt == 0 or num_bboxes == 0: # No ground truth or boxes, return empty assignment max_overlaps = decoded_bboxes.new_zeros((num_bboxes, )) if num_gt == 0: # No truth, assign everything to background assigned_gt_inds[:] = 0 assigned_labels = decoded_bboxes.new_full((num_bboxes, ), -1, dtype=torch.long) return AssignResult( num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) prior_center = priors[:, :2] if isinstance(gt_bboxes, BaseBoxes): is_in_gts = gt_bboxes.find_inside_points(prior_center) else: # Tensor boxes will be treated as horizontal boxes by defaults lt_ = prior_center[:, None] - gt_bboxes[:, :2] rb_ = gt_bboxes[:, 2:] - prior_center[:, None] deltas = torch.cat([lt_, rb_], dim=-1) is_in_gts = deltas.min(dim=-1).values > 0 valid_mask = is_in_gts.sum(dim=1) > 0 valid_decoded_bbox = decoded_bboxes[valid_mask] valid_pred_scores = pred_scores[valid_mask] num_valid = valid_decoded_bbox.size(0) if num_valid == 0: # No ground truth or boxes, return empty assignment max_overlaps = decoded_bboxes.new_zeros((num_bboxes, )) assigned_labels = decoded_bboxes.new_full((num_bboxes, ), -1, dtype=torch.long) return AssignResult( num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) if hasattr(gt_instances, 'masks'): gt_center = center_of_mass(gt_instances.masks, eps=EPS) elif isinstance(gt_bboxes, BaseBoxes): gt_center = gt_bboxes.centers else: # Tensor boxes will be treated as horizontal boxes by defaults gt_center = (gt_bboxes[:, :2] + gt_bboxes[:, 2:]) / 2.0 valid_prior = priors[valid_mask] strides = valid_prior[:, 2] distance = (valid_prior[:, None, :2] - gt_center[None, :, :] ).pow(2).sum(-1).sqrt() / strides[:, None] soft_center_prior = torch.pow(10, distance - self.soft_center_radius) pairwise_ious = self.iou_calculator(valid_decoded_bbox, gt_bboxes) iou_cost = -torch.log(pairwise_ious + EPS) * self.iou_weight gt_onehot_label = ( F.one_hot(gt_labels.to(torch.int64), pred_scores.shape[-1]).float().unsqueeze(0).repeat( num_valid, 1, 1)) valid_pred_scores = valid_pred_scores.unsqueeze(1).repeat(1, num_gt, 1) soft_label = gt_onehot_label * pairwise_ious[..., None] scale_factor = soft_label - valid_pred_scores.sigmoid() soft_cls_cost = F.binary_cross_entropy_with_logits( valid_pred_scores, soft_label, reduction='none') * scale_factor.abs().pow(2.0) soft_cls_cost = soft_cls_cost.sum(dim=-1) cost_matrix = soft_cls_cost + iou_cost + soft_center_prior matched_pred_ious, matched_gt_inds = self.dynamic_k_matching( cost_matrix, pairwise_ious, num_gt, valid_mask) # convert to AssignResult format assigned_gt_inds[valid_mask] = matched_gt_inds + 1 assigned_labels = assigned_gt_inds.new_full((num_bboxes, ), -1) assigned_labels[valid_mask] = gt_labels[matched_gt_inds].long() max_overlaps = assigned_gt_inds.new_full((num_bboxes, ), -INF, dtype=torch.float32) max_overlaps[valid_mask] = matched_pred_ious return AssignResult( num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) def dynamic_k_matching(self, cost: Tensor, pairwise_ious: Tensor, num_gt: int, valid_mask: Tensor) -> Tuple[Tensor, Tensor]: """Use IoU and matching cost to calculate the dynamic top-k positive targets. Same as SimOTA. Args: cost (Tensor): Cost matrix. pairwise_ious (Tensor): Pairwise iou matrix. num_gt (int): Number of gt. valid_mask (Tensor): Mask for valid bboxes. Returns: tuple: matched ious and gt indexes. """ matching_matrix = torch.zeros_like(cost, dtype=torch.uint8) # select candidate topk ious for dynamic-k calculation candidate_topk = min(self.topk, pairwise_ious.size(0)) topk_ious, _ = torch.topk(pairwise_ious, candidate_topk, dim=0) # calculate dynamic k for each gt dynamic_ks = torch.clamp(topk_ious.sum(0).int(), min=1) for gt_idx in range(num_gt): _, pos_idx = torch.topk( cost[:, gt_idx], k=dynamic_ks[gt_idx], largest=False) matching_matrix[:, gt_idx][pos_idx] = 1 del topk_ious, dynamic_ks, pos_idx prior_match_gt_mask = matching_matrix.sum(1) > 1 if prior_match_gt_mask.sum() > 0: cost_min, cost_argmin = torch.min( cost[prior_match_gt_mask, :], dim=1) matching_matrix[prior_match_gt_mask, :] *= 0 matching_matrix[prior_match_gt_mask, cost_argmin] = 1 # get foreground mask inside box and center prior fg_mask_inboxes = matching_matrix.sum(1) > 0 valid_mask[valid_mask.clone()] = fg_mask_inboxes matched_gt_inds = matching_matrix[fg_mask_inboxes, :].argmax(1) matched_pred_ious = (matching_matrix * pairwise_ious).sum(1)[fg_mask_inboxes] return matched_pred_ious, matched_gt_inds ================================================ FILE: mmdet/models/task_modules/assigners/grid_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple, Union import torch from mmengine.structures import InstanceData from mmdet.registry import TASK_UTILS from mmdet.utils import ConfigType from .assign_result import AssignResult from .base_assigner import BaseAssigner @TASK_UTILS.register_module() class GridAssigner(BaseAssigner): """Assign a corresponding gt bbox or background to each bbox. Each proposals will be assigned with `-1`, `0`, or a positive integer indicating the ground truth index. - -1: don't care - 0: negative sample, no assigned gt - positive integer: positive sample, index (1-based) of assigned gt Args: pos_iou_thr (float): IoU threshold for positive bboxes. neg_iou_thr (float or tuple[float, float]): IoU threshold for negative bboxes. min_pos_iou (float): Minimum iou for a bbox to be considered as a positive bbox. Positive samples can have smaller IoU than pos_iou_thr due to the 4th step (assign max IoU sample to each gt). Defaults to 0. gt_max_assign_all (bool): Whether to assign all bboxes with the same highest overlap with some gt to that gt. iou_calculator (:obj:`ConfigDict` or dict): Config of overlaps Calculator. """ def __init__( self, pos_iou_thr: float, neg_iou_thr: Union[float, Tuple[float, float]], min_pos_iou: float = .0, gt_max_assign_all: bool = True, iou_calculator: ConfigType = dict(type='BboxOverlaps2D') ) -> None: self.pos_iou_thr = pos_iou_thr self.neg_iou_thr = neg_iou_thr self.min_pos_iou = min_pos_iou self.gt_max_assign_all = gt_max_assign_all self.iou_calculator = TASK_UTILS.build(iou_calculator) def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, **kwargs) -> AssignResult: """Assign gt to bboxes. The process is very much like the max iou assigner, except that positive samples are constrained within the cell that the gt boxes fell in. This method assign a gt bbox to every bbox (proposal/anchor), each bbox will be assigned with -1, 0, or a positive number. -1 means don't care, 0 means negative sample, positive number is the index (1-based) of assigned gt. The assignment is done in following steps, the order matters. 1. assign every bbox to -1 2. assign proposals whose iou with all gts <= neg_iou_thr to 0 3. for each bbox within a cell, if the iou with its nearest gt > pos_iou_thr and the center of that gt falls inside the cell, assign it to that bbox 4. for each gt bbox, assign its nearest proposals within the cell the gt bbox falls in to itself. Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: :obj:`AssignResult`: The assign result. """ gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels priors = pred_instances.priors responsible_flags = pred_instances.responsible_flags num_gts, num_priors = gt_bboxes.size(0), priors.size(0) # compute iou between all gt and priors overlaps = self.iou_calculator(gt_bboxes, priors) # 1. assign -1 by default assigned_gt_inds = overlaps.new_full((num_priors, ), -1, dtype=torch.long) if num_gts == 0 or num_priors == 0: # No ground truth or priors, return empty assignment max_overlaps = overlaps.new_zeros((num_priors, )) if num_gts == 0: # No truth, assign everything to background assigned_gt_inds[:] = 0 assigned_labels = overlaps.new_full((num_priors, ), -1, dtype=torch.long) return AssignResult( num_gts, assigned_gt_inds, max_overlaps, labels=assigned_labels) # 2. assign negative: below # for each anchor, which gt best overlaps with it # for each anchor, the max iou of all gts # shape of max_overlaps == argmax_overlaps == num_priors max_overlaps, argmax_overlaps = overlaps.max(dim=0) if isinstance(self.neg_iou_thr, float): assigned_gt_inds[(max_overlaps >= 0) & (max_overlaps <= self.neg_iou_thr)] = 0 elif isinstance(self.neg_iou_thr, (tuple, list)): assert len(self.neg_iou_thr) == 2 assigned_gt_inds[(max_overlaps > self.neg_iou_thr[0]) & (max_overlaps <= self.neg_iou_thr[1])] = 0 # 3. assign positive: falls into responsible cell and above # positive IOU threshold, the order matters. # the prior condition of comparison is to filter out all # unrelated anchors, i.e. not responsible_flags overlaps[:, ~responsible_flags.type(torch.bool)] = -1. # calculate max_overlaps again, but this time we only consider IOUs # for anchors responsible for prediction max_overlaps, argmax_overlaps = overlaps.max(dim=0) # for each gt, which anchor best overlaps with it # for each gt, the max iou of all proposals # shape of gt_max_overlaps == gt_argmax_overlaps == num_gts gt_max_overlaps, gt_argmax_overlaps = overlaps.max(dim=1) pos_inds = (max_overlaps > self.pos_iou_thr) & responsible_flags.type( torch.bool) assigned_gt_inds[pos_inds] = argmax_overlaps[pos_inds] + 1 # 4. assign positive to max overlapped anchors within responsible cell for i in range(num_gts): if gt_max_overlaps[i] > self.min_pos_iou: if self.gt_max_assign_all: max_iou_inds = (overlaps[i, :] == gt_max_overlaps[i]) & \ responsible_flags.type(torch.bool) assigned_gt_inds[max_iou_inds] = i + 1 elif responsible_flags[gt_argmax_overlaps[i]]: assigned_gt_inds[gt_argmax_overlaps[i]] = i + 1 # assign labels of positive anchors assigned_labels = assigned_gt_inds.new_full((num_priors, ), -1) pos_inds = torch.nonzero( assigned_gt_inds > 0, as_tuple=False).squeeze() if pos_inds.numel() > 0: assigned_labels[pos_inds] = gt_labels[assigned_gt_inds[pos_inds] - 1] return AssignResult( num_gts, assigned_gt_inds, max_overlaps, labels=assigned_labels) ================================================ FILE: mmdet/models/task_modules/assigners/hungarian_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Union import torch from mmengine import ConfigDict from mmengine.structures import InstanceData from scipy.optimize import linear_sum_assignment from torch import Tensor from mmdet.registry import TASK_UTILS from .assign_result import AssignResult from .base_assigner import BaseAssigner @TASK_UTILS.register_module() class HungarianAssigner(BaseAssigner): """Computes one-to-one matching between predictions and ground truth. This class computes an assignment between the targets and the predictions based on the costs. The costs are weighted sum of some components. For DETR the costs are weighted sum of classification cost, regression L1 cost and regression iou cost. The targets don't include the no_object, so generally there are more predictions than targets. After the one-to-one matching, the un-matched are treated as backgrounds. Thus each query prediction will be assigned with `0` or a positive integer indicating the ground truth index: - 0: negative sample, no assigned gt - positive integer: positive sample, index (1-based) of assigned gt Args: match_costs (:obj:`ConfigDict` or dict or \ List[Union[:obj:`ConfigDict`, dict]]): Match cost configs. """ def __init__( self, match_costs: Union[List[Union[dict, ConfigDict]], dict, ConfigDict] ) -> None: if isinstance(match_costs, dict): match_costs = [match_costs] elif isinstance(match_costs, list): assert len(match_costs) > 0, \ 'match_costs must not be a empty list.' self.match_costs = [ TASK_UTILS.build(match_cost) for match_cost in match_costs ] def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, img_meta: Optional[dict] = None, **kwargs) -> AssignResult: """Computes one-to-one matching based on the weighted costs. This method assign each query prediction to a ground truth or background. The `assigned_gt_inds` with -1 means don't care, 0 means negative sample, and positive number is the index (1-based) of assigned gt. The assignment is done in the following steps, the order matters. 1. assign every prediction to -1 2. compute the weighted costs 3. do Hungarian matching on CPU based on the costs 4. assign all to 0 (background) first, then for each matched pair between predictions and gts, treat this prediction as foreground and assign the corresponding gt index (plus 1) to it. Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. It may includes ``masks``, with shape (n, h, w) or (n, l). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), ``labels``, with shape (k, ) and ``masks``, with shape (k, h, w) or (k, l). img_meta (dict): Image information. Returns: :obj:`AssignResult`: The assigned result. """ assert isinstance(gt_instances.labels, Tensor) num_gts, num_preds = len(gt_instances), len(pred_instances) gt_labels = gt_instances.labels device = gt_labels.device # 1. assign -1 by default assigned_gt_inds = torch.full((num_preds, ), -1, dtype=torch.long, device=device) assigned_labels = torch.full((num_preds, ), -1, dtype=torch.long, device=device) if num_gts == 0 or num_preds == 0: # No ground truth or boxes, return empty assignment if num_gts == 0: # No ground truth, assign all to background assigned_gt_inds[:] = 0 return AssignResult( num_gts=num_gts, gt_inds=assigned_gt_inds, max_overlaps=None, labels=assigned_labels) # 2. compute weighted cost cost_list = [] for match_cost in self.match_costs: cost = match_cost( pred_instances=pred_instances, gt_instances=gt_instances, img_meta=img_meta) cost_list.append(cost) cost = torch.stack(cost_list).sum(dim=0) # 3. do Hungarian matching on CPU using linear_sum_assignment cost = cost.detach().cpu() if linear_sum_assignment is None: raise ImportError('Please run "pip install scipy" ' 'to install scipy first.') matched_row_inds, matched_col_inds = linear_sum_assignment(cost) matched_row_inds = torch.from_numpy(matched_row_inds).to(device) matched_col_inds = torch.from_numpy(matched_col_inds).to(device) # 4. assign backgrounds and foregrounds # assign all indices to backgrounds first assigned_gt_inds[:] = 0 # assign foregrounds based on matching results assigned_gt_inds[matched_row_inds] = matched_col_inds + 1 assigned_labels[matched_row_inds] = gt_labels[matched_col_inds] return AssignResult( num_gts=num_gts, gt_inds=assigned_gt_inds, max_overlaps=None, labels=assigned_labels) ================================================ FILE: mmdet/models/task_modules/assigners/iou2d_calculator.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import bbox_overlaps, get_box_tensor def cast_tensor_type(x, scale=1., dtype=None): if dtype == 'fp16': # scale is for preventing overflows x = (x / scale).half() return x @TASK_UTILS.register_module() class BboxOverlaps2D: """2D Overlaps (e.g. IoUs, GIoUs) Calculator.""" def __init__(self, scale=1., dtype=None): self.scale = scale self.dtype = dtype def __call__(self, bboxes1, bboxes2, mode='iou', is_aligned=False): """Calculate IoU between 2D bboxes. Args: bboxes1 (Tensor or :obj:`BaseBoxes`): bboxes have shape (m, 4) in format, or shape (m, 5) in format. bboxes2 (Tensor or :obj:`BaseBoxes`): bboxes have shape (m, 4) in format, shape (m, 5) in format, or be empty. If ``is_aligned `` is ``True``, then m and n must be equal. mode (str): "iou" (intersection over union), "iof" (intersection over foreground), or "giou" (generalized intersection over union). is_aligned (bool, optional): If True, then m and n must be equal. Default False. Returns: Tensor: shape (m, n) if ``is_aligned `` is False else shape (m,) """ bboxes1 = get_box_tensor(bboxes1) bboxes2 = get_box_tensor(bboxes2) assert bboxes1.size(-1) in [0, 4, 5] assert bboxes2.size(-1) in [0, 4, 5] if bboxes2.size(-1) == 5: bboxes2 = bboxes2[..., :4] if bboxes1.size(-1) == 5: bboxes1 = bboxes1[..., :4] if self.dtype == 'fp16': # change tensor type to save cpu and cuda memory and keep speed bboxes1 = cast_tensor_type(bboxes1, self.scale, self.dtype) bboxes2 = cast_tensor_type(bboxes2, self.scale, self.dtype) overlaps = bbox_overlaps(bboxes1, bboxes2, mode, is_aligned) if not overlaps.is_cuda and overlaps.dtype == torch.float16: # resume cpu float32 overlaps = overlaps.float() return overlaps return bbox_overlaps(bboxes1, bboxes2, mode, is_aligned) def __repr__(self): """str: a string describing the module""" repr_str = self.__class__.__name__ + f'(' \ f'scale={self.scale}, dtype={self.dtype})' return repr_str ================================================ FILE: mmdet/models/task_modules/assigners/match_cost.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import abstractmethod from typing import Optional, Union import torch import torch.nn.functional as F from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import bbox_overlaps, bbox_xyxy_to_cxcywh class BaseMatchCost: """Base match cost class. Args: weight (Union[float, int]): Cost weight. Defaults to 1. """ def __init__(self, weight: Union[float, int] = 1.) -> None: self.weight = weight @abstractmethod def __call__(self, pred_instances: InstanceData, gt_instances: InstanceData, img_meta: Optional[dict] = None, **kwargs) -> Tensor: """Compute match cost. Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). img_meta (dict, optional): Image information. Returns: Tensor: Match Cost matrix of shape (num_preds, num_gts). """ pass @TASK_UTILS.register_module() class BBoxL1Cost(BaseMatchCost): """BBoxL1Cost. Note: ``bboxes`` in ``InstanceData`` passed in is of format 'xyxy' and its coordinates are unnormalized. Args: box_format (str, optional): 'xyxy' for DETR, 'xywh' for Sparse_RCNN. Defaults to 'xyxy'. weight (Union[float, int]): Cost weight. Defaults to 1. Examples: >>> from mmdet.models.task_modules.assigners. ... match_costs.match_cost import BBoxL1Cost >>> import torch >>> self = BBoxL1Cost() >>> bbox_pred = torch.rand(1, 4) >>> gt_bboxes= torch.FloatTensor([[0, 0, 2, 4], [1, 2, 3, 4]]) >>> factor = torch.tensor([10, 8, 10, 8]) >>> self(bbox_pred, gt_bboxes, factor) tensor([[1.6172, 1.6422]]) """ def __init__(self, box_format: str = 'xyxy', weight: Union[float, int] = 1.) -> None: super().__init__(weight=weight) assert box_format in ['xyxy', 'xywh'] self.box_format = box_format def __call__(self, pred_instances: InstanceData, gt_instances: InstanceData, img_meta: Optional[dict] = None, **kwargs) -> Tensor: """Compute match cost. Args: pred_instances (:obj:`InstanceData`): ``bboxes`` inside is predicted boxes with unnormalized coordinate (x, y, x, y). gt_instances (:obj:`InstanceData`): ``bboxes`` inside is gt bboxes with unnormalized coordinate (x, y, x, y). img_meta (Optional[dict]): Image information. Defaults to None. Returns: Tensor: Match Cost matrix of shape (num_preds, num_gts). """ pred_bboxes = pred_instances.bboxes gt_bboxes = gt_instances.bboxes # convert box format if self.box_format == 'xywh': gt_bboxes = bbox_xyxy_to_cxcywh(gt_bboxes) pred_bboxes = bbox_xyxy_to_cxcywh(pred_bboxes) # normalized img_h, img_w = img_meta['img_shape'] factor = gt_bboxes.new_tensor([img_w, img_h, img_w, img_h]).unsqueeze(0) gt_bboxes = gt_bboxes / factor pred_bboxes = pred_bboxes / factor bbox_cost = torch.cdist(pred_bboxes, gt_bboxes, p=1) return bbox_cost * self.weight @TASK_UTILS.register_module() class IoUCost(BaseMatchCost): """IoUCost. Note: ``bboxes`` in ``InstanceData`` passed in is of format 'xyxy' and its coordinates are unnormalized. Args: iou_mode (str): iou mode such as 'iou', 'giou'. Defaults to 'giou'. weight (Union[float, int]): Cost weight. Defaults to 1. Examples: >>> from mmdet.models.task_modules.assigners. ... match_costs.match_cost import IoUCost >>> import torch >>> self = IoUCost() >>> bboxes = torch.FloatTensor([[1,1, 2, 2], [2, 2, 3, 4]]) >>> gt_bboxes = torch.FloatTensor([[0, 0, 2, 4], [1, 2, 3, 4]]) >>> self(bboxes, gt_bboxes) tensor([[-0.1250, 0.1667], [ 0.1667, -0.5000]]) """ def __init__(self, iou_mode: str = 'giou', weight: Union[float, int] = 1.): super().__init__(weight=weight) self.iou_mode = iou_mode def __call__(self, pred_instances: InstanceData, gt_instances: InstanceData, img_meta: Optional[dict] = None, **kwargs): """Compute match cost. Args: pred_instances (:obj:`InstanceData`): ``bboxes`` inside is predicted boxes with unnormalized coordinate (x, y, x, y). gt_instances (:obj:`InstanceData`): ``bboxes`` inside is gt bboxes with unnormalized coordinate (x, y, x, y). img_meta (Optional[dict]): Image information. Defaults to None. Returns: Tensor: Match Cost matrix of shape (num_preds, num_gts). """ pred_bboxes = pred_instances.bboxes gt_bboxes = gt_instances.bboxes overlaps = bbox_overlaps( pred_bboxes, gt_bboxes, mode=self.iou_mode, is_aligned=False) # The 1 is a constant that doesn't change the matching, so omitted. iou_cost = -overlaps return iou_cost * self.weight @TASK_UTILS.register_module() class ClassificationCost(BaseMatchCost): """ClsSoftmaxCost. Args: weight (Union[float, int]): Cost weight. Defaults to 1. Examples: >>> from mmdet.models.task_modules.assigners. ... match_costs.match_cost import ClassificationCost >>> import torch >>> self = ClassificationCost() >>> cls_pred = torch.rand(4, 3) >>> gt_labels = torch.tensor([0, 1, 2]) >>> factor = torch.tensor([10, 8, 10, 8]) >>> self(cls_pred, gt_labels) tensor([[-0.3430, -0.3525, -0.3045], [-0.3077, -0.2931, -0.3992], [-0.3664, -0.3455, -0.2881], [-0.3343, -0.2701, -0.3956]]) """ def __init__(self, weight: Union[float, int] = 1) -> None: super().__init__(weight=weight) def __call__(self, pred_instances: InstanceData, gt_instances: InstanceData, img_meta: Optional[dict] = None, **kwargs) -> Tensor: """Compute match cost. Args: pred_instances (:obj:`InstanceData`): ``scores`` inside is predicted classification logits, of shape (num_queries, num_class). gt_instances (:obj:`InstanceData`): ``labels`` inside should have shape (num_gt, ). img_meta (Optional[dict]): _description_. Defaults to None. Returns: Tensor: Match Cost matrix of shape (num_preds, num_gts). """ pred_scores = pred_instances.scores gt_labels = gt_instances.labels pred_scores = pred_scores.softmax(-1) cls_cost = -pred_scores[:, gt_labels] return cls_cost * self.weight @TASK_UTILS.register_module() class FocalLossCost(BaseMatchCost): """FocalLossCost. Args: alpha (Union[float, int]): focal_loss alpha. Defaults to 0.25. gamma (Union[float, int]): focal_loss gamma. Defaults to 2. eps (float): Defaults to 1e-12. binary_input (bool): Whether the input is binary. Currently, binary_input = True is for masks input, binary_input = False is for label input. Defaults to False. weight (Union[float, int]): Cost weight. Defaults to 1. """ def __init__(self, alpha: Union[float, int] = 0.25, gamma: Union[float, int] = 2, eps: float = 1e-12, binary_input: bool = False, weight: Union[float, int] = 1.) -> None: super().__init__(weight=weight) self.alpha = alpha self.gamma = gamma self.eps = eps self.binary_input = binary_input def _focal_loss_cost(self, cls_pred: Tensor, gt_labels: Tensor) -> Tensor: """ Args: cls_pred (Tensor): Predicted classification logits, shape (num_queries, num_class). gt_labels (Tensor): Label of `gt_bboxes`, shape (num_gt,). Returns: torch.Tensor: cls_cost value with weight """ cls_pred = cls_pred.sigmoid() neg_cost = -(1 - cls_pred + self.eps).log() * ( 1 - self.alpha) * cls_pred.pow(self.gamma) pos_cost = -(cls_pred + self.eps).log() * self.alpha * ( 1 - cls_pred).pow(self.gamma) cls_cost = pos_cost[:, gt_labels] - neg_cost[:, gt_labels] return cls_cost * self.weight def _mask_focal_loss_cost(self, cls_pred, gt_labels) -> Tensor: """ Args: cls_pred (Tensor): Predicted classification logits. in shape (num_queries, d1, ..., dn), dtype=torch.float32. gt_labels (Tensor): Ground truth in shape (num_gt, d1, ..., dn), dtype=torch.long. Labels should be binary. Returns: Tensor: Focal cost matrix with weight in shape\ (num_queries, num_gt). """ cls_pred = cls_pred.flatten(1) gt_labels = gt_labels.flatten(1).float() n = cls_pred.shape[1] cls_pred = cls_pred.sigmoid() neg_cost = -(1 - cls_pred + self.eps).log() * ( 1 - self.alpha) * cls_pred.pow(self.gamma) pos_cost = -(cls_pred + self.eps).log() * self.alpha * ( 1 - cls_pred).pow(self.gamma) cls_cost = torch.einsum('nc,mc->nm', pos_cost, gt_labels) + \ torch.einsum('nc,mc->nm', neg_cost, (1 - gt_labels)) return cls_cost / n * self.weight def __call__(self, pred_instances: InstanceData, gt_instances: InstanceData, img_meta: Optional[dict] = None, **kwargs) -> Tensor: """Compute match cost. Args: pred_instances (:obj:`InstanceData`): Predicted instances which must contain ``scores`` or ``masks``. gt_instances (:obj:`InstanceData`): Ground truth which must contain ``labels`` or ``mask``. img_meta (Optional[dict]): Image information. Defaults to None. Returns: Tensor: Match Cost matrix of shape (num_preds, num_gts). """ if self.binary_input: pred_masks = pred_instances.masks gt_masks = gt_instances.masks return self._mask_focal_loss_cost(pred_masks, gt_masks) else: pred_scores = pred_instances.scores gt_labels = gt_instances.labels return self._focal_loss_cost(pred_scores, gt_labels) @TASK_UTILS.register_module() class DiceCost(BaseMatchCost): """Cost of mask assignments based on dice losses. Args: pred_act (bool): Whether to apply sigmoid to mask_pred. Defaults to False. eps (float): Defaults to 1e-3. naive_dice (bool): If True, use the naive dice loss in which the power of the number in the denominator is the first power. If False, use the second power that is adopted by K-Net and SOLO. Defaults to True. weight (Union[float, int]): Cost weight. Defaults to 1. """ def __init__(self, pred_act: bool = False, eps: float = 1e-3, naive_dice: bool = True, weight: Union[float, int] = 1.) -> None: super().__init__(weight=weight) self.pred_act = pred_act self.eps = eps self.naive_dice = naive_dice def _binary_mask_dice_loss(self, mask_preds: Tensor, gt_masks: Tensor) -> Tensor: """ Args: mask_preds (Tensor): Mask prediction in shape (num_queries, *). gt_masks (Tensor): Ground truth in shape (num_gt, *) store 0 or 1, 0 for negative class and 1 for positive class. Returns: Tensor: Dice cost matrix in shape (num_queries, num_gt). """ mask_preds = mask_preds.flatten(1) gt_masks = gt_masks.flatten(1).float() numerator = 2 * torch.einsum('nc,mc->nm', mask_preds, gt_masks) if self.naive_dice: denominator = mask_preds.sum(-1)[:, None] + \ gt_masks.sum(-1)[None, :] else: denominator = mask_preds.pow(2).sum(1)[:, None] + \ gt_masks.pow(2).sum(1)[None, :] loss = 1 - (numerator + self.eps) / (denominator + self.eps) return loss def __call__(self, pred_instances: InstanceData, gt_instances: InstanceData, img_meta: Optional[dict] = None, **kwargs) -> Tensor: """Compute match cost. Args: pred_instances (:obj:`InstanceData`): Predicted instances which must contain ``masks``. gt_instances (:obj:`InstanceData`): Ground truth which must contain ``mask``. img_meta (Optional[dict]): Image information. Defaults to None. Returns: Tensor: Match Cost matrix of shape (num_preds, num_gts). """ pred_masks = pred_instances.masks gt_masks = gt_instances.masks if self.pred_act: pred_masks = pred_masks.sigmoid() dice_cost = self._binary_mask_dice_loss(pred_masks, gt_masks) return dice_cost * self.weight @TASK_UTILS.register_module() class CrossEntropyLossCost(BaseMatchCost): """CrossEntropyLossCost. Args: use_sigmoid (bool): Whether the prediction uses sigmoid of softmax. Defaults to True. weight (Union[float, int]): Cost weight. Defaults to 1. """ def __init__(self, use_sigmoid: bool = True, weight: Union[float, int] = 1.) -> None: super().__init__(weight=weight) self.use_sigmoid = use_sigmoid def _binary_cross_entropy(self, cls_pred: Tensor, gt_labels: Tensor) -> Tensor: """ Args: cls_pred (Tensor): The prediction with shape (num_queries, 1, *) or (num_queries, *). gt_labels (Tensor): The learning label of prediction with shape (num_gt, *). Returns: Tensor: Cross entropy cost matrix in shape (num_queries, num_gt). """ cls_pred = cls_pred.flatten(1).float() gt_labels = gt_labels.flatten(1).float() n = cls_pred.shape[1] pos = F.binary_cross_entropy_with_logits( cls_pred, torch.ones_like(cls_pred), reduction='none') neg = F.binary_cross_entropy_with_logits( cls_pred, torch.zeros_like(cls_pred), reduction='none') cls_cost = torch.einsum('nc,mc->nm', pos, gt_labels) + \ torch.einsum('nc,mc->nm', neg, 1 - gt_labels) cls_cost = cls_cost / n return cls_cost def __call__(self, pred_instances: InstanceData, gt_instances: InstanceData, img_meta: Optional[dict] = None, **kwargs) -> Tensor: """Compute match cost. Args: pred_instances (:obj:`InstanceData`): Predicted instances which must contain ``scores`` or ``masks``. gt_instances (:obj:`InstanceData`): Ground truth which must contain ``labels`` or ``masks``. img_meta (Optional[dict]): Image information. Defaults to None. Returns: Tensor: Match Cost matrix of shape (num_preds, num_gts). """ pred_masks = pred_instances.masks gt_masks = gt_instances.masks if self.use_sigmoid: cls_cost = self._binary_cross_entropy(pred_masks, gt_masks) else: raise NotImplementedError return cls_cost * self.weight ================================================ FILE: mmdet/models/task_modules/assigners/max_iou_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Union import torch from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import TASK_UTILS from .assign_result import AssignResult from .base_assigner import BaseAssigner @TASK_UTILS.register_module() class MaxIoUAssigner(BaseAssigner): """Assign a corresponding gt bbox or background to each bbox. Each proposals will be assigned with `-1`, or a semi-positive integer indicating the ground truth index. - -1: negative sample, no assigned gt - semi-positive integer: positive sample, index (0-based) of assigned gt Args: pos_iou_thr (float): IoU threshold for positive bboxes. neg_iou_thr (float or tuple): IoU threshold for negative bboxes. min_pos_iou (float): Minimum iou for a bbox to be considered as a positive bbox. Positive samples can have smaller IoU than pos_iou_thr due to the 4th step (assign max IoU sample to each gt). `min_pos_iou` is set to avoid assigning bboxes that have extremely small iou with GT as positive samples. It brings about 0.3 mAP improvements in 1x schedule but does not affect the performance of 3x schedule. More comparisons can be found in `PR #7464 `_. gt_max_assign_all (bool): Whether to assign all bboxes with the same highest overlap with some gt to that gt. ignore_iof_thr (float): IoF threshold for ignoring bboxes (if `gt_bboxes_ignore` is specified). Negative values mean not ignoring any bboxes. ignore_wrt_candidates (bool): Whether to compute the iof between `bboxes` and `gt_bboxes_ignore`, or the contrary. match_low_quality (bool): Whether to allow low quality matches. This is usually allowed for RPN and single stage detectors, but not allowed in the second stage. Details are demonstrated in Step 4. gpu_assign_thr (int): The upper bound of the number of GT for GPU assign. When the number of gt is above this threshold, will assign on CPU device. Negative values mean not assign on CPU. iou_calculator (dict): Config of overlaps Calculator. """ def __init__(self, pos_iou_thr: float, neg_iou_thr: Union[float, tuple], min_pos_iou: float = .0, gt_max_assign_all: bool = True, ignore_iof_thr: float = -1, ignore_wrt_candidates: bool = True, match_low_quality: bool = True, gpu_assign_thr: float = -1, iou_calculator: dict = dict(type='BboxOverlaps2D')): self.pos_iou_thr = pos_iou_thr self.neg_iou_thr = neg_iou_thr self.min_pos_iou = min_pos_iou self.gt_max_assign_all = gt_max_assign_all self.ignore_iof_thr = ignore_iof_thr self.ignore_wrt_candidates = ignore_wrt_candidates self.gpu_assign_thr = gpu_assign_thr self.match_low_quality = match_low_quality self.iou_calculator = TASK_UTILS.build(iou_calculator) def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, **kwargs) -> AssignResult: """Assign gt to bboxes. This method assign a gt bbox to every bbox (proposal/anchor), each bbox will be assigned with -1, or a semi-positive number. -1 means negative sample, semi-positive number is the index (0-based) of assigned gt. The assignment is done in following steps, the order matters. 1. assign every bbox to the background 2. assign proposals whose iou with all gts < neg_iou_thr to 0 3. for each bbox, if the iou with its nearest gt >= pos_iou_thr, assign it to that bbox 4. for each gt bbox, assign its nearest proposals (may be more than one) to itself Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: :obj:`AssignResult`: The assign result. Example: >>> from mmengine.structures import InstanceData >>> self = MaxIoUAssigner(0.5, 0.5) >>> pred_instances = InstanceData() >>> pred_instances.priors = torch.Tensor([[0, 0, 10, 10], ... [10, 10, 20, 20]]) >>> gt_instances = InstanceData() >>> gt_instances.bboxes = torch.Tensor([[0, 0, 10, 9]]) >>> gt_instances.labels = torch.Tensor([0]) >>> assign_result = self.assign(pred_instances, gt_instances) >>> expected_gt_inds = torch.LongTensor([1, 0]) >>> assert torch.all(assign_result.gt_inds == expected_gt_inds) """ gt_bboxes = gt_instances.bboxes priors = pred_instances.priors gt_labels = gt_instances.labels if gt_instances_ignore is not None: gt_bboxes_ignore = gt_instances_ignore.bboxes else: gt_bboxes_ignore = None assign_on_cpu = True if (self.gpu_assign_thr > 0) and ( gt_bboxes.shape[0] > self.gpu_assign_thr) else False # compute overlap and assign gt on CPU when number of GT is large if assign_on_cpu: device = priors.device priors = priors.cpu() gt_bboxes = gt_bboxes.cpu() gt_labels = gt_labels.cpu() if gt_bboxes_ignore is not None: gt_bboxes_ignore = gt_bboxes_ignore.cpu() overlaps = self.iou_calculator(gt_bboxes, priors) if (self.ignore_iof_thr > 0 and gt_bboxes_ignore is not None and gt_bboxes_ignore.numel() > 0 and priors.numel() > 0): if self.ignore_wrt_candidates: ignore_overlaps = self.iou_calculator( priors, gt_bboxes_ignore, mode='iof') ignore_max_overlaps, _ = ignore_overlaps.max(dim=1) else: ignore_overlaps = self.iou_calculator( gt_bboxes_ignore, priors, mode='iof') ignore_max_overlaps, _ = ignore_overlaps.max(dim=0) overlaps[:, ignore_max_overlaps > self.ignore_iof_thr] = -1 assign_result = self.assign_wrt_overlaps(overlaps, gt_labels) if assign_on_cpu: assign_result.gt_inds = assign_result.gt_inds.to(device) assign_result.max_overlaps = assign_result.max_overlaps.to(device) if assign_result.labels is not None: assign_result.labels = assign_result.labels.to(device) return assign_result def assign_wrt_overlaps(self, overlaps: Tensor, gt_labels: Tensor) -> AssignResult: """Assign w.r.t. the overlaps of priors with gts. Args: overlaps (Tensor): Overlaps between k gt_bboxes and n bboxes, shape(k, n). gt_labels (Tensor): Labels of k gt_bboxes, shape (k, ). Returns: :obj:`AssignResult`: The assign result. """ num_gts, num_bboxes = overlaps.size(0), overlaps.size(1) # 1. assign -1 by default assigned_gt_inds = overlaps.new_full((num_bboxes, ), -1, dtype=torch.long) if num_gts == 0 or num_bboxes == 0: # No ground truth or boxes, return empty assignment max_overlaps = overlaps.new_zeros((num_bboxes, )) assigned_labels = overlaps.new_full((num_bboxes, ), -1, dtype=torch.long) if num_gts == 0: # No truth, assign everything to background assigned_gt_inds[:] = 0 return AssignResult( num_gts=num_gts, gt_inds=assigned_gt_inds, max_overlaps=max_overlaps, labels=assigned_labels) # for each anchor, which gt best overlaps with it # for each anchor, the max iou of all gts max_overlaps, argmax_overlaps = overlaps.max(dim=0) # for each gt, which anchor best overlaps with it # for each gt, the max iou of all proposals gt_max_overlaps, gt_argmax_overlaps = overlaps.max(dim=1) # 2. assign negative: below # the negative inds are set to be 0 if isinstance(self.neg_iou_thr, float): assigned_gt_inds[(max_overlaps >= 0) & (max_overlaps < self.neg_iou_thr)] = 0 elif isinstance(self.neg_iou_thr, tuple): assert len(self.neg_iou_thr) == 2 assigned_gt_inds[(max_overlaps >= self.neg_iou_thr[0]) & (max_overlaps < self.neg_iou_thr[1])] = 0 # 3. assign positive: above positive IoU threshold pos_inds = max_overlaps >= self.pos_iou_thr assigned_gt_inds[pos_inds] = argmax_overlaps[pos_inds] + 1 if self.match_low_quality: # Low-quality matching will overwrite the assigned_gt_inds assigned # in Step 3. Thus, the assigned gt might not be the best one for # prediction. # For example, if bbox A has 0.9 and 0.8 iou with GT bbox 1 & 2, # bbox 1 will be assigned as the best target for bbox A in step 3. # However, if GT bbox 2's gt_argmax_overlaps = A, bbox A's # assigned_gt_inds will be overwritten to be bbox 2. # This might be the reason that it is not used in ROI Heads. for i in range(num_gts): if gt_max_overlaps[i] >= self.min_pos_iou: if self.gt_max_assign_all: max_iou_inds = overlaps[i, :] == gt_max_overlaps[i] assigned_gt_inds[max_iou_inds] = i + 1 else: assigned_gt_inds[gt_argmax_overlaps[i]] = i + 1 assigned_labels = assigned_gt_inds.new_full((num_bboxes, ), -1) pos_inds = torch.nonzero( assigned_gt_inds > 0, as_tuple=False).squeeze() if pos_inds.numel() > 0: assigned_labels[pos_inds] = gt_labels[assigned_gt_inds[pos_inds] - 1] return AssignResult( num_gts=num_gts, gt_inds=assigned_gt_inds, max_overlaps=max_overlaps, labels=assigned_labels) ================================================ FILE: mmdet/models/task_modules/assigners/multi_instance_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch from mmengine.structures import InstanceData from mmdet.registry import TASK_UTILS from .assign_result import AssignResult from .max_iou_assigner import MaxIoUAssigner @TASK_UTILS.register_module() class MultiInstanceAssigner(MaxIoUAssigner): """Assign a corresponding gt bbox or background to each proposal bbox. If we need to use a proposal box to generate multiple predict boxes, `MultiInstanceAssigner` can assign multiple gt to each proposal box. Args: num_instance (int): How many bboxes are predicted by each proposal box. """ def __init__(self, num_instance: int = 2, **kwargs): super().__init__(**kwargs) self.num_instance = num_instance def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, **kwargs) -> AssignResult: """Assign gt to bboxes. This method assign gt bboxes to every bbox (proposal/anchor), each bbox is assigned a set of gts, and the number of gts in this set is defined by `self.num_instance`. Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: :obj:`AssignResult`: The assign result. """ gt_bboxes = gt_instances.bboxes priors = pred_instances.priors # Set the FG label to 1 and add ignored annotations gt_labels = gt_instances.labels + 1 if gt_instances_ignore is not None: gt_bboxes_ignore = gt_instances_ignore.bboxes if hasattr(gt_instances_ignore, 'labels'): gt_labels_ignore = gt_instances_ignore.labels else: gt_labels_ignore = torch.ones_like(gt_bboxes_ignore)[:, 0] * -1 else: gt_bboxes_ignore = None gt_labels_ignore = None assign_on_cpu = True if (self.gpu_assign_thr > 0) and ( gt_bboxes.shape[0] > self.gpu_assign_thr) else False # compute overlap and assign gt on CPU when number of GT is large if assign_on_cpu: device = priors.device priors = priors.cpu() gt_bboxes = gt_bboxes.cpu() gt_labels = gt_labels.cpu() if gt_bboxes_ignore is not None: gt_bboxes_ignore = gt_bboxes_ignore.cpu() gt_labels_ignore = gt_labels_ignore.cpu() if gt_bboxes_ignore is not None: all_bboxes = torch.cat([gt_bboxes, gt_bboxes_ignore], dim=0) all_labels = torch.cat([gt_labels, gt_labels_ignore], dim=0) else: all_bboxes = gt_bboxes all_labels = gt_labels all_priors = torch.cat([priors, all_bboxes], dim=0) overlaps_normal = self.iou_calculator( all_priors, all_bboxes, mode='iou') overlaps_ignore = self.iou_calculator( all_priors, all_bboxes, mode='iof') gt_ignore_mask = all_labels.eq(-1).repeat(all_priors.shape[0], 1) overlaps_normal = overlaps_normal * ~gt_ignore_mask overlaps_ignore = overlaps_ignore * gt_ignore_mask overlaps_normal, overlaps_normal_indices = overlaps_normal.sort( descending=True, dim=1) overlaps_ignore, overlaps_ignore_indices = overlaps_ignore.sort( descending=True, dim=1) # select the roi with the higher score max_overlaps_normal = overlaps_normal[:, :self.num_instance].flatten() gt_assignment_normal = overlaps_normal_indices[:, :self. num_instance].flatten() max_overlaps_ignore = overlaps_ignore[:, :self.num_instance].flatten() gt_assignment_ignore = overlaps_ignore_indices[:, :self. num_instance].flatten() # ignore or not ignore_assign_mask = (max_overlaps_normal < self.pos_iou_thr) * ( max_overlaps_ignore > max_overlaps_normal) overlaps = (max_overlaps_normal * ~ignore_assign_mask) + ( max_overlaps_ignore * ignore_assign_mask) gt_assignment = (gt_assignment_normal * ~ignore_assign_mask) + ( gt_assignment_ignore * ignore_assign_mask) assigned_labels = all_labels[gt_assignment] fg_mask = (overlaps >= self.pos_iou_thr) * (assigned_labels != -1) bg_mask = (overlaps < self.neg_iou_thr) * (overlaps >= 0) assigned_labels[fg_mask] = 1 assigned_labels[bg_mask] = 0 overlaps = overlaps.reshape(-1, self.num_instance) gt_assignment = gt_assignment.reshape(-1, self.num_instance) assigned_labels = assigned_labels.reshape(-1, self.num_instance) assign_result = AssignResult( num_gts=all_bboxes.size(0), gt_inds=gt_assignment, max_overlaps=overlaps, labels=assigned_labels) if assign_on_cpu: assign_result.gt_inds = assign_result.gt_inds.to(device) assign_result.max_overlaps = assign_result.max_overlaps.to(device) if assign_result.labels is not None: assign_result.labels = assign_result.labels.to(device) return assign_result ================================================ FILE: mmdet/models/task_modules/assigners/point_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch from mmengine.structures import InstanceData from mmdet.registry import TASK_UTILS from .assign_result import AssignResult from .base_assigner import BaseAssigner @TASK_UTILS.register_module() class PointAssigner(BaseAssigner): """Assign a corresponding gt bbox or background to each point. Each proposals will be assigned with `0`, or a positive integer indicating the ground truth index. - 0: negative sample, no assigned gt - positive integer: positive sample, index (1-based) of assigned gt """ def __init__(self, scale: int = 4, pos_num: int = 3) -> None: self.scale = scale self.pos_num = pos_num def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, **kwargs) -> AssignResult: """Assign gt to points. This method assign a gt bbox to every points set, each points set will be assigned with the background_label (-1), or a label number. -1 is background, and semi-positive number is the index (0-based) of assigned gt. The assignment is done in following steps, the order matters. 1. assign every points to the background_label (-1) 2. A point is assigned to some gt bbox if (i) the point is within the k closest points to the gt bbox (ii) the distance between this point and the gt is smaller than other gt bboxes Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: :obj:`AssignResult`: The assign result. """ gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels # points to be assigned, shape(n, 3) while last # dimension stands for (x, y, stride). points = pred_instances.priors num_points = points.shape[0] num_gts = gt_bboxes.shape[0] if num_gts == 0 or num_points == 0: # If no truth assign everything to the background assigned_gt_inds = points.new_full((num_points, ), 0, dtype=torch.long) assigned_labels = points.new_full((num_points, ), -1, dtype=torch.long) return AssignResult( num_gts=num_gts, gt_inds=assigned_gt_inds, max_overlaps=None, labels=assigned_labels) points_xy = points[:, :2] points_stride = points[:, 2] points_lvl = torch.log2( points_stride).int() # [3...,4...,5...,6...,7...] lvl_min, lvl_max = points_lvl.min(), points_lvl.max() # assign gt box gt_bboxes_xy = (gt_bboxes[:, :2] + gt_bboxes[:, 2:]) / 2 gt_bboxes_wh = (gt_bboxes[:, 2:] - gt_bboxes[:, :2]).clamp(min=1e-6) scale = self.scale gt_bboxes_lvl = ((torch.log2(gt_bboxes_wh[:, 0] / scale) + torch.log2(gt_bboxes_wh[:, 1] / scale)) / 2).int() gt_bboxes_lvl = torch.clamp(gt_bboxes_lvl, min=lvl_min, max=lvl_max) # stores the assigned gt index of each point assigned_gt_inds = points.new_zeros((num_points, ), dtype=torch.long) # stores the assigned gt dist (to this point) of each point assigned_gt_dist = points.new_full((num_points, ), float('inf')) points_range = torch.arange(points.shape[0]) for idx in range(num_gts): gt_lvl = gt_bboxes_lvl[idx] # get the index of points in this level lvl_idx = gt_lvl == points_lvl points_index = points_range[lvl_idx] # get the points in this level lvl_points = points_xy[lvl_idx, :] # get the center point of gt gt_point = gt_bboxes_xy[[idx], :] # get width and height of gt gt_wh = gt_bboxes_wh[[idx], :] # compute the distance between gt center and # all points in this level points_gt_dist = ((lvl_points - gt_point) / gt_wh).norm(dim=1) # find the nearest k points to gt center in this level min_dist, min_dist_index = torch.topk( points_gt_dist, self.pos_num, largest=False) # the index of nearest k points to gt center in this level min_dist_points_index = points_index[min_dist_index] # The less_than_recorded_index stores the index # of min_dist that is less then the assigned_gt_dist. Where # assigned_gt_dist stores the dist from previous assigned gt # (if exist) to each point. less_than_recorded_index = min_dist < assigned_gt_dist[ min_dist_points_index] # The min_dist_points_index stores the index of points satisfy: # (1) it is k nearest to current gt center in this level. # (2) it is closer to current gt center than other gt center. min_dist_points_index = min_dist_points_index[ less_than_recorded_index] # assign the result assigned_gt_inds[min_dist_points_index] = idx + 1 assigned_gt_dist[min_dist_points_index] = min_dist[ less_than_recorded_index] assigned_labels = assigned_gt_inds.new_full((num_points, ), -1) pos_inds = torch.nonzero( assigned_gt_inds > 0, as_tuple=False).squeeze() if pos_inds.numel() > 0: assigned_labels[pos_inds] = gt_labels[assigned_gt_inds[pos_inds] - 1] return AssignResult( num_gts=num_gts, gt_inds=assigned_gt_inds, max_overlaps=None, labels=assigned_labels) ================================================ FILE: mmdet/models/task_modules/assigners/region_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Tuple import torch from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import TASK_UTILS from ..prior_generators import anchor_inside_flags from .assign_result import AssignResult from .base_assigner import BaseAssigner def calc_region( bbox: Tensor, ratio: float, stride: int, featmap_size: Optional[Tuple[int, int]] = None) -> Tuple[Tensor]: """Calculate region of the box defined by the ratio, the ratio is from the center of the box to every edge.""" # project bbox on the feature f_bbox = bbox / stride x1 = torch.round((1 - ratio) * f_bbox[0] + ratio * f_bbox[2]) y1 = torch.round((1 - ratio) * f_bbox[1] + ratio * f_bbox[3]) x2 = torch.round(ratio * f_bbox[0] + (1 - ratio) * f_bbox[2]) y2 = torch.round(ratio * f_bbox[1] + (1 - ratio) * f_bbox[3]) if featmap_size is not None: x1 = x1.clamp(min=0, max=featmap_size[1]) y1 = y1.clamp(min=0, max=featmap_size[0]) x2 = x2.clamp(min=0, max=featmap_size[1]) y2 = y2.clamp(min=0, max=featmap_size[0]) return (x1, y1, x2, y2) def anchor_ctr_inside_region_flags(anchors: Tensor, stride: int, region: Tuple[Tensor]) -> Tensor: """Get the flag indicate whether anchor centers are inside regions.""" x1, y1, x2, y2 = region f_anchors = anchors / stride x = (f_anchors[:, 0] + f_anchors[:, 2]) * 0.5 y = (f_anchors[:, 1] + f_anchors[:, 3]) * 0.5 flags = (x >= x1) & (x <= x2) & (y >= y1) & (y <= y2) return flags @TASK_UTILS.register_module() class RegionAssigner(BaseAssigner): """Assign a corresponding gt bbox or background to each bbox. Each proposals will be assigned with `-1`, `0`, or a positive integer indicating the ground truth index. - -1: don't care - 0: negative sample, no assigned gt - positive integer: positive sample, index (1-based) of assigned gt Args: center_ratio (float): ratio of the region in the center of the bbox to define positive sample. ignore_ratio (float): ratio of the region to define ignore samples. """ def __init__(self, center_ratio: float = 0.2, ignore_ratio: float = 0.5) -> None: self.center_ratio = center_ratio self.ignore_ratio = ignore_ratio def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, img_meta: dict, featmap_sizes: List[Tuple[int, int]], num_level_anchors: List[int], anchor_scale: int, anchor_strides: List[int], gt_instances_ignore: Optional[InstanceData] = None, allowed_border: int = 0) -> AssignResult: """Assign gt to anchors. This method assign a gt bbox to every bbox (proposal/anchor), each bbox will be assigned with -1, 0, or a positive number. -1 means don't care, 0 means negative sample, positive number is the index (1-based) of assigned gt. The assignment is done in following steps, and the order matters. 1. Assign every anchor to 0 (negative) 2. (For each gt_bboxes) Compute ignore flags based on ignore_region then assign -1 to anchors w.r.t. ignore flags 3. (For each gt_bboxes) Compute pos flags based on center_region then assign gt_bboxes to anchors w.r.t. pos flags 4. (For each gt_bboxes) Compute ignore flags based on adjacent anchor level then assign -1 to anchors w.r.t. ignore flags 5. Assign anchor outside of image to -1 Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). img_meta (dict): Meta info of image. featmap_sizes (list[tuple[int, int]]): Feature map size each level. num_level_anchors (list[int]): The number of anchors in each level. anchor_scale (int): Scale of the anchor. anchor_strides (list[int]): Stride of the anchor. gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. allowed_border (int, optional): The border to allow the valid anchor. Defaults to 0. Returns: :obj:`AssignResult`: The assign result. """ if gt_instances_ignore is not None: raise NotImplementedError num_gts = len(gt_instances) num_bboxes = len(pred_instances) gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels flat_anchors = pred_instances.priors flat_valid_flags = pred_instances.valid_flags mlvl_anchors = torch.split(flat_anchors, num_level_anchors) if num_gts == 0 or num_bboxes == 0: # No ground truth or boxes, return empty assignment max_overlaps = gt_bboxes.new_zeros((num_bboxes, )) assigned_gt_inds = gt_bboxes.new_zeros((num_bboxes, ), dtype=torch.long) assigned_labels = gt_bboxes.new_full((num_bboxes, ), -1, dtype=torch.long) return AssignResult( num_gts=num_gts, gt_inds=assigned_gt_inds, max_overlaps=max_overlaps, labels=assigned_labels) num_lvls = len(mlvl_anchors) r1 = (1 - self.center_ratio) / 2 r2 = (1 - self.ignore_ratio) / 2 scale = torch.sqrt((gt_bboxes[:, 2] - gt_bboxes[:, 0]) * (gt_bboxes[:, 3] - gt_bboxes[:, 1])) min_anchor_size = scale.new_full( (1, ), float(anchor_scale * anchor_strides[0])) target_lvls = torch.floor( torch.log2(scale) - torch.log2(min_anchor_size) + 0.5) target_lvls = target_lvls.clamp(min=0, max=num_lvls - 1).long() # 1. assign 0 (negative) by default mlvl_assigned_gt_inds = [] mlvl_ignore_flags = [] for lvl in range(num_lvls): assigned_gt_inds = gt_bboxes.new_full((num_level_anchors[lvl], ), 0, dtype=torch.long) ignore_flags = torch.zeros_like(assigned_gt_inds) mlvl_assigned_gt_inds.append(assigned_gt_inds) mlvl_ignore_flags.append(ignore_flags) for gt_id in range(num_gts): lvl = target_lvls[gt_id].item() featmap_size = featmap_sizes[lvl] stride = anchor_strides[lvl] anchors = mlvl_anchors[lvl] gt_bbox = gt_bboxes[gt_id, :4] # Compute regions ignore_region = calc_region(gt_bbox, r2, stride, featmap_size) ctr_region = calc_region(gt_bbox, r1, stride, featmap_size) # 2. Assign -1 to ignore flags ignore_flags = anchor_ctr_inside_region_flags( anchors, stride, ignore_region) mlvl_assigned_gt_inds[lvl][ignore_flags] = -1 # 3. Assign gt_bboxes to pos flags pos_flags = anchor_ctr_inside_region_flags(anchors, stride, ctr_region) mlvl_assigned_gt_inds[lvl][pos_flags] = gt_id + 1 # 4. Assign -1 to ignore adjacent lvl if lvl > 0: d_lvl = lvl - 1 d_anchors = mlvl_anchors[d_lvl] d_featmap_size = featmap_sizes[d_lvl] d_stride = anchor_strides[d_lvl] d_ignore_region = calc_region(gt_bbox, r2, d_stride, d_featmap_size) ignore_flags = anchor_ctr_inside_region_flags( d_anchors, d_stride, d_ignore_region) mlvl_ignore_flags[d_lvl][ignore_flags] = 1 if lvl < num_lvls - 1: u_lvl = lvl + 1 u_anchors = mlvl_anchors[u_lvl] u_featmap_size = featmap_sizes[u_lvl] u_stride = anchor_strides[u_lvl] u_ignore_region = calc_region(gt_bbox, r2, u_stride, u_featmap_size) ignore_flags = anchor_ctr_inside_region_flags( u_anchors, u_stride, u_ignore_region) mlvl_ignore_flags[u_lvl][ignore_flags] = 1 # 4. (cont.) Assign -1 to ignore adjacent lvl for lvl in range(num_lvls): ignore_flags = mlvl_ignore_flags[lvl] mlvl_assigned_gt_inds[lvl][ignore_flags == 1] = -1 # 5. Assign -1 to anchor outside of image flat_assigned_gt_inds = torch.cat(mlvl_assigned_gt_inds) assert (flat_assigned_gt_inds.shape[0] == flat_anchors.shape[0] == flat_valid_flags.shape[0]) inside_flags = anchor_inside_flags(flat_anchors, flat_valid_flags, img_meta['img_shape'], allowed_border) outside_flags = ~inside_flags flat_assigned_gt_inds[outside_flags] = -1 assigned_labels = torch.zeros_like(flat_assigned_gt_inds) pos_flags = flat_assigned_gt_inds > 0 assigned_labels[pos_flags] = gt_labels[flat_assigned_gt_inds[pos_flags] - 1] return AssignResult( num_gts=num_gts, gt_inds=flat_assigned_gt_inds, max_overlaps=None, labels=assigned_labels) ================================================ FILE: mmdet/models/task_modules/assigners/sim_ota_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple import torch import torch.nn.functional as F from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import TASK_UTILS from mmdet.utils import ConfigType from .assign_result import AssignResult from .base_assigner import BaseAssigner INF = 100000.0 EPS = 1.0e-7 @TASK_UTILS.register_module() class SimOTAAssigner(BaseAssigner): """Computes matching between predictions and ground truth. Args: center_radius (float): Ground truth center size to judge whether a prior is in center. Defaults to 2.5. candidate_topk (int): The candidate top-k which used to get top-k ious to calculate dynamic-k. Defaults to 10. iou_weight (float): The scale factor for regression iou cost. Defaults to 3.0. cls_weight (float): The scale factor for classification cost. Defaults to 1.0. iou_calculator (ConfigType): Config of overlaps Calculator. Defaults to dict(type='BboxOverlaps2D'). """ def __init__(self, center_radius: float = 2.5, candidate_topk: int = 10, iou_weight: float = 3.0, cls_weight: float = 1.0, iou_calculator: ConfigType = dict(type='BboxOverlaps2D')): self.center_radius = center_radius self.candidate_topk = candidate_topk self.iou_weight = iou_weight self.cls_weight = cls_weight self.iou_calculator = TASK_UTILS.build(iou_calculator) def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, **kwargs) -> AssignResult: """Assign gt to priors using SimOTA. Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: obj:`AssignResult`: The assigned result. """ gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels num_gt = gt_bboxes.size(0) decoded_bboxes = pred_instances.bboxes pred_scores = pred_instances.scores priors = pred_instances.priors num_bboxes = decoded_bboxes.size(0) # assign 0 by default assigned_gt_inds = decoded_bboxes.new_full((num_bboxes, ), 0, dtype=torch.long) if num_gt == 0 or num_bboxes == 0: # No ground truth or boxes, return empty assignment max_overlaps = decoded_bboxes.new_zeros((num_bboxes, )) assigned_labels = decoded_bboxes.new_full((num_bboxes, ), -1, dtype=torch.long) return AssignResult( num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) valid_mask, is_in_boxes_and_center = self.get_in_gt_and_in_center_info( priors, gt_bboxes) valid_decoded_bbox = decoded_bboxes[valid_mask] valid_pred_scores = pred_scores[valid_mask] num_valid = valid_decoded_bbox.size(0) if num_valid == 0: # No valid bboxes, return empty assignment max_overlaps = decoded_bboxes.new_zeros((num_bboxes, )) assigned_labels = decoded_bboxes.new_full((num_bboxes, ), -1, dtype=torch.long) return AssignResult( num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) pairwise_ious = self.iou_calculator(valid_decoded_bbox, gt_bboxes) iou_cost = -torch.log(pairwise_ious + EPS) gt_onehot_label = ( F.one_hot(gt_labels.to(torch.int64), pred_scores.shape[-1]).float().unsqueeze(0).repeat( num_valid, 1, 1)) valid_pred_scores = valid_pred_scores.unsqueeze(1).repeat(1, num_gt, 1) # disable AMP autocast and calculate BCE with FP32 to avoid overflow with torch.cuda.amp.autocast(enabled=False): cls_cost = ( F.binary_cross_entropy( valid_pred_scores.to(dtype=torch.float32), gt_onehot_label, reduction='none', ).sum(-1).to(dtype=valid_pred_scores.dtype)) cost_matrix = ( cls_cost * self.cls_weight + iou_cost * self.iou_weight + (~is_in_boxes_and_center) * INF) matched_pred_ious, matched_gt_inds = \ self.dynamic_k_matching( cost_matrix, pairwise_ious, num_gt, valid_mask) # convert to AssignResult format assigned_gt_inds[valid_mask] = matched_gt_inds + 1 assigned_labels = assigned_gt_inds.new_full((num_bboxes, ), -1) assigned_labels[valid_mask] = gt_labels[matched_gt_inds].long() max_overlaps = assigned_gt_inds.new_full((num_bboxes, ), -INF, dtype=torch.float32) max_overlaps[valid_mask] = matched_pred_ious return AssignResult( num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) def get_in_gt_and_in_center_info( self, priors: Tensor, gt_bboxes: Tensor) -> Tuple[Tensor, Tensor]: """Get the information of which prior is in gt bboxes and gt center priors.""" num_gt = gt_bboxes.size(0) repeated_x = priors[:, 0].unsqueeze(1).repeat(1, num_gt) repeated_y = priors[:, 1].unsqueeze(1).repeat(1, num_gt) repeated_stride_x = priors[:, 2].unsqueeze(1).repeat(1, num_gt) repeated_stride_y = priors[:, 3].unsqueeze(1).repeat(1, num_gt) # is prior centers in gt bboxes, shape: [n_prior, n_gt] l_ = repeated_x - gt_bboxes[:, 0] t_ = repeated_y - gt_bboxes[:, 1] r_ = gt_bboxes[:, 2] - repeated_x b_ = gt_bboxes[:, 3] - repeated_y deltas = torch.stack([l_, t_, r_, b_], dim=1) is_in_gts = deltas.min(dim=1).values > 0 is_in_gts_all = is_in_gts.sum(dim=1) > 0 # is prior centers in gt centers gt_cxs = (gt_bboxes[:, 0] + gt_bboxes[:, 2]) / 2.0 gt_cys = (gt_bboxes[:, 1] + gt_bboxes[:, 3]) / 2.0 ct_box_l = gt_cxs - self.center_radius * repeated_stride_x ct_box_t = gt_cys - self.center_radius * repeated_stride_y ct_box_r = gt_cxs + self.center_radius * repeated_stride_x ct_box_b = gt_cys + self.center_radius * repeated_stride_y cl_ = repeated_x - ct_box_l ct_ = repeated_y - ct_box_t cr_ = ct_box_r - repeated_x cb_ = ct_box_b - repeated_y ct_deltas = torch.stack([cl_, ct_, cr_, cb_], dim=1) is_in_cts = ct_deltas.min(dim=1).values > 0 is_in_cts_all = is_in_cts.sum(dim=1) > 0 # in boxes or in centers, shape: [num_priors] is_in_gts_or_centers = is_in_gts_all | is_in_cts_all # both in boxes and centers, shape: [num_fg, num_gt] is_in_boxes_and_centers = ( is_in_gts[is_in_gts_or_centers, :] & is_in_cts[is_in_gts_or_centers, :]) return is_in_gts_or_centers, is_in_boxes_and_centers def dynamic_k_matching(self, cost: Tensor, pairwise_ious: Tensor, num_gt: int, valid_mask: Tensor) -> Tuple[Tensor, Tensor]: """Use IoU and matching cost to calculate the dynamic top-k positive targets.""" matching_matrix = torch.zeros_like(cost, dtype=torch.uint8) # select candidate topk ious for dynamic-k calculation candidate_topk = min(self.candidate_topk, pairwise_ious.size(0)) topk_ious, _ = torch.topk(pairwise_ious, candidate_topk, dim=0) # calculate dynamic k for each gt dynamic_ks = torch.clamp(topk_ious.sum(0).int(), min=1) for gt_idx in range(num_gt): _, pos_idx = torch.topk( cost[:, gt_idx], k=dynamic_ks[gt_idx], largest=False) matching_matrix[:, gt_idx][pos_idx] = 1 del topk_ious, dynamic_ks, pos_idx prior_match_gt_mask = matching_matrix.sum(1) > 1 if prior_match_gt_mask.sum() > 0: cost_min, cost_argmin = torch.min( cost[prior_match_gt_mask, :], dim=1) matching_matrix[prior_match_gt_mask, :] *= 0 matching_matrix[prior_match_gt_mask, cost_argmin] = 1 # get foreground mask inside box and center prior fg_mask_inboxes = matching_matrix.sum(1) > 0 valid_mask[valid_mask.clone()] = fg_mask_inboxes matched_gt_inds = matching_matrix[fg_mask_inboxes, :].argmax(1) matched_pred_ious = (matching_matrix * pairwise_ious).sum(1)[fg_mask_inboxes] return matched_pred_ious, matched_gt_inds ================================================ FILE: mmdet/models/task_modules/assigners/task_aligned_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch from mmengine.structures import InstanceData from mmdet.registry import TASK_UTILS from mmdet.utils import ConfigType from .assign_result import AssignResult from .base_assigner import BaseAssigner INF = 100000000 @TASK_UTILS.register_module() class TaskAlignedAssigner(BaseAssigner): """Task aligned assigner used in the paper: `TOOD: Task-aligned One-stage Object Detection. `_. Assign a corresponding gt bbox or background to each predicted bbox. Each bbox will be assigned with `0` or a positive integer indicating the ground truth index. - 0: negative sample, no assigned gt - positive integer: positive sample, index (1-based) of assigned gt Args: topk (int): number of bbox selected in each level iou_calculator (:obj:`ConfigDict` or dict): Config dict for iou calculator. Defaults to ``dict(type='BboxOverlaps2D')`` """ def __init__(self, topk: int, iou_calculator: ConfigType = dict(type='BboxOverlaps2D')): assert topk >= 1 self.topk = topk self.iou_calculator = TASK_UTILS.build(iou_calculator) def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, alpha: int = 1, beta: int = 6) -> AssignResult: """Assign gt to bboxes. The assignment is done in following steps 1. compute alignment metric between all bbox (bbox of all pyramid levels) and gt 2. select top-k bbox as candidates for each gt 3. limit the positive sample's center in gt (because the anchor-free detector only can predict positive distance) Args: pred_instances (:obj:`InstaceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors, points, or bboxes predicted by the model, shape(n, 4). gt_instances (:obj:`InstaceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. gt_instances_ignore (:obj:`InstaceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. alpha (int): Hyper-parameters related to alignment_metrics. Defaults to 1. beta (int): Hyper-parameters related to alignment_metrics. Defaults to 6. Returns: :obj:`TaskAlignedAssignResult`: The assign result. """ priors = pred_instances.priors decode_bboxes = pred_instances.bboxes pred_scores = pred_instances.scores gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels priors = priors[:, :4] num_gt, num_bboxes = gt_bboxes.size(0), priors.size(0) # compute alignment metric between all bbox and gt overlaps = self.iou_calculator(decode_bboxes, gt_bboxes).detach() bbox_scores = pred_scores[:, gt_labels].detach() # assign 0 by default assigned_gt_inds = priors.new_full((num_bboxes, ), 0, dtype=torch.long) assign_metrics = priors.new_zeros((num_bboxes, )) if num_gt == 0 or num_bboxes == 0: # No ground truth or boxes, return empty assignment max_overlaps = priors.new_zeros((num_bboxes, )) if num_gt == 0: # No gt boxes, assign everything to background assigned_gt_inds[:] = 0 assigned_labels = priors.new_full((num_bboxes, ), -1, dtype=torch.long) assign_result = AssignResult( num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) assign_result.assign_metrics = assign_metrics return assign_result # select top-k bboxes as candidates for each gt alignment_metrics = bbox_scores**alpha * overlaps**beta topk = min(self.topk, alignment_metrics.size(0)) _, candidate_idxs = alignment_metrics.topk(topk, dim=0, largest=True) candidate_metrics = alignment_metrics[candidate_idxs, torch.arange(num_gt)] is_pos = candidate_metrics > 0 # limit the positive sample's center in gt priors_cx = (priors[:, 0] + priors[:, 2]) / 2.0 priors_cy = (priors[:, 1] + priors[:, 3]) / 2.0 for gt_idx in range(num_gt): candidate_idxs[:, gt_idx] += gt_idx * num_bboxes ep_priors_cx = priors_cx.view(1, -1).expand( num_gt, num_bboxes).contiguous().view(-1) ep_priors_cy = priors_cy.view(1, -1).expand( num_gt, num_bboxes).contiguous().view(-1) candidate_idxs = candidate_idxs.view(-1) # calculate the left, top, right, bottom distance between positive # bbox center and gt side l_ = ep_priors_cx[candidate_idxs].view(-1, num_gt) - gt_bboxes[:, 0] t_ = ep_priors_cy[candidate_idxs].view(-1, num_gt) - gt_bboxes[:, 1] r_ = gt_bboxes[:, 2] - ep_priors_cx[candidate_idxs].view(-1, num_gt) b_ = gt_bboxes[:, 3] - ep_priors_cy[candidate_idxs].view(-1, num_gt) is_in_gts = torch.stack([l_, t_, r_, b_], dim=1).min(dim=1)[0] > 0.01 is_pos = is_pos & is_in_gts # if an anchor box is assigned to multiple gts, # the one with the highest iou will be selected. overlaps_inf = torch.full_like(overlaps, -INF).t().contiguous().view(-1) index = candidate_idxs.view(-1)[is_pos.view(-1)] overlaps_inf[index] = overlaps.t().contiguous().view(-1)[index] overlaps_inf = overlaps_inf.view(num_gt, -1).t() max_overlaps, argmax_overlaps = overlaps_inf.max(dim=1) assigned_gt_inds[ max_overlaps != -INF] = argmax_overlaps[max_overlaps != -INF] + 1 assign_metrics[max_overlaps != -INF] = alignment_metrics[ max_overlaps != -INF, argmax_overlaps[max_overlaps != -INF]] assigned_labels = assigned_gt_inds.new_full((num_bboxes, ), -1) pos_inds = torch.nonzero( assigned_gt_inds > 0, as_tuple=False).squeeze() if pos_inds.numel() > 0: assigned_labels[pos_inds] = gt_labels[assigned_gt_inds[pos_inds] - 1] assign_result = AssignResult( num_gt, assigned_gt_inds, max_overlaps, labels=assigned_labels) assign_result.assign_metrics = assign_metrics return assign_result ================================================ FILE: mmdet/models/task_modules/assigners/uniform_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch from mmengine.structures import InstanceData from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import bbox_xyxy_to_cxcywh from mmdet.utils import ConfigType from .assign_result import AssignResult from .base_assigner import BaseAssigner @TASK_UTILS.register_module() class UniformAssigner(BaseAssigner): """Uniform Matching between the priors and gt boxes, which can achieve balance in positive priors, and gt_bboxes_ignore was not considered for now. Args: pos_ignore_thr (float): the threshold to ignore positive priors neg_ignore_thr (float): the threshold to ignore negative priors match_times(int): Number of positive priors for each gt box. Defaults to 4. iou_calculator (:obj:`ConfigDict` or dict): Config dict for iou calculator. Defaults to ``dict(type='BboxOverlaps2D')`` """ def __init__(self, pos_ignore_thr: float, neg_ignore_thr: float, match_times: int = 4, iou_calculator: ConfigType = dict(type='BboxOverlaps2D')): self.match_times = match_times self.pos_ignore_thr = pos_ignore_thr self.neg_ignore_thr = neg_ignore_thr self.iou_calculator = TASK_UTILS.build(iou_calculator) def assign( self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None ) -> AssignResult: """Assign gt to priors. The assignment is done in following steps 1. assign -1 by default 2. compute the L1 cost between boxes. Note that we use priors and predict boxes both 3. compute the ignore indexes use gt_bboxes and predict boxes 4. compute the ignore indexes of positive sample use priors and predict boxes Args: pred_instances (:obj:`InstaceData`): Instances of model predictions. It includes ``priors``, and the priors can be priors, points, or bboxes predicted by the model, shape(n, 4). gt_instances (:obj:`InstaceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. gt_instances_ignore (:obj:`InstaceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: :obj:`AssignResult`: The assign result. """ gt_bboxes = gt_instances.bboxes gt_labels = gt_instances.labels priors = pred_instances.priors bbox_pred = pred_instances.decoder_priors num_gts, num_bboxes = gt_bboxes.size(0), bbox_pred.size(0) # 1. assign -1 by default assigned_gt_inds = bbox_pred.new_full((num_bboxes, ), 0, dtype=torch.long) assigned_labels = bbox_pred.new_full((num_bboxes, ), -1, dtype=torch.long) if num_gts == 0 or num_bboxes == 0: # No ground truth or boxes, return empty assignment if num_gts == 0: # No ground truth, assign all to background assigned_gt_inds[:] = 0 assign_result = AssignResult( num_gts, assigned_gt_inds, None, labels=assigned_labels) assign_result.set_extra_property( 'pos_idx', bbox_pred.new_empty(0, dtype=torch.bool)) assign_result.set_extra_property('pos_predicted_boxes', bbox_pred.new_empty((0, 4))) assign_result.set_extra_property('target_boxes', bbox_pred.new_empty((0, 4))) return assign_result # 2. Compute the L1 cost between boxes # Note that we use priors and predict boxes both cost_bbox = torch.cdist( bbox_xyxy_to_cxcywh(bbox_pred), bbox_xyxy_to_cxcywh(gt_bboxes), p=1) cost_bbox_priors = torch.cdist( bbox_xyxy_to_cxcywh(priors), bbox_xyxy_to_cxcywh(gt_bboxes), p=1) # We found that topk function has different results in cpu and # cuda mode. In order to ensure consistency with the source code, # we also use cpu mode. # TODO: Check whether the performance of cpu and cuda are the same. C = cost_bbox.cpu() C1 = cost_bbox_priors.cpu() # self.match_times x n index = torch.topk( C, # c=b,n,x c[i]=n,x k=self.match_times, dim=0, largest=False)[1] # self.match_times x n index1 = torch.topk(C1, k=self.match_times, dim=0, largest=False)[1] # (self.match_times*2) x n indexes = torch.cat((index, index1), dim=1).reshape(-1).to(bbox_pred.device) pred_overlaps = self.iou_calculator(bbox_pred, gt_bboxes) anchor_overlaps = self.iou_calculator(priors, gt_bboxes) pred_max_overlaps, _ = pred_overlaps.max(dim=1) anchor_max_overlaps, _ = anchor_overlaps.max(dim=0) # 3. Compute the ignore indexes use gt_bboxes and predict boxes ignore_idx = pred_max_overlaps > self.neg_ignore_thr assigned_gt_inds[ignore_idx] = -1 # 4. Compute the ignore indexes of positive sample use priors # and predict boxes pos_gt_index = torch.arange( 0, C1.size(1), device=bbox_pred.device).repeat(self.match_times * 2) pos_ious = anchor_overlaps[indexes, pos_gt_index] pos_ignore_idx = pos_ious < self.pos_ignore_thr pos_gt_index_with_ignore = pos_gt_index + 1 pos_gt_index_with_ignore[pos_ignore_idx] = -1 assigned_gt_inds[indexes] = pos_gt_index_with_ignore if gt_labels is not None: assigned_labels = assigned_gt_inds.new_full((num_bboxes, ), -1) pos_inds = torch.nonzero( assigned_gt_inds > 0, as_tuple=False).squeeze() if pos_inds.numel() > 0: assigned_labels[pos_inds] = gt_labels[ assigned_gt_inds[pos_inds] - 1] else: assigned_labels = None assign_result = AssignResult( num_gts, assigned_gt_inds, anchor_max_overlaps, labels=assigned_labels) assign_result.set_extra_property('pos_idx', ~pos_ignore_idx) assign_result.set_extra_property('pos_predicted_boxes', bbox_pred[indexes]) assign_result.set_extra_property('target_boxes', gt_bboxes[pos_gt_index]) return assign_result ================================================ FILE: mmdet/models/task_modules/builder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings from mmdet.registry import TASK_UTILS PRIOR_GENERATORS = TASK_UTILS ANCHOR_GENERATORS = TASK_UTILS BBOX_ASSIGNERS = TASK_UTILS BBOX_SAMPLERS = TASK_UTILS BBOX_CODERS = TASK_UTILS MATCH_COSTS = TASK_UTILS IOU_CALCULATORS = TASK_UTILS def build_bbox_coder(cfg, **default_args): """Builder of box coder.""" warnings.warn('``build_sampler`` would be deprecated soon, please use ' '``mmdet.registry.TASK_UTILS.build()`` ') return TASK_UTILS.build(cfg, default_args=default_args) def build_iou_calculator(cfg, default_args=None): """Builder of IoU calculator.""" warnings.warn( '``build_iou_calculator`` would be deprecated soon, please use ' '``mmdet.registry.TASK_UTILS.build()`` ') return TASK_UTILS.build(cfg, default_args=default_args) def build_match_cost(cfg, default_args=None): """Builder of IoU calculator.""" warnings.warn('``build_match_cost`` would be deprecated soon, please use ' '``mmdet.registry.TASK_UTILS.build()`` ') return TASK_UTILS.build(cfg, default_args=default_args) def build_assigner(cfg, **default_args): """Builder of box assigner.""" warnings.warn('``build_assigner`` would be deprecated soon, please use ' '``mmdet.registry.TASK_UTILS.build()`` ') return TASK_UTILS.build(cfg, default_args=default_args) def build_sampler(cfg, **default_args): """Builder of box sampler.""" warnings.warn('``build_sampler`` would be deprecated soon, please use ' '``mmdet.registry.TASK_UTILS.build()`` ') return TASK_UTILS.build(cfg, default_args=default_args) def build_prior_generator(cfg, default_args=None): warnings.warn( '``build_prior_generator`` would be deprecated soon, please use ' '``mmdet.registry.TASK_UTILS.build()`` ') return TASK_UTILS.build(cfg, default_args=default_args) def build_anchor_generator(cfg, default_args=None): warnings.warn( '``build_anchor_generator`` would be deprecated soon, please use ' '``mmdet.registry.TASK_UTILS.build()`` ') return TASK_UTILS.build(cfg, default_args=default_args) ================================================ FILE: mmdet/models/task_modules/coders/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .base_bbox_coder import BaseBBoxCoder from .bucketing_bbox_coder import BucketingBBoxCoder from .delta_xywh_bbox_coder import DeltaXYWHBBoxCoder from .distance_point_bbox_coder import DistancePointBBoxCoder from .legacy_delta_xywh_bbox_coder import LegacyDeltaXYWHBBoxCoder from .pseudo_bbox_coder import PseudoBBoxCoder from .tblr_bbox_coder import TBLRBBoxCoder from .yolo_bbox_coder import YOLOBBoxCoder __all__ = [ 'BaseBBoxCoder', 'PseudoBBoxCoder', 'DeltaXYWHBBoxCoder', 'LegacyDeltaXYWHBBoxCoder', 'TBLRBBoxCoder', 'YOLOBBoxCoder', 'BucketingBBoxCoder', 'DistancePointBBoxCoder' ] ================================================ FILE: mmdet/models/task_modules/coders/base_bbox_coder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod class BaseBBoxCoder(metaclass=ABCMeta): """Base bounding box coder. Args: use_box_type (bool): Whether to warp decoded boxes with the box type data structure. Defaults to False. """ # The size of the last of dimension of the encoded tensor. encode_size = 4 def __init__(self, use_box_type: bool = False, **kwargs): self.use_box_type = use_box_type @abstractmethod def encode(self, bboxes, gt_bboxes): """Encode deltas between bboxes and ground truth boxes.""" @abstractmethod def decode(self, bboxes, bboxes_pred): """Decode the predicted bboxes according to prediction and base boxes.""" ================================================ FILE: mmdet/models/task_modules/coders/bucketing_bbox_coder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch import torch.nn.functional as F from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import HorizontalBoxes, bbox_rescale, get_box_tensor from .base_bbox_coder import BaseBBoxCoder @TASK_UTILS.register_module() class BucketingBBoxCoder(BaseBBoxCoder): """Bucketing BBox Coder for Side-Aware Boundary Localization (SABL). Boundary Localization with Bucketing and Bucketing Guided Rescoring are implemented here. Please refer to https://arxiv.org/abs/1912.04260 for more details. Args: num_buckets (int): Number of buckets. scale_factor (int): Scale factor of proposals to generate buckets. offset_topk (int): Topk buckets are used to generate bucket fine regression targets. Defaults to 2. offset_upperbound (float): Offset upperbound to generate bucket fine regression targets. To avoid too large offset displacements. Defaults to 1.0. cls_ignore_neighbor (bool): Ignore second nearest bucket or Not. Defaults to True. clip_border (bool, optional): Whether clip the objects outside the border of the image. Defaults to True. """ def __init__(self, num_buckets, scale_factor, offset_topk=2, offset_upperbound=1.0, cls_ignore_neighbor=True, clip_border=True, **kwargs): super().__init__(**kwargs) self.num_buckets = num_buckets self.scale_factor = scale_factor self.offset_topk = offset_topk self.offset_upperbound = offset_upperbound self.cls_ignore_neighbor = cls_ignore_neighbor self.clip_border = clip_border def encode(self, bboxes, gt_bboxes): """Get bucketing estimation and fine regression targets during training. Args: bboxes (torch.Tensor or :obj:`BaseBoxes`): source boxes, e.g., object proposals. gt_bboxes (torch.Tensor or :obj:`BaseBoxes`): target of the transformation, e.g., ground truth boxes. Returns: encoded_bboxes(tuple[Tensor]): bucketing estimation and fine regression targets and weights """ bboxes = get_box_tensor(bboxes) gt_bboxes = get_box_tensor(gt_bboxes) assert bboxes.size(0) == gt_bboxes.size(0) assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 encoded_bboxes = bbox2bucket(bboxes, gt_bboxes, self.num_buckets, self.scale_factor, self.offset_topk, self.offset_upperbound, self.cls_ignore_neighbor) return encoded_bboxes def decode(self, bboxes, pred_bboxes, max_shape=None): """Apply transformation `pred_bboxes` to `boxes`. Args: boxes (torch.Tensor or :obj:`BaseBoxes`): Basic boxes. pred_bboxes (torch.Tensor): Predictions for bucketing estimation and fine regression max_shape (tuple[int], optional): Maximum shape of boxes. Defaults to None. Returns: Union[torch.Tensor, :obj:`BaseBoxes`]: Decoded boxes. """ bboxes = get_box_tensor(bboxes) assert len(pred_bboxes) == 2 cls_preds, offset_preds = pred_bboxes assert cls_preds.size(0) == bboxes.size(0) and offset_preds.size( 0) == bboxes.size(0) bboxes, loc_confidence = bucket2bbox(bboxes, cls_preds, offset_preds, self.num_buckets, self.scale_factor, max_shape, self.clip_border) if self.use_box_type: bboxes = HorizontalBoxes(bboxes, clone=False) return bboxes, loc_confidence def generat_buckets(proposals, num_buckets, scale_factor=1.0): """Generate buckets w.r.t bucket number and scale factor of proposals. Args: proposals (Tensor): Shape (n, 4) num_buckets (int): Number of buckets. scale_factor (float): Scale factor to rescale proposals. Returns: tuple[Tensor]: (bucket_w, bucket_h, l_buckets, r_buckets, t_buckets, d_buckets) - bucket_w: Width of buckets on x-axis. Shape (n, ). - bucket_h: Height of buckets on y-axis. Shape (n, ). - l_buckets: Left buckets. Shape (n, ceil(side_num/2)). - r_buckets: Right buckets. Shape (n, ceil(side_num/2)). - t_buckets: Top buckets. Shape (n, ceil(side_num/2)). - d_buckets: Down buckets. Shape (n, ceil(side_num/2)). """ proposals = bbox_rescale(proposals, scale_factor) # number of buckets in each side side_num = int(np.ceil(num_buckets / 2.0)) pw = proposals[..., 2] - proposals[..., 0] ph = proposals[..., 3] - proposals[..., 1] px1 = proposals[..., 0] py1 = proposals[..., 1] px2 = proposals[..., 2] py2 = proposals[..., 3] bucket_w = pw / num_buckets bucket_h = ph / num_buckets # left buckets l_buckets = px1[:, None] + (0.5 + torch.arange( 0, side_num).to(proposals).float())[None, :] * bucket_w[:, None] # right buckets r_buckets = px2[:, None] - (0.5 + torch.arange( 0, side_num).to(proposals).float())[None, :] * bucket_w[:, None] # top buckets t_buckets = py1[:, None] + (0.5 + torch.arange( 0, side_num).to(proposals).float())[None, :] * bucket_h[:, None] # down buckets d_buckets = py2[:, None] - (0.5 + torch.arange( 0, side_num).to(proposals).float())[None, :] * bucket_h[:, None] return bucket_w, bucket_h, l_buckets, r_buckets, t_buckets, d_buckets def bbox2bucket(proposals, gt, num_buckets, scale_factor, offset_topk=2, offset_upperbound=1.0, cls_ignore_neighbor=True): """Generate buckets estimation and fine regression targets. Args: proposals (Tensor): Shape (n, 4) gt (Tensor): Shape (n, 4) num_buckets (int): Number of buckets. scale_factor (float): Scale factor to rescale proposals. offset_topk (int): Topk buckets are used to generate bucket fine regression targets. Defaults to 2. offset_upperbound (float): Offset allowance to generate bucket fine regression targets. To avoid too large offset displacements. Defaults to 1.0. cls_ignore_neighbor (bool): Ignore second nearest bucket or Not. Defaults to True. Returns: tuple[Tensor]: (offsets, offsets_weights, bucket_labels, cls_weights). - offsets: Fine regression targets. \ Shape (n, num_buckets*2). - offsets_weights: Fine regression weights. \ Shape (n, num_buckets*2). - bucket_labels: Bucketing estimation labels. \ Shape (n, num_buckets*2). - cls_weights: Bucketing estimation weights. \ Shape (n, num_buckets*2). """ assert proposals.size() == gt.size() # generate buckets proposals = proposals.float() gt = gt.float() (bucket_w, bucket_h, l_buckets, r_buckets, t_buckets, d_buckets) = generat_buckets(proposals, num_buckets, scale_factor) gx1 = gt[..., 0] gy1 = gt[..., 1] gx2 = gt[..., 2] gy2 = gt[..., 3] # generate offset targets and weights # offsets from buckets to gts l_offsets = (l_buckets - gx1[:, None]) / bucket_w[:, None] r_offsets = (r_buckets - gx2[:, None]) / bucket_w[:, None] t_offsets = (t_buckets - gy1[:, None]) / bucket_h[:, None] d_offsets = (d_buckets - gy2[:, None]) / bucket_h[:, None] # select top-k nearest buckets l_topk, l_label = l_offsets.abs().topk( offset_topk, dim=1, largest=False, sorted=True) r_topk, r_label = r_offsets.abs().topk( offset_topk, dim=1, largest=False, sorted=True) t_topk, t_label = t_offsets.abs().topk( offset_topk, dim=1, largest=False, sorted=True) d_topk, d_label = d_offsets.abs().topk( offset_topk, dim=1, largest=False, sorted=True) offset_l_weights = l_offsets.new_zeros(l_offsets.size()) offset_r_weights = r_offsets.new_zeros(r_offsets.size()) offset_t_weights = t_offsets.new_zeros(t_offsets.size()) offset_d_weights = d_offsets.new_zeros(d_offsets.size()) inds = torch.arange(0, proposals.size(0)).to(proposals).long() # generate offset weights of top-k nearest buckets for k in range(offset_topk): if k >= 1: offset_l_weights[inds, l_label[:, k]] = (l_topk[:, k] < offset_upperbound).float() offset_r_weights[inds, r_label[:, k]] = (r_topk[:, k] < offset_upperbound).float() offset_t_weights[inds, t_label[:, k]] = (t_topk[:, k] < offset_upperbound).float() offset_d_weights[inds, d_label[:, k]] = (d_topk[:, k] < offset_upperbound).float() else: offset_l_weights[inds, l_label[:, k]] = 1.0 offset_r_weights[inds, r_label[:, k]] = 1.0 offset_t_weights[inds, t_label[:, k]] = 1.0 offset_d_weights[inds, d_label[:, k]] = 1.0 offsets = torch.cat([l_offsets, r_offsets, t_offsets, d_offsets], dim=-1) offsets_weights = torch.cat([ offset_l_weights, offset_r_weights, offset_t_weights, offset_d_weights ], dim=-1) # generate bucket labels and weight side_num = int(np.ceil(num_buckets / 2.0)) labels = torch.stack( [l_label[:, 0], r_label[:, 0], t_label[:, 0], d_label[:, 0]], dim=-1) batch_size = labels.size(0) bucket_labels = F.one_hot(labels.view(-1), side_num).view(batch_size, -1).float() bucket_cls_l_weights = (l_offsets.abs() < 1).float() bucket_cls_r_weights = (r_offsets.abs() < 1).float() bucket_cls_t_weights = (t_offsets.abs() < 1).float() bucket_cls_d_weights = (d_offsets.abs() < 1).float() bucket_cls_weights = torch.cat([ bucket_cls_l_weights, bucket_cls_r_weights, bucket_cls_t_weights, bucket_cls_d_weights ], dim=-1) # ignore second nearest buckets for cls if necessary if cls_ignore_neighbor: bucket_cls_weights = (~((bucket_cls_weights == 1) & (bucket_labels == 0))).float() else: bucket_cls_weights[:] = 1.0 return offsets, offsets_weights, bucket_labels, bucket_cls_weights def bucket2bbox(proposals, cls_preds, offset_preds, num_buckets, scale_factor=1.0, max_shape=None, clip_border=True): """Apply bucketing estimation (cls preds) and fine regression (offset preds) to generate det bboxes. Args: proposals (Tensor): Boxes to be transformed. Shape (n, 4) cls_preds (Tensor): bucketing estimation. Shape (n, num_buckets*2). offset_preds (Tensor): fine regression. Shape (n, num_buckets*2). num_buckets (int): Number of buckets. scale_factor (float): Scale factor to rescale proposals. max_shape (tuple[int, int]): Maximum bounds for boxes. specifies (H, W) clip_border (bool, optional): Whether clip the objects outside the border of the image. Defaults to True. Returns: tuple[Tensor]: (bboxes, loc_confidence). - bboxes: predicted bboxes. Shape (n, 4) - loc_confidence: localization confidence of predicted bboxes. Shape (n,). """ side_num = int(np.ceil(num_buckets / 2.0)) cls_preds = cls_preds.view(-1, side_num) offset_preds = offset_preds.view(-1, side_num) scores = F.softmax(cls_preds, dim=1) score_topk, score_label = scores.topk(2, dim=1, largest=True, sorted=True) rescaled_proposals = bbox_rescale(proposals, scale_factor) pw = rescaled_proposals[..., 2] - rescaled_proposals[..., 0] ph = rescaled_proposals[..., 3] - rescaled_proposals[..., 1] px1 = rescaled_proposals[..., 0] py1 = rescaled_proposals[..., 1] px2 = rescaled_proposals[..., 2] py2 = rescaled_proposals[..., 3] bucket_w = pw / num_buckets bucket_h = ph / num_buckets score_inds_l = score_label[0::4, 0] score_inds_r = score_label[1::4, 0] score_inds_t = score_label[2::4, 0] score_inds_d = score_label[3::4, 0] l_buckets = px1 + (0.5 + score_inds_l.float()) * bucket_w r_buckets = px2 - (0.5 + score_inds_r.float()) * bucket_w t_buckets = py1 + (0.5 + score_inds_t.float()) * bucket_h d_buckets = py2 - (0.5 + score_inds_d.float()) * bucket_h offsets = offset_preds.view(-1, 4, side_num) inds = torch.arange(proposals.size(0)).to(proposals).long() l_offsets = offsets[:, 0, :][inds, score_inds_l] r_offsets = offsets[:, 1, :][inds, score_inds_r] t_offsets = offsets[:, 2, :][inds, score_inds_t] d_offsets = offsets[:, 3, :][inds, score_inds_d] x1 = l_buckets - l_offsets * bucket_w x2 = r_buckets - r_offsets * bucket_w y1 = t_buckets - t_offsets * bucket_h y2 = d_buckets - d_offsets * bucket_h if clip_border and max_shape is not None: x1 = x1.clamp(min=0, max=max_shape[1] - 1) y1 = y1.clamp(min=0, max=max_shape[0] - 1) x2 = x2.clamp(min=0, max=max_shape[1] - 1) y2 = y2.clamp(min=0, max=max_shape[0] - 1) bboxes = torch.cat([x1[:, None], y1[:, None], x2[:, None], y2[:, None]], dim=-1) # bucketing guided rescoring loc_confidence = score_topk[:, 0] top2_neighbor_inds = (score_label[:, 0] - score_label[:, 1]).abs() == 1 loc_confidence += score_topk[:, 1] * top2_neighbor_inds.float() loc_confidence = loc_confidence.view(-1, 4).mean(dim=1) return bboxes, loc_confidence ================================================ FILE: mmdet/models/task_modules/coders/delta_xywh_bbox_coder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings import numpy as np import torch from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import HorizontalBoxes, get_box_tensor from .base_bbox_coder import BaseBBoxCoder @TASK_UTILS.register_module() class DeltaXYWHBBoxCoder(BaseBBoxCoder): """Delta XYWH BBox coder. Following the practice in `R-CNN `_, this coder encodes bbox (x1, y1, x2, y2) into delta (dx, dy, dw, dh) and decodes delta (dx, dy, dw, dh) back to original bbox (x1, y1, x2, y2). Args: target_means (Sequence[float]): Denormalizing means of target for delta coordinates target_stds (Sequence[float]): Denormalizing standard deviation of target for delta coordinates clip_border (bool, optional): Whether clip the objects outside the border of the image. Defaults to True. add_ctr_clamp (bool): Whether to add center clamp, when added, the predicted box is clamped is its center is too far away from the original anchor's center. Only used by YOLOF. Default False. ctr_clamp (int): the maximum pixel shift to clamp. Only used by YOLOF. Default 32. """ def __init__(self, target_means=(0., 0., 0., 0.), target_stds=(1., 1., 1., 1.), clip_border=True, add_ctr_clamp=False, ctr_clamp=32, **kwargs): super().__init__(**kwargs) self.means = target_means self.stds = target_stds self.clip_border = clip_border self.add_ctr_clamp = add_ctr_clamp self.ctr_clamp = ctr_clamp def encode(self, bboxes, gt_bboxes): """Get box regression transformation deltas that can be used to transform the ``bboxes`` into the ``gt_bboxes``. Args: bboxes (torch.Tensor or :obj:`BaseBoxes`): Source boxes, e.g., object proposals. gt_bboxes (torch.Tensor or :obj:`BaseBoxes`): Target of the transformation, e.g., ground-truth boxes. Returns: torch.Tensor: Box transformation deltas """ bboxes = get_box_tensor(bboxes) gt_bboxes = get_box_tensor(gt_bboxes) assert bboxes.size(0) == gt_bboxes.size(0) assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 encoded_bboxes = bbox2delta(bboxes, gt_bboxes, self.means, self.stds) return encoded_bboxes def decode(self, bboxes, pred_bboxes, max_shape=None, wh_ratio_clip=16 / 1000): """Apply transformation `pred_bboxes` to `boxes`. Args: bboxes (torch.Tensor or :obj:`BaseBoxes`): Basic boxes. Shape (B, N, 4) or (N, 4) pred_bboxes (Tensor): Encoded offsets with respect to each roi. Has shape (B, N, num_classes * 4) or (B, N, 4) or (N, num_classes * 4) or (N, 4). Note N = num_anchors * W * H when rois is a grid of anchors.Offset encoding follows [1]_. max_shape (Sequence[int] or torch.Tensor or Sequence[ Sequence[int]],optional): Maximum bounds for boxes, specifies (H, W, C) or (H, W). If bboxes shape is (B, N, 4), then the max_shape should be a Sequence[Sequence[int]] and the length of max_shape should also be B. wh_ratio_clip (float, optional): The allowed ratio between width and height. Returns: Union[torch.Tensor, :obj:`BaseBoxes`]: Decoded boxes. """ bboxes = get_box_tensor(bboxes) assert pred_bboxes.size(0) == bboxes.size(0) if pred_bboxes.ndim == 3: assert pred_bboxes.size(1) == bboxes.size(1) if pred_bboxes.ndim == 2 and not torch.onnx.is_in_onnx_export(): # single image decode decoded_bboxes = delta2bbox(bboxes, pred_bboxes, self.means, self.stds, max_shape, wh_ratio_clip, self.clip_border, self.add_ctr_clamp, self.ctr_clamp) else: if pred_bboxes.ndim == 3 and not torch.onnx.is_in_onnx_export(): warnings.warn( 'DeprecationWarning: onnx_delta2bbox is deprecated ' 'in the case of batch decoding and non-ONNX, ' 'please use “delta2bbox” instead. In order to improve ' 'the decoding speed, the batch function will no ' 'longer be supported. ') decoded_bboxes = onnx_delta2bbox(bboxes, pred_bboxes, self.means, self.stds, max_shape, wh_ratio_clip, self.clip_border, self.add_ctr_clamp, self.ctr_clamp) if self.use_box_type: assert decoded_bboxes.size(-1) == 4, \ ('Cannot warp decoded boxes with box type when decoded boxes' 'have shape of (N, num_classes * 4)') decoded_bboxes = HorizontalBoxes(decoded_bboxes) return decoded_bboxes def bbox2delta(proposals, gt, means=(0., 0., 0., 0.), stds=(1., 1., 1., 1.)): """Compute deltas of proposals w.r.t. gt. We usually compute the deltas of x, y, w, h of proposals w.r.t ground truth bboxes to get regression target. This is the inverse function of :func:`delta2bbox`. Args: proposals (Tensor): Boxes to be transformed, shape (N, ..., 4) gt (Tensor): Gt bboxes to be used as base, shape (N, ..., 4) means (Sequence[float]): Denormalizing means for delta coordinates stds (Sequence[float]): Denormalizing standard deviation for delta coordinates Returns: Tensor: deltas with shape (N, 4), where columns represent dx, dy, dw, dh. """ assert proposals.size() == gt.size() proposals = proposals.float() gt = gt.float() px = (proposals[..., 0] + proposals[..., 2]) * 0.5 py = (proposals[..., 1] + proposals[..., 3]) * 0.5 pw = proposals[..., 2] - proposals[..., 0] ph = proposals[..., 3] - proposals[..., 1] gx = (gt[..., 0] + gt[..., 2]) * 0.5 gy = (gt[..., 1] + gt[..., 3]) * 0.5 gw = gt[..., 2] - gt[..., 0] gh = gt[..., 3] - gt[..., 1] dx = (gx - px) / pw dy = (gy - py) / ph dw = torch.log(gw / pw) dh = torch.log(gh / ph) deltas = torch.stack([dx, dy, dw, dh], dim=-1) means = deltas.new_tensor(means).unsqueeze(0) stds = deltas.new_tensor(stds).unsqueeze(0) deltas = deltas.sub_(means).div_(stds) return deltas def delta2bbox(rois, deltas, means=(0., 0., 0., 0.), stds=(1., 1., 1., 1.), max_shape=None, wh_ratio_clip=16 / 1000, clip_border=True, add_ctr_clamp=False, ctr_clamp=32): """Apply deltas to shift/scale base boxes. Typically the rois are anchor or proposed bounding boxes and the deltas are network outputs used to shift/scale those boxes. This is the inverse function of :func:`bbox2delta`. Args: rois (Tensor): Boxes to be transformed. Has shape (N, 4). deltas (Tensor): Encoded offsets relative to each roi. Has shape (N, num_classes * 4) or (N, 4). Note N = num_base_anchors * W * H, when rois is a grid of anchors. Offset encoding follows [1]_. means (Sequence[float]): Denormalizing means for delta coordinates. Default (0., 0., 0., 0.). stds (Sequence[float]): Denormalizing standard deviation for delta coordinates. Default (1., 1., 1., 1.). max_shape (tuple[int, int]): Maximum bounds for boxes, specifies (H, W). Default None. wh_ratio_clip (float): Maximum aspect ratio for boxes. Default 16 / 1000. clip_border (bool, optional): Whether clip the objects outside the border of the image. Default True. add_ctr_clamp (bool): Whether to add center clamp. When set to True, the center of the prediction bounding box will be clamped to avoid being too far away from the center of the anchor. Only used by YOLOF. Default False. ctr_clamp (int): the maximum pixel shift to clamp. Only used by YOLOF. Default 32. Returns: Tensor: Boxes with shape (N, num_classes * 4) or (N, 4), where 4 represent tl_x, tl_y, br_x, br_y. References: .. [1] https://arxiv.org/abs/1311.2524 Example: >>> rois = torch.Tensor([[ 0., 0., 1., 1.], >>> [ 0., 0., 1., 1.], >>> [ 0., 0., 1., 1.], >>> [ 5., 5., 5., 5.]]) >>> deltas = torch.Tensor([[ 0., 0., 0., 0.], >>> [ 1., 1., 1., 1.], >>> [ 0., 0., 2., -1.], >>> [ 0.7, -1.9, -0.5, 0.3]]) >>> delta2bbox(rois, deltas, max_shape=(32, 32, 3)) tensor([[0.0000, 0.0000, 1.0000, 1.0000], [0.1409, 0.1409, 2.8591, 2.8591], [0.0000, 0.3161, 4.1945, 0.6839], [5.0000, 5.0000, 5.0000, 5.0000]]) """ num_bboxes, num_classes = deltas.size(0), deltas.size(1) // 4 if num_bboxes == 0: return deltas deltas = deltas.reshape(-1, 4) means = deltas.new_tensor(means).view(1, -1) stds = deltas.new_tensor(stds).view(1, -1) denorm_deltas = deltas * stds + means dxy = denorm_deltas[:, :2] dwh = denorm_deltas[:, 2:] # Compute width/height of each roi rois_ = rois.repeat(1, num_classes).reshape(-1, 4) pxy = ((rois_[:, :2] + rois_[:, 2:]) * 0.5) pwh = (rois_[:, 2:] - rois_[:, :2]) dxy_wh = pwh * dxy max_ratio = np.abs(np.log(wh_ratio_clip)) if add_ctr_clamp: dxy_wh = torch.clamp(dxy_wh, max=ctr_clamp, min=-ctr_clamp) dwh = torch.clamp(dwh, max=max_ratio) else: dwh = dwh.clamp(min=-max_ratio, max=max_ratio) gxy = pxy + dxy_wh gwh = pwh * dwh.exp() x1y1 = gxy - (gwh * 0.5) x2y2 = gxy + (gwh * 0.5) bboxes = torch.cat([x1y1, x2y2], dim=-1) if clip_border and max_shape is not None: bboxes[..., 0::2].clamp_(min=0, max=max_shape[1]) bboxes[..., 1::2].clamp_(min=0, max=max_shape[0]) bboxes = bboxes.reshape(num_bboxes, -1) return bboxes def onnx_delta2bbox(rois, deltas, means=(0., 0., 0., 0.), stds=(1., 1., 1., 1.), max_shape=None, wh_ratio_clip=16 / 1000, clip_border=True, add_ctr_clamp=False, ctr_clamp=32): """Apply deltas to shift/scale base boxes. Typically the rois are anchor or proposed bounding boxes and the deltas are network outputs used to shift/scale those boxes. This is the inverse function of :func:`bbox2delta`. Args: rois (Tensor): Boxes to be transformed. Has shape (N, 4) or (B, N, 4) deltas (Tensor): Encoded offsets with respect to each roi. Has shape (B, N, num_classes * 4) or (B, N, 4) or (N, num_classes * 4) or (N, 4). Note N = num_anchors * W * H when rois is a grid of anchors.Offset encoding follows [1]_. means (Sequence[float]): Denormalizing means for delta coordinates. Default (0., 0., 0., 0.). stds (Sequence[float]): Denormalizing standard deviation for delta coordinates. Default (1., 1., 1., 1.). max_shape (Sequence[int] or torch.Tensor or Sequence[ Sequence[int]],optional): Maximum bounds for boxes, specifies (H, W, C) or (H, W). If rois shape is (B, N, 4), then the max_shape should be a Sequence[Sequence[int]] and the length of max_shape should also be B. Default None. wh_ratio_clip (float): Maximum aspect ratio for boxes. Default 16 / 1000. clip_border (bool, optional): Whether clip the objects outside the border of the image. Default True. add_ctr_clamp (bool): Whether to add center clamp, when added, the predicted box is clamped is its center is too far away from the original anchor's center. Only used by YOLOF. Default False. ctr_clamp (int): the maximum pixel shift to clamp. Only used by YOLOF. Default 32. Returns: Tensor: Boxes with shape (B, N, num_classes * 4) or (B, N, 4) or (N, num_classes * 4) or (N, 4), where 4 represent tl_x, tl_y, br_x, br_y. References: .. [1] https://arxiv.org/abs/1311.2524 Example: >>> rois = torch.Tensor([[ 0., 0., 1., 1.], >>> [ 0., 0., 1., 1.], >>> [ 0., 0., 1., 1.], >>> [ 5., 5., 5., 5.]]) >>> deltas = torch.Tensor([[ 0., 0., 0., 0.], >>> [ 1., 1., 1., 1.], >>> [ 0., 0., 2., -1.], >>> [ 0.7, -1.9, -0.5, 0.3]]) >>> delta2bbox(rois, deltas, max_shape=(32, 32, 3)) tensor([[0.0000, 0.0000, 1.0000, 1.0000], [0.1409, 0.1409, 2.8591, 2.8591], [0.0000, 0.3161, 4.1945, 0.6839], [5.0000, 5.0000, 5.0000, 5.0000]]) """ means = deltas.new_tensor(means).view(1, -1).repeat(1, deltas.size(-1) // 4) stds = deltas.new_tensor(stds).view(1, -1).repeat(1, deltas.size(-1) // 4) denorm_deltas = deltas * stds + means dx = denorm_deltas[..., 0::4] dy = denorm_deltas[..., 1::4] dw = denorm_deltas[..., 2::4] dh = denorm_deltas[..., 3::4] x1, y1 = rois[..., 0], rois[..., 1] x2, y2 = rois[..., 2], rois[..., 3] # Compute center of each roi px = ((x1 + x2) * 0.5).unsqueeze(-1).expand_as(dx) py = ((y1 + y2) * 0.5).unsqueeze(-1).expand_as(dy) # Compute width/height of each roi pw = (x2 - x1).unsqueeze(-1).expand_as(dw) ph = (y2 - y1).unsqueeze(-1).expand_as(dh) dx_width = pw * dx dy_height = ph * dy max_ratio = np.abs(np.log(wh_ratio_clip)) if add_ctr_clamp: dx_width = torch.clamp(dx_width, max=ctr_clamp, min=-ctr_clamp) dy_height = torch.clamp(dy_height, max=ctr_clamp, min=-ctr_clamp) dw = torch.clamp(dw, max=max_ratio) dh = torch.clamp(dh, max=max_ratio) else: dw = dw.clamp(min=-max_ratio, max=max_ratio) dh = dh.clamp(min=-max_ratio, max=max_ratio) # Use exp(network energy) to enlarge/shrink each roi gw = pw * dw.exp() gh = ph * dh.exp() # Use network energy to shift the center of each roi gx = px + dx_width gy = py + dy_height # Convert center-xy/width/height to top-left, bottom-right x1 = gx - gw * 0.5 y1 = gy - gh * 0.5 x2 = gx + gw * 0.5 y2 = gy + gh * 0.5 bboxes = torch.stack([x1, y1, x2, y2], dim=-1).view(deltas.size()) if clip_border and max_shape is not None: # clip bboxes with dynamic `min` and `max` for onnx if torch.onnx.is_in_onnx_export(): from mmdet.core.export import dynamic_clip_for_onnx x1, y1, x2, y2 = dynamic_clip_for_onnx(x1, y1, x2, y2, max_shape) bboxes = torch.stack([x1, y1, x2, y2], dim=-1).view(deltas.size()) return bboxes if not isinstance(max_shape, torch.Tensor): max_shape = x1.new_tensor(max_shape) max_shape = max_shape[..., :2].type_as(x1) if max_shape.ndim == 2: assert bboxes.ndim == 3 assert max_shape.size(0) == bboxes.size(0) min_xy = x1.new_tensor(0) max_xy = torch.cat( [max_shape] * (deltas.size(-1) // 2), dim=-1).flip(-1).unsqueeze(-2) bboxes = torch.where(bboxes < min_xy, min_xy, bboxes) bboxes = torch.where(bboxes > max_xy, max_xy, bboxes) return bboxes ================================================ FILE: mmdet/models/task_modules/coders/distance_point_bbox_coder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import (HorizontalBoxes, bbox2distance, distance2bbox, get_box_tensor) from .base_bbox_coder import BaseBBoxCoder @TASK_UTILS.register_module() class DistancePointBBoxCoder(BaseBBoxCoder): """Distance Point BBox coder. This coder encodes gt bboxes (x1, y1, x2, y2) into (top, bottom, left, right) and decode it back to the original. Args: clip_border (bool, optional): Whether clip the objects outside the border of the image. Defaults to True. """ def __init__(self, clip_border=True, **kwargs): super().__init__(**kwargs) self.clip_border = clip_border def encode(self, points, gt_bboxes, max_dis=None, eps=0.1): """Encode bounding box to distances. Args: points (Tensor): Shape (N, 2), The format is [x, y]. gt_bboxes (Tensor or :obj:`BaseBoxes`): Shape (N, 4), The format is "xyxy" max_dis (float): Upper bound of the distance. Default None. eps (float): a small value to ensure target < max_dis, instead <=. Default 0.1. Returns: Tensor: Box transformation deltas. The shape is (N, 4). """ gt_bboxes = get_box_tensor(gt_bboxes) assert points.size(0) == gt_bboxes.size(0) assert points.size(-1) == 2 assert gt_bboxes.size(-1) == 4 return bbox2distance(points, gt_bboxes, max_dis, eps) def decode(self, points, pred_bboxes, max_shape=None): """Decode distance prediction to bounding box. Args: points (Tensor): Shape (B, N, 2) or (N, 2). pred_bboxes (Tensor): Distance from the given point to 4 boundaries (left, top, right, bottom). Shape (B, N, 4) or (N, 4) max_shape (Sequence[int] or torch.Tensor or Sequence[ Sequence[int]],optional): Maximum bounds for boxes, specifies (H, W, C) or (H, W). If priors shape is (B, N, 4), then the max_shape should be a Sequence[Sequence[int]], and the length of max_shape should also be B. Default None. Returns: Union[Tensor, :obj:`BaseBoxes`]: Boxes with shape (N, 4) or (B, N, 4) """ assert points.size(0) == pred_bboxes.size(0) assert points.size(-1) == 2 assert pred_bboxes.size(-1) == 4 if self.clip_border is False: max_shape = None bboxes = distance2bbox(points, pred_bboxes, max_shape) if self.use_box_type: bboxes = HorizontalBoxes(bboxes) return bboxes ================================================ FILE: mmdet/models/task_modules/coders/legacy_delta_xywh_bbox_coder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import HorizontalBoxes, get_box_tensor from .base_bbox_coder import BaseBBoxCoder @TASK_UTILS.register_module() class LegacyDeltaXYWHBBoxCoder(BaseBBoxCoder): """Legacy Delta XYWH BBox coder used in MMDet V1.x. Following the practice in R-CNN [1]_, this coder encodes bbox (x1, y1, x2, y2) into delta (dx, dy, dw, dh) and decodes delta (dx, dy, dw, dh) back to original bbox (x1, y1, x2, y2). Note: The main difference between :class`LegacyDeltaXYWHBBoxCoder` and :class:`DeltaXYWHBBoxCoder` is whether ``+ 1`` is used during width and height calculation. We suggest to only use this coder when testing with MMDet V1.x models. References: .. [1] https://arxiv.org/abs/1311.2524 Args: target_means (Sequence[float]): denormalizing means of target for delta coordinates target_stds (Sequence[float]): denormalizing standard deviation of target for delta coordinates """ def __init__(self, target_means=(0., 0., 0., 0.), target_stds=(1., 1., 1., 1.), **kwargs): super().__init__(**kwargs) self.means = target_means self.stds = target_stds def encode(self, bboxes, gt_bboxes): """Get box regression transformation deltas that can be used to transform the ``bboxes`` into the ``gt_bboxes``. Args: bboxes (torch.Tensor or :obj:`BaseBoxes`): source boxes, e.g., object proposals. gt_bboxes (torch.Tensor or :obj:`BaseBoxes`): target of the transformation, e.g., ground-truth boxes. Returns: torch.Tensor: Box transformation deltas """ bboxes = get_box_tensor(bboxes) gt_bboxes = get_box_tensor(gt_bboxes) assert bboxes.size(0) == gt_bboxes.size(0) assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 encoded_bboxes = legacy_bbox2delta(bboxes, gt_bboxes, self.means, self.stds) return encoded_bboxes def decode(self, bboxes, pred_bboxes, max_shape=None, wh_ratio_clip=16 / 1000): """Apply transformation `pred_bboxes` to `boxes`. Args: boxes (torch.Tensor or :obj:`BaseBoxes`): Basic boxes. pred_bboxes (torch.Tensor): Encoded boxes with shape max_shape (tuple[int], optional): Maximum shape of boxes. Defaults to None. wh_ratio_clip (float, optional): The allowed ratio between width and height. Returns: Union[torch.Tensor, :obj:`BaseBoxes`]: Decoded boxes. """ bboxes = get_box_tensor(bboxes) assert pred_bboxes.size(0) == bboxes.size(0) decoded_bboxes = legacy_delta2bbox(bboxes, pred_bboxes, self.means, self.stds, max_shape, wh_ratio_clip) if self.use_box_type: assert decoded_bboxes.size(-1) == 4, \ ('Cannot warp decoded boxes with box type when decoded boxes' 'have shape of (N, num_classes * 4)') decoded_bboxes = HorizontalBoxes(decoded_bboxes) return decoded_bboxes def legacy_bbox2delta(proposals, gt, means=(0., 0., 0., 0.), stds=(1., 1., 1., 1.)): """Compute deltas of proposals w.r.t. gt in the MMDet V1.x manner. We usually compute the deltas of x, y, w, h of proposals w.r.t ground truth bboxes to get regression target. This is the inverse function of `delta2bbox()` Args: proposals (Tensor): Boxes to be transformed, shape (N, ..., 4) gt (Tensor): Gt bboxes to be used as base, shape (N, ..., 4) means (Sequence[float]): Denormalizing means for delta coordinates stds (Sequence[float]): Denormalizing standard deviation for delta coordinates Returns: Tensor: deltas with shape (N, 4), where columns represent dx, dy, dw, dh. """ assert proposals.size() == gt.size() proposals = proposals.float() gt = gt.float() px = (proposals[..., 0] + proposals[..., 2]) * 0.5 py = (proposals[..., 1] + proposals[..., 3]) * 0.5 pw = proposals[..., 2] - proposals[..., 0] + 1.0 ph = proposals[..., 3] - proposals[..., 1] + 1.0 gx = (gt[..., 0] + gt[..., 2]) * 0.5 gy = (gt[..., 1] + gt[..., 3]) * 0.5 gw = gt[..., 2] - gt[..., 0] + 1.0 gh = gt[..., 3] - gt[..., 1] + 1.0 dx = (gx - px) / pw dy = (gy - py) / ph dw = torch.log(gw / pw) dh = torch.log(gh / ph) deltas = torch.stack([dx, dy, dw, dh], dim=-1) means = deltas.new_tensor(means).unsqueeze(0) stds = deltas.new_tensor(stds).unsqueeze(0) deltas = deltas.sub_(means).div_(stds) return deltas def legacy_delta2bbox(rois, deltas, means=(0., 0., 0., 0.), stds=(1., 1., 1., 1.), max_shape=None, wh_ratio_clip=16 / 1000): """Apply deltas to shift/scale base boxes in the MMDet V1.x manner. Typically the rois are anchor or proposed bounding boxes and the deltas are network outputs used to shift/scale those boxes. This is the inverse function of `bbox2delta()` Args: rois (Tensor): Boxes to be transformed. Has shape (N, 4) deltas (Tensor): Encoded offsets with respect to each roi. Has shape (N, 4 * num_classes). Note N = num_anchors * W * H when rois is a grid of anchors. Offset encoding follows [1]_. means (Sequence[float]): Denormalizing means for delta coordinates stds (Sequence[float]): Denormalizing standard deviation for delta coordinates max_shape (tuple[int, int]): Maximum bounds for boxes. specifies (H, W) wh_ratio_clip (float): Maximum aspect ratio for boxes. Returns: Tensor: Boxes with shape (N, 4), where columns represent tl_x, tl_y, br_x, br_y. References: .. [1] https://arxiv.org/abs/1311.2524 Example: >>> rois = torch.Tensor([[ 0., 0., 1., 1.], >>> [ 0., 0., 1., 1.], >>> [ 0., 0., 1., 1.], >>> [ 5., 5., 5., 5.]]) >>> deltas = torch.Tensor([[ 0., 0., 0., 0.], >>> [ 1., 1., 1., 1.], >>> [ 0., 0., 2., -1.], >>> [ 0.7, -1.9, -0.5, 0.3]]) >>> legacy_delta2bbox(rois, deltas, max_shape=(32, 32)) tensor([[0.0000, 0.0000, 1.5000, 1.5000], [0.0000, 0.0000, 5.2183, 5.2183], [0.0000, 0.1321, 7.8891, 0.8679], [5.3967, 2.4251, 6.0033, 3.7749]]) """ means = deltas.new_tensor(means).repeat(1, deltas.size(1) // 4) stds = deltas.new_tensor(stds).repeat(1, deltas.size(1) // 4) denorm_deltas = deltas * stds + means dx = denorm_deltas[:, 0::4] dy = denorm_deltas[:, 1::4] dw = denorm_deltas[:, 2::4] dh = denorm_deltas[:, 3::4] max_ratio = np.abs(np.log(wh_ratio_clip)) dw = dw.clamp(min=-max_ratio, max=max_ratio) dh = dh.clamp(min=-max_ratio, max=max_ratio) # Compute center of each roi px = ((rois[:, 0] + rois[:, 2]) * 0.5).unsqueeze(1).expand_as(dx) py = ((rois[:, 1] + rois[:, 3]) * 0.5).unsqueeze(1).expand_as(dy) # Compute width/height of each roi pw = (rois[:, 2] - rois[:, 0] + 1.0).unsqueeze(1).expand_as(dw) ph = (rois[:, 3] - rois[:, 1] + 1.0).unsqueeze(1).expand_as(dh) # Use exp(network energy) to enlarge/shrink each roi gw = pw * dw.exp() gh = ph * dh.exp() # Use network energy to shift the center of each roi gx = px + pw * dx gy = py + ph * dy # Convert center-xy/width/height to top-left, bottom-right # The true legacy box coder should +- 0.5 here. # However, current implementation improves the performance when testing # the models trained in MMDetection 1.X (~0.5 bbox AP, 0.2 mask AP) x1 = gx - gw * 0.5 y1 = gy - gh * 0.5 x2 = gx + gw * 0.5 y2 = gy + gh * 0.5 if max_shape is not None: x1 = x1.clamp(min=0, max=max_shape[1] - 1) y1 = y1.clamp(min=0, max=max_shape[0] - 1) x2 = x2.clamp(min=0, max=max_shape[1] - 1) y2 = y2.clamp(min=0, max=max_shape[0] - 1) bboxes = torch.stack([x1, y1, x2, y2], dim=-1).view_as(deltas) return bboxes ================================================ FILE: mmdet/models/task_modules/coders/pseudo_bbox_coder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import HorizontalBoxes, get_box_tensor from .base_bbox_coder import BaseBBoxCoder @TASK_UTILS.register_module() class PseudoBBoxCoder(BaseBBoxCoder): """Pseudo bounding box coder.""" def __init__(self, **kwargs): super().__init__(**kwargs) def encode(self, bboxes, gt_bboxes): """torch.Tensor: return the given ``bboxes``""" gt_bboxes = get_box_tensor(gt_bboxes) return gt_bboxes def decode(self, bboxes, pred_bboxes): """torch.Tensor: return the given ``pred_bboxes``""" if self.use_box_type: pred_bboxes = HorizontalBoxes(pred_bboxes) return pred_bboxes ================================================ FILE: mmdet/models/task_modules/coders/tblr_bbox_coder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import HorizontalBoxes, get_box_tensor from .base_bbox_coder import BaseBBoxCoder @TASK_UTILS.register_module() class TBLRBBoxCoder(BaseBBoxCoder): """TBLR BBox coder. Following the practice in `FSAF `_, this coder encodes gt bboxes (x1, y1, x2, y2) into (top, bottom, left, right) and decode it back to the original. Args: normalizer (list | float): Normalization factor to be divided with when coding the coordinates. If it is a list, it should have length of 4 indicating normalization factor in tblr dims. Otherwise it is a unified float factor for all dims. Default: 4.0 clip_border (bool, optional): Whether clip the objects outside the border of the image. Defaults to True. """ def __init__(self, normalizer=4.0, clip_border=True, **kwargs): super().__init__(**kwargs) self.normalizer = normalizer self.clip_border = clip_border def encode(self, bboxes, gt_bboxes): """Get box regression transformation deltas that can be used to transform the ``bboxes`` into the ``gt_bboxes`` in the (top, left, bottom, right) order. Args: bboxes (torch.Tensor or :obj:`BaseBoxes`): source boxes, e.g., object proposals. gt_bboxes (torch.Tensor or :obj:`BaseBoxes`): target of the transformation, e.g., ground truth boxes. Returns: torch.Tensor: Box transformation deltas """ bboxes = get_box_tensor(bboxes) gt_bboxes = get_box_tensor(gt_bboxes) assert bboxes.size(0) == gt_bboxes.size(0) assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 encoded_bboxes = bboxes2tblr( bboxes, gt_bboxes, normalizer=self.normalizer) return encoded_bboxes def decode(self, bboxes, pred_bboxes, max_shape=None): """Apply transformation `pred_bboxes` to `boxes`. Args: bboxes (torch.Tensor or :obj:`BaseBoxes`): Basic boxes.Shape (B, N, 4) or (N, 4) pred_bboxes (torch.Tensor): Encoded boxes with shape (B, N, 4) or (N, 4) max_shape (Sequence[int] or torch.Tensor or Sequence[ Sequence[int]],optional): Maximum bounds for boxes, specifies (H, W, C) or (H, W). If bboxes shape is (B, N, 4), then the max_shape should be a Sequence[Sequence[int]] and the length of max_shape should also be B. Returns: Union[torch.Tensor, :obj:`BaseBoxes`]: Decoded boxes. """ bboxes = get_box_tensor(bboxes) decoded_bboxes = tblr2bboxes( bboxes, pred_bboxes, normalizer=self.normalizer, max_shape=max_shape, clip_border=self.clip_border) if self.use_box_type: decoded_bboxes = HorizontalBoxes(decoded_bboxes) return decoded_bboxes def bboxes2tblr(priors, gts, normalizer=4.0, normalize_by_wh=True): """Encode ground truth boxes to tblr coordinate. It first convert the gt coordinate to tblr format, (top, bottom, left, right), relative to prior box centers. The tblr coordinate may be normalized by the side length of prior bboxes if `normalize_by_wh` is specified as True, and it is then normalized by the `normalizer` factor. Args: priors (Tensor): Prior boxes in point form Shape: (num_proposals,4). gts (Tensor): Coords of ground truth for each prior in point-form Shape: (num_proposals, 4). normalizer (Sequence[float] | float): normalization parameter of encoded boxes. If it is a list, it has to have length = 4. Default: 4.0 normalize_by_wh (bool): Whether to normalize tblr coordinate by the side length (wh) of prior bboxes. Return: encoded boxes (Tensor), Shape: (num_proposals, 4) """ # dist b/t match center and prior's center if not isinstance(normalizer, float): normalizer = torch.tensor(normalizer, device=priors.device) assert len(normalizer) == 4, 'Normalizer must have length = 4' assert priors.size(0) == gts.size(0) prior_centers = (priors[:, 0:2] + priors[:, 2:4]) / 2 xmin, ymin, xmax, ymax = gts.split(1, dim=1) top = prior_centers[:, 1].unsqueeze(1) - ymin bottom = ymax - prior_centers[:, 1].unsqueeze(1) left = prior_centers[:, 0].unsqueeze(1) - xmin right = xmax - prior_centers[:, 0].unsqueeze(1) loc = torch.cat((top, bottom, left, right), dim=1) if normalize_by_wh: # Normalize tblr by anchor width and height wh = priors[:, 2:4] - priors[:, 0:2] w, h = torch.split(wh, 1, dim=1) loc[:, :2] /= h # tb is normalized by h loc[:, 2:] /= w # lr is normalized by w # Normalize tblr by the given normalization factor return loc / normalizer def tblr2bboxes(priors, tblr, normalizer=4.0, normalize_by_wh=True, max_shape=None, clip_border=True): """Decode tblr outputs to prediction boxes. The process includes 3 steps: 1) De-normalize tblr coordinates by multiplying it with `normalizer`; 2) De-normalize tblr coordinates by the prior bbox width and height if `normalize_by_wh` is `True`; 3) Convert tblr (top, bottom, left, right) pair relative to the center of priors back to (xmin, ymin, xmax, ymax) coordinate. Args: priors (Tensor): Prior boxes in point form (x0, y0, x1, y1) Shape: (N,4) or (B, N, 4). tblr (Tensor): Coords of network output in tblr form Shape: (N, 4) or (B, N, 4). normalizer (Sequence[float] | float): Normalization parameter of encoded boxes. By list, it represents the normalization factors at tblr dims. By float, it is the unified normalization factor at all dims. Default: 4.0 normalize_by_wh (bool): Whether the tblr coordinates have been normalized by the side length (wh) of prior bboxes. max_shape (Sequence[int] or torch.Tensor or Sequence[ Sequence[int]],optional): Maximum bounds for boxes, specifies (H, W, C) or (H, W). If priors shape is (B, N, 4), then the max_shape should be a Sequence[Sequence[int]] and the length of max_shape should also be B. clip_border (bool, optional): Whether clip the objects outside the border of the image. Defaults to True. Return: encoded boxes (Tensor): Boxes with shape (N, 4) or (B, N, 4) """ if not isinstance(normalizer, float): normalizer = torch.tensor(normalizer, device=priors.device) assert len(normalizer) == 4, 'Normalizer must have length = 4' assert priors.size(0) == tblr.size(0) if priors.ndim == 3: assert priors.size(1) == tblr.size(1) loc_decode = tblr * normalizer prior_centers = (priors[..., 0:2] + priors[..., 2:4]) / 2 if normalize_by_wh: wh = priors[..., 2:4] - priors[..., 0:2] w, h = torch.split(wh, 1, dim=-1) # Inplace operation with slice would failed for exporting to ONNX th = h * loc_decode[..., :2] # tb tw = w * loc_decode[..., 2:] # lr loc_decode = torch.cat([th, tw], dim=-1) # Cannot be exported using onnx when loc_decode.split(1, dim=-1) top, bottom, left, right = loc_decode.split((1, 1, 1, 1), dim=-1) xmin = prior_centers[..., 0].unsqueeze(-1) - left xmax = prior_centers[..., 0].unsqueeze(-1) + right ymin = prior_centers[..., 1].unsqueeze(-1) - top ymax = prior_centers[..., 1].unsqueeze(-1) + bottom bboxes = torch.cat((xmin, ymin, xmax, ymax), dim=-1) if clip_border and max_shape is not None: # clip bboxes with dynamic `min` and `max` for onnx if torch.onnx.is_in_onnx_export(): from mmdet.core.export import dynamic_clip_for_onnx xmin, ymin, xmax, ymax = dynamic_clip_for_onnx( xmin, ymin, xmax, ymax, max_shape) bboxes = torch.cat([xmin, ymin, xmax, ymax], dim=-1) return bboxes if not isinstance(max_shape, torch.Tensor): max_shape = priors.new_tensor(max_shape) max_shape = max_shape[..., :2].type_as(priors) if max_shape.ndim == 2: assert bboxes.ndim == 3 assert max_shape.size(0) == bboxes.size(0) min_xy = priors.new_tensor(0) max_xy = torch.cat([max_shape, max_shape], dim=-1).flip(-1).unsqueeze(-2) bboxes = torch.where(bboxes < min_xy, min_xy, bboxes) bboxes = torch.where(bboxes > max_xy, max_xy, bboxes) return bboxes ================================================ FILE: mmdet/models/task_modules/coders/yolo_bbox_coder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import HorizontalBoxes, get_box_tensor from .base_bbox_coder import BaseBBoxCoder @TASK_UTILS.register_module() class YOLOBBoxCoder(BaseBBoxCoder): """YOLO BBox coder. Following `YOLO `_, this coder divide image into grids, and encode bbox (x1, y1, x2, y2) into (cx, cy, dw, dh). cx, cy in [0., 1.], denotes relative center position w.r.t the center of bboxes. dw, dh are the same as :obj:`DeltaXYWHBBoxCoder`. Args: eps (float): Min value of cx, cy when encoding. """ def __init__(self, eps=1e-6, **kwargs): super().__init__(**kwargs) self.eps = eps def encode(self, bboxes, gt_bboxes, stride): """Get box regression transformation deltas that can be used to transform the ``bboxes`` into the ``gt_bboxes``. Args: bboxes (torch.Tensor or :obj:`BaseBoxes`): Source boxes, e.g., anchors. gt_bboxes (torch.Tensor or :obj:`BaseBoxes`): Target of the transformation, e.g., ground-truth boxes. stride (torch.Tensor | int): Stride of bboxes. Returns: torch.Tensor: Box transformation deltas """ bboxes = get_box_tensor(bboxes) gt_bboxes = get_box_tensor(gt_bboxes) assert bboxes.size(0) == gt_bboxes.size(0) assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 x_center_gt = (gt_bboxes[..., 0] + gt_bboxes[..., 2]) * 0.5 y_center_gt = (gt_bboxes[..., 1] + gt_bboxes[..., 3]) * 0.5 w_gt = gt_bboxes[..., 2] - gt_bboxes[..., 0] h_gt = gt_bboxes[..., 3] - gt_bboxes[..., 1] x_center = (bboxes[..., 0] + bboxes[..., 2]) * 0.5 y_center = (bboxes[..., 1] + bboxes[..., 3]) * 0.5 w = bboxes[..., 2] - bboxes[..., 0] h = bboxes[..., 3] - bboxes[..., 1] w_target = torch.log((w_gt / w).clamp(min=self.eps)) h_target = torch.log((h_gt / h).clamp(min=self.eps)) x_center_target = ((x_center_gt - x_center) / stride + 0.5).clamp( self.eps, 1 - self.eps) y_center_target = ((y_center_gt - y_center) / stride + 0.5).clamp( self.eps, 1 - self.eps) encoded_bboxes = torch.stack( [x_center_target, y_center_target, w_target, h_target], dim=-1) return encoded_bboxes def decode(self, bboxes, pred_bboxes, stride): """Apply transformation `pred_bboxes` to `boxes`. Args: boxes (torch.Tensor or :obj:`BaseBoxes`): Basic boxes, e.g. anchors. pred_bboxes (torch.Tensor): Encoded boxes with shape stride (torch.Tensor | int): Strides of bboxes. Returns: Union[torch.Tensor, :obj:`BaseBoxes`]: Decoded boxes. """ bboxes = get_box_tensor(bboxes) assert pred_bboxes.size(-1) == bboxes.size(-1) == 4 xy_centers = (bboxes[..., :2] + bboxes[..., 2:]) * 0.5 + ( pred_bboxes[..., :2] - 0.5) * stride whs = (bboxes[..., 2:] - bboxes[..., :2]) * 0.5 * pred_bboxes[..., 2:].exp() decoded_bboxes = torch.stack( (xy_centers[..., 0] - whs[..., 0], xy_centers[..., 1] - whs[..., 1], xy_centers[..., 0] + whs[..., 0], xy_centers[..., 1] + whs[..., 1]), dim=-1) if self.use_box_type: decoded_bboxes = HorizontalBoxes(decoded_bboxes) return decoded_bboxes ================================================ FILE: mmdet/models/task_modules/prior_generators/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .anchor_generator import (AnchorGenerator, LegacyAnchorGenerator, SSDAnchorGenerator, YOLOAnchorGenerator) from .point_generator import MlvlPointGenerator, PointGenerator from .utils import anchor_inside_flags, calc_region __all__ = [ 'AnchorGenerator', 'LegacyAnchorGenerator', 'anchor_inside_flags', 'PointGenerator', 'calc_region', 'YOLOAnchorGenerator', 'MlvlPointGenerator', 'SSDAnchorGenerator' ] ================================================ FILE: mmdet/models/task_modules/prior_generators/anchor_generator.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings from typing import List, Optional, Tuple, Union import numpy as np import torch from mmengine.utils import is_tuple_of from torch import Tensor from torch.nn.modules.utils import _pair from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import HorizontalBoxes DeviceType = Union[str, torch.device] @TASK_UTILS.register_module() class AnchorGenerator: """Standard anchor generator for 2D anchor-based detectors. Args: strides (list[int] | list[tuple[int, int]]): Strides of anchors in multiple feature levels in order (w, h). ratios (list[float]): The list of ratios between the height and width of anchors in a single level. scales (list[int], Optional): Anchor scales for anchors in a single level. It cannot be set at the same time if `octave_base_scale` and `scales_per_octave` are set. base_sizes (list[int], Optional): The basic sizes of anchors in multiple levels. If None is given, strides will be used as base_sizes. (If strides are non square, the shortest stride is taken.) scale_major (bool): Whether to multiply scales first when generating base anchors. If true, the anchors in the same row will have the same scales. By default it is True in V2.0 octave_base_scale (int, Optional): The base scale of octave. scales_per_octave (int, Optional): Number of scales for each octave. `octave_base_scale` and `scales_per_octave` are usually used in retinanet and the `scales` should be None when they are set. centers (list[tuple[float]], Optional): The centers of the anchor relative to the feature grid center in multiple feature levels. By default it is set to be None and not used. If a list of tuple of float is given, they will be used to shift the centers of anchors. center_offset (float): The offset of center in proportion to anchors' width and height. By default it is 0 in V2.0. use_box_type (bool): Whether to warp anchors with the box type data structure. Defaults to False. Examples: >>> from mmdet.models.task_modules. ... prior_generators import AnchorGenerator >>> self = AnchorGenerator([16], [1.], [1.], [9]) >>> all_anchors = self.grid_priors([(2, 2)], device='cpu') >>> print(all_anchors) [tensor([[-4.5000, -4.5000, 4.5000, 4.5000], [11.5000, -4.5000, 20.5000, 4.5000], [-4.5000, 11.5000, 4.5000, 20.5000], [11.5000, 11.5000, 20.5000, 20.5000]])] >>> self = AnchorGenerator([16, 32], [1.], [1.], [9, 18]) >>> all_anchors = self.grid_priors([(2, 2), (1, 1)], device='cpu') >>> print(all_anchors) [tensor([[-4.5000, -4.5000, 4.5000, 4.5000], [11.5000, -4.5000, 20.5000, 4.5000], [-4.5000, 11.5000, 4.5000, 20.5000], [11.5000, 11.5000, 20.5000, 20.5000]]), \ tensor([[-9., -9., 9., 9.]])] """ def __init__(self, strides: Union[List[int], List[Tuple[int, int]]], ratios: List[float], scales: Optional[List[int]] = None, base_sizes: Optional[List[int]] = None, scale_major: bool = True, octave_base_scale: Optional[int] = None, scales_per_octave: Optional[int] = None, centers: Optional[List[Tuple[float, float]]] = None, center_offset: float = 0., use_box_type: bool = False) -> None: # check center and center_offset if center_offset != 0: assert centers is None, 'center cannot be set when center_offset' \ f'!=0, {centers} is given.' if not (0 <= center_offset <= 1): raise ValueError('center_offset should be in range [0, 1], ' f'{center_offset} is given.') if centers is not None: assert len(centers) == len(strides), \ 'The number of strides should be the same as centers, got ' \ f'{strides} and {centers}' # calculate base sizes of anchors self.strides = [_pair(stride) for stride in strides] self.base_sizes = [min(stride) for stride in self.strides ] if base_sizes is None else base_sizes assert len(self.base_sizes) == len(self.strides), \ 'The number of strides should be the same as base sizes, got ' \ f'{self.strides} and {self.base_sizes}' # calculate scales of anchors assert ((octave_base_scale is not None and scales_per_octave is not None) ^ (scales is not None)), \ 'scales and octave_base_scale with scales_per_octave cannot' \ ' be set at the same time' if scales is not None: self.scales = torch.Tensor(scales) elif octave_base_scale is not None and scales_per_octave is not None: octave_scales = np.array( [2**(i / scales_per_octave) for i in range(scales_per_octave)]) scales = octave_scales * octave_base_scale self.scales = torch.Tensor(scales) else: raise ValueError('Either scales or octave_base_scale with ' 'scales_per_octave should be set') self.octave_base_scale = octave_base_scale self.scales_per_octave = scales_per_octave self.ratios = torch.Tensor(ratios) self.scale_major = scale_major self.centers = centers self.center_offset = center_offset self.base_anchors = self.gen_base_anchors() self.use_box_type = use_box_type @property def num_base_anchors(self) -> List[int]: """list[int]: total number of base anchors in a feature grid""" return self.num_base_priors @property def num_base_priors(self) -> List[int]: """list[int]: The number of priors (anchors) at a point on the feature grid""" return [base_anchors.size(0) for base_anchors in self.base_anchors] @property def num_levels(self) -> int: """int: number of feature levels that the generator will be applied""" return len(self.strides) def gen_base_anchors(self) -> List[Tensor]: """Generate base anchors. Returns: list(torch.Tensor): Base anchors of a feature grid in multiple \ feature levels. """ multi_level_base_anchors = [] for i, base_size in enumerate(self.base_sizes): center = None if self.centers is not None: center = self.centers[i] multi_level_base_anchors.append( self.gen_single_level_base_anchors( base_size, scales=self.scales, ratios=self.ratios, center=center)) return multi_level_base_anchors def gen_single_level_base_anchors(self, base_size: Union[int, float], scales: Tensor, ratios: Tensor, center: Optional[Tuple[float]] = None) \ -> Tensor: """Generate base anchors of a single level. Args: base_size (int | float): Basic size of an anchor. scales (torch.Tensor): Scales of the anchor. ratios (torch.Tensor): The ratio between the height and width of anchors in a single level. center (tuple[float], optional): The center of the base anchor related to a single feature grid. Defaults to None. Returns: torch.Tensor: Anchors in a single-level feature maps. """ w = base_size h = base_size if center is None: x_center = self.center_offset * w y_center = self.center_offset * h else: x_center, y_center = center h_ratios = torch.sqrt(ratios) w_ratios = 1 / h_ratios if self.scale_major: ws = (w * w_ratios[:, None] * scales[None, :]).view(-1) hs = (h * h_ratios[:, None] * scales[None, :]).view(-1) else: ws = (w * scales[:, None] * w_ratios[None, :]).view(-1) hs = (h * scales[:, None] * h_ratios[None, :]).view(-1) # use float anchor and the anchor's center is aligned with the # pixel center base_anchors = [ x_center - 0.5 * ws, y_center - 0.5 * hs, x_center + 0.5 * ws, y_center + 0.5 * hs ] base_anchors = torch.stack(base_anchors, dim=-1) return base_anchors def _meshgrid(self, x: Tensor, y: Tensor, row_major: bool = True) -> Tuple[Tensor]: """Generate mesh grid of x and y. Args: x (torch.Tensor): Grids of x dimension. y (torch.Tensor): Grids of y dimension. row_major (bool): Whether to return y grids first. Defaults to True. Returns: tuple[torch.Tensor]: The mesh grids of x and y. """ # use shape instead of len to keep tracing while exporting to onnx xx = x.repeat(y.shape[0]) yy = y.view(-1, 1).repeat(1, x.shape[0]).view(-1) if row_major: return xx, yy else: return yy, xx def grid_priors(self, featmap_sizes: List[Tuple], dtype: torch.dtype = torch.float32, device: DeviceType = 'cuda') -> List[Tensor]: """Generate grid anchors in multiple feature levels. Args: featmap_sizes (list[tuple]): List of feature map sizes in multiple feature levels. dtype (:obj:`torch.dtype`): Dtype of priors. Defaults to torch.float32. device (str | torch.device): The device where the anchors will be put on. Return: list[torch.Tensor]: Anchors in multiple feature levels. \ The sizes of each tensor should be [N, 4], where \ N = width * height * num_base_anchors, width and height \ are the sizes of the corresponding feature level, \ num_base_anchors is the number of anchors for that level. """ assert self.num_levels == len(featmap_sizes) multi_level_anchors = [] for i in range(self.num_levels): anchors = self.single_level_grid_priors( featmap_sizes[i], level_idx=i, dtype=dtype, device=device) multi_level_anchors.append(anchors) return multi_level_anchors def single_level_grid_priors(self, featmap_size: Tuple[int, int], level_idx: int, dtype: torch.dtype = torch.float32, device: DeviceType = 'cuda') -> Tensor: """Generate grid anchors of a single level. Note: This function is usually called by method ``self.grid_priors``. Args: featmap_size (tuple[int, int]): Size of the feature maps. level_idx (int): The index of corresponding feature map level. dtype (obj:`torch.dtype`): Date type of points.Defaults to ``torch.float32``. device (str | torch.device): The device the tensor will be put on. Defaults to 'cuda'. Returns: torch.Tensor: Anchors in the overall feature maps. """ base_anchors = self.base_anchors[level_idx].to(device).to(dtype) feat_h, feat_w = featmap_size stride_w, stride_h = self.strides[level_idx] # First create Range with the default dtype, than convert to # target `dtype` for onnx exporting. shift_x = torch.arange(0, feat_w, device=device).to(dtype) * stride_w shift_y = torch.arange(0, feat_h, device=device).to(dtype) * stride_h shift_xx, shift_yy = self._meshgrid(shift_x, shift_y) shifts = torch.stack([shift_xx, shift_yy, shift_xx, shift_yy], dim=-1) # first feat_w elements correspond to the first row of shifts # add A anchors (1, A, 4) to K shifts (K, 1, 4) to get # shifted anchors (K, A, 4), reshape to (K*A, 4) all_anchors = base_anchors[None, :, :] + shifts[:, None, :] all_anchors = all_anchors.view(-1, 4) # first A rows correspond to A anchors of (0, 0) in feature map, # then (0, 1), (0, 2), ... if self.use_box_type: all_anchors = HorizontalBoxes(all_anchors) return all_anchors def sparse_priors(self, prior_idxs: Tensor, featmap_size: Tuple[int, int], level_idx: int, dtype: torch.dtype = torch.float32, device: DeviceType = 'cuda') -> Tensor: """Generate sparse anchors according to the ``prior_idxs``. Args: prior_idxs (Tensor): The index of corresponding anchors in the feature map. featmap_size (tuple[int, int]): feature map size arrange as (h, w). level_idx (int): The level index of corresponding feature map. dtype (obj:`torch.dtype`): Date type of points.Defaults to ``torch.float32``. device (str | torch.device): The device where the points is located. Returns: Tensor: Anchor with shape (N, 4), N should be equal to the length of ``prior_idxs``. """ height, width = featmap_size num_base_anchors = self.num_base_anchors[level_idx] base_anchor_id = prior_idxs % num_base_anchors x = (prior_idxs // num_base_anchors) % width * self.strides[level_idx][0] y = (prior_idxs // width // num_base_anchors) % height * self.strides[level_idx][1] priors = torch.stack([x, y, x, y], 1).to(dtype).to(device) + \ self.base_anchors[level_idx][base_anchor_id, :].to(device) return priors def grid_anchors(self, featmap_sizes: List[Tuple], device: DeviceType = 'cuda') -> List[Tensor]: """Generate grid anchors in multiple feature levels. Args: featmap_sizes (list[tuple]): List of feature map sizes in multiple feature levels. device (str | torch.device): Device where the anchors will be put on. Return: list[torch.Tensor]: Anchors in multiple feature levels. \ The sizes of each tensor should be [N, 4], where \ N = width * height * num_base_anchors, width and height \ are the sizes of the corresponding feature level, \ num_base_anchors is the number of anchors for that level. """ warnings.warn('``grid_anchors`` would be deprecated soon. ' 'Please use ``grid_priors`` ') assert self.num_levels == len(featmap_sizes) multi_level_anchors = [] for i in range(self.num_levels): anchors = self.single_level_grid_anchors( self.base_anchors[i].to(device), featmap_sizes[i], self.strides[i], device=device) multi_level_anchors.append(anchors) return multi_level_anchors def single_level_grid_anchors(self, base_anchors: Tensor, featmap_size: Tuple[int, int], stride: Tuple[int, int] = (16, 16), device: DeviceType = 'cuda') -> Tensor: """Generate grid anchors of a single level. Note: This function is usually called by method ``self.grid_anchors``. Args: base_anchors (torch.Tensor): The base anchors of a feature grid. featmap_size (tuple[int]): Size of the feature maps. stride (tuple[int, int]): Stride of the feature map in order (w, h). Defaults to (16, 16). device (str | torch.device): Device the tensor will be put on. Defaults to 'cuda'. Returns: torch.Tensor: Anchors in the overall feature maps. """ warnings.warn( '``single_level_grid_anchors`` would be deprecated soon. ' 'Please use ``single_level_grid_priors`` ') # keep featmap_size as Tensor instead of int, so that we # can convert to ONNX correctly feat_h, feat_w = featmap_size shift_x = torch.arange(0, feat_w, device=device) * stride[0] shift_y = torch.arange(0, feat_h, device=device) * stride[1] shift_xx, shift_yy = self._meshgrid(shift_x, shift_y) shifts = torch.stack([shift_xx, shift_yy, shift_xx, shift_yy], dim=-1) shifts = shifts.type_as(base_anchors) # first feat_w elements correspond to the first row of shifts # add A anchors (1, A, 4) to K shifts (K, 1, 4) to get # shifted anchors (K, A, 4), reshape to (K*A, 4) all_anchors = base_anchors[None, :, :] + shifts[:, None, :] all_anchors = all_anchors.view(-1, 4) # first A rows correspond to A anchors of (0, 0) in feature map, # then (0, 1), (0, 2), ... return all_anchors def valid_flags(self, featmap_sizes: List[Tuple[int, int]], pad_shape: Tuple, device: DeviceType = 'cuda') -> List[Tensor]: """Generate valid flags of anchors in multiple feature levels. Args: featmap_sizes (list(tuple[int, int])): List of feature map sizes in multiple feature levels. pad_shape (tuple): The padded shape of the image. device (str | torch.device): Device where the anchors will be put on. Return: list(torch.Tensor): Valid flags of anchors in multiple levels. """ assert self.num_levels == len(featmap_sizes) multi_level_flags = [] for i in range(self.num_levels): anchor_stride = self.strides[i] feat_h, feat_w = featmap_sizes[i] h, w = pad_shape[:2] valid_feat_h = min(int(np.ceil(h / anchor_stride[1])), feat_h) valid_feat_w = min(int(np.ceil(w / anchor_stride[0])), feat_w) flags = self.single_level_valid_flags((feat_h, feat_w), (valid_feat_h, valid_feat_w), self.num_base_anchors[i], device=device) multi_level_flags.append(flags) return multi_level_flags def single_level_valid_flags(self, featmap_size: Tuple[int, int], valid_size: Tuple[int, int], num_base_anchors: int, device: DeviceType = 'cuda') -> Tensor: """Generate the valid flags of anchor in a single feature map. Args: featmap_size (tuple[int]): The size of feature maps, arrange as (h, w). valid_size (tuple[int]): The valid size of the feature maps. num_base_anchors (int): The number of base anchors. device (str | torch.device): Device where the flags will be put on. Defaults to 'cuda'. Returns: torch.Tensor: The valid flags of each anchor in a single level \ feature map. """ feat_h, feat_w = featmap_size valid_h, valid_w = valid_size assert valid_h <= feat_h and valid_w <= feat_w valid_x = torch.zeros(feat_w, dtype=torch.bool, device=device) valid_y = torch.zeros(feat_h, dtype=torch.bool, device=device) valid_x[:valid_w] = 1 valid_y[:valid_h] = 1 valid_xx, valid_yy = self._meshgrid(valid_x, valid_y) valid = valid_xx & valid_yy valid = valid[:, None].expand(valid.size(0), num_base_anchors).contiguous().view(-1) return valid def __repr__(self) -> str: """str: a string that describes the module""" indent_str = ' ' repr_str = self.__class__.__name__ + '(\n' repr_str += f'{indent_str}strides={self.strides},\n' repr_str += f'{indent_str}ratios={self.ratios},\n' repr_str += f'{indent_str}scales={self.scales},\n' repr_str += f'{indent_str}base_sizes={self.base_sizes},\n' repr_str += f'{indent_str}scale_major={self.scale_major},\n' repr_str += f'{indent_str}octave_base_scale=' repr_str += f'{self.octave_base_scale},\n' repr_str += f'{indent_str}scales_per_octave=' repr_str += f'{self.scales_per_octave},\n' repr_str += f'{indent_str}num_levels={self.num_levels}\n' repr_str += f'{indent_str}centers={self.centers},\n' repr_str += f'{indent_str}center_offset={self.center_offset})' return repr_str @TASK_UTILS.register_module() class SSDAnchorGenerator(AnchorGenerator): """Anchor generator for SSD. Args: strides (list[int] | list[tuple[int, int]]): Strides of anchors in multiple feature levels. ratios (list[float]): The list of ratios between the height and width of anchors in a single level. min_sizes (list[float]): The list of minimum anchor sizes on each level. max_sizes (list[float]): The list of maximum anchor sizes on each level. basesize_ratio_range (tuple(float)): Ratio range of anchors. Being used when not setting min_sizes and max_sizes. input_size (int): Size of feature map, 300 for SSD300, 512 for SSD512. Being used when not setting min_sizes and max_sizes. scale_major (bool): Whether to multiply scales first when generating base anchors. If true, the anchors in the same row will have the same scales. It is always set to be False in SSD. use_box_type (bool): Whether to warp anchors with the box type data structure. Defaults to False. """ def __init__(self, strides: Union[List[int], List[Tuple[int, int]]], ratios: List[float], min_sizes: Optional[List[float]] = None, max_sizes: Optional[List[float]] = None, basesize_ratio_range: Tuple[float] = (0.15, 0.9), input_size: int = 300, scale_major: bool = True, use_box_type: bool = False) -> None: assert len(strides) == len(ratios) assert not (min_sizes is None) ^ (max_sizes is None) self.strides = [_pair(stride) for stride in strides] self.centers = [(stride[0] / 2., stride[1] / 2.) for stride in self.strides] if min_sizes is None and max_sizes is None: # use hard code to generate SSD anchors self.input_size = input_size assert is_tuple_of(basesize_ratio_range, float) self.basesize_ratio_range = basesize_ratio_range # calculate anchor ratios and sizes min_ratio, max_ratio = basesize_ratio_range min_ratio = int(min_ratio * 100) max_ratio = int(max_ratio * 100) step = int(np.floor(max_ratio - min_ratio) / (self.num_levels - 2)) min_sizes = [] max_sizes = [] for ratio in range(int(min_ratio), int(max_ratio) + 1, step): min_sizes.append(int(self.input_size * ratio / 100)) max_sizes.append(int(self.input_size * (ratio + step) / 100)) if self.input_size == 300: if basesize_ratio_range[0] == 0.15: # SSD300 COCO min_sizes.insert(0, int(self.input_size * 7 / 100)) max_sizes.insert(0, int(self.input_size * 15 / 100)) elif basesize_ratio_range[0] == 0.2: # SSD300 VOC min_sizes.insert(0, int(self.input_size * 10 / 100)) max_sizes.insert(0, int(self.input_size * 20 / 100)) else: raise ValueError( 'basesize_ratio_range[0] should be either 0.15' 'or 0.2 when input_size is 300, got ' f'{basesize_ratio_range[0]}.') elif self.input_size == 512: if basesize_ratio_range[0] == 0.1: # SSD512 COCO min_sizes.insert(0, int(self.input_size * 4 / 100)) max_sizes.insert(0, int(self.input_size * 10 / 100)) elif basesize_ratio_range[0] == 0.15: # SSD512 VOC min_sizes.insert(0, int(self.input_size * 7 / 100)) max_sizes.insert(0, int(self.input_size * 15 / 100)) else: raise ValueError( 'When not setting min_sizes and max_sizes,' 'basesize_ratio_range[0] should be either 0.1' 'or 0.15 when input_size is 512, got' f' {basesize_ratio_range[0]}.') else: raise ValueError( 'Only support 300 or 512 in SSDAnchorGenerator when ' 'not setting min_sizes and max_sizes, ' f'got {self.input_size}.') assert len(min_sizes) == len(max_sizes) == len(strides) anchor_ratios = [] anchor_scales = [] for k in range(len(self.strides)): scales = [1., np.sqrt(max_sizes[k] / min_sizes[k])] anchor_ratio = [1.] for r in ratios[k]: anchor_ratio += [1 / r, r] # 4 or 6 ratio anchor_ratios.append(torch.Tensor(anchor_ratio)) anchor_scales.append(torch.Tensor(scales)) self.base_sizes = min_sizes self.scales = anchor_scales self.ratios = anchor_ratios self.scale_major = scale_major self.center_offset = 0 self.base_anchors = self.gen_base_anchors() self.use_box_type = use_box_type def gen_base_anchors(self) -> List[Tensor]: """Generate base anchors. Returns: list(torch.Tensor): Base anchors of a feature grid in multiple \ feature levels. """ multi_level_base_anchors = [] for i, base_size in enumerate(self.base_sizes): base_anchors = self.gen_single_level_base_anchors( base_size, scales=self.scales[i], ratios=self.ratios[i], center=self.centers[i]) indices = list(range(len(self.ratios[i]))) indices.insert(1, len(indices)) base_anchors = torch.index_select(base_anchors, 0, torch.LongTensor(indices)) multi_level_base_anchors.append(base_anchors) return multi_level_base_anchors def __repr__(self) -> str: """str: a string that describes the module""" indent_str = ' ' repr_str = self.__class__.__name__ + '(\n' repr_str += f'{indent_str}strides={self.strides},\n' repr_str += f'{indent_str}scales={self.scales},\n' repr_str += f'{indent_str}scale_major={self.scale_major},\n' repr_str += f'{indent_str}input_size={self.input_size},\n' repr_str += f'{indent_str}scales={self.scales},\n' repr_str += f'{indent_str}ratios={self.ratios},\n' repr_str += f'{indent_str}num_levels={self.num_levels},\n' repr_str += f'{indent_str}base_sizes={self.base_sizes},\n' repr_str += f'{indent_str}basesize_ratio_range=' repr_str += f'{self.basesize_ratio_range})' return repr_str @TASK_UTILS.register_module() class LegacyAnchorGenerator(AnchorGenerator): """Legacy anchor generator used in MMDetection V1.x. Note: Difference to the V2.0 anchor generator: 1. The center offset of V1.x anchors are set to be 0.5 rather than 0. 2. The width/height are minused by 1 when calculating the anchors' \ centers and corners to meet the V1.x coordinate system. 3. The anchors' corners are quantized. Args: strides (list[int] | list[tuple[int]]): Strides of anchors in multiple feature levels. ratios (list[float]): The list of ratios between the height and width of anchors in a single level. scales (list[int] | None): Anchor scales for anchors in a single level. It cannot be set at the same time if `octave_base_scale` and `scales_per_octave` are set. base_sizes (list[int]): The basic sizes of anchors in multiple levels. If None is given, strides will be used to generate base_sizes. scale_major (bool): Whether to multiply scales first when generating base anchors. If true, the anchors in the same row will have the same scales. By default it is True in V2.0 octave_base_scale (int): The base scale of octave. scales_per_octave (int): Number of scales for each octave. `octave_base_scale` and `scales_per_octave` are usually used in retinanet and the `scales` should be None when they are set. centers (list[tuple[float, float]] | None): The centers of the anchor relative to the feature grid center in multiple feature levels. By default it is set to be None and not used. It a list of float is given, this list will be used to shift the centers of anchors. center_offset (float): The offset of center in proportion to anchors' width and height. By default it is 0.5 in V2.0 but it should be 0.5 in v1.x models. use_box_type (bool): Whether to warp anchors with the box type data structure. Defaults to False. Examples: >>> from mmdet.models.task_modules. ... prior_generators import LegacyAnchorGenerator >>> self = LegacyAnchorGenerator( >>> [16], [1.], [1.], [9], center_offset=0.5) >>> all_anchors = self.grid_anchors(((2, 2),), device='cpu') >>> print(all_anchors) [tensor([[ 0., 0., 8., 8.], [16., 0., 24., 8.], [ 0., 16., 8., 24.], [16., 16., 24., 24.]])] """ def gen_single_level_base_anchors(self, base_size: Union[int, float], scales: Tensor, ratios: Tensor, center: Optional[Tuple[float]] = None) \ -> Tensor: """Generate base anchors of a single level. Note: The width/height of anchors are minused by 1 when calculating \ the centers and corners to meet the V1.x coordinate system. Args: base_size (int | float): Basic size of an anchor. scales (torch.Tensor): Scales of the anchor. ratios (torch.Tensor): The ratio between the height. and width of anchors in a single level. center (tuple[float], optional): The center of the base anchor related to a single feature grid. Defaults to None. Returns: torch.Tensor: Anchors in a single-level feature map. """ w = base_size h = base_size if center is None: x_center = self.center_offset * (w - 1) y_center = self.center_offset * (h - 1) else: x_center, y_center = center h_ratios = torch.sqrt(ratios) w_ratios = 1 / h_ratios if self.scale_major: ws = (w * w_ratios[:, None] * scales[None, :]).view(-1) hs = (h * h_ratios[:, None] * scales[None, :]).view(-1) else: ws = (w * scales[:, None] * w_ratios[None, :]).view(-1) hs = (h * scales[:, None] * h_ratios[None, :]).view(-1) # use float anchor and the anchor's center is aligned with the # pixel center base_anchors = [ x_center - 0.5 * (ws - 1), y_center - 0.5 * (hs - 1), x_center + 0.5 * (ws - 1), y_center + 0.5 * (hs - 1) ] base_anchors = torch.stack(base_anchors, dim=-1).round() return base_anchors @TASK_UTILS.register_module() class LegacySSDAnchorGenerator(SSDAnchorGenerator, LegacyAnchorGenerator): """Legacy anchor generator used in MMDetection V1.x. The difference between `LegacySSDAnchorGenerator` and `SSDAnchorGenerator` can be found in `LegacyAnchorGenerator`. """ def __init__(self, strides: Union[List[int], List[Tuple[int, int]]], ratios: List[float], basesize_ratio_range: Tuple[float], input_size: int = 300, scale_major: bool = True, use_box_type: bool = False) -> None: super(LegacySSDAnchorGenerator, self).__init__( strides=strides, ratios=ratios, basesize_ratio_range=basesize_ratio_range, input_size=input_size, scale_major=scale_major, use_box_type=use_box_type) self.centers = [((stride - 1) / 2., (stride - 1) / 2.) for stride in strides] self.base_anchors = self.gen_base_anchors() @TASK_UTILS.register_module() class YOLOAnchorGenerator(AnchorGenerator): """Anchor generator for YOLO. Args: strides (list[int] | list[tuple[int, int]]): Strides of anchors in multiple feature levels. base_sizes (list[list[tuple[int, int]]]): The basic sizes of anchors in multiple levels. """ def __init__(self, strides: Union[List[int], List[Tuple[int, int]]], base_sizes: List[List[Tuple[int, int]]], use_box_type: bool = False) -> None: self.strides = [_pair(stride) for stride in strides] self.centers = [(stride[0] / 2., stride[1] / 2.) for stride in self.strides] self.base_sizes = [] num_anchor_per_level = len(base_sizes[0]) for base_sizes_per_level in base_sizes: assert num_anchor_per_level == len(base_sizes_per_level) self.base_sizes.append( [_pair(base_size) for base_size in base_sizes_per_level]) self.base_anchors = self.gen_base_anchors() self.use_box_type = use_box_type @property def num_levels(self) -> int: """int: number of feature levels that the generator will be applied""" return len(self.base_sizes) def gen_base_anchors(self) -> List[Tensor]: """Generate base anchors. Returns: list(torch.Tensor): Base anchors of a feature grid in multiple \ feature levels. """ multi_level_base_anchors = [] for i, base_sizes_per_level in enumerate(self.base_sizes): center = None if self.centers is not None: center = self.centers[i] multi_level_base_anchors.append( self.gen_single_level_base_anchors(base_sizes_per_level, center)) return multi_level_base_anchors def gen_single_level_base_anchors(self, base_sizes_per_level: List[Tuple[int]], center: Optional[Tuple[float]] = None) \ -> Tensor: """Generate base anchors of a single level. Args: base_sizes_per_level (list[tuple[int]]): Basic sizes of anchors. center (tuple[float], optional): The center of the base anchor related to a single feature grid. Defaults to None. Returns: torch.Tensor: Anchors in a single-level feature maps. """ x_center, y_center = center base_anchors = [] for base_size in base_sizes_per_level: w, h = base_size # use float anchor and the anchor's center is aligned with the # pixel center base_anchor = torch.Tensor([ x_center - 0.5 * w, y_center - 0.5 * h, x_center + 0.5 * w, y_center + 0.5 * h ]) base_anchors.append(base_anchor) base_anchors = torch.stack(base_anchors, dim=0) return base_anchors ================================================ FILE: mmdet/models/task_modules/prior_generators/point_generator.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple, Union import numpy as np import torch from torch import Tensor from torch.nn.modules.utils import _pair from mmdet.registry import TASK_UTILS DeviceType = Union[str, torch.device] @TASK_UTILS.register_module() class PointGenerator: def _meshgrid(self, x: Tensor, y: Tensor, row_major: bool = True) -> Tuple[Tensor, Tensor]: """Generate mesh grid of x and y. Args: x (torch.Tensor): Grids of x dimension. y (torch.Tensor): Grids of y dimension. row_major (bool): Whether to return y grids first. Defaults to True. Returns: tuple[torch.Tensor]: The mesh grids of x and y. """ xx = x.repeat(len(y)) yy = y.view(-1, 1).repeat(1, len(x)).view(-1) if row_major: return xx, yy else: return yy, xx def grid_points(self, featmap_size: Tuple[int, int], stride=16, device: DeviceType = 'cuda') -> Tensor: """Generate grid points of a single level. Args: featmap_size (tuple[int, int]): Size of the feature maps. stride (int): The stride of corresponding feature map. device (str | torch.device): The device the tensor will be put on. Defaults to 'cuda'. Returns: torch.Tensor: grid point in a feature map. """ feat_h, feat_w = featmap_size shift_x = torch.arange(0., feat_w, device=device) * stride shift_y = torch.arange(0., feat_h, device=device) * stride shift_xx, shift_yy = self._meshgrid(shift_x, shift_y) stride = shift_x.new_full((shift_xx.shape[0], ), stride) shifts = torch.stack([shift_xx, shift_yy, stride], dim=-1) all_points = shifts.to(device) return all_points def valid_flags(self, featmap_size: Tuple[int, int], valid_size: Tuple[int, int], device: DeviceType = 'cuda') -> Tensor: """Generate valid flags of anchors in a feature map. Args: featmap_sizes (list(tuple[int, int])): List of feature map sizes in multiple feature levels. valid_shape (tuple[int, int]): The valid shape of the image. device (str | torch.device): Device where the anchors will be put on. Return: torch.Tensor: Valid flags of anchors in a level. """ feat_h, feat_w = featmap_size valid_h, valid_w = valid_size assert valid_h <= feat_h and valid_w <= feat_w valid_x = torch.zeros(feat_w, dtype=torch.bool, device=device) valid_y = torch.zeros(feat_h, dtype=torch.bool, device=device) valid_x[:valid_w] = 1 valid_y[:valid_h] = 1 valid_xx, valid_yy = self._meshgrid(valid_x, valid_y) valid = valid_xx & valid_yy return valid @TASK_UTILS.register_module() class MlvlPointGenerator: """Standard points generator for multi-level (Mlvl) feature maps in 2D points-based detectors. Args: strides (list[int] | list[tuple[int, int]]): Strides of anchors in multiple feature levels in order (w, h). offset (float): The offset of points, the value is normalized with corresponding stride. Defaults to 0.5. """ def __init__(self, strides: Union[List[int], List[Tuple[int, int]]], offset: float = 0.5) -> None: self.strides = [_pair(stride) for stride in strides] self.offset = offset @property def num_levels(self) -> int: """int: number of feature levels that the generator will be applied""" return len(self.strides) @property def num_base_priors(self) -> List[int]: """list[int]: The number of priors (points) at a point on the feature grid""" return [1 for _ in range(len(self.strides))] def _meshgrid(self, x: Tensor, y: Tensor, row_major: bool = True) -> Tuple[Tensor, Tensor]: yy, xx = torch.meshgrid(y, x) if row_major: # warning .flatten() would cause error in ONNX exporting # have to use reshape here return xx.reshape(-1), yy.reshape(-1) else: return yy.reshape(-1), xx.reshape(-1) def grid_priors(self, featmap_sizes: List[Tuple], dtype: torch.dtype = torch.float32, device: DeviceType = 'cuda', with_stride: bool = False) -> List[Tensor]: """Generate grid points of multiple feature levels. Args: featmap_sizes (list[tuple]): List of feature map sizes in multiple feature levels, each size arrange as as (h, w). dtype (:obj:`dtype`): Dtype of priors. Defaults to torch.float32. device (str | torch.device): The device where the anchors will be put on. with_stride (bool): Whether to concatenate the stride to the last dimension of points. Return: list[torch.Tensor]: Points of multiple feature levels. The sizes of each tensor should be (N, 2) when with stride is ``False``, where N = width * height, width and height are the sizes of the corresponding feature level, and the last dimension 2 represent (coord_x, coord_y), otherwise the shape should be (N, 4), and the last dimension 4 represent (coord_x, coord_y, stride_w, stride_h). """ assert self.num_levels == len(featmap_sizes) multi_level_priors = [] for i in range(self.num_levels): priors = self.single_level_grid_priors( featmap_sizes[i], level_idx=i, dtype=dtype, device=device, with_stride=with_stride) multi_level_priors.append(priors) return multi_level_priors def single_level_grid_priors(self, featmap_size: Tuple[int], level_idx: int, dtype: torch.dtype = torch.float32, device: DeviceType = 'cuda', with_stride: bool = False) -> Tensor: """Generate grid Points of a single level. Note: This function is usually called by method ``self.grid_priors``. Args: featmap_size (tuple[int]): Size of the feature maps, arrange as (h, w). level_idx (int): The index of corresponding feature map level. dtype (:obj:`dtype`): Dtype of priors. Defaults to torch.float32. device (str | torch.device): The device the tensor will be put on. Defaults to 'cuda'. with_stride (bool): Concatenate the stride to the last dimension of points. Return: Tensor: Points of single feature levels. The shape of tensor should be (N, 2) when with stride is ``False``, where N = width * height, width and height are the sizes of the corresponding feature level, and the last dimension 2 represent (coord_x, coord_y), otherwise the shape should be (N, 4), and the last dimension 4 represent (coord_x, coord_y, stride_w, stride_h). """ feat_h, feat_w = featmap_size stride_w, stride_h = self.strides[level_idx] shift_x = (torch.arange(0, feat_w, device=device) + self.offset) * stride_w # keep featmap_size as Tensor instead of int, so that we # can convert to ONNX correctly shift_x = shift_x.to(dtype) shift_y = (torch.arange(0, feat_h, device=device) + self.offset) * stride_h # keep featmap_size as Tensor instead of int, so that we # can convert to ONNX correctly shift_y = shift_y.to(dtype) shift_xx, shift_yy = self._meshgrid(shift_x, shift_y) if not with_stride: shifts = torch.stack([shift_xx, shift_yy], dim=-1) else: # use `shape[0]` instead of `len(shift_xx)` for ONNX export stride_w = shift_xx.new_full((shift_xx.shape[0], ), stride_w).to(dtype) stride_h = shift_xx.new_full((shift_yy.shape[0], ), stride_h).to(dtype) shifts = torch.stack([shift_xx, shift_yy, stride_w, stride_h], dim=-1) all_points = shifts.to(device) return all_points def valid_flags(self, featmap_sizes: List[Tuple[int, int]], pad_shape: Tuple[int], device: DeviceType = 'cuda') -> List[Tensor]: """Generate valid flags of points of multiple feature levels. Args: featmap_sizes (list(tuple)): List of feature map sizes in multiple feature levels, each size arrange as as (h, w). pad_shape (tuple(int)): The padded shape of the image, arrange as (h, w). device (str | torch.device): The device where the anchors will be put on. Return: list(torch.Tensor): Valid flags of points of multiple levels. """ assert self.num_levels == len(featmap_sizes) multi_level_flags = [] for i in range(self.num_levels): point_stride = self.strides[i] feat_h, feat_w = featmap_sizes[i] h, w = pad_shape[:2] valid_feat_h = min(int(np.ceil(h / point_stride[1])), feat_h) valid_feat_w = min(int(np.ceil(w / point_stride[0])), feat_w) flags = self.single_level_valid_flags((feat_h, feat_w), (valid_feat_h, valid_feat_w), device=device) multi_level_flags.append(flags) return multi_level_flags def single_level_valid_flags(self, featmap_size: Tuple[int, int], valid_size: Tuple[int, int], device: DeviceType = 'cuda') -> Tensor: """Generate the valid flags of points of a single feature map. Args: featmap_size (tuple[int]): The size of feature maps, arrange as as (h, w). valid_size (tuple[int]): The valid size of the feature maps. The size arrange as as (h, w). device (str | torch.device): The device where the flags will be put on. Defaults to 'cuda'. Returns: torch.Tensor: The valid flags of each points in a single level \ feature map. """ feat_h, feat_w = featmap_size valid_h, valid_w = valid_size assert valid_h <= feat_h and valid_w <= feat_w valid_x = torch.zeros(feat_w, dtype=torch.bool, device=device) valid_y = torch.zeros(feat_h, dtype=torch.bool, device=device) valid_x[:valid_w] = 1 valid_y[:valid_h] = 1 valid_xx, valid_yy = self._meshgrid(valid_x, valid_y) valid = valid_xx & valid_yy return valid def sparse_priors(self, prior_idxs: Tensor, featmap_size: Tuple[int], level_idx: int, dtype: torch.dtype = torch.float32, device: DeviceType = 'cuda') -> Tensor: """Generate sparse points according to the ``prior_idxs``. Args: prior_idxs (Tensor): The index of corresponding anchors in the feature map. featmap_size (tuple[int]): feature map size arrange as (w, h). level_idx (int): The level index of corresponding feature map. dtype (obj:`torch.dtype`): Date type of points. Defaults to ``torch.float32``. device (str | torch.device): The device where the points is located. Returns: Tensor: Anchor with shape (N, 2), N should be equal to the length of ``prior_idxs``. And last dimension 2 represent (coord_x, coord_y). """ height, width = featmap_size x = (prior_idxs % width + self.offset) * self.strides[level_idx][0] y = ((prior_idxs // width) % height + self.offset) * self.strides[level_idx][1] prioris = torch.stack([x, y], 1).to(dtype) prioris = prioris.to(device) return prioris ================================================ FILE: mmdet/models/task_modules/prior_generators/utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple import torch from torch import Tensor from mmdet.structures.bbox import BaseBoxes def anchor_inside_flags(flat_anchors: Tensor, valid_flags: Tensor, img_shape: Tuple[int], allowed_border: int = 0) -> Tensor: """Check whether the anchors are inside the border. Args: flat_anchors (torch.Tensor): Flatten anchors, shape (n, 4). valid_flags (torch.Tensor): An existing valid flags of anchors. img_shape (tuple(int)): Shape of current image. allowed_border (int): The border to allow the valid anchor. Defaults to 0. Returns: torch.Tensor: Flags indicating whether the anchors are inside a \ valid range. """ img_h, img_w = img_shape[:2] if allowed_border >= 0: if isinstance(flat_anchors, BaseBoxes): inside_flags = valid_flags & \ flat_anchors.is_inside([img_h, img_w], all_inside=True, allowed_border=allowed_border) else: inside_flags = valid_flags & \ (flat_anchors[:, 0] >= -allowed_border) & \ (flat_anchors[:, 1] >= -allowed_border) & \ (flat_anchors[:, 2] < img_w + allowed_border) & \ (flat_anchors[:, 3] < img_h + allowed_border) else: inside_flags = valid_flags return inside_flags def calc_region(bbox: Tensor, ratio: float, featmap_size: Optional[Tuple] = None) -> Tuple[int]: """Calculate a proportional bbox region. The bbox center are fixed and the new h' and w' is h * ratio and w * ratio. Args: bbox (Tensor): Bboxes to calculate regions, shape (n, 4). ratio (float): Ratio of the output region. featmap_size (tuple, Optional): Feature map size in (height, width) order used for clipping the boundary. Defaults to None. Returns: tuple: x1, y1, x2, y2 """ x1 = torch.round((1 - ratio) * bbox[0] + ratio * bbox[2]).long() y1 = torch.round((1 - ratio) * bbox[1] + ratio * bbox[3]).long() x2 = torch.round(ratio * bbox[0] + (1 - ratio) * bbox[2]).long() y2 = torch.round(ratio * bbox[1] + (1 - ratio) * bbox[3]).long() if featmap_size is not None: x1 = x1.clamp(min=0, max=featmap_size[1]) y1 = y1.clamp(min=0, max=featmap_size[0]) x2 = x2.clamp(min=0, max=featmap_size[1]) y2 = y2.clamp(min=0, max=featmap_size[0]) return (x1, y1, x2, y2) ================================================ FILE: mmdet/models/task_modules/samplers/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .base_sampler import BaseSampler from .combined_sampler import CombinedSampler from .instance_balanced_pos_sampler import InstanceBalancedPosSampler from .iou_balanced_neg_sampler import IoUBalancedNegSampler from .mask_pseudo_sampler import MaskPseudoSampler from .mask_sampling_result import MaskSamplingResult from .multi_instance_random_sampler import MultiInsRandomSampler from .multi_instance_sampling_result import MultiInstanceSamplingResult from .ohem_sampler import OHEMSampler from .pseudo_sampler import PseudoSampler from .random_sampler import RandomSampler from .sampling_result import SamplingResult from .score_hlr_sampler import ScoreHLRSampler __all__ = [ 'BaseSampler', 'PseudoSampler', 'RandomSampler', 'InstanceBalancedPosSampler', 'IoUBalancedNegSampler', 'CombinedSampler', 'OHEMSampler', 'SamplingResult', 'ScoreHLRSampler', 'MaskPseudoSampler', 'MaskSamplingResult', 'MultiInstanceSamplingResult', 'MultiInsRandomSampler' ] ================================================ FILE: mmdet/models/task_modules/samplers/base_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod import torch from mmengine.structures import InstanceData from mmdet.structures.bbox import BaseBoxes, cat_boxes from ..assigners import AssignResult from .sampling_result import SamplingResult class BaseSampler(metaclass=ABCMeta): """Base class of samplers. Args: num (int): Number of samples pos_fraction (float): Fraction of positive samples neg_pos_up (int): Upper bound number of negative and positive samples. Defaults to -1. add_gt_as_proposals (bool): Whether to add ground truth boxes as proposals. Defaults to True. """ def __init__(self, num: int, pos_fraction: float, neg_pos_ub: int = -1, add_gt_as_proposals: bool = True, **kwargs) -> None: self.num = num self.pos_fraction = pos_fraction self.neg_pos_ub = neg_pos_ub self.add_gt_as_proposals = add_gt_as_proposals self.pos_sampler = self self.neg_sampler = self @abstractmethod def _sample_pos(self, assign_result: AssignResult, num_expected: int, **kwargs): """Sample positive samples.""" pass @abstractmethod def _sample_neg(self, assign_result: AssignResult, num_expected: int, **kwargs): """Sample negative samples.""" pass def sample(self, assign_result: AssignResult, pred_instances: InstanceData, gt_instances: InstanceData, **kwargs) -> SamplingResult: """Sample positive and negative bboxes. This is a simple implementation of bbox sampling given candidates, assigning results and ground truth bboxes. Args: assign_result (:obj:`AssignResult`): Assigning results. pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). Returns: :obj:`SamplingResult`: Sampling result. Example: >>> from mmengine.structures import InstanceData >>> from mmdet.models.task_modules.samplers import RandomSampler, >>> from mmdet.models.task_modules.assigners import AssignResult >>> from mmdet.models.task_modules.samplers. ... sampling_result import ensure_rng, random_boxes >>> rng = ensure_rng(None) >>> assign_result = AssignResult.random(rng=rng) >>> pred_instances = InstanceData() >>> pred_instances.priors = random_boxes(assign_result.num_preds, ... rng=rng) >>> gt_instances = InstanceData() >>> gt_instances.bboxes = random_boxes(assign_result.num_gts, ... rng=rng) >>> gt_instances.labels = torch.randint( ... 0, 5, (assign_result.num_gts,), dtype=torch.long) >>> self = RandomSampler(num=32, pos_fraction=0.5, neg_pos_ub=-1, >>> add_gt_as_proposals=False) >>> self = self.sample(assign_result, pred_instances, gt_instances) """ gt_bboxes = gt_instances.bboxes priors = pred_instances.priors gt_labels = gt_instances.labels if len(priors.shape) < 2: priors = priors[None, :] gt_flags = priors.new_zeros((priors.shape[0], ), dtype=torch.uint8) if self.add_gt_as_proposals and len(gt_bboxes) > 0: # When `gt_bboxes` and `priors` are all box type, convert # `gt_bboxes` type to `priors` type. if (isinstance(gt_bboxes, BaseBoxes) and isinstance(priors, BaseBoxes)): gt_bboxes_ = gt_bboxes.convert_to(type(priors)) else: gt_bboxes_ = gt_bboxes priors = cat_boxes([gt_bboxes_, priors], dim=0) assign_result.add_gt_(gt_labels) gt_ones = priors.new_ones(gt_bboxes_.shape[0], dtype=torch.uint8) gt_flags = torch.cat([gt_ones, gt_flags]) num_expected_pos = int(self.num * self.pos_fraction) pos_inds = self.pos_sampler._sample_pos( assign_result, num_expected_pos, bboxes=priors, **kwargs) # We found that sampled indices have duplicated items occasionally. # (may be a bug of PyTorch) pos_inds = pos_inds.unique() num_sampled_pos = pos_inds.numel() num_expected_neg = self.num - num_sampled_pos if self.neg_pos_ub >= 0: _pos = max(1, num_sampled_pos) neg_upper_bound = int(self.neg_pos_ub * _pos) if num_expected_neg > neg_upper_bound: num_expected_neg = neg_upper_bound neg_inds = self.neg_sampler._sample_neg( assign_result, num_expected_neg, bboxes=priors, **kwargs) neg_inds = neg_inds.unique() sampling_result = SamplingResult( pos_inds=pos_inds, neg_inds=neg_inds, priors=priors, gt_bboxes=gt_bboxes, assign_result=assign_result, gt_flags=gt_flags) return sampling_result ================================================ FILE: mmdet/models/task_modules/samplers/combined_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.registry import TASK_UTILS from .base_sampler import BaseSampler @TASK_UTILS.register_module() class CombinedSampler(BaseSampler): """A sampler that combines positive sampler and negative sampler.""" def __init__(self, pos_sampler, neg_sampler, **kwargs): super(CombinedSampler, self).__init__(**kwargs) self.pos_sampler = TASK_UTILS.build(pos_sampler, default_args=kwargs) self.neg_sampler = TASK_UTILS.build(neg_sampler, default_args=kwargs) def _sample_pos(self, **kwargs): """Sample positive samples.""" raise NotImplementedError def _sample_neg(self, **kwargs): """Sample negative samples.""" raise NotImplementedError ================================================ FILE: mmdet/models/task_modules/samplers/instance_balanced_pos_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from mmdet.registry import TASK_UTILS from .random_sampler import RandomSampler @TASK_UTILS.register_module() class InstanceBalancedPosSampler(RandomSampler): """Instance balanced sampler that samples equal number of positive samples for each instance.""" def _sample_pos(self, assign_result, num_expected, **kwargs): """Sample positive boxes. Args: assign_result (:obj:`AssignResult`): The assigned results of boxes. num_expected (int): The number of expected positive samples Returns: Tensor or ndarray: sampled indices. """ pos_inds = torch.nonzero(assign_result.gt_inds > 0, as_tuple=False) if pos_inds.numel() != 0: pos_inds = pos_inds.squeeze(1) if pos_inds.numel() <= num_expected: return pos_inds else: unique_gt_inds = assign_result.gt_inds[pos_inds].unique() num_gts = len(unique_gt_inds) num_per_gt = int(round(num_expected / float(num_gts)) + 1) sampled_inds = [] for i in unique_gt_inds: inds = torch.nonzero( assign_result.gt_inds == i.item(), as_tuple=False) if inds.numel() != 0: inds = inds.squeeze(1) else: continue if len(inds) > num_per_gt: inds = self.random_choice(inds, num_per_gt) sampled_inds.append(inds) sampled_inds = torch.cat(sampled_inds) if len(sampled_inds) < num_expected: num_extra = num_expected - len(sampled_inds) extra_inds = np.array( list(set(pos_inds.cpu()) - set(sampled_inds.cpu()))) if len(extra_inds) > num_extra: extra_inds = self.random_choice(extra_inds, num_extra) extra_inds = torch.from_numpy(extra_inds).to( assign_result.gt_inds.device).long() sampled_inds = torch.cat([sampled_inds, extra_inds]) elif len(sampled_inds) > num_expected: sampled_inds = self.random_choice(sampled_inds, num_expected) return sampled_inds ================================================ FILE: mmdet/models/task_modules/samplers/iou_balanced_neg_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from mmdet.registry import TASK_UTILS from .random_sampler import RandomSampler @TASK_UTILS.register_module() class IoUBalancedNegSampler(RandomSampler): """IoU Balanced Sampling. arXiv: https://arxiv.org/pdf/1904.02701.pdf (CVPR 2019) Sampling proposals according to their IoU. `floor_fraction` of needed RoIs are sampled from proposals whose IoU are lower than `floor_thr` randomly. The others are sampled from proposals whose IoU are higher than `floor_thr`. These proposals are sampled from some bins evenly, which are split by `num_bins` via IoU evenly. Args: num (int): number of proposals. pos_fraction (float): fraction of positive proposals. floor_thr (float): threshold (minimum) IoU for IoU balanced sampling, set to -1 if all using IoU balanced sampling. floor_fraction (float): sampling fraction of proposals under floor_thr. num_bins (int): number of bins in IoU balanced sampling. """ def __init__(self, num, pos_fraction, floor_thr=-1, floor_fraction=0, num_bins=3, **kwargs): super(IoUBalancedNegSampler, self).__init__(num, pos_fraction, **kwargs) assert floor_thr >= 0 or floor_thr == -1 assert 0 <= floor_fraction <= 1 assert num_bins >= 1 self.floor_thr = floor_thr self.floor_fraction = floor_fraction self.num_bins = num_bins def sample_via_interval(self, max_overlaps, full_set, num_expected): """Sample according to the iou interval. Args: max_overlaps (torch.Tensor): IoU between bounding boxes and ground truth boxes. full_set (set(int)): A full set of indices of boxes。 num_expected (int): Number of expected samples。 Returns: np.ndarray: Indices of samples """ max_iou = max_overlaps.max() iou_interval = (max_iou - self.floor_thr) / self.num_bins per_num_expected = int(num_expected / self.num_bins) sampled_inds = [] for i in range(self.num_bins): start_iou = self.floor_thr + i * iou_interval end_iou = self.floor_thr + (i + 1) * iou_interval tmp_set = set( np.where( np.logical_and(max_overlaps >= start_iou, max_overlaps < end_iou))[0]) tmp_inds = list(tmp_set & full_set) if len(tmp_inds) > per_num_expected: tmp_sampled_set = self.random_choice(tmp_inds, per_num_expected) else: tmp_sampled_set = np.array(tmp_inds, dtype=np.int64) sampled_inds.append(tmp_sampled_set) sampled_inds = np.concatenate(sampled_inds) if len(sampled_inds) < num_expected: num_extra = num_expected - len(sampled_inds) extra_inds = np.array(list(full_set - set(sampled_inds))) if len(extra_inds) > num_extra: extra_inds = self.random_choice(extra_inds, num_extra) sampled_inds = np.concatenate([sampled_inds, extra_inds]) return sampled_inds def _sample_neg(self, assign_result, num_expected, **kwargs): """Sample negative boxes. Args: assign_result (:obj:`AssignResult`): The assigned results of boxes. num_expected (int): The number of expected negative samples Returns: Tensor or ndarray: sampled indices. """ neg_inds = torch.nonzero(assign_result.gt_inds == 0, as_tuple=False) if neg_inds.numel() != 0: neg_inds = neg_inds.squeeze(1) if len(neg_inds) <= num_expected: return neg_inds else: max_overlaps = assign_result.max_overlaps.cpu().numpy() # balance sampling for negative samples neg_set = set(neg_inds.cpu().numpy()) if self.floor_thr > 0: floor_set = set( np.where( np.logical_and(max_overlaps >= 0, max_overlaps < self.floor_thr))[0]) iou_sampling_set = set( np.where(max_overlaps >= self.floor_thr)[0]) elif self.floor_thr == 0: floor_set = set(np.where(max_overlaps == 0)[0]) iou_sampling_set = set( np.where(max_overlaps > self.floor_thr)[0]) else: floor_set = set() iou_sampling_set = set( np.where(max_overlaps > self.floor_thr)[0]) # for sampling interval calculation self.floor_thr = 0 floor_neg_inds = list(floor_set & neg_set) iou_sampling_neg_inds = list(iou_sampling_set & neg_set) num_expected_iou_sampling = int(num_expected * (1 - self.floor_fraction)) if len(iou_sampling_neg_inds) > num_expected_iou_sampling: if self.num_bins >= 2: iou_sampled_inds = self.sample_via_interval( max_overlaps, set(iou_sampling_neg_inds), num_expected_iou_sampling) else: iou_sampled_inds = self.random_choice( iou_sampling_neg_inds, num_expected_iou_sampling) else: iou_sampled_inds = np.array( iou_sampling_neg_inds, dtype=np.int64) num_expected_floor = num_expected - len(iou_sampled_inds) if len(floor_neg_inds) > num_expected_floor: sampled_floor_inds = self.random_choice( floor_neg_inds, num_expected_floor) else: sampled_floor_inds = np.array(floor_neg_inds, dtype=np.int64) sampled_inds = np.concatenate( (sampled_floor_inds, iou_sampled_inds)) if len(sampled_inds) < num_expected: num_extra = num_expected - len(sampled_inds) extra_inds = np.array(list(neg_set - set(sampled_inds))) if len(extra_inds) > num_extra: extra_inds = self.random_choice(extra_inds, num_extra) sampled_inds = np.concatenate((sampled_inds, extra_inds)) sampled_inds = torch.from_numpy(sampled_inds).long().to( assign_result.gt_inds.device) return sampled_inds ================================================ FILE: mmdet/models/task_modules/samplers/mask_pseudo_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """copy from https://github.com/ZwwWayne/K-Net/blob/main/knet/det/mask_pseudo_sampler.py.""" import torch from mmengine.structures import InstanceData from mmdet.registry import TASK_UTILS from ..assigners import AssignResult from .base_sampler import BaseSampler from .mask_sampling_result import MaskSamplingResult @TASK_UTILS.register_module() class MaskPseudoSampler(BaseSampler): """A pseudo sampler that does not do sampling actually.""" def __init__(self, **kwargs): pass def _sample_pos(self, **kwargs): """Sample positive samples.""" raise NotImplementedError def _sample_neg(self, **kwargs): """Sample negative samples.""" raise NotImplementedError def sample(self, assign_result: AssignResult, pred_instances: InstanceData, gt_instances: InstanceData, *args, **kwargs): """Directly returns the positive and negative indices of samples. Args: assign_result (:obj:`AssignResult`): Mask assigning results. pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``scores`` and ``masks`` predicted by the model. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``labels`` and ``masks`` attributes. Returns: :obj:`SamplingResult`: sampler results """ pred_masks = pred_instances.masks gt_masks = gt_instances.masks pos_inds = torch.nonzero( assign_result.gt_inds > 0, as_tuple=False).squeeze(-1).unique() neg_inds = torch.nonzero( assign_result.gt_inds == 0, as_tuple=False).squeeze(-1).unique() gt_flags = pred_masks.new_zeros(pred_masks.shape[0], dtype=torch.uint8) sampling_result = MaskSamplingResult( pos_inds=pos_inds, neg_inds=neg_inds, masks=pred_masks, gt_masks=gt_masks, assign_result=assign_result, gt_flags=gt_flags, avg_factor_with_neg=False) return sampling_result ================================================ FILE: mmdet/models/task_modules/samplers/mask_sampling_result.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """copy from https://github.com/ZwwWayne/K-Net/blob/main/knet/det/mask_pseudo_sampler.py.""" import torch from torch import Tensor from ..assigners import AssignResult from .sampling_result import SamplingResult class MaskSamplingResult(SamplingResult): """Mask sampling result.""" def __init__(self, pos_inds: Tensor, neg_inds: Tensor, masks: Tensor, gt_masks: Tensor, assign_result: AssignResult, gt_flags: Tensor, avg_factor_with_neg: bool = True) -> None: self.pos_inds = pos_inds self.neg_inds = neg_inds self.num_pos = max(pos_inds.numel(), 1) self.num_neg = max(neg_inds.numel(), 1) self.avg_factor = self.num_pos + self.num_neg \ if avg_factor_with_neg else self.num_pos self.pos_masks = masks[pos_inds] self.neg_masks = masks[neg_inds] self.pos_is_gt = gt_flags[pos_inds] self.num_gts = gt_masks.shape[0] self.pos_assigned_gt_inds = assign_result.gt_inds[pos_inds] - 1 if gt_masks.numel() == 0: # hack for index error case assert self.pos_assigned_gt_inds.numel() == 0 self.pos_gt_masks = torch.empty_like(gt_masks) else: self.pos_gt_masks = gt_masks[self.pos_assigned_gt_inds, :] @property def masks(self) -> Tensor: """torch.Tensor: concatenated positive and negative masks.""" return torch.cat([self.pos_masks, self.neg_masks]) def __nice__(self) -> str: data = self.info.copy() data['pos_masks'] = data.pop('pos_masks').shape data['neg_masks'] = data.pop('neg_masks').shape parts = [f"'{k}': {v!r}" for k, v in sorted(data.items())] body = ' ' + ',\n '.join(parts) return '{\n' + body + '\n}' @property def info(self) -> dict: """Returns a dictionary of info about the object.""" return { 'pos_inds': self.pos_inds, 'neg_inds': self.neg_inds, 'pos_masks': self.pos_masks, 'neg_masks': self.neg_masks, 'pos_is_gt': self.pos_is_gt, 'num_gts': self.num_gts, 'pos_assigned_gt_inds': self.pos_assigned_gt_inds, } ================================================ FILE: mmdet/models/task_modules/samplers/multi_instance_random_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Union import torch from mmengine.structures import InstanceData from numpy import ndarray from torch import Tensor from mmdet.registry import TASK_UTILS from ..assigners import AssignResult from .multi_instance_sampling_result import MultiInstanceSamplingResult from .random_sampler import RandomSampler @TASK_UTILS.register_module() class MultiInsRandomSampler(RandomSampler): """Random sampler for multi instance. Note: Multi-instance means to predict multiple detection boxes with one proposal box. `AssignResult` may assign multiple gt boxes to each proposal box, in this case `RandomSampler` should be replaced by `MultiInsRandomSampler` """ def _sample_pos(self, assign_result: AssignResult, num_expected: int, **kwargs) -> Union[Tensor, ndarray]: """Randomly sample some positive samples. Args: assign_result (:obj:`AssignResult`): Bbox assigning results. num_expected (int): The number of expected positive samples Returns: Tensor or ndarray: sampled indices. """ pos_inds = torch.nonzero( assign_result.labels[:, 0] > 0, as_tuple=False) if pos_inds.numel() != 0: pos_inds = pos_inds.squeeze(1) if pos_inds.numel() <= num_expected: return pos_inds else: return self.random_choice(pos_inds, num_expected) def _sample_neg(self, assign_result: AssignResult, num_expected: int, **kwargs) -> Union[Tensor, ndarray]: """Randomly sample some negative samples. Args: assign_result (:obj:`AssignResult`): Bbox assigning results. num_expected (int): The number of expected positive samples Returns: Tensor or ndarray: sampled indices. """ neg_inds = torch.nonzero( assign_result.labels[:, 0] == 0, as_tuple=False) if neg_inds.numel() != 0: neg_inds = neg_inds.squeeze(1) if len(neg_inds) <= num_expected: return neg_inds else: return self.random_choice(neg_inds, num_expected) def sample(self, assign_result: AssignResult, pred_instances: InstanceData, gt_instances: InstanceData, **kwargs) -> MultiInstanceSamplingResult: """Sample positive and negative bboxes. Args: assign_result (:obj:`AssignResult`): Assigning results from MultiInstanceAssigner. pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). Returns: :obj:`MultiInstanceSamplingResult`: Sampling result. """ assert 'batch_gt_instances_ignore' in kwargs, \ 'batch_gt_instances_ignore is necessary for MultiInsRandomSampler' gt_bboxes = gt_instances.bboxes ignore_bboxes = kwargs['batch_gt_instances_ignore'].bboxes gt_and_ignore_bboxes = torch.cat([gt_bboxes, ignore_bboxes], dim=0) priors = pred_instances.priors if len(priors.shape) < 2: priors = priors[None, :] priors = priors[:, :4] gt_flags = priors.new_zeros((priors.shape[0], ), dtype=torch.uint8) priors = torch.cat([priors, gt_and_ignore_bboxes], dim=0) gt_ones = priors.new_ones( gt_and_ignore_bboxes.shape[0], dtype=torch.uint8) gt_flags = torch.cat([gt_flags, gt_ones]) num_expected_pos = int(self.num * self.pos_fraction) pos_inds = self.pos_sampler._sample_pos(assign_result, num_expected_pos) # We found that sampled indices have duplicated items occasionally. # (may be a bug of PyTorch) pos_inds = pos_inds.unique() num_sampled_pos = pos_inds.numel() num_expected_neg = self.num - num_sampled_pos if self.neg_pos_ub >= 0: _pos = max(1, num_sampled_pos) neg_upper_bound = int(self.neg_pos_ub * _pos) if num_expected_neg > neg_upper_bound: num_expected_neg = neg_upper_bound neg_inds = self.neg_sampler._sample_neg(assign_result, num_expected_neg) neg_inds = neg_inds.unique() sampling_result = MultiInstanceSamplingResult( pos_inds=pos_inds, neg_inds=neg_inds, priors=priors, gt_and_ignore_bboxes=gt_and_ignore_bboxes, assign_result=assign_result, gt_flags=gt_flags) return sampling_result ================================================ FILE: mmdet/models/task_modules/samplers/multi_instance_sampling_result.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch from torch import Tensor from ..assigners import AssignResult from .sampling_result import SamplingResult class MultiInstanceSamplingResult(SamplingResult): """Bbox sampling result. Further encapsulation of SamplingResult. Three attributes neg_assigned_gt_inds, neg_gt_labels, and neg_gt_bboxes have been added for SamplingResult. Args: pos_inds (Tensor): Indices of positive samples. neg_inds (Tensor): Indices of negative samples. priors (Tensor): The priors can be anchors or points, or the bboxes predicted by the previous stage. gt_and_ignore_bboxes (Tensor): Ground truth and ignore bboxes. assign_result (:obj:`AssignResult`): Assigning results. gt_flags (Tensor): The Ground truth flags. avg_factor_with_neg (bool): If True, ``avg_factor`` equal to the number of total priors; Otherwise, it is the number of positive priors. Defaults to True. """ def __init__(self, pos_inds: Tensor, neg_inds: Tensor, priors: Tensor, gt_and_ignore_bboxes: Tensor, assign_result: AssignResult, gt_flags: Tensor, avg_factor_with_neg: bool = True) -> None: self.neg_assigned_gt_inds = assign_result.gt_inds[neg_inds] self.neg_gt_labels = assign_result.labels[neg_inds] if gt_and_ignore_bboxes.numel() == 0: self.neg_gt_bboxes = torch.empty_like(gt_and_ignore_bboxes).view( -1, 4) else: if len(gt_and_ignore_bboxes.shape) < 2: gt_and_ignore_bboxes = gt_and_ignore_bboxes.view(-1, 4) self.neg_gt_bboxes = gt_and_ignore_bboxes[ self.neg_assigned_gt_inds.long(), :] # To resist the minus 1 operation in `SamplingResult.init()`. assign_result.gt_inds += 1 super().__init__( pos_inds=pos_inds, neg_inds=neg_inds, priors=priors, gt_bboxes=gt_and_ignore_bboxes, assign_result=assign_result, gt_flags=gt_flags, avg_factor_with_neg=avg_factor_with_neg) ================================================ FILE: mmdet/models/task_modules/samplers/ohem_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import bbox2roi from .base_sampler import BaseSampler @TASK_UTILS.register_module() class OHEMSampler(BaseSampler): r"""Online Hard Example Mining Sampler described in `Training Region-based Object Detectors with Online Hard Example Mining `_. """ def __init__(self, num, pos_fraction, context, neg_pos_ub=-1, add_gt_as_proposals=True, loss_key='loss_cls', **kwargs): super(OHEMSampler, self).__init__(num, pos_fraction, neg_pos_ub, add_gt_as_proposals) self.context = context if not hasattr(self.context, 'num_stages'): self.bbox_head = self.context.bbox_head else: self.bbox_head = self.context.bbox_head[self.context.current_stage] self.loss_key = loss_key def hard_mining(self, inds, num_expected, bboxes, labels, feats): with torch.no_grad(): rois = bbox2roi([bboxes]) if not hasattr(self.context, 'num_stages'): bbox_results = self.context._bbox_forward(feats, rois) else: bbox_results = self.context._bbox_forward( self.context.current_stage, feats, rois) cls_score = bbox_results['cls_score'] loss = self.bbox_head.loss( cls_score=cls_score, bbox_pred=None, rois=rois, labels=labels, label_weights=cls_score.new_ones(cls_score.size(0)), bbox_targets=None, bbox_weights=None, reduction_override='none')[self.loss_key] _, topk_loss_inds = loss.topk(num_expected) return inds[topk_loss_inds] def _sample_pos(self, assign_result, num_expected, bboxes=None, feats=None, **kwargs): """Sample positive boxes. Args: assign_result (:obj:`AssignResult`): Assigned results num_expected (int): Number of expected positive samples bboxes (torch.Tensor, optional): Boxes. Defaults to None. feats (list[torch.Tensor], optional): Multi-level features. Defaults to None. Returns: torch.Tensor: Indices of positive samples """ # Sample some hard positive samples pos_inds = torch.nonzero(assign_result.gt_inds > 0, as_tuple=False) if pos_inds.numel() != 0: pos_inds = pos_inds.squeeze(1) if pos_inds.numel() <= num_expected: return pos_inds else: return self.hard_mining(pos_inds, num_expected, bboxes[pos_inds], assign_result.labels[pos_inds], feats) def _sample_neg(self, assign_result, num_expected, bboxes=None, feats=None, **kwargs): """Sample negative boxes. Args: assign_result (:obj:`AssignResult`): Assigned results num_expected (int): Number of expected negative samples bboxes (torch.Tensor, optional): Boxes. Defaults to None. feats (list[torch.Tensor], optional): Multi-level features. Defaults to None. Returns: torch.Tensor: Indices of negative samples """ # Sample some hard negative samples neg_inds = torch.nonzero(assign_result.gt_inds == 0, as_tuple=False) if neg_inds.numel() != 0: neg_inds = neg_inds.squeeze(1) if len(neg_inds) <= num_expected: return neg_inds else: neg_labels = assign_result.labels.new_empty( neg_inds.size(0)).fill_(self.bbox_head.num_classes) return self.hard_mining(neg_inds, num_expected, bboxes[neg_inds], neg_labels, feats) ================================================ FILE: mmdet/models/task_modules/samplers/pseudo_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch from mmengine.structures import InstanceData from mmdet.registry import TASK_UTILS from ..assigners import AssignResult from .base_sampler import BaseSampler from .sampling_result import SamplingResult @TASK_UTILS.register_module() class PseudoSampler(BaseSampler): """A pseudo sampler that does not do sampling actually.""" def __init__(self, **kwargs): pass def _sample_pos(self, **kwargs): """Sample positive samples.""" raise NotImplementedError def _sample_neg(self, **kwargs): """Sample negative samples.""" raise NotImplementedError def sample(self, assign_result: AssignResult, pred_instances: InstanceData, gt_instances: InstanceData, *args, **kwargs): """Directly returns the positive and negative indices of samples. Args: assign_result (:obj:`AssignResult`): Bbox assigning results. pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors, points, or bboxes predicted by the model, shape(n, 4). gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes`` and ``labels`` attributes. Returns: :obj:`SamplingResult`: sampler results """ gt_bboxes = gt_instances.bboxes priors = pred_instances.priors pos_inds = torch.nonzero( assign_result.gt_inds > 0, as_tuple=False).squeeze(-1).unique() neg_inds = torch.nonzero( assign_result.gt_inds == 0, as_tuple=False).squeeze(-1).unique() gt_flags = priors.new_zeros(priors.shape[0], dtype=torch.uint8) sampling_result = SamplingResult( pos_inds=pos_inds, neg_inds=neg_inds, priors=priors, gt_bboxes=gt_bboxes, assign_result=assign_result, gt_flags=gt_flags, avg_factor_with_neg=False) return sampling_result ================================================ FILE: mmdet/models/task_modules/samplers/random_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Union import torch from numpy import ndarray from torch import Tensor from mmdet.registry import TASK_UTILS from ..assigners import AssignResult from .base_sampler import BaseSampler @TASK_UTILS.register_module() class RandomSampler(BaseSampler): """Random sampler. Args: num (int): Number of samples pos_fraction (float): Fraction of positive samples neg_pos_up (int): Upper bound number of negative and positive samples. Defaults to -1. add_gt_as_proposals (bool): Whether to add ground truth boxes as proposals. Defaults to True. """ def __init__(self, num: int, pos_fraction: float, neg_pos_ub: int = -1, add_gt_as_proposals: bool = True, **kwargs): from .sampling_result import ensure_rng super().__init__( num=num, pos_fraction=pos_fraction, neg_pos_ub=neg_pos_ub, add_gt_as_proposals=add_gt_as_proposals) self.rng = ensure_rng(kwargs.get('rng', None)) def random_choice(self, gallery: Union[Tensor, ndarray, list], num: int) -> Union[Tensor, ndarray]: """Random select some elements from the gallery. If `gallery` is a Tensor, the returned indices will be a Tensor; If `gallery` is a ndarray or list, the returned indices will be a ndarray. Args: gallery (Tensor | ndarray | list): indices pool. num (int): expected sample num. Returns: Tensor or ndarray: sampled indices. """ assert len(gallery) >= num is_tensor = isinstance(gallery, torch.Tensor) if not is_tensor: if torch.cuda.is_available(): device = torch.cuda.current_device() else: device = 'cpu' gallery = torch.tensor(gallery, dtype=torch.long, device=device) # This is a temporary fix. We can revert the following code # when PyTorch fixes the abnormal return of torch.randperm. # See: https://github.com/open-mmlab/mmdetection/pull/5014 perm = torch.randperm(gallery.numel())[:num].to(device=gallery.device) rand_inds = gallery[perm] if not is_tensor: rand_inds = rand_inds.cpu().numpy() return rand_inds def _sample_pos(self, assign_result: AssignResult, num_expected: int, **kwargs) -> Union[Tensor, ndarray]: """Randomly sample some positive samples. Args: assign_result (:obj:`AssignResult`): Bbox assigning results. num_expected (int): The number of expected positive samples Returns: Tensor or ndarray: sampled indices. """ pos_inds = torch.nonzero(assign_result.gt_inds > 0, as_tuple=False) if pos_inds.numel() != 0: pos_inds = pos_inds.squeeze(1) if pos_inds.numel() <= num_expected: return pos_inds else: return self.random_choice(pos_inds, num_expected) def _sample_neg(self, assign_result: AssignResult, num_expected: int, **kwargs) -> Union[Tensor, ndarray]: """Randomly sample some negative samples. Args: assign_result (:obj:`AssignResult`): Bbox assigning results. num_expected (int): The number of expected positive samples Returns: Tensor or ndarray: sampled indices. """ neg_inds = torch.nonzero(assign_result.gt_inds == 0, as_tuple=False) if neg_inds.numel() != 0: neg_inds = neg_inds.squeeze(1) if len(neg_inds) <= num_expected: return neg_inds else: return self.random_choice(neg_inds, num_expected) ================================================ FILE: mmdet/models/task_modules/samplers/sampling_result.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings import numpy as np import torch from torch import Tensor from mmdet.structures.bbox import BaseBoxes, cat_boxes from mmdet.utils import util_mixins from mmdet.utils.util_random import ensure_rng from ..assigners import AssignResult def random_boxes(num=1, scale=1, rng=None): """Simple version of ``kwimage.Boxes.random`` Returns: Tensor: shape (n, 4) in x1, y1, x2, y2 format. References: https://gitlab.kitware.com/computer-vision/kwimage/blob/master/kwimage/structs/boxes.py#L1390 Example: >>> num = 3 >>> scale = 512 >>> rng = 0 >>> boxes = random_boxes(num, scale, rng) >>> print(boxes) tensor([[280.9925, 278.9802, 308.6148, 366.1769], [216.9113, 330.6978, 224.0446, 456.5878], [405.3632, 196.3221, 493.3953, 270.7942]]) """ rng = ensure_rng(rng) tlbr = rng.rand(num, 4).astype(np.float32) tl_x = np.minimum(tlbr[:, 0], tlbr[:, 2]) tl_y = np.minimum(tlbr[:, 1], tlbr[:, 3]) br_x = np.maximum(tlbr[:, 0], tlbr[:, 2]) br_y = np.maximum(tlbr[:, 1], tlbr[:, 3]) tlbr[:, 0] = tl_x * scale tlbr[:, 1] = tl_y * scale tlbr[:, 2] = br_x * scale tlbr[:, 3] = br_y * scale boxes = torch.from_numpy(tlbr) return boxes class SamplingResult(util_mixins.NiceRepr): """Bbox sampling result. Args: pos_inds (Tensor): Indices of positive samples. neg_inds (Tensor): Indices of negative samples. priors (Tensor): The priors can be anchors or points, or the bboxes predicted by the previous stage. gt_bboxes (Tensor): Ground truth of bboxes. assign_result (:obj:`AssignResult`): Assigning results. gt_flags (Tensor): The Ground truth flags. avg_factor_with_neg (bool): If True, ``avg_factor`` equal to the number of total priors; Otherwise, it is the number of positive priors. Defaults to True. Example: >>> # xdoctest: +IGNORE_WANT >>> from mmdet.models.task_modules.samplers.sampling_result import * # NOQA >>> self = SamplingResult.random(rng=10) >>> print(f'self = {self}') self = """ def __init__(self, pos_inds: Tensor, neg_inds: Tensor, priors: Tensor, gt_bboxes: Tensor, assign_result: AssignResult, gt_flags: Tensor, avg_factor_with_neg: bool = True) -> None: self.pos_inds = pos_inds self.neg_inds = neg_inds self.num_pos = max(pos_inds.numel(), 1) self.num_neg = max(neg_inds.numel(), 1) self.avg_factor_with_neg = avg_factor_with_neg self.avg_factor = self.num_pos + self.num_neg \ if avg_factor_with_neg else self.num_pos self.pos_priors = priors[pos_inds] self.neg_priors = priors[neg_inds] self.pos_is_gt = gt_flags[pos_inds] self.num_gts = gt_bboxes.shape[0] self.pos_assigned_gt_inds = assign_result.gt_inds[pos_inds] - 1 self.pos_gt_labels = assign_result.labels[pos_inds] box_dim = gt_bboxes.box_dim if isinstance(gt_bboxes, BaseBoxes) else 4 if gt_bboxes.numel() == 0: # hack for index error case assert self.pos_assigned_gt_inds.numel() == 0 self.pos_gt_bboxes = gt_bboxes.view(-1, box_dim) else: if len(gt_bboxes.shape) < 2: gt_bboxes = gt_bboxes.view(-1, box_dim) self.pos_gt_bboxes = gt_bboxes[self.pos_assigned_gt_inds.long()] @property def priors(self): """torch.Tensor: concatenated positive and negative priors""" return cat_boxes([self.pos_priors, self.neg_priors]) @property def bboxes(self): """torch.Tensor: concatenated positive and negative boxes""" warnings.warn('DeprecationWarning: bboxes is deprecated, ' 'please use "priors" instead') return self.priors @property def pos_bboxes(self): warnings.warn('DeprecationWarning: pos_bboxes is deprecated, ' 'please use "pos_priors" instead') return self.pos_priors @property def neg_bboxes(self): warnings.warn('DeprecationWarning: neg_bboxes is deprecated, ' 'please use "neg_priors" instead') return self.neg_priors def to(self, device): """Change the device of the data inplace. Example: >>> self = SamplingResult.random() >>> print(f'self = {self.to(None)}') >>> # xdoctest: +REQUIRES(--gpu) >>> print(f'self = {self.to(0)}') """ _dict = self.__dict__ for key, value in _dict.items(): if isinstance(value, (torch.Tensor, BaseBoxes)): _dict[key] = value.to(device) return self def __nice__(self): data = self.info.copy() data['pos_priors'] = data.pop('pos_priors').shape data['neg_priors'] = data.pop('neg_priors').shape parts = [f"'{k}': {v!r}" for k, v in sorted(data.items())] body = ' ' + ',\n '.join(parts) return '{\n' + body + '\n}' @property def info(self): """Returns a dictionary of info about the object.""" return { 'pos_inds': self.pos_inds, 'neg_inds': self.neg_inds, 'pos_priors': self.pos_priors, 'neg_priors': self.neg_priors, 'pos_is_gt': self.pos_is_gt, 'num_gts': self.num_gts, 'pos_assigned_gt_inds': self.pos_assigned_gt_inds, 'num_pos': self.num_pos, 'num_neg': self.num_neg, 'avg_factor': self.avg_factor } @classmethod def random(cls, rng=None, **kwargs): """ Args: rng (None | int | numpy.random.RandomState): seed or state. kwargs (keyword arguments): - num_preds: Number of predicted boxes. - num_gts: Number of true boxes. - p_ignore (float): Probability of a predicted box assigned to an ignored truth. - p_assigned (float): probability of a predicted box not being assigned. Returns: :obj:`SamplingResult`: Randomly generated sampling result. Example: >>> from mmdet.models.task_modules.samplers.sampling_result import * # NOQA >>> self = SamplingResult.random() >>> print(self.__dict__) """ from mmengine.structures import InstanceData from mmdet.models.task_modules.assigners import AssignResult from mmdet.models.task_modules.samplers import RandomSampler rng = ensure_rng(rng) # make probabilistic? num = 32 pos_fraction = 0.5 neg_pos_ub = -1 assign_result = AssignResult.random(rng=rng, **kwargs) # Note we could just compute an assignment priors = random_boxes(assign_result.num_preds, rng=rng) gt_bboxes = random_boxes(assign_result.num_gts, rng=rng) gt_labels = torch.randint( 0, 5, (assign_result.num_gts, ), dtype=torch.long) pred_instances = InstanceData() pred_instances.priors = priors gt_instances = InstanceData() gt_instances.bboxes = gt_bboxes gt_instances.labels = gt_labels add_gt_as_proposals = True sampler = RandomSampler( num, pos_fraction, neg_pos_ub=neg_pos_ub, add_gt_as_proposals=add_gt_as_proposals, rng=rng) self = sampler.sample( assign_result=assign_result, pred_instances=pred_instances, gt_instances=gt_instances) return self ================================================ FILE: mmdet/models/task_modules/samplers/score_hlr_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Union import torch from mmcv.ops import nms_match from mmengine.structures import InstanceData from numpy import ndarray from torch import Tensor from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import bbox2roi from ..assigners import AssignResult from .base_sampler import BaseSampler from .sampling_result import SamplingResult @TASK_UTILS.register_module() class ScoreHLRSampler(BaseSampler): r"""Importance-based Sample Reweighting (ISR_N), described in `Prime Sample Attention in Object Detection `_. Score hierarchical local rank (HLR) differentiates with RandomSampler in negative part. It firstly computes Score-HLR in a two-step way, then linearly maps score hlr to the loss weights. Args: num (int): Total number of sampled RoIs. pos_fraction (float): Fraction of positive samples. context (:obj:`BaseRoIHead`): RoI head that the sampler belongs to. neg_pos_ub (int): Upper bound of the ratio of num negative to num positive, -1 means no upper bound. Defaults to -1. add_gt_as_proposals (bool): Whether to add ground truth as proposals. Defaults to True. k (float): Power of the non-linear mapping. Defaults to 0.5 bias (float): Shift of the non-linear mapping. Defaults to 0. score_thr (float): Minimum score that a negative sample is to be considered as valid bbox. Defaults to 0.05. iou_thr (float): IoU threshold for NMS match. Defaults to 0.5. """ def __init__(self, num: int, pos_fraction: float, context, neg_pos_ub: int = -1, add_gt_as_proposals: bool = True, k: float = 0.5, bias: float = 0, score_thr: float = 0.05, iou_thr: float = 0.5, **kwargs) -> None: super().__init__( num=num, pos_fraction=pos_fraction, neg_pos_ub=neg_pos_ub, add_gt_as_proposals=add_gt_as_proposals) self.k = k self.bias = bias self.score_thr = score_thr self.iou_thr = iou_thr self.context = context # context of cascade detectors is a list, so distinguish them here. if not hasattr(context, 'num_stages'): self.bbox_roi_extractor = context.bbox_roi_extractor self.bbox_head = context.bbox_head self.with_shared_head = context.with_shared_head if self.with_shared_head: self.shared_head = context.shared_head else: self.bbox_roi_extractor = context.bbox_roi_extractor[ context.current_stage] self.bbox_head = context.bbox_head[context.current_stage] @staticmethod def random_choice(gallery: Union[Tensor, ndarray, list], num: int) -> Union[Tensor, ndarray]: """Randomly select some elements from the gallery. If `gallery` is a Tensor, the returned indices will be a Tensor; If `gallery` is a ndarray or list, the returned indices will be a ndarray. Args: gallery (Tensor or ndarray or list): indices pool. num (int): expected sample num. Returns: Tensor or ndarray: sampled indices. """ assert len(gallery) >= num is_tensor = isinstance(gallery, torch.Tensor) if not is_tensor: if torch.cuda.is_available(): device = torch.cuda.current_device() else: device = 'cpu' gallery = torch.tensor(gallery, dtype=torch.long, device=device) perm = torch.randperm(gallery.numel(), device=gallery.device)[:num] rand_inds = gallery[perm] if not is_tensor: rand_inds = rand_inds.cpu().numpy() return rand_inds def _sample_pos(self, assign_result: AssignResult, num_expected: int, **kwargs) -> Union[Tensor, ndarray]: """Randomly sample some positive samples. Args: assign_result (:obj:`AssignResult`): Bbox assigning results. num_expected (int): The number of expected positive samples Returns: Tensor or ndarray: sampled indices. """ pos_inds = torch.nonzero(assign_result.gt_inds > 0).flatten() if pos_inds.numel() <= num_expected: return pos_inds else: return self.random_choice(pos_inds, num_expected) def _sample_neg(self, assign_result: AssignResult, num_expected: int, bboxes: Tensor, feats: Tensor, **kwargs) -> Union[Tensor, ndarray]: """Sample negative samples. Score-HLR sampler is done in the following steps: 1. Take the maximum positive score prediction of each negative samples as s_i. 2. Filter out negative samples whose s_i <= score_thr, the left samples are called valid samples. 3. Use NMS-Match to divide valid samples into different groups, samples in the same group will greatly overlap with each other 4. Rank the matched samples in two-steps to get Score-HLR. (1) In the same group, rank samples with their scores. (2) In the same score rank across different groups, rank samples with their scores again. 5. Linearly map Score-HLR to the final label weights. Args: assign_result (:obj:`AssignResult`): result of assigner. num_expected (int): Expected number of samples. bboxes (Tensor): bbox to be sampled. feats (Tensor): Features come from FPN. Returns: Tensor or ndarray: sampled indices. """ neg_inds = torch.nonzero(assign_result.gt_inds == 0).flatten() num_neg = neg_inds.size(0) if num_neg == 0: return neg_inds, None with torch.no_grad(): neg_bboxes = bboxes[neg_inds] neg_rois = bbox2roi([neg_bboxes]) bbox_result = self.context._bbox_forward(feats, neg_rois) cls_score, bbox_pred = bbox_result['cls_score'], bbox_result[ 'bbox_pred'] ori_loss = self.bbox_head.loss( cls_score=cls_score, bbox_pred=None, rois=None, labels=neg_inds.new_full((num_neg, ), self.bbox_head.num_classes), label_weights=cls_score.new_ones(num_neg), bbox_targets=None, bbox_weights=None, reduction_override='none')['loss_cls'] # filter out samples with the max score lower than score_thr max_score, argmax_score = cls_score.softmax(-1)[:, :-1].max(-1) valid_inds = (max_score > self.score_thr).nonzero().view(-1) invalid_inds = (max_score <= self.score_thr).nonzero().view(-1) num_valid = valid_inds.size(0) num_invalid = invalid_inds.size(0) num_expected = min(num_neg, num_expected) num_hlr = min(num_valid, num_expected) num_rand = num_expected - num_hlr if num_valid > 0: valid_rois = neg_rois[valid_inds] valid_max_score = max_score[valid_inds] valid_argmax_score = argmax_score[valid_inds] valid_bbox_pred = bbox_pred[valid_inds] # valid_bbox_pred shape: [num_valid, #num_classes, 4] valid_bbox_pred = valid_bbox_pred.view( valid_bbox_pred.size(0), -1, 4) selected_bbox_pred = valid_bbox_pred[range(num_valid), valid_argmax_score] pred_bboxes = self.bbox_head.bbox_coder.decode( valid_rois[:, 1:], selected_bbox_pred) pred_bboxes_with_score = torch.cat( [pred_bboxes, valid_max_score[:, None]], -1) group = nms_match(pred_bboxes_with_score, self.iou_thr) # imp: importance imp = cls_score.new_zeros(num_valid) for g in group: g_score = valid_max_score[g] # g_score has already sorted rank = g_score.new_tensor(range(g_score.size(0))) imp[g] = num_valid - rank + g_score _, imp_rank_inds = imp.sort(descending=True) _, imp_rank = imp_rank_inds.sort() hlr_inds = imp_rank_inds[:num_expected] if num_rand > 0: rand_inds = torch.randperm(num_invalid)[:num_rand] select_inds = torch.cat( [valid_inds[hlr_inds], invalid_inds[rand_inds]]) else: select_inds = valid_inds[hlr_inds] neg_label_weights = cls_score.new_ones(num_expected) up_bound = max(num_expected, num_valid) imp_weights = (up_bound - imp_rank[hlr_inds].float()) / up_bound neg_label_weights[:num_hlr] = imp_weights neg_label_weights[num_hlr:] = imp_weights.min() neg_label_weights = (self.bias + (1 - self.bias) * neg_label_weights).pow( self.k) ori_selected_loss = ori_loss[select_inds] new_loss = ori_selected_loss * neg_label_weights norm_ratio = ori_selected_loss.sum() / new_loss.sum() neg_label_weights *= norm_ratio else: neg_label_weights = cls_score.new_ones(num_expected) select_inds = torch.randperm(num_neg)[:num_expected] return neg_inds[select_inds], neg_label_weights def sample(self, assign_result: AssignResult, pred_instances: InstanceData, gt_instances: InstanceData, **kwargs) -> SamplingResult: """Sample positive and negative bboxes. This is a simple implementation of bbox sampling given candidates, assigning results and ground truth bboxes. Args: assign_result (:obj:`AssignResult`): Assigning results. pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). Returns: :obj:`SamplingResult`: Sampling result. """ gt_bboxes = gt_instances.bboxes priors = pred_instances.priors gt_labels = gt_instances.labels gt_flags = priors.new_zeros((priors.shape[0], ), dtype=torch.uint8) if self.add_gt_as_proposals and len(gt_bboxes) > 0: priors = torch.cat([gt_bboxes, priors], dim=0) assign_result.add_gt_(gt_labels) gt_ones = priors.new_ones(gt_bboxes.shape[0], dtype=torch.uint8) gt_flags = torch.cat([gt_ones, gt_flags]) num_expected_pos = int(self.num * self.pos_fraction) pos_inds = self.pos_sampler._sample_pos( assign_result, num_expected_pos, bboxes=priors, **kwargs) num_sampled_pos = pos_inds.numel() num_expected_neg = self.num - num_sampled_pos if self.neg_pos_ub >= 0: _pos = max(1, num_sampled_pos) neg_upper_bound = int(self.neg_pos_ub * _pos) if num_expected_neg > neg_upper_bound: num_expected_neg = neg_upper_bound neg_inds, neg_label_weights = self.neg_sampler._sample_neg( assign_result, num_expected_neg, bboxes=priors, **kwargs) sampling_result = SamplingResult( pos_inds=pos_inds, neg_inds=neg_inds, priors=priors, gt_bboxes=gt_bboxes, assign_result=assign_result, gt_flags=gt_flags) return sampling_result, neg_label_weights ================================================ FILE: mmdet/models/test_time_augs/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .det_tta import DetTTAModel from .merge_augs import (merge_aug_bboxes, merge_aug_masks, merge_aug_proposals, merge_aug_results, merge_aug_scores) __all__ = [ 'merge_aug_bboxes', 'merge_aug_masks', 'merge_aug_proposals', 'merge_aug_scores', 'merge_aug_results', 'DetTTAModel' ] ================================================ FILE: mmdet/models/test_time_augs/det_tta.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple import torch from mmcv.ops import batched_nms from mmengine.model import BaseTTAModel from mmengine.registry import MODELS from mmengine.structures import InstanceData from torch import Tensor from mmdet.structures import DetDataSample from mmdet.structures.bbox import bbox_flip @MODELS.register_module() class DetTTAModel(BaseTTAModel): """Merge augmented detection results, only bboxes corresponding score under flipping and multi-scale resizing can be processed now. Examples: >>> tta_model = dict( >>> type='DetTTAModel', >>> tta_cfg=dict(nms=dict( >>> type='nms', >>> iou_threshold=0.5), >>> max_per_img=100)) >>> >>> tta_pipeline = [ >>> dict(type='LoadImageFromFile', >>> file_client_args=dict(backend='disk')), >>> dict( >>> type='TestTimeAug', >>> transforms=[[ >>> dict(type='Resize', >>> scale=(1333, 800), >>> keep_ratio=True), >>> ], [ >>> dict(type='RandomFlip', prob=1.), >>> dict(type='RandomFlip', prob=0.) >>> ], [ >>> dict( >>> type='PackDetInputs', >>> meta_keys=('img_id', 'img_path', 'ori_shape', >>> 'img_shape', 'scale_factor', 'flip', >>> 'flip_direction')) >>> ]])] """ def __init__(self, tta_cfg=None, **kwargs): super().__init__(**kwargs) self.tta_cfg = tta_cfg def merge_aug_bboxes(self, aug_bboxes: List[Tensor], aug_scores: List[Tensor], img_metas: List[str]) -> Tuple[Tensor, Tensor]: """Merge augmented detection bboxes and scores. Args: aug_bboxes (list[Tensor]): shape (n, 4*#class) aug_scores (list[Tensor] or None): shape (n, #class) Returns: tuple[Tensor]: ``bboxes`` with shape (n,4), where 4 represent (tl_x, tl_y, br_x, br_y) and ``scores`` with shape (n,). """ recovered_bboxes = [] for bboxes, img_info in zip(aug_bboxes, img_metas): ori_shape = img_info['ori_shape'] flip = img_info['flip'] flip_direction = img_info['flip_direction'] if flip: bboxes = bbox_flip( bboxes=bboxes, img_shape=ori_shape, direction=flip_direction) recovered_bboxes.append(bboxes) bboxes = torch.cat(recovered_bboxes, dim=0) if aug_scores is None: return bboxes else: scores = torch.cat(aug_scores, dim=0) return bboxes, scores def merge_preds(self, data_samples_list: List[List[DetDataSample]]): """Merge batch predictions of enhanced data. Args: data_samples_list (List[List[DetDataSample]]): List of predictions of all enhanced data. The outer list indicates images, and the inner list corresponds to the different views of one image. Each element of the inner list is a ``DetDataSample``. Returns: List[DetDataSample]: Merged batch prediction. """ merged_data_samples = [] for data_samples in data_samples_list: merged_data_samples.append(self._merge_single_sample(data_samples)) return merged_data_samples def _merge_single_sample( self, data_samples: List[DetDataSample]) -> DetDataSample: """Merge predictions which come form the different views of one image to one prediction. Args: data_samples (List[DetDataSample]): List of predictions of enhanced data which come form one image. Returns: List[DetDataSample]: Merged prediction. """ aug_bboxes = [] aug_scores = [] aug_labels = [] img_metas = [] # TODO: support instance segmentation TTA assert data_samples[0].pred_instances.get('masks', None) is None, \ 'TTA of instance segmentation does not support now.' for data_sample in data_samples: aug_bboxes.append(data_sample.pred_instances.bboxes) aug_scores.append(data_sample.pred_instances.scores) aug_labels.append(data_sample.pred_instances.labels) img_metas.append(data_sample.metainfo) merged_bboxes, merged_scores = self.merge_aug_bboxes( aug_bboxes, aug_scores, img_metas) merged_labels = torch.cat(aug_labels, dim=0) if merged_bboxes.numel() == 0: return data_samples[0] det_bboxes, keep_idxs = batched_nms(merged_bboxes, merged_scores, merged_labels, self.tta_cfg.nms) det_bboxes = det_bboxes[:self.tta_cfg.max_per_img] det_labels = merged_labels[keep_idxs][:self.tta_cfg.max_per_img] results = InstanceData() _det_bboxes = det_bboxes.clone() results.bboxes = _det_bboxes[:, :-1] results.scores = _det_bboxes[:, -1] results.labels = det_labels det_results = data_samples[0] det_results.pred_instances = results return det_results ================================================ FILE: mmdet/models/test_time_augs/merge_augs.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import warnings from typing import List, Optional, Union import numpy as np import torch from mmcv.ops import nms from mmengine.config import ConfigDict from torch import Tensor from mmdet.structures.bbox import bbox_mapping_back # TODO remove this, never be used in mmdet def merge_aug_proposals(aug_proposals, img_metas, cfg): """Merge augmented proposals (multiscale, flip, etc.) Args: aug_proposals (list[Tensor]): proposals from different testing schemes, shape (n, 5). Note that they are not rescaled to the original image size. img_metas (list[dict]): list of image info dict where each dict has: 'img_shape', 'scale_factor', 'flip', and may also contain 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. For details on the values of these keys see `mmdet/datasets/pipelines/formatting.py:Collect`. cfg (dict): rpn test config. Returns: Tensor: shape (n, 4), proposals corresponding to original image scale. """ cfg = copy.deepcopy(cfg) # deprecate arguments warning if 'nms' not in cfg or 'max_num' in cfg or 'nms_thr' in cfg: warnings.warn( 'In rpn_proposal or test_cfg, ' 'nms_thr has been moved to a dict named nms as ' 'iou_threshold, max_num has been renamed as max_per_img, ' 'name of original arguments and the way to specify ' 'iou_threshold of NMS will be deprecated.') if 'nms' not in cfg: cfg.nms = ConfigDict(dict(type='nms', iou_threshold=cfg.nms_thr)) if 'max_num' in cfg: if 'max_per_img' in cfg: assert cfg.max_num == cfg.max_per_img, f'You set max_num and ' \ f'max_per_img at the same time, but get {cfg.max_num} ' \ f'and {cfg.max_per_img} respectively' \ f'Please delete max_num which will be deprecated.' else: cfg.max_per_img = cfg.max_num if 'nms_thr' in cfg: assert cfg.nms.iou_threshold == cfg.nms_thr, f'You set ' \ f'iou_threshold in nms and ' \ f'nms_thr at the same time, but get ' \ f'{cfg.nms.iou_threshold} and {cfg.nms_thr}' \ f' respectively. Please delete the nms_thr ' \ f'which will be deprecated.' recovered_proposals = [] for proposals, img_info in zip(aug_proposals, img_metas): img_shape = img_info['img_shape'] scale_factor = img_info['scale_factor'] flip = img_info['flip'] flip_direction = img_info['flip_direction'] _proposals = proposals.clone() _proposals[:, :4] = bbox_mapping_back(_proposals[:, :4], img_shape, scale_factor, flip, flip_direction) recovered_proposals.append(_proposals) aug_proposals = torch.cat(recovered_proposals, dim=0) merged_proposals, _ = nms(aug_proposals[:, :4].contiguous(), aug_proposals[:, -1].contiguous(), cfg.nms.iou_threshold) scores = merged_proposals[:, 4] _, order = scores.sort(0, descending=True) num = min(cfg.max_per_img, merged_proposals.shape[0]) order = order[:num] merged_proposals = merged_proposals[order, :] return merged_proposals # TODO remove this, never be used in mmdet def merge_aug_bboxes(aug_bboxes, aug_scores, img_metas, rcnn_test_cfg): """Merge augmented detection bboxes and scores. Args: aug_bboxes (list[Tensor]): shape (n, 4*#class) aug_scores (list[Tensor] or None): shape (n, #class) img_shapes (list[Tensor]): shape (3, ). rcnn_test_cfg (dict): rcnn test config. Returns: tuple: (bboxes, scores) """ recovered_bboxes = [] for bboxes, img_info in zip(aug_bboxes, img_metas): img_shape = img_info[0]['img_shape'] scale_factor = img_info[0]['scale_factor'] flip = img_info[0]['flip'] flip_direction = img_info[0]['flip_direction'] bboxes = bbox_mapping_back(bboxes, img_shape, scale_factor, flip, flip_direction) recovered_bboxes.append(bboxes) bboxes = torch.stack(recovered_bboxes).mean(dim=0) if aug_scores is None: return bboxes else: scores = torch.stack(aug_scores).mean(dim=0) return bboxes, scores def merge_aug_results(aug_batch_results, aug_batch_img_metas): """Merge augmented detection results, only bboxes corresponding score under flipping and multi-scale resizing can be processed now. Args: aug_batch_results (list[list[[obj:`InstanceData`]]): Detection results of multiple images with different augmentations. The outer list indicate the augmentation . The inter list indicate the batch dimension. Each item usually contains the following keys. - scores (Tensor): Classification scores, in shape (num_instance,) - labels (Tensor): Labels of bboxes, in shape (num_instances,). - bboxes (Tensor): In shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). aug_batch_img_metas (list[list[dict]]): The outer list indicates test-time augs (multiscale, flip, etc.) and the inner list indicates images in a batch. Each dict in the list contains information of an image in the batch. Returns: batch_results (list[obj:`InstanceData`]): Same with the input `aug_results` except that all bboxes have been mapped to the original scale. """ num_augs = len(aug_batch_results) num_imgs = len(aug_batch_results[0]) batch_results = [] aug_batch_results = copy.deepcopy(aug_batch_results) for img_id in range(num_imgs): aug_results = [] for aug_id in range(num_augs): img_metas = aug_batch_img_metas[aug_id][img_id] results = aug_batch_results[aug_id][img_id] img_shape = img_metas['img_shape'] scale_factor = img_metas['scale_factor'] flip = img_metas['flip'] flip_direction = img_metas['flip_direction'] bboxes = bbox_mapping_back(results.bboxes, img_shape, scale_factor, flip, flip_direction) results.bboxes = bboxes aug_results.append(results) merged_aug_results = results.cat(aug_results) batch_results.append(merged_aug_results) return batch_results def merge_aug_scores(aug_scores): """Merge augmented bbox scores.""" if isinstance(aug_scores[0], torch.Tensor): return torch.mean(torch.stack(aug_scores), dim=0) else: return np.mean(aug_scores, axis=0) def merge_aug_masks(aug_masks: List[Tensor], img_metas: dict, weights: Optional[Union[list, Tensor]] = None) -> Tensor: """Merge augmented mask prediction. Args: aug_masks (list[Tensor]): each has shape (n, c, h, w). img_metas (dict): Image information. weights (list or Tensor): Weight of each aug_masks, the length should be n. Returns: Tensor: has shape (n, c, h, w) """ recovered_masks = [] for i, mask in enumerate(aug_masks): if weights is not None: assert len(weights) == len(aug_masks) weight = weights[i] else: weight = 1 flip = img_metas.get('filp', False) if flip: flip_direction = img_metas['flip_direction'] if flip_direction == 'horizontal': mask = mask[:, :, :, ::-1] elif flip_direction == 'vertical': mask = mask[:, :, ::-1, :] elif flip_direction == 'diagonal': mask = mask[:, :, :, ::-1] mask = mask[:, :, ::-1, :] else: raise ValueError( f"Invalid flipping direction '{flip_direction}'") recovered_masks.append(mask[None, :] * weight) merged_masks = torch.cat(recovered_masks, 0).mean(dim=0) if weights is not None: merged_masks = merged_masks * len(weights) / sum(weights) return merged_masks ================================================ FILE: mmdet/models/utils/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .gaussian_target import (gather_feat, gaussian_radius, gen_gaussian_target, get_local_maximum, get_topk_from_heatmap, transpose_and_gather_feat) from .make_divisible import make_divisible from .misc import (aligned_bilinear, center_of_mass, empty_instances, filter_gt_instances, filter_scores_and_topk, flip_tensor, generate_coordinate, images_to_levels, interpolate_as, levels_to_images, mask2ndarray, multi_apply, relative_coordinate_maps, rename_loss_dict, reweight_loss_dict, samplelist_boxtype2tensor, select_single_mlvl, sigmoid_geometric_mean, unfold_wo_center, unmap, unpack_gt_instances) from .panoptic_gt_processing import preprocess_panoptic_gt from .point_sample import (get_uncertain_point_coords_with_randomness, get_uncertainty) __all__ = [ 'gaussian_radius', 'gen_gaussian_target', 'make_divisible', 'get_local_maximum', 'get_topk_from_heatmap', 'transpose_and_gather_feat', 'interpolate_as', 'sigmoid_geometric_mean', 'gather_feat', 'preprocess_panoptic_gt', 'get_uncertain_point_coords_with_randomness', 'get_uncertainty', 'unpack_gt_instances', 'empty_instances', 'center_of_mass', 'filter_scores_and_topk', 'flip_tensor', 'generate_coordinate', 'levels_to_images', 'mask2ndarray', 'multi_apply', 'select_single_mlvl', 'unmap', 'images_to_levels', 'samplelist_boxtype2tensor', 'filter_gt_instances', 'rename_loss_dict', 'reweight_loss_dict', 'relative_coordinate_maps', 'aligned_bilinear', 'unfold_wo_center' ] ================================================ FILE: mmdet/models/utils/gaussian_target.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from math import sqrt import torch import torch.nn.functional as F def gaussian2D(radius, sigma=1, dtype=torch.float32, device='cpu'): """Generate 2D gaussian kernel. Args: radius (int): Radius of gaussian kernel. sigma (int): Sigma of gaussian function. Default: 1. dtype (torch.dtype): Dtype of gaussian tensor. Default: torch.float32. device (str): Device of gaussian tensor. Default: 'cpu'. Returns: h (Tensor): Gaussian kernel with a ``(2 * radius + 1) * (2 * radius + 1)`` shape. """ x = torch.arange( -radius, radius + 1, dtype=dtype, device=device).view(1, -1) y = torch.arange( -radius, radius + 1, dtype=dtype, device=device).view(-1, 1) h = (-(x * x + y * y) / (2 * sigma * sigma)).exp() h[h < torch.finfo(h.dtype).eps * h.max()] = 0 return h def gen_gaussian_target(heatmap, center, radius, k=1): """Generate 2D gaussian heatmap. Args: heatmap (Tensor): Input heatmap, the gaussian kernel will cover on it and maintain the max value. center (list[int]): Coord of gaussian kernel's center. radius (int): Radius of gaussian kernel. k (int): Coefficient of gaussian kernel. Default: 1. Returns: out_heatmap (Tensor): Updated heatmap covered by gaussian kernel. """ diameter = 2 * radius + 1 gaussian_kernel = gaussian2D( radius, sigma=diameter / 6, dtype=heatmap.dtype, device=heatmap.device) x, y = center height, width = heatmap.shape[:2] left, right = min(x, radius), min(width - x, radius + 1) top, bottom = min(y, radius), min(height - y, radius + 1) masked_heatmap = heatmap[y - top:y + bottom, x - left:x + right] masked_gaussian = gaussian_kernel[radius - top:radius + bottom, radius - left:radius + right] out_heatmap = heatmap torch.max( masked_heatmap, masked_gaussian * k, out=out_heatmap[y - top:y + bottom, x - left:x + right]) return out_heatmap def gaussian_radius(det_size, min_overlap): r"""Generate 2D gaussian radius. This function is modified from the `official github repo `_. Given ``min_overlap``, radius could computed by a quadratic equation according to Vieta's formulas. There are 3 cases for computing gaussian radius, details are following: - Explanation of figure: ``lt`` and ``br`` indicates the left-top and bottom-right corner of ground truth box. ``x`` indicates the generated corner at the limited position when ``radius=r``. - Case1: one corner is inside the gt box and the other is outside. .. code:: text |< width >| lt-+----------+ - | | | ^ +--x----------+--+ | | | | | | | | height | | overlap | | | | | | | | | | v +--+---------br--+ - | | | +----------+--x To ensure IoU of generated box and gt box is larger than ``min_overlap``: .. math:: \cfrac{(w-r)*(h-r)}{w*h+(w+h)r-r^2} \ge {iou} \quad\Rightarrow\quad {r^2-(w+h)r+\cfrac{1-iou}{1+iou}*w*h} \ge 0 \\ {a} = 1,\quad{b} = {-(w+h)},\quad{c} = {\cfrac{1-iou}{1+iou}*w*h} {r} \le \cfrac{-b-\sqrt{b^2-4*a*c}}{2*a} - Case2: both two corners are inside the gt box. .. code:: text |< width >| lt-+----------+ - | | | ^ +--x-------+ | | | | | | |overlap| | height | | | | | +-------x--+ | | | v +----------+-br - To ensure IoU of generated box and gt box is larger than ``min_overlap``: .. math:: \cfrac{(w-2*r)*(h-2*r)}{w*h} \ge {iou} \quad\Rightarrow\quad {4r^2-2(w+h)r+(1-iou)*w*h} \ge 0 \\ {a} = 4,\quad {b} = {-2(w+h)},\quad {c} = {(1-iou)*w*h} {r} \le \cfrac{-b-\sqrt{b^2-4*a*c}}{2*a} - Case3: both two corners are outside the gt box. .. code:: text |< width >| x--+----------------+ | | | +-lt-------------+ | - | | | | ^ | | | | | | overlap | | height | | | | | | | | v | +------------br--+ - | | | +----------------+--x To ensure IoU of generated box and gt box is larger than ``min_overlap``: .. math:: \cfrac{w*h}{(w+2*r)*(h+2*r)} \ge {iou} \quad\Rightarrow\quad {4*iou*r^2+2*iou*(w+h)r+(iou-1)*w*h} \le 0 \\ {a} = {4*iou},\quad {b} = {2*iou*(w+h)},\quad {c} = {(iou-1)*w*h} \\ {r} \le \cfrac{-b+\sqrt{b^2-4*a*c}}{2*a} Args: det_size (list[int]): Shape of object. min_overlap (float): Min IoU with ground truth for boxes generated by keypoints inside the gaussian kernel. Returns: radius (int): Radius of gaussian kernel. """ height, width = det_size a1 = 1 b1 = (height + width) c1 = width * height * (1 - min_overlap) / (1 + min_overlap) sq1 = sqrt(b1**2 - 4 * a1 * c1) r1 = (b1 - sq1) / (2 * a1) a2 = 4 b2 = 2 * (height + width) c2 = (1 - min_overlap) * width * height sq2 = sqrt(b2**2 - 4 * a2 * c2) r2 = (b2 - sq2) / (2 * a2) a3 = 4 * min_overlap b3 = -2 * min_overlap * (height + width) c3 = (min_overlap - 1) * width * height sq3 = sqrt(b3**2 - 4 * a3 * c3) r3 = (b3 + sq3) / (2 * a3) return min(r1, r2, r3) def get_local_maximum(heat, kernel=3): """Extract local maximum pixel with given kernel. Args: heat (Tensor): Target heatmap. kernel (int): Kernel size of max pooling. Default: 3. Returns: heat (Tensor): A heatmap where local maximum pixels maintain its own value and other positions are 0. """ pad = (kernel - 1) // 2 hmax = F.max_pool2d(heat, kernel, stride=1, padding=pad) keep = (hmax == heat).float() return heat * keep def get_topk_from_heatmap(scores, k=20): """Get top k positions from heatmap. Args: scores (Tensor): Target heatmap with shape [batch, num_classes, height, width]. k (int): Target number. Default: 20. Returns: tuple[torch.Tensor]: Scores, indexes, categories and coords of topk keypoint. Containing following Tensors: - topk_scores (Tensor): Max scores of each topk keypoint. - topk_inds (Tensor): Indexes of each topk keypoint. - topk_clses (Tensor): Categories of each topk keypoint. - topk_ys (Tensor): Y-coord of each topk keypoint. - topk_xs (Tensor): X-coord of each topk keypoint. """ batch, _, height, width = scores.size() topk_scores, topk_inds = torch.topk(scores.view(batch, -1), k) topk_clses = topk_inds // (height * width) topk_inds = topk_inds % (height * width) topk_ys = topk_inds // width topk_xs = (topk_inds % width).int().float() return topk_scores, topk_inds, topk_clses, topk_ys, topk_xs def gather_feat(feat, ind, mask=None): """Gather feature according to index. Args: feat (Tensor): Target feature map. ind (Tensor): Target coord index. mask (Tensor | None): Mask of feature map. Default: None. Returns: feat (Tensor): Gathered feature. """ dim = feat.size(2) ind = ind.unsqueeze(2).repeat(1, 1, dim) feat = feat.gather(1, ind) if mask is not None: mask = mask.unsqueeze(2).expand_as(feat) feat = feat[mask] feat = feat.view(-1, dim) return feat def transpose_and_gather_feat(feat, ind): """Transpose and gather feature according to index. Args: feat (Tensor): Target feature map. ind (Tensor): Target coord index. Returns: feat (Tensor): Transposed and gathered feature. """ feat = feat.permute(0, 2, 3, 1).contiguous() feat = feat.view(feat.size(0), -1, feat.size(3)) feat = gather_feat(feat, ind) return feat ================================================ FILE: mmdet/models/utils/make_divisible.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. def make_divisible(value, divisor, min_value=None, min_ratio=0.9): """Make divisible function. This function rounds the channel number to the nearest value that can be divisible by the divisor. It is taken from the original tf repo. It ensures that all layers have a channel number that is divisible by divisor. It can be seen here: https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py # noqa Args: value (int): The original channel number. divisor (int): The divisor to fully divide the channel number. min_value (int): The minimum value of the output channel. Default: None, means that the minimum value equal to the divisor. min_ratio (float): The minimum ratio of the rounded channel number to the original channel number. Default: 0.9. Returns: int: The modified output channel number. """ if min_value is None: min_value = divisor new_value = max(min_value, int(value + divisor / 2) // divisor * divisor) # Make sure that round down does not go down by more than (1-min_ratio). if new_value < min_ratio * value: new_value += divisor return new_value ================================================ FILE: mmdet/models/utils/misc.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from functools import partial from typing import List, Sequence, Tuple, Union import numpy as np import torch from mmengine.structures import InstanceData from mmengine.utils import digit_version from six.moves import map, zip from torch import Tensor from torch.autograd import Function from torch.nn import functional as F from mmdet.structures import SampleList from mmdet.structures.bbox import BaseBoxes, get_box_type, stack_boxes from mmdet.structures.mask import BitmapMasks, PolygonMasks from mmdet.utils import OptInstanceList class SigmoidGeometricMean(Function): """Forward and backward function of geometric mean of two sigmoid functions. This implementation with analytical gradient function substitutes the autograd function of (x.sigmoid() * y.sigmoid()).sqrt(). The original implementation incurs none during gradient backprapagation if both x and y are very small values. """ @staticmethod def forward(ctx, x, y): x_sigmoid = x.sigmoid() y_sigmoid = y.sigmoid() z = (x_sigmoid * y_sigmoid).sqrt() ctx.save_for_backward(x_sigmoid, y_sigmoid, z) return z @staticmethod def backward(ctx, grad_output): x_sigmoid, y_sigmoid, z = ctx.saved_tensors grad_x = grad_output * z * (1 - x_sigmoid) / 2 grad_y = grad_output * z * (1 - y_sigmoid) / 2 return grad_x, grad_y sigmoid_geometric_mean = SigmoidGeometricMean.apply def interpolate_as(source, target, mode='bilinear', align_corners=False): """Interpolate the `source` to the shape of the `target`. The `source` must be a Tensor, but the `target` can be a Tensor or a np.ndarray with the shape (..., target_h, target_w). Args: source (Tensor): A 3D/4D Tensor with the shape (N, H, W) or (N, C, H, W). target (Tensor | np.ndarray): The interpolation target with the shape (..., target_h, target_w). mode (str): Algorithm used for interpolation. The options are the same as those in F.interpolate(). Default: ``'bilinear'``. align_corners (bool): The same as the argument in F.interpolate(). Returns: Tensor: The interpolated source Tensor. """ assert len(target.shape) >= 2 def _interpolate_as(source, target, mode='bilinear', align_corners=False): """Interpolate the `source` (4D) to the shape of the `target`.""" target_h, target_w = target.shape[-2:] source_h, source_w = source.shape[-2:] if target_h != source_h or target_w != source_w: source = F.interpolate( source, size=(target_h, target_w), mode=mode, align_corners=align_corners) return source if len(source.shape) == 3: source = source[:, None, :, :] source = _interpolate_as(source, target, mode, align_corners) return source[:, 0, :, :] else: return _interpolate_as(source, target, mode, align_corners) def unpack_gt_instances(batch_data_samples: SampleList) -> tuple: """Unpack ``gt_instances``, ``gt_instances_ignore`` and ``img_metas`` based on ``batch_data_samples`` Args: batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: tuple: - batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes`` and ``labels`` attributes. - batch_gt_instances_ignore (list[:obj:`InstanceData`]): Batch of gt_instances_ignore. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. - batch_img_metas (list[dict]): Meta information of each image, e.g., image size, scaling factor, etc. """ batch_gt_instances = [] batch_gt_instances_ignore = [] batch_img_metas = [] for data_sample in batch_data_samples: batch_img_metas.append(data_sample.metainfo) batch_gt_instances.append(data_sample.gt_instances) if 'ignored_instances' in data_sample: batch_gt_instances_ignore.append(data_sample.ignored_instances) else: batch_gt_instances_ignore.append(None) return batch_gt_instances, batch_gt_instances_ignore, batch_img_metas def empty_instances(batch_img_metas: List[dict], device: torch.device, task_type: str, instance_results: OptInstanceList = None, mask_thr_binary: Union[int, float] = 0, box_type: Union[str, type] = 'hbox', use_box_type: bool = False, num_classes: int = 80, score_per_cls: bool = False) -> List[InstanceData]: """Handle predicted instances when RoI is empty. Note: If ``instance_results`` is not None, it will be modified in place internally, and then return ``instance_results`` Args: batch_img_metas (list[dict]): List of image information. device (torch.device): Device of tensor. task_type (str): Expected returned task type. it currently supports bbox and mask. instance_results (list[:obj:`InstanceData`]): List of instance results. mask_thr_binary (int, float): mask binarization threshold. Defaults to 0. box_type (str or type): The empty box type. Defaults to `hbox`. use_box_type (bool): Whether to warp boxes with the box type. Defaults to False. num_classes (int): num_classes of bbox_head. Defaults to 80. score_per_cls (bool): Whether to generate classwise score for the empty instance. ``score_per_cls`` will be True when the model needs to produce raw results without nms. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image """ assert task_type in ('bbox', 'mask'), 'Only support bbox and mask,' \ f' but got {task_type}' if instance_results is not None: assert len(instance_results) == len(batch_img_metas) results_list = [] for img_id in range(len(batch_img_metas)): if instance_results is not None: results = instance_results[img_id] assert isinstance(results, InstanceData) else: results = InstanceData() if task_type == 'bbox': _, box_type = get_box_type(box_type) bboxes = torch.zeros(0, box_type.box_dim, device=device) if use_box_type: bboxes = box_type(bboxes, clone=False) results.bboxes = bboxes score_shape = (0, num_classes + 1) if score_per_cls else (0, ) results.scores = torch.zeros(score_shape, device=device) results.labels = torch.zeros((0, ), device=device, dtype=torch.long) else: # TODO: Handle the case where rescale is false img_h, img_w = batch_img_metas[img_id]['ori_shape'][:2] # the type of `im_mask` will be torch.bool or torch.uint8, # where uint8 if for visualization and debugging. im_mask = torch.zeros( 0, img_h, img_w, device=device, dtype=torch.bool if mask_thr_binary >= 0 else torch.uint8) results.masks = im_mask results_list.append(results) return results_list def multi_apply(func, *args, **kwargs): """Apply function to a list of arguments. Note: This function applies the ``func`` to multiple inputs and map the multiple outputs of the ``func`` into different list. Each list contains the same type of outputs corresponding to different inputs. Args: func (Function): A function that will be applied to a list of arguments Returns: tuple(list): A tuple containing multiple list, each list contains \ a kind of returned results by the function """ pfunc = partial(func, **kwargs) if kwargs else func map_results = map(pfunc, *args) return tuple(map(list, zip(*map_results))) def unmap(data, count, inds, fill=0): """Unmap a subset of item (data) back to the original set of items (of size count)""" if data.dim() == 1: ret = data.new_full((count, ), fill) ret[inds.type(torch.bool)] = data else: new_size = (count, ) + data.size()[1:] ret = data.new_full(new_size, fill) ret[inds.type(torch.bool), :] = data return ret def mask2ndarray(mask): """Convert Mask to ndarray.. Args: mask (:obj:`BitmapMasks` or :obj:`PolygonMasks` or torch.Tensor or np.ndarray): The mask to be converted. Returns: np.ndarray: Ndarray mask of shape (n, h, w) that has been converted """ if isinstance(mask, (BitmapMasks, PolygonMasks)): mask = mask.to_ndarray() elif isinstance(mask, torch.Tensor): mask = mask.detach().cpu().numpy() elif not isinstance(mask, np.ndarray): raise TypeError(f'Unsupported {type(mask)} data type') return mask def flip_tensor(src_tensor, flip_direction): """flip tensor base on flip_direction. Args: src_tensor (Tensor): input feature map, shape (B, C, H, W). flip_direction (str): The flipping direction. Options are 'horizontal', 'vertical', 'diagonal'. Returns: out_tensor (Tensor): Flipped tensor. """ assert src_tensor.ndim == 4 valid_directions = ['horizontal', 'vertical', 'diagonal'] assert flip_direction in valid_directions if flip_direction == 'horizontal': out_tensor = torch.flip(src_tensor, [3]) elif flip_direction == 'vertical': out_tensor = torch.flip(src_tensor, [2]) else: out_tensor = torch.flip(src_tensor, [2, 3]) return out_tensor def select_single_mlvl(mlvl_tensors, batch_id, detach=True): """Extract a multi-scale single image tensor from a multi-scale batch tensor based on batch index. Note: The default value of detach is True, because the proposal gradient needs to be detached during the training of the two-stage model. E.g Cascade Mask R-CNN. Args: mlvl_tensors (list[Tensor]): Batch tensor for all scale levels, each is a 4D-tensor. batch_id (int): Batch index. detach (bool): Whether detach gradient. Default True. Returns: list[Tensor]: Multi-scale single image tensor. """ assert isinstance(mlvl_tensors, (list, tuple)) num_levels = len(mlvl_tensors) if detach: mlvl_tensor_list = [ mlvl_tensors[i][batch_id].detach() for i in range(num_levels) ] else: mlvl_tensor_list = [ mlvl_tensors[i][batch_id] for i in range(num_levels) ] return mlvl_tensor_list def filter_scores_and_topk(scores, score_thr, topk, results=None): """Filter results using score threshold and topk candidates. Args: scores (Tensor): The scores, shape (num_bboxes, K). score_thr (float): The score filter threshold. topk (int): The number of topk candidates. results (dict or list or Tensor, Optional): The results to which the filtering rule is to be applied. The shape of each item is (num_bboxes, N). Returns: tuple: Filtered results - scores (Tensor): The scores after being filtered, \ shape (num_bboxes_filtered, ). - labels (Tensor): The class labels, shape \ (num_bboxes_filtered, ). - anchor_idxs (Tensor): The anchor indexes, shape \ (num_bboxes_filtered, ). - filtered_results (dict or list or Tensor, Optional): \ The filtered results. The shape of each item is \ (num_bboxes_filtered, N). """ valid_mask = scores > score_thr scores = scores[valid_mask] valid_idxs = torch.nonzero(valid_mask) num_topk = min(topk, valid_idxs.size(0)) # torch.sort is actually faster than .topk (at least on GPUs) scores, idxs = scores.sort(descending=True) scores = scores[:num_topk] topk_idxs = valid_idxs[idxs[:num_topk]] keep_idxs, labels = topk_idxs.unbind(dim=1) filtered_results = None if results is not None: if isinstance(results, dict): filtered_results = {k: v[keep_idxs] for k, v in results.items()} elif isinstance(results, list): filtered_results = [result[keep_idxs] for result in results] elif isinstance(results, torch.Tensor): filtered_results = results[keep_idxs] else: raise NotImplementedError(f'Only supports dict or list or Tensor, ' f'but get {type(results)}.') return scores, labels, keep_idxs, filtered_results def center_of_mass(mask, esp=1e-6): """Calculate the centroid coordinates of the mask. Args: mask (Tensor): The mask to be calculated, shape (h, w). esp (float): Avoid dividing by zero. Default: 1e-6. Returns: tuple[Tensor]: the coordinates of the center point of the mask. - center_h (Tensor): the center point of the height. - center_w (Tensor): the center point of the width. """ h, w = mask.shape grid_h = torch.arange(h, device=mask.device)[:, None] grid_w = torch.arange(w, device=mask.device) normalizer = mask.sum().float().clamp(min=esp) center_h = (mask * grid_h).sum() / normalizer center_w = (mask * grid_w).sum() / normalizer return center_h, center_w def generate_coordinate(featmap_sizes, device='cuda'): """Generate the coordinate. Args: featmap_sizes (tuple): The feature to be calculated, of shape (N, C, W, H). device (str): The device where the feature will be put on. Returns: coord_feat (Tensor): The coordinate feature, of shape (N, 2, W, H). """ x_range = torch.linspace(-1, 1, featmap_sizes[-1], device=device) y_range = torch.linspace(-1, 1, featmap_sizes[-2], device=device) y, x = torch.meshgrid(y_range, x_range) y = y.expand([featmap_sizes[0], 1, -1, -1]) x = x.expand([featmap_sizes[0], 1, -1, -1]) coord_feat = torch.cat([x, y], 1) return coord_feat def levels_to_images(mlvl_tensor: List[torch.Tensor]) -> List[torch.Tensor]: """Concat multi-level feature maps by image. [feature_level0, feature_level1...] -> [feature_image0, feature_image1...] Convert the shape of each element in mlvl_tensor from (N, C, H, W) to (N, H*W , C), then split the element to N elements with shape (H*W, C), and concat elements in same image of all level along first dimension. Args: mlvl_tensor (list[Tensor]): list of Tensor which collect from corresponding level. Each element is of shape (N, C, H, W) Returns: list[Tensor]: A list that contains N tensors and each tensor is of shape (num_elements, C) """ batch_size = mlvl_tensor[0].size(0) batch_list = [[] for _ in range(batch_size)] channels = mlvl_tensor[0].size(1) for t in mlvl_tensor: t = t.permute(0, 2, 3, 1) t = t.view(batch_size, -1, channels).contiguous() for img in range(batch_size): batch_list[img].append(t[img]) return [torch.cat(item, 0) for item in batch_list] def images_to_levels(target, num_levels): """Convert targets by image to targets by feature level. [target_img0, target_img1] -> [target_level0, target_level1, ...] """ target = stack_boxes(target, 0) level_targets = [] start = 0 for n in num_levels: end = start + n # level_targets.append(target[:, start:end].squeeze(0)) level_targets.append(target[:, start:end]) start = end return level_targets def samplelist_boxtype2tensor(batch_data_samples: SampleList) -> SampleList: for data_samples in batch_data_samples: if 'gt_instances' in data_samples: bboxes = data_samples.gt_instances.get('bboxes', None) if isinstance(bboxes, BaseBoxes): data_samples.gt_instances.bboxes = bboxes.tensor if 'pred_instances' in data_samples: bboxes = data_samples.pred_instances.get('bboxes', None) if isinstance(bboxes, BaseBoxes): data_samples.pred_instances.bboxes = bboxes.tensor if 'ignored_instances' in data_samples: bboxes = data_samples.ignored_instances.get('bboxes', None) if isinstance(bboxes, BaseBoxes): data_samples.ignored_instances.bboxes = bboxes.tensor _torch_version_div_indexing = ( 'parrots' not in torch.__version__ and digit_version(torch.__version__) >= digit_version('1.8')) def floordiv(dividend, divisor, rounding_mode='trunc'): if _torch_version_div_indexing: return torch.div(dividend, divisor, rounding_mode=rounding_mode) else: return dividend // divisor def _filter_gt_instances_by_score(batch_data_samples: SampleList, score_thr: float) -> SampleList: """Filter ground truth (GT) instances by score. Args: batch_data_samples (SampleList): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. score_thr (float): The score filter threshold. Returns: SampleList: The Data Samples filtered by score. """ for data_samples in batch_data_samples: assert 'scores' in data_samples.gt_instances, \ 'there does not exit scores in instances' if data_samples.gt_instances.bboxes.shape[0] > 0: data_samples.gt_instances = data_samples.gt_instances[ data_samples.gt_instances.scores > score_thr] return batch_data_samples def _filter_gt_instances_by_size(batch_data_samples: SampleList, wh_thr: tuple) -> SampleList: """Filter ground truth (GT) instances by size. Args: batch_data_samples (SampleList): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. wh_thr (tuple): Minimum width and height of bbox. Returns: SampleList: The Data Samples filtered by score. """ for data_samples in batch_data_samples: bboxes = data_samples.gt_instances.bboxes if bboxes.shape[0] > 0: w = bboxes[:, 2] - bboxes[:, 0] h = bboxes[:, 3] - bboxes[:, 1] data_samples.gt_instances = data_samples.gt_instances[ (w > wh_thr[0]) & (h > wh_thr[1])] return batch_data_samples def filter_gt_instances(batch_data_samples: SampleList, score_thr: float = None, wh_thr: tuple = None): """Filter ground truth (GT) instances by score and/or size. Args: batch_data_samples (SampleList): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. score_thr (float): The score filter threshold. wh_thr (tuple): Minimum width and height of bbox. Returns: SampleList: The Data Samples filtered by score and/or size. """ if score_thr is not None: batch_data_samples = _filter_gt_instances_by_score( batch_data_samples, score_thr) if wh_thr is not None: batch_data_samples = _filter_gt_instances_by_size( batch_data_samples, wh_thr) return batch_data_samples def rename_loss_dict(prefix: str, losses: dict) -> dict: """Rename the key names in loss dict by adding a prefix. Args: prefix (str): The prefix for loss components. losses (dict): A dictionary of loss components. Returns: dict: A dictionary of loss components with prefix. """ return {prefix + k: v for k, v in losses.items()} def reweight_loss_dict(losses: dict, weight: float) -> dict: """Reweight losses in the dict by weight. Args: losses (dict): A dictionary of loss components. weight (float): Weight for loss components. Returns: dict: A dictionary of weighted loss components. """ for name, loss in losses.items(): if 'loss' in name: if isinstance(loss, Sequence): losses[name] = [item * weight for item in loss] else: losses[name] = loss * weight return losses def relative_coordinate_maps( locations: Tensor, centers: Tensor, strides: Tensor, size_of_interest: int, feat_sizes: Tuple[int], ) -> Tensor: """Generate the relative coordinate maps with feat_stride. Args: locations (Tensor): The prior location of mask feature map. It has shape (num_priors, 2). centers (Tensor): The prior points of a object in all feature pyramid. It has shape (num_pos, 2) strides (Tensor): The prior strides of a object in all feature pyramid. It has shape (num_pos, 1) size_of_interest (int): The size of the region used in rel coord. feat_sizes (Tuple[int]): The feature size H and W, which has 2 dims. Returns: rel_coord_feat (Tensor): The coordinate feature of shape (num_pos, 2, H, W). """ H, W = feat_sizes rel_coordinates = centers.reshape(-1, 1, 2) - locations.reshape(1, -1, 2) rel_coordinates = rel_coordinates.permute(0, 2, 1).float() rel_coordinates = rel_coordinates / ( strides[:, None, None] * size_of_interest) return rel_coordinates.reshape(-1, 2, H, W) def aligned_bilinear(tensor: Tensor, factor: int) -> Tensor: """aligned bilinear, used in original implement in CondInst: https://github.com/aim-uofa/AdelaiDet/blob/\ c0b2092ce72442b0f40972f7c6dda8bb52c46d16/adet/utils/comm.py#L23 """ assert tensor.dim() == 4 assert factor >= 1 assert int(factor) == factor if factor == 1: return tensor h, w = tensor.size()[2:] tensor = F.pad(tensor, pad=(0, 1, 0, 1), mode='replicate') oh = factor * h + 1 ow = factor * w + 1 tensor = F.interpolate( tensor, size=(oh, ow), mode='bilinear', align_corners=True) tensor = F.pad( tensor, pad=(factor // 2, 0, factor // 2, 0), mode='replicate') return tensor[:, :, :oh - 1, :ow - 1] def unfold_wo_center(x, kernel_size: int, dilation: int) -> Tensor: """unfold_wo_center, used in original implement in BoxInst: https://github.com/aim-uofa/AdelaiDet/blob/\ 4a3a1f7372c35b48ebf5f6adc59f135a0fa28d60/\ adet/modeling/condinst/condinst.py#L53 """ assert x.dim() == 4 assert kernel_size % 2 == 1 # using SAME padding padding = (kernel_size + (dilation - 1) * (kernel_size - 1)) // 2 unfolded_x = F.unfold( x, kernel_size=kernel_size, padding=padding, dilation=dilation) unfolded_x = unfolded_x.reshape( x.size(0), x.size(1), -1, x.size(2), x.size(3)) # remove the center pixels size = kernel_size**2 unfolded_x = torch.cat( (unfolded_x[:, :, :size // 2], unfolded_x[:, :, size // 2 + 1:]), dim=2) return unfolded_x ================================================ FILE: mmdet/models/utils/panoptic_gt_processing.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple import torch from torch import Tensor def preprocess_panoptic_gt(gt_labels: Tensor, gt_masks: Tensor, gt_semantic_seg: Tensor, num_things: int, num_stuff: int) -> Tuple[Tensor, Tensor]: """Preprocess the ground truth for a image. Args: gt_labels (Tensor): Ground truth labels of each bbox, with shape (num_gts, ). gt_masks (BitmapMasks): Ground truth masks of each instances of a image, shape (num_gts, h, w). gt_semantic_seg (Tensor | None): Ground truth of semantic segmentation with the shape (1, h, w). [0, num_thing_class - 1] means things, [num_thing_class, num_class-1] means stuff, 255 means VOID. It's None when training instance segmentation. Returns: tuple[Tensor, Tensor]: a tuple containing the following targets. - labels (Tensor): Ground truth class indices for a image, with shape (n, ), n is the sum of number of stuff type and number of instance in a image. - masks (Tensor): Ground truth mask for a image, with shape (n, h, w). Contains stuff and things when training panoptic segmentation, and things only when training instance segmentation. """ num_classes = num_things + num_stuff things_masks = gt_masks.to_tensor( dtype=torch.bool, device=gt_labels.device) if gt_semantic_seg is None: masks = things_masks.long() return gt_labels, masks things_labels = gt_labels gt_semantic_seg = gt_semantic_seg.squeeze(0) semantic_labels = torch.unique( gt_semantic_seg, sorted=False, return_inverse=False, return_counts=False) stuff_masks_list = [] stuff_labels_list = [] for label in semantic_labels: if label < num_things or label >= num_classes: continue stuff_mask = gt_semantic_seg == label stuff_masks_list.append(stuff_mask) stuff_labels_list.append(label) if len(stuff_masks_list) > 0: stuff_masks = torch.stack(stuff_masks_list, dim=0) stuff_labels = torch.stack(stuff_labels_list, dim=0) labels = torch.cat([things_labels, stuff_labels], dim=0) masks = torch.cat([things_masks, stuff_masks], dim=0) else: labels = things_labels masks = things_masks masks = masks.long() return labels, masks ================================================ FILE: mmdet/models/utils/point_sample.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch from mmcv.ops import point_sample from torch import Tensor def get_uncertainty(mask_preds: Tensor, labels: Tensor) -> Tensor: """Estimate uncertainty based on pred logits. We estimate uncertainty as L1 distance between 0.0 and the logits prediction in 'mask_preds' for the foreground class in `classes`. Args: mask_preds (Tensor): mask predication logits, shape (num_rois, num_classes, mask_height, mask_width). labels (Tensor): Either predicted or ground truth label for each predicted mask, of length num_rois. Returns: scores (Tensor): Uncertainty scores with the most uncertain locations having the highest uncertainty score, shape (num_rois, 1, mask_height, mask_width) """ if mask_preds.shape[1] == 1: gt_class_logits = mask_preds.clone() else: inds = torch.arange(mask_preds.shape[0], device=mask_preds.device) gt_class_logits = mask_preds[inds, labels].unsqueeze(1) return -torch.abs(gt_class_logits) def get_uncertain_point_coords_with_randomness( mask_preds: Tensor, labels: Tensor, num_points: int, oversample_ratio: float, importance_sample_ratio: float) -> Tensor: """Get ``num_points`` most uncertain points with random points during train. Sample points in [0, 1] x [0, 1] coordinate space based on their uncertainty. The uncertainties are calculated for each point using 'get_uncertainty()' function that takes point's logit prediction as input. Args: mask_preds (Tensor): A tensor of shape (num_rois, num_classes, mask_height, mask_width) for class-specific or class-agnostic prediction. labels (Tensor): The ground truth class for each instance. num_points (int): The number of points to sample. oversample_ratio (float): Oversampling parameter. importance_sample_ratio (float): Ratio of points that are sampled via importnace sampling. Returns: point_coords (Tensor): A tensor of shape (num_rois, num_points, 2) that contains the coordinates sampled points. """ assert oversample_ratio >= 1 assert 0 <= importance_sample_ratio <= 1 batch_size = mask_preds.shape[0] num_sampled = int(num_points * oversample_ratio) point_coords = torch.rand( batch_size, num_sampled, 2, device=mask_preds.device) point_logits = point_sample(mask_preds, point_coords) # It is crucial to calculate uncertainty based on the sampled # prediction value for the points. Calculating uncertainties of the # coarse predictions first and sampling them for points leads to # incorrect results. To illustrate this: assume uncertainty func( # logits)=-abs(logits), a sampled point between two coarse # predictions with -1 and 1 logits has 0 logits, and therefore 0 # uncertainty value. However, if we calculate uncertainties for the # coarse predictions first, both will have -1 uncertainty, # and sampled point will get -1 uncertainty. point_uncertainties = get_uncertainty(point_logits, labels) num_uncertain_points = int(importance_sample_ratio * num_points) num_random_points = num_points - num_uncertain_points idx = torch.topk( point_uncertainties[:, 0, :], k=num_uncertain_points, dim=1)[1] shift = num_sampled * torch.arange( batch_size, dtype=torch.long, device=mask_preds.device) idx += shift[:, None] point_coords = point_coords.view(-1, 2)[idx.view(-1), :].view( batch_size, num_uncertain_points, 2) if num_random_points > 0: rand_roi_coords = torch.rand( batch_size, num_random_points, 2, device=mask_preds.device) point_coords = torch.cat((point_coords, rand_roi_coords), dim=1) return point_coords ================================================ FILE: mmdet/registry.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """MMDetection provides 17 registry nodes to support using modules across projects. Each node is a child of the root registry in MMEngine. More details can be found at https://mmengine.readthedocs.io/en/latest/tutorials/registry.html. """ from mmengine.registry import DATA_SAMPLERS as MMENGINE_DATA_SAMPLERS from mmengine.registry import DATASETS as MMENGINE_DATASETS from mmengine.registry import EVALUATOR as MMENGINE_EVALUATOR from mmengine.registry import HOOKS as MMENGINE_HOOKS from mmengine.registry import LOG_PROCESSORS as MMENGINE_LOG_PROCESSORS from mmengine.registry import LOOPS as MMENGINE_LOOPS from mmengine.registry import METRICS as MMENGINE_METRICS from mmengine.registry import MODEL_WRAPPERS as MMENGINE_MODEL_WRAPPERS from mmengine.registry import MODELS as MMENGINE_MODELS from mmengine.registry import \ OPTIM_WRAPPER_CONSTRUCTORS as MMENGINE_OPTIM_WRAPPER_CONSTRUCTORS from mmengine.registry import OPTIM_WRAPPERS as MMENGINE_OPTIM_WRAPPERS from mmengine.registry import OPTIMIZERS as MMENGINE_OPTIMIZERS from mmengine.registry import PARAM_SCHEDULERS as MMENGINE_PARAM_SCHEDULERS from mmengine.registry import \ RUNNER_CONSTRUCTORS as MMENGINE_RUNNER_CONSTRUCTORS from mmengine.registry import RUNNERS as MMENGINE_RUNNERS from mmengine.registry import TASK_UTILS as MMENGINE_TASK_UTILS from mmengine.registry import TRANSFORMS as MMENGINE_TRANSFORMS from mmengine.registry import VISBACKENDS as MMENGINE_VISBACKENDS from mmengine.registry import VISUALIZERS as MMENGINE_VISUALIZERS from mmengine.registry import \ WEIGHT_INITIALIZERS as MMENGINE_WEIGHT_INITIALIZERS from mmengine.registry import Registry # manage all kinds of runners like `EpochBasedRunner` and `IterBasedRunner` RUNNERS = Registry( 'runner', parent=MMENGINE_RUNNERS, locations=['mmdet.engine.runner']) # manage runner constructors that define how to initialize runners RUNNER_CONSTRUCTORS = Registry( 'runner constructor', parent=MMENGINE_RUNNER_CONSTRUCTORS, locations=['mmdet.engine.runner']) # manage all kinds of loops like `EpochBasedTrainLoop` LOOPS = Registry( 'loop', parent=MMENGINE_LOOPS, locations=['mmdet.engine.runner']) # manage all kinds of hooks like `CheckpointHook` HOOKS = Registry( 'hook', parent=MMENGINE_HOOKS, locations=['mmdet.engine.hooks']) # manage data-related modules DATASETS = Registry( 'dataset', parent=MMENGINE_DATASETS, locations=['mmdet.datasets']) DATA_SAMPLERS = Registry( 'data sampler', parent=MMENGINE_DATA_SAMPLERS, locations=['mmdet.datasets.samplers']) TRANSFORMS = Registry( 'transform', parent=MMENGINE_TRANSFORMS, locations=['mmdet.datasets.transforms']) # manage all kinds of modules inheriting `nn.Module` MODELS = Registry('model', parent=MMENGINE_MODELS, locations=['mmdet.models']) # manage all kinds of model wrappers like 'MMDistributedDataParallel' MODEL_WRAPPERS = Registry( 'model_wrapper', parent=MMENGINE_MODEL_WRAPPERS, locations=['mmdet.models']) # manage all kinds of weight initialization modules like `Uniform` WEIGHT_INITIALIZERS = Registry( 'weight initializer', parent=MMENGINE_WEIGHT_INITIALIZERS, locations=['mmdet.models']) # manage all kinds of optimizers like `SGD` and `Adam` OPTIMIZERS = Registry( 'optimizer', parent=MMENGINE_OPTIMIZERS, locations=['mmdet.engine.optimizers']) # manage optimizer wrapper OPTIM_WRAPPERS = Registry( 'optim_wrapper', parent=MMENGINE_OPTIM_WRAPPERS, locations=['mmdet.engine.optimizers']) # manage constructors that customize the optimization hyperparameters. OPTIM_WRAPPER_CONSTRUCTORS = Registry( 'optimizer constructor', parent=MMENGINE_OPTIM_WRAPPER_CONSTRUCTORS, locations=['mmdet.engine.optimizers']) # manage all kinds of parameter schedulers like `MultiStepLR` PARAM_SCHEDULERS = Registry( 'parameter scheduler', parent=MMENGINE_PARAM_SCHEDULERS, locations=['mmdet.engine.schedulers']) # manage all kinds of metrics METRICS = Registry( 'metric', parent=MMENGINE_METRICS, locations=['mmdet.evaluation']) # manage evaluator EVALUATOR = Registry( 'evaluator', parent=MMENGINE_EVALUATOR, locations=['mmdet.evaluation']) # manage task-specific modules like anchor generators and box coders TASK_UTILS = Registry( 'task util', parent=MMENGINE_TASK_UTILS, locations=['mmdet.models']) # manage visualizer VISUALIZERS = Registry( 'visualizer', parent=MMENGINE_VISUALIZERS, locations=['mmdet.visualization']) # manage visualizer backend VISBACKENDS = Registry( 'vis_backend', parent=MMENGINE_VISBACKENDS, locations=['mmdet.visualization']) # manage logprocessor LOG_PROCESSORS = Registry( 'log_processor', parent=MMENGINE_LOG_PROCESSORS, # TODO: update the location when mmdet has its own log processor locations=['mmdet.engine']) ================================================ FILE: mmdet/structures/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .det_data_sample import DetDataSample, OptSampleList, SampleList __all__ = ['DetDataSample', 'SampleList', 'OptSampleList'] ================================================ FILE: mmdet/structures/bbox/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .base_boxes import BaseBoxes from .bbox_overlaps import bbox_overlaps from .box_type import (autocast_box_type, convert_box_type, get_box_type, register_box, register_box_converter) from .horizontal_boxes import HorizontalBoxes from .transforms import (bbox2corner, bbox2distance, bbox2result, bbox2roi, bbox_cxcywh_to_xyxy, bbox_flip, bbox_mapping, bbox_mapping_back, bbox_project, bbox_rescale, bbox_xyxy_to_cxcywh, cat_boxes, corner2bbox, distance2bbox, empty_box_as, find_inside_bboxes, get_box_tensor, get_box_wh, roi2bbox, scale_boxes, stack_boxes) __all__ = [ 'bbox_overlaps', 'bbox_flip', 'bbox_mapping', 'bbox_mapping_back', 'bbox2roi', 'roi2bbox', 'bbox2result', 'distance2bbox', 'bbox2distance', 'bbox_rescale', 'bbox_cxcywh_to_xyxy', 'bbox_xyxy_to_cxcywh', 'find_inside_bboxes', 'bbox2corner', 'corner2bbox', 'bbox_project', 'BaseBoxes', 'convert_box_type', 'get_box_type', 'register_box', 'register_box_converter', 'HorizontalBoxes', 'autocast_box_type', 'cat_boxes', 'stack_boxes', 'scale_boxes', 'get_box_wh', 'get_box_tensor', 'empty_box_as' ] ================================================ FILE: mmdet/structures/bbox/base_boxes.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from abc import ABCMeta, abstractmethod, abstractproperty, abstractstaticmethod from typing import List, Optional, Sequence, Tuple, Type, TypeVar, Union import numpy as np import torch from torch import BoolTensor, Tensor from mmdet.structures.mask.structures import BitmapMasks, PolygonMasks T = TypeVar('T') DeviceType = Union[str, torch.device] IndexType = Union[slice, int, list, torch.LongTensor, torch.cuda.LongTensor, torch.BoolTensor, torch.cuda.BoolTensor, np.ndarray] MaskType = Union[BitmapMasks, PolygonMasks] class BaseBoxes(metaclass=ABCMeta): """The base class for 2D box types. The functions of ``BaseBoxes`` lie in three fields: - Verify the boxes shape. - Support tensor-like operations. - Define abstract functions for 2D boxes. In ``__init__`` , ``BaseBoxes`` verifies the validity of the data shape w.r.t ``box_dim``. The tensor with the dimension >= 2 and the length of the last dimension being ``box_dim`` will be regarded as valid. ``BaseBoxes`` will restore them at the field ``tensor``. It's necessary to override ``box_dim`` in subclass to guarantee the data shape is correct. There are many basic tensor-like functions implemented in ``BaseBoxes``. In most cases, users can operate ``BaseBoxes`` instance like a normal tensor. To protect the validity of data shape, All tensor-like functions cannot modify the last dimension of ``self.tensor``. When creating a new box type, users need to inherit from ``BaseBoxes`` and override abstract methods and specify the ``box_dim``. Then, register the new box type by using the decorator ``register_box_type``. Args: data (Tensor or np.ndarray or Sequence): The box data with shape (..., box_dim). dtype (torch.dtype, Optional): data type of boxes. Defaults to None. device (str or torch.device, Optional): device of boxes. Default to None. clone (bool): Whether clone ``boxes`` or not. Defaults to True. """ # Used to verify the last dimension length # Should override it in subclass. box_dim: int = 0 def __init__(self, data: Union[Tensor, np.ndarray, Sequence], dtype: Optional[torch.dtype] = None, device: Optional[DeviceType] = None, clone: bool = True) -> None: if isinstance(data, (np.ndarray, Tensor, Sequence)): data = torch.as_tensor(data) else: raise TypeError('boxes should be Tensor, ndarray, or Sequence, ', f'but got {type(data)}') if device is not None or dtype is not None: data = data.to(dtype=dtype, device=device) # Clone the data to avoid potential bugs if clone: data = data.clone() # handle the empty input like [] if data.numel() == 0: data = data.reshape((-1, self.box_dim)) assert data.dim() >= 2 and data.size(-1) == self.box_dim, \ ('The boxes dimension must >= 2 and the length of the last ' f'dimension must be {self.box_dim}, but got boxes with ' f'shape {data.shape}.') self.tensor = data def convert_to(self, dst_type: Union[str, type]) -> 'BaseBoxes': """Convert self to another box type. Args: dst_type (str or type): destination box type. Returns: :obj:`BaseBoxes`: destination box type object . """ from .box_type import convert_box_type return convert_box_type(self, dst_type=dst_type) def empty_boxes(self: T, dtype: Optional[torch.dtype] = None, device: Optional[DeviceType] = None) -> T: """Create empty box. Args: dtype (torch.dtype, Optional): data type of boxes. device (str or torch.device, Optional): device of boxes. Returns: T: empty boxes with shape of (0, box_dim). """ empty_box = self.tensor.new_zeros( 0, self.box_dim, dtype=dtype, device=device) return type(self)(empty_box, clone=False) def fake_boxes(self: T, sizes: Tuple[int], fill: float = 0, dtype: Optional[torch.dtype] = None, device: Optional[DeviceType] = None) -> T: """Create fake boxes with specific sizes and fill values. Args: sizes (Tuple[int]): The size of fake boxes. The last value must be equal with ``self.box_dim``. fill (float): filling value. Defaults to 0. dtype (torch.dtype, Optional): data type of boxes. device (str or torch.device, Optional): device of boxes. Returns: T: Fake boxes with shape of ``sizes``. """ fake_boxes = self.tensor.new_full( sizes, fill, dtype=dtype, device=device) return type(self)(fake_boxes, clone=False) def __getitem__(self: T, index: IndexType) -> T: """Rewrite getitem to protect the last dimension shape.""" boxes = self.tensor if isinstance(index, np.ndarray): index = torch.as_tensor(index, device=self.device) if isinstance(index, Tensor) and index.dtype == torch.bool: assert index.dim() < boxes.dim() elif isinstance(index, tuple): assert len(index) < boxes.dim() # `Ellipsis`(...) is commonly used in index like [None, ...]. # When `Ellipsis` is in index, it must be the last item. if Ellipsis in index: assert index[-1] is Ellipsis boxes = boxes[index] if boxes.dim() == 1: boxes = boxes.reshape(1, -1) return type(self)(boxes, clone=False) def __setitem__(self: T, index: IndexType, values: Union[Tensor, T]) -> T: """Rewrite setitem to protect the last dimension shape.""" assert type(values) is type(self), \ 'The value to be set must be the same box type as self' values = values.tensor if isinstance(index, np.ndarray): index = torch.as_tensor(index, device=self.device) if isinstance(index, Tensor) and index.dtype == torch.bool: assert index.dim() < self.tensor.dim() elif isinstance(index, tuple): assert len(index) < self.tensor.dim() # `Ellipsis`(...) is commonly used in index like [None, ...]. # When `Ellipsis` is in index, it must be the last item. if Ellipsis in index: assert index[-1] is Ellipsis self.tensor[index] = values def __len__(self) -> int: """Return the length of self.tensor first dimension.""" return self.tensor.size(0) def __deepcopy__(self, memo): """Only clone the ``self.tensor`` when applying deepcopy.""" cls = self.__class__ other = cls.__new__(cls) memo[id(self)] = other other.tensor = self.tensor.clone() return other def __repr__(self) -> str: """Return a strings that describes the object.""" return self.__class__.__name__ + '(\n' + str(self.tensor) + ')' def new_tensor(self, *args, **kwargs) -> Tensor: """Reload ``new_tensor`` from self.tensor.""" return self.tensor.new_tensor(*args, **kwargs) def new_full(self, *args, **kwargs) -> Tensor: """Reload ``new_full`` from self.tensor.""" return self.tensor.new_full(*args, **kwargs) def new_empty(self, *args, **kwargs) -> Tensor: """Reload ``new_empty`` from self.tensor.""" return self.tensor.new_empty(*args, **kwargs) def new_ones(self, *args, **kwargs) -> Tensor: """Reload ``new_ones`` from self.tensor.""" return self.tensor.new_ones(*args, **kwargs) def new_zeros(self, *args, **kwargs) -> Tensor: """Reload ``new_zeros`` from self.tensor.""" return self.tensor.new_zeros(*args, **kwargs) def size(self, dim: Optional[int] = None) -> Union[int, torch.Size]: """Reload new_zeros from self.tensor.""" # self.tensor.size(dim) cannot work when dim=None. return self.tensor.size() if dim is None else self.tensor.size(dim) def dim(self) -> int: """Reload ``dim`` from self.tensor.""" return self.tensor.dim() @property def device(self) -> torch.device: """Reload ``device`` from self.tensor.""" return self.tensor.device @property def dtype(self) -> torch.dtype: """Reload ``dtype`` from self.tensor.""" return self.tensor.dtype @property def shape(self) -> torch.Size: return self.tensor.shape def numel(self) -> int: """Reload ``numel`` from self.tensor.""" return self.tensor.numel() def numpy(self) -> np.ndarray: """Reload ``numpy`` from self.tensor.""" return self.tensor.numpy() def to(self: T, *args, **kwargs) -> T: """Reload ``to`` from self.tensor.""" return type(self)(self.tensor.to(*args, **kwargs), clone=False) def cpu(self: T) -> T: """Reload ``cpu`` from self.tensor.""" return type(self)(self.tensor.cpu(), clone=False) def cuda(self: T, *args, **kwargs) -> T: """Reload ``cuda`` from self.tensor.""" return type(self)(self.tensor.cuda(*args, **kwargs), clone=False) def clone(self: T) -> T: """Reload ``clone`` from self.tensor.""" return type(self)(self.tensor) def detach(self: T) -> T: """Reload ``detach`` from self.tensor.""" return type(self)(self.tensor.detach(), clone=False) def view(self: T, *shape: Tuple[int]) -> T: """Reload ``view`` from self.tensor.""" return type(self)(self.tensor.view(shape), clone=False) def reshape(self: T, *shape: Tuple[int]) -> T: """Reload ``reshape`` from self.tensor.""" return type(self)(self.tensor.reshape(shape), clone=False) def expand(self: T, *sizes: Tuple[int]) -> T: """Reload ``expand`` from self.tensor.""" return type(self)(self.tensor.expand(sizes), clone=False) def repeat(self: T, *sizes: Tuple[int]) -> T: """Reload ``repeat`` from self.tensor.""" return type(self)(self.tensor.repeat(sizes), clone=False) def transpose(self: T, dim0: int, dim1: int) -> T: """Reload ``transpose`` from self.tensor.""" ndim = self.tensor.dim() assert dim0 != -1 and dim0 != ndim - 1 assert dim1 != -1 and dim1 != ndim - 1 return type(self)(self.tensor.transpose(dim0, dim1), clone=False) def permute(self: T, *dims: Tuple[int]) -> T: """Reload ``permute`` from self.tensor.""" assert dims[-1] == -1 or dims[-1] == self.tensor.dim() - 1 return type(self)(self.tensor.permute(dims), clone=False) def split(self: T, split_size_or_sections: Union[int, Sequence[int]], dim: int = 0) -> List[T]: """Reload ``split`` from self.tensor.""" assert dim != -1 and dim != self.tensor.dim() - 1 boxes_list = self.tensor.split(split_size_or_sections, dim=dim) return [type(self)(boxes, clone=False) for boxes in boxes_list] def chunk(self: T, chunks: int, dim: int = 0) -> List[T]: """Reload ``chunk`` from self.tensor.""" assert dim != -1 and dim != self.tensor.dim() - 1 boxes_list = self.tensor.chunk(chunks, dim=dim) return [type(self)(boxes, clone=False) for boxes in boxes_list] def unbind(self: T, dim: int = 0) -> T: """Reload ``unbind`` from self.tensor.""" assert dim != -1 and dim != self.tensor.dim() - 1 boxes_list = self.tensor.unbind(dim=dim) return [type(self)(boxes, clone=False) for boxes in boxes_list] def flatten(self: T, start_dim: int = 0, end_dim: int = -2) -> T: """Reload ``flatten`` from self.tensor.""" assert end_dim != -1 and end_dim != self.tensor.dim() - 1 return type(self)(self.tensor.flatten(start_dim, end_dim), clone=False) def squeeze(self: T, dim: Optional[int] = None) -> T: """Reload ``squeeze`` from self.tensor.""" boxes = self.tensor.squeeze() if dim is None else \ self.tensor.squeeze(dim) return type(self)(boxes, clone=False) def unsqueeze(self: T, dim: int) -> T: """Reload ``unsqueeze`` from self.tensor.""" assert dim != -1 and dim != self.tensor.dim() return type(self)(self.tensor.unsqueeze(dim), clone=False) @classmethod def cat(cls: Type[T], box_list: Sequence[T], dim: int = 0) -> T: """Cancatenates a box instance list into one single box instance. Similar to ``torch.cat``. Args: box_list (Sequence[T]): A sequence of box instances. dim (int): The dimension over which the box are concatenated. Defaults to 0. Returns: T: Concatenated box instance. """ assert isinstance(box_list, Sequence) if len(box_list) == 0: raise ValueError('box_list should not be a empty list.') assert dim != -1 and dim != box_list[0].dim() - 1 assert all(isinstance(boxes, cls) for boxes in box_list) th_box_list = [boxes.tensor for boxes in box_list] return cls(torch.cat(th_box_list, dim=dim), clone=False) @classmethod def stack(cls: Type[T], box_list: Sequence[T], dim: int = 0) -> T: """Concatenates a sequence of tensors along a new dimension. Similar to ``torch.stack``. Args: box_list (Sequence[T]): A sequence of box instances. dim (int): Dimension to insert. Defaults to 0. Returns: T: Concatenated box instance. """ assert isinstance(box_list, Sequence) if len(box_list) == 0: raise ValueError('box_list should not be a empty list.') assert dim != -1 and dim != box_list[0].dim() assert all(isinstance(boxes, cls) for boxes in box_list) th_box_list = [boxes.tensor for boxes in box_list] return cls(torch.stack(th_box_list, dim=dim), clone=False) @abstractproperty def centers(self) -> Tensor: """Return a tensor representing the centers of boxes.""" pass @abstractproperty def areas(self) -> Tensor: """Return a tensor representing the areas of boxes.""" pass @abstractproperty def widths(self) -> Tensor: """Return a tensor representing the widths of boxes.""" pass @abstractproperty def heights(self) -> Tensor: """Return a tensor representing the heights of boxes.""" pass @abstractmethod def flip_(self, img_shape: Tuple[int, int], direction: str = 'horizontal') -> None: """Flip boxes horizontally or vertically in-place. Args: img_shape (Tuple[int, int]): A tuple of image height and width. direction (str): Flip direction, options are "horizontal", "vertical" and "diagonal". Defaults to "horizontal" """ pass @abstractmethod def translate_(self, distances: Tuple[float, float]) -> None: """Translate boxes in-place. Args: distances (Tuple[float, float]): translate distances. The first is horizontal distance and the second is vertical distance. """ pass @abstractmethod def clip_(self, img_shape: Tuple[int, int]) -> None: """Clip boxes according to the image shape in-place. Args: img_shape (Tuple[int, int]): A tuple of image height and width. """ pass @abstractmethod def rotate_(self, center: Tuple[float, float], angle: float) -> None: """Rotate all boxes in-place. Args: center (Tuple[float, float]): Rotation origin. angle (float): Rotation angle represented in degrees. Positive values mean clockwise rotation. """ pass @abstractmethod def project_(self, homography_matrix: Union[Tensor, np.ndarray]) -> None: """Geometric transformat boxes in-place. Args: homography_matrix (Tensor or np.ndarray]): Shape (3, 3) for geometric transformation. """ pass @abstractmethod def rescale_(self, scale_factor: Tuple[float, float]) -> None: """Rescale boxes w.r.t. rescale_factor in-place. Note: Both ``rescale_`` and ``resize_`` will enlarge or shrink boxes w.r.t ``scale_facotr``. The difference is that ``resize_`` only changes the width and the height of boxes, but ``rescale_`` also rescales the box centers simultaneously. Args: scale_factor (Tuple[float, float]): factors for scaling boxes. The length should be 2. """ pass @abstractmethod def resize_(self, scale_factor: Tuple[float, float]) -> None: """Resize the box width and height w.r.t scale_factor in-place. Note: Both ``rescale_`` and ``resize_`` will enlarge or shrink boxes w.r.t ``scale_facotr``. The difference is that ``resize_`` only changes the width and the height of boxes, but ``rescale_`` also rescales the box centers simultaneously. Args: scale_factor (Tuple[float, float]): factors for scaling box shapes. The length should be 2. """ pass @abstractmethod def is_inside(self, img_shape: Tuple[int, int], all_inside: bool = False, allowed_border: int = 0) -> BoolTensor: """Find boxes inside the image. Args: img_shape (Tuple[int, int]): A tuple of image height and width. all_inside (bool): Whether the boxes are all inside the image or part inside the image. Defaults to False. allowed_border (int): Boxes that extend beyond the image shape boundary by more than ``allowed_border`` are considered "outside" Defaults to 0. Returns: BoolTensor: A BoolTensor indicating whether the box is inside the image. Assuming the original boxes have shape (m, n, box_dim), the output has shape (m, n). """ pass @abstractmethod def find_inside_points(self, points: Tensor, is_aligned: bool = False) -> BoolTensor: """Find inside box points. Boxes dimension must be 2. Args: points (Tensor): Points coordinates. Has shape of (m, 2). is_aligned (bool): Whether ``points`` has been aligned with boxes or not. If True, the length of boxes and ``points`` should be the same. Defaults to False. Returns: BoolTensor: A BoolTensor indicating whether a point is inside boxes. Assuming the boxes has shape of (n, box_dim), if ``is_aligned`` is False. The index has shape of (m, n). If ``is_aligned`` is True, m should be equal to n and the index has shape of (m, ). """ pass @abstractstaticmethod def overlaps(boxes1: 'BaseBoxes', boxes2: 'BaseBoxes', mode: str = 'iou', is_aligned: bool = False, eps: float = 1e-6) -> Tensor: """Calculate overlap between two set of boxes with their types converted to the present box type. Args: boxes1 (:obj:`BaseBoxes`): BaseBoxes with shape of (m, box_dim) or empty. boxes2 (:obj:`BaseBoxes`): BaseBoxes with shape of (n, box_dim) or empty. mode (str): "iou" (intersection over union), "iof" (intersection over foreground). Defaults to "iou". is_aligned (bool): If True, then m and n must be equal. Defaults to False. eps (float): A value added to the denominator for numerical stability. Defaults to 1e-6. Returns: Tensor: shape (m, n) if ``is_aligned`` is False else shape (m,) """ pass @abstractstaticmethod def from_instance_masks(masks: MaskType) -> 'BaseBoxes': """Create boxes from instance masks. Args: masks (:obj:`BitmapMasks` or :obj:`PolygonMasks`): BitmapMasks or PolygonMasks instance with length of n. Returns: :obj:`BaseBoxes`: Converted boxes with shape of (n, box_dim). """ pass ================================================ FILE: mmdet/structures/bbox/bbox_overlaps.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch def fp16_clamp(x, min=None, max=None): if not x.is_cuda and x.dtype == torch.float16: # clamp for cpu float16, tensor fp16 has no clamp implementation return x.float().clamp(min, max).half() return x.clamp(min, max) def bbox_overlaps(bboxes1, bboxes2, mode='iou', is_aligned=False, eps=1e-6): """Calculate overlap between two set of bboxes. FP16 Contributed by https://github.com/open-mmlab/mmdetection/pull/4889 Note: Assume bboxes1 is M x 4, bboxes2 is N x 4, when mode is 'iou', there are some new generated variable when calculating IOU using bbox_overlaps function: 1) is_aligned is False area1: M x 1 area2: N x 1 lt: M x N x 2 rb: M x N x 2 wh: M x N x 2 overlap: M x N x 1 union: M x N x 1 ious: M x N x 1 Total memory: S = (9 x N x M + N + M) * 4 Byte, When using FP16, we can reduce: R = (9 x N x M + N + M) * 4 / 2 Byte R large than (N + M) * 4 * 2 is always true when N and M >= 1. Obviously, N + M <= N * M < 3 * N * M, when N >=2 and M >=2, N + 1 < 3 * N, when N or M is 1. Given M = 40 (ground truth), N = 400000 (three anchor boxes in per grid, FPN, R-CNNs), R = 275 MB (one times) A special case (dense detection), M = 512 (ground truth), R = 3516 MB = 3.43 GB When the batch size is B, reduce: B x R Therefore, CUDA memory runs out frequently. Experiments on GeForce RTX 2080Ti (11019 MiB): | dtype | M | N | Use | Real | Ideal | |:----:|:----:|:----:|:----:|:----:|:----:| | FP32 | 512 | 400000 | 8020 MiB | -- | -- | | FP16 | 512 | 400000 | 4504 MiB | 3516 MiB | 3516 MiB | | FP32 | 40 | 400000 | 1540 MiB | -- | -- | | FP16 | 40 | 400000 | 1264 MiB | 276MiB | 275 MiB | 2) is_aligned is True area1: N x 1 area2: N x 1 lt: N x 2 rb: N x 2 wh: N x 2 overlap: N x 1 union: N x 1 ious: N x 1 Total memory: S = 11 x N * 4 Byte When using FP16, we can reduce: R = 11 x N * 4 / 2 Byte So do the 'giou' (large than 'iou'). Time-wise, FP16 is generally faster than FP32. When gpu_assign_thr is not -1, it takes more time on cpu but not reduce memory. There, we can reduce half the memory and keep the speed. If ``is_aligned`` is ``False``, then calculate the overlaps between each bbox of bboxes1 and bboxes2, otherwise the overlaps between each aligned pair of bboxes1 and bboxes2. Args: bboxes1 (Tensor): shape (B, m, 4) in format or empty. bboxes2 (Tensor): shape (B, n, 4) in format or empty. B indicates the batch dim, in shape (B1, B2, ..., Bn). If ``is_aligned`` is ``True``, then m and n must be equal. mode (str): "iou" (intersection over union), "iof" (intersection over foreground) or "giou" (generalized intersection over union). Default "iou". is_aligned (bool, optional): If True, then m and n must be equal. Default False. eps (float, optional): A value added to the denominator for numerical stability. Default 1e-6. Returns: Tensor: shape (m, n) if ``is_aligned`` is False else shape (m,) Example: >>> bboxes1 = torch.FloatTensor([ >>> [0, 0, 10, 10], >>> [10, 10, 20, 20], >>> [32, 32, 38, 42], >>> ]) >>> bboxes2 = torch.FloatTensor([ >>> [0, 0, 10, 20], >>> [0, 10, 10, 19], >>> [10, 10, 20, 20], >>> ]) >>> overlaps = bbox_overlaps(bboxes1, bboxes2) >>> assert overlaps.shape == (3, 3) >>> overlaps = bbox_overlaps(bboxes1, bboxes2, is_aligned=True) >>> assert overlaps.shape == (3, ) Example: >>> empty = torch.empty(0, 4) >>> nonempty = torch.FloatTensor([[0, 0, 10, 9]]) >>> assert tuple(bbox_overlaps(empty, nonempty).shape) == (0, 1) >>> assert tuple(bbox_overlaps(nonempty, empty).shape) == (1, 0) >>> assert tuple(bbox_overlaps(empty, empty).shape) == (0, 0) """ assert mode in ['iou', 'iof', 'giou'], f'Unsupported mode {mode}' # Either the boxes are empty or the length of boxes' last dimension is 4 assert (bboxes1.size(-1) == 4 or bboxes1.size(0) == 0) assert (bboxes2.size(-1) == 4 or bboxes2.size(0) == 0) # Batch dim must be the same # Batch dim: (B1, B2, ... Bn) assert bboxes1.shape[:-2] == bboxes2.shape[:-2] batch_shape = bboxes1.shape[:-2] rows = bboxes1.size(-2) cols = bboxes2.size(-2) if is_aligned: assert rows == cols if rows * cols == 0: if is_aligned: return bboxes1.new(batch_shape + (rows, )) else: return bboxes1.new(batch_shape + (rows, cols)) area1 = (bboxes1[..., 2] - bboxes1[..., 0]) * ( bboxes1[..., 3] - bboxes1[..., 1]) area2 = (bboxes2[..., 2] - bboxes2[..., 0]) * ( bboxes2[..., 3] - bboxes2[..., 1]) if is_aligned: lt = torch.max(bboxes1[..., :2], bboxes2[..., :2]) # [B, rows, 2] rb = torch.min(bboxes1[..., 2:], bboxes2[..., 2:]) # [B, rows, 2] wh = fp16_clamp(rb - lt, min=0) overlap = wh[..., 0] * wh[..., 1] if mode in ['iou', 'giou']: union = area1 + area2 - overlap else: union = area1 if mode == 'giou': enclosed_lt = torch.min(bboxes1[..., :2], bboxes2[..., :2]) enclosed_rb = torch.max(bboxes1[..., 2:], bboxes2[..., 2:]) else: lt = torch.max(bboxes1[..., :, None, :2], bboxes2[..., None, :, :2]) # [B, rows, cols, 2] rb = torch.min(bboxes1[..., :, None, 2:], bboxes2[..., None, :, 2:]) # [B, rows, cols, 2] wh = fp16_clamp(rb - lt, min=0) overlap = wh[..., 0] * wh[..., 1] if mode in ['iou', 'giou']: union = area1[..., None] + area2[..., None, :] - overlap else: union = area1[..., None] if mode == 'giou': enclosed_lt = torch.min(bboxes1[..., :, None, :2], bboxes2[..., None, :, :2]) enclosed_rb = torch.max(bboxes1[..., :, None, 2:], bboxes2[..., None, :, 2:]) eps = union.new_tensor([eps]) union = torch.max(union, eps) ious = overlap / union if mode in ['iou', 'iof']: return ious # calculate gious enclose_wh = fp16_clamp(enclosed_rb - enclosed_lt, min=0) enclose_area = enclose_wh[..., 0] * enclose_wh[..., 1] enclose_area = torch.max(enclose_area, eps) gious = ious - (enclose_area - union) / enclose_area return gious ================================================ FILE: mmdet/structures/bbox/box_type.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Callable, Optional, Tuple, Type, Union import numpy as np import torch from torch import Tensor from .base_boxes import BaseBoxes BoxType = Union[np.ndarray, Tensor, BaseBoxes] box_types: dict = {} _box_type_to_name: dict = {} box_converters: dict = {} def _register_box(name: str, box_type: Type, force: bool = False) -> None: """Register a box type. Args: name (str): The name of box type. box_type (type): Box mode class to be registered. force (bool): Whether to override an existing class with the same name. Defaults to False. """ assert issubclass(box_type, BaseBoxes) name = name.lower() if not force and (name in box_types or box_type in _box_type_to_name): raise KeyError(f'box type {name} has been registered') elif name in box_types: _box_type = box_types.pop(name) _box_type_to_name.pop(_box_type) elif box_type in _box_type_to_name: _name = _box_type_to_name.pop(box_type) box_types.pop(_name) box_types[name] = box_type _box_type_to_name[box_type] = name def register_box(name: str, box_type: Type = None, force: bool = False) -> Union[Type, Callable]: """Register a box type. A record will be added to ``bbox_types``, whose key is the box type name and value is the box type itself. Simultaneously, a reverse dictionary ``_box_type_to_name`` will be updated. It can be used as a decorator or a normal function. Args: name (str): The name of box type. bbox_type (type, Optional): Box type class to be registered. Defaults to None. force (bool): Whether to override the existing box type with the same name. Defaults to False. Examples: >>> from mmdet.structures.bbox import register_box >>> from mmdet.structures.bbox import BaseBoxes >>> # as a decorator >>> @register_box('hbox') >>> class HorizontalBoxes(BaseBoxes): >>> pass >>> # as a normal function >>> class RotatedBoxes(BaseBoxes): >>> pass >>> register_box('rbox', RotatedBoxes) """ if not isinstance(force, bool): raise TypeError(f'force must be a boolean, but got {type(force)}') # use it as a normal method: register_box(name, box_type=BoxCls) if box_type is not None: _register_box(name=name, box_type=box_type, force=force) return box_type # use it as a decorator: @register_box(name) def _register(cls): _register_box(name=name, box_type=cls, force=force) return cls return _register def _register_box_converter(src_type: Union[str, type], dst_type: Union[str, type], converter: Callable, force: bool = False) -> None: """Register a box converter. Args: src_type (str or type): source box type name or class. dst_type (str or type): destination box type name or class. converter (Callable): Convert function. force (bool): Whether to override the existing box type with the same name. Defaults to False. """ assert callable(converter) src_type_name, _ = get_box_type(src_type) dst_type_name, _ = get_box_type(dst_type) converter_name = src_type_name + '2' + dst_type_name if not force and converter_name in box_converters: raise KeyError(f'The box converter from {src_type_name} to ' f'{dst_type_name} has been registered.') box_converters[converter_name] = converter def register_box_converter(src_type: Union[str, type], dst_type: Union[str, type], converter: Optional[Callable] = None, force: bool = False) -> Callable: """Register a box converter. A record will be added to ``box_converter``, whose key is '{src_type_name}2{dst_type_name}' and value is the convert function. It can be used as a decorator or a normal function. Args: src_type (str or type): source box type name or class. dst_type (str or type): destination box type name or class. converter (Callable): Convert function. Defaults to None. force (bool): Whether to override the existing box type with the same name. Defaults to False. Examples: >>> from mmdet.structures.bbox import register_box_converter >>> # as a decorator >>> @register_box_converter('hbox', 'rbox') >>> def converter_A(boxes): >>> pass >>> # as a normal function >>> def converter_B(boxes): >>> pass >>> register_box_converter('rbox', 'hbox', converter_B) """ if not isinstance(force, bool): raise TypeError(f'force must be a boolean, but got {type(force)}') # use it as a normal method: # register_box_converter(src_type, dst_type, converter=Func) if converter is not None: _register_box_converter( src_type=src_type, dst_type=dst_type, converter=converter, force=force) return converter # use it as a decorator: @register_box_converter(name) def _register(func): _register_box_converter( src_type=src_type, dst_type=dst_type, converter=func, force=force) return func return _register def get_box_type(box_type: Union[str, type]) -> Tuple[str, type]: """get both box type name and class. Args: box_type (str or type): Single box type name or class. Returns: Tuple[str, type]: A tuple of box type name and class. """ if isinstance(box_type, str): type_name = box_type.lower() assert type_name in box_types, \ f"Box type {type_name} hasn't been registered in box_types." type_cls = box_types[type_name] elif issubclass(box_type, BaseBoxes): assert box_type in _box_type_to_name, \ f"Box type {box_type} hasn't been registered in box_types." type_name = _box_type_to_name[box_type] type_cls = box_type else: raise KeyError('box_type must be a str or class inheriting from ' f'BaseBoxes, but got {type(box_type)}.') return type_name, type_cls def convert_box_type(boxes: BoxType, *, src_type: Union[str, type] = None, dst_type: Union[str, type] = None) -> BoxType: """Convert boxes from source type to destination type. If ``boxes`` is a instance of BaseBoxes, the ``src_type`` will be set as the type of ``boxes``. Args: boxes (np.ndarray or Tensor or :obj:`BaseBoxes`): boxes need to convert. src_type (str or type, Optional): source box type. Defaults to None. dst_type (str or type, Optional): destination box type. Defaults to None. Returns: Union[np.ndarray, Tensor, :obj:`BaseBoxes`]: Converted boxes. It's type is consistent with the input's type. """ assert dst_type is not None dst_type_name, dst_type_cls = get_box_type(dst_type) is_box_cls = False is_numpy = False if isinstance(boxes, BaseBoxes): src_type_name, _ = get_box_type(type(boxes)) is_box_cls = True elif isinstance(boxes, (Tensor, np.ndarray)): assert src_type is not None src_type_name, _ = get_box_type(src_type) if isinstance(boxes, np.ndarray): is_numpy = True else: raise TypeError('boxes must be a instance of BaseBoxes, Tensor or ' f'ndarray, but get {type(boxes)}.') if src_type_name == dst_type_name: return boxes converter_name = src_type_name + '2' + dst_type_name assert converter_name in box_converters, \ "Convert function hasn't been registered in box_converters." converter = box_converters[converter_name] if is_box_cls: boxes = converter(boxes.tensor) return dst_type_cls(boxes) elif is_numpy: boxes = converter(torch.from_numpy(boxes)) return boxes.numpy() else: return converter(boxes) def autocast_box_type(dst_box_type='hbox') -> Callable: """A decorator which automatically casts results['gt_bboxes'] to the destination box type. It commenly used in mmdet.datasets.transforms to make the transforms up- compatible with the np.ndarray type of results['gt_bboxes']. The speed of processing of np.ndarray and BaseBoxes data are the same: - np.ndarray: 0.0509 img/s - BaseBoxes: 0.0551 img/s Args: dst_box_type (str): Destination box type. """ _, box_type_cls = get_box_type(dst_box_type) def decorator(func: Callable) -> Callable: def wrapper(self, results: dict, *args, **kwargs) -> dict: if ('gt_bboxes' not in results or isinstance(results['gt_bboxes'], BaseBoxes)): return func(self, results) elif isinstance(results['gt_bboxes'], np.ndarray): results['gt_bboxes'] = box_type_cls( results['gt_bboxes'], clone=False) if 'mix_results' in results: for res in results['mix_results']: if isinstance(res['gt_bboxes'], np.ndarray): res['gt_bboxes'] = box_type_cls( res['gt_bboxes'], clone=False) _results = func(self, results, *args, **kwargs) # In some cases, the function will process gt_bboxes in-place # Simultaneously convert inputting and outputting gt_bboxes # back to np.ndarray if isinstance(_results, dict) and 'gt_bboxes' in _results: if isinstance(_results['gt_bboxes'], BaseBoxes): _results['gt_bboxes'] = _results['gt_bboxes'].numpy() if isinstance(results['gt_bboxes'], BaseBoxes): results['gt_bboxes'] = results['gt_bboxes'].numpy() return _results else: raise TypeError( "auto_box_type requires results['gt_bboxes'] to " 'be BaseBoxes or np.ndarray, but got ' f"{type(results['gt_bboxes'])}") return wrapper return decorator ================================================ FILE: mmdet/structures/bbox/horizontal_boxes.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple, TypeVar, Union import cv2 import numpy as np import torch from torch import BoolTensor, Tensor from mmdet.structures.mask.structures import BitmapMasks, PolygonMasks from .base_boxes import BaseBoxes from .bbox_overlaps import bbox_overlaps from .box_type import register_box T = TypeVar('T') DeviceType = Union[str, torch.device] MaskType = Union[BitmapMasks, PolygonMasks] @register_box(name='hbox') class HorizontalBoxes(BaseBoxes): """The horizontal box class used in MMDetection by default. The ``box_dim`` of ``HorizontalBoxes`` is 4, which means the length of the last dimension of the data should be 4. Two modes of box data are supported in ``HorizontalBoxes``: - 'xyxy': Each row of data indicates (x1, y1, x2, y2), which are the coordinates of the left-top and right-bottom points. - 'cxcywh': Each row of data indicates (x, y, w, h), where (x, y) are the coordinates of the box centers and (w, h) are the width and height. ``HorizontalBoxes`` only restores 'xyxy' mode of data. If the the data is in 'cxcywh' mode, users need to input ``in_mode='cxcywh'`` and The code will convert the 'cxcywh' data to 'xyxy' automatically. Args: data (Tensor or np.ndarray or Sequence): The box data with shape of (..., 4). dtype (torch.dtype, Optional): data type of boxes. Defaults to None. device (str or torch.device, Optional): device of boxes. Default to None. clone (bool): Whether clone ``boxes`` or not. Defaults to True. mode (str, Optional): the mode of boxes. If it is 'cxcywh', the `data` will be converted to 'xyxy' mode. Defaults to None. """ box_dim: int = 4 def __init__(self, data: Union[Tensor, np.ndarray], dtype: torch.dtype = None, device: DeviceType = None, clone: bool = True, in_mode: Optional[str] = None) -> None: super().__init__(data=data, dtype=dtype, device=device, clone=clone) if isinstance(in_mode, str): if in_mode not in ('xyxy', 'cxcywh'): raise ValueError(f'Get invalid mode {in_mode}.') if in_mode == 'cxcywh': self.tensor = self.cxcywh_to_xyxy(self.tensor) @staticmethod def cxcywh_to_xyxy(boxes: Tensor) -> Tensor: """Convert box coordinates from (cx, cy, w, h) to (x1, y1, x2, y2). Args: boxes (Tensor): cxcywh boxes tensor with shape of (..., 4). Returns: Tensor: xyxy boxes tensor with shape of (..., 4). """ ctr, wh = boxes.split((2, 2), dim=-1) return torch.cat([(ctr - wh / 2), (ctr + wh / 2)], dim=-1) @staticmethod def xyxy_to_cxcywh(boxes: Tensor) -> Tensor: """Convert box coordinates from (x1, y1, x2, y2) to (cx, cy, w, h). Args: boxes (Tensor): xyxy boxes tensor with shape of (..., 4). Returns: Tensor: cxcywh boxes tensor with shape of (..., 4). """ xy1, xy2 = boxes.split((2, 2), dim=-1) return torch.cat([(xy2 + xy1) / 2, (xy2 - xy1)], dim=-1) @property def cxcywh(self) -> Tensor: """Return a tensor representing the cxcywh boxes.""" return self.xyxy_to_cxcywh(self.tensor) @property def centers(self) -> Tensor: """Return a tensor representing the centers of boxes.""" boxes = self.tensor return (boxes[..., :2] + boxes[..., 2:]) / 2 @property def areas(self) -> Tensor: """Return a tensor representing the areas of boxes.""" boxes = self.tensor return (boxes[..., 2] - boxes[..., 0]) * ( boxes[..., 3] - boxes[..., 1]) @property def widths(self) -> Tensor: """Return a tensor representing the widths of boxes.""" boxes = self.tensor return boxes[..., 2] - boxes[..., 0] @property def heights(self) -> Tensor: """Return a tensor representing the heights of boxes.""" boxes = self.tensor return boxes[..., 3] - boxes[..., 1] def flip_(self, img_shape: Tuple[int, int], direction: str = 'horizontal') -> None: """Flip boxes horizontally or vertically in-place. Args: img_shape (Tuple[int, int]): A tuple of image height and width. direction (str): Flip direction, options are "horizontal", "vertical" and "diagonal". Defaults to "horizontal" """ assert direction in ['horizontal', 'vertical', 'diagonal'] flipped = self.tensor boxes = flipped.clone() if direction == 'horizontal': flipped[..., 0] = img_shape[1] - boxes[..., 2] flipped[..., 2] = img_shape[1] - boxes[..., 0] elif direction == 'vertical': flipped[..., 1] = img_shape[0] - boxes[..., 3] flipped[..., 3] = img_shape[0] - boxes[..., 1] else: flipped[..., 0] = img_shape[1] - boxes[..., 2] flipped[..., 1] = img_shape[0] - boxes[..., 3] flipped[..., 2] = img_shape[1] - boxes[..., 0] flipped[..., 3] = img_shape[0] - boxes[..., 1] def translate_(self, distances: Tuple[float, float]) -> None: """Translate boxes in-place. Args: distances (Tuple[float, float]): translate distances. The first is horizontal distance and the second is vertical distance. """ boxes = self.tensor assert len(distances) == 2 self.tensor = boxes + boxes.new_tensor(distances).repeat(2) def clip_(self, img_shape: Tuple[int, int]) -> None: """Clip boxes according to the image shape in-place. Args: img_shape (Tuple[int, int]): A tuple of image height and width. """ boxes = self.tensor boxes[..., 0::2] = boxes[..., 0::2].clamp(0, img_shape[1]) boxes[..., 1::2] = boxes[..., 1::2].clamp(0, img_shape[0]) def rotate_(self, center: Tuple[float, float], angle: float) -> None: """Rotate all boxes in-place. Args: center (Tuple[float, float]): Rotation origin. angle (float): Rotation angle represented in degrees. Positive values mean clockwise rotation. """ boxes = self.tensor rotation_matrix = boxes.new_tensor( cv2.getRotationMatrix2D(center, -angle, 1)) corners = self.hbox2corner(boxes) corners = torch.cat( [corners, corners.new_ones(*corners.shape[:-1], 1)], dim=-1) corners_T = torch.transpose(corners, -1, -2) corners_T = torch.matmul(rotation_matrix, corners_T) corners = torch.transpose(corners_T, -1, -2) self.tensor = self.corner2hbox(corners) def project_(self, homography_matrix: Union[Tensor, np.ndarray]) -> None: """Geometric transformat boxes in-place. Args: homography_matrix (Tensor or np.ndarray]): Shape (3, 3) for geometric transformation. """ boxes = self.tensor if isinstance(homography_matrix, np.ndarray): homography_matrix = boxes.new_tensor(homography_matrix) corners = self.hbox2corner(boxes) corners = torch.cat( [corners, corners.new_ones(*corners.shape[:-1], 1)], dim=-1) corners_T = torch.transpose(corners, -1, -2) corners_T = torch.matmul(homography_matrix, corners_T) corners = torch.transpose(corners_T, -1, -2) # Convert to homogeneous coordinates by normalization corners = corners[..., :2] / corners[..., 2:3] self.tensor = self.corner2hbox(corners) @staticmethod def hbox2corner(boxes: Tensor) -> Tensor: """Convert box coordinates from (x1, y1, x2, y2) to corners ((x1, y1), (x2, y1), (x1, y2), (x2, y2)). Args: boxes (Tensor): Horizontal box tensor with shape of (..., 4). Returns: Tensor: Corner tensor with shape of (..., 4, 2). """ x1, y1, x2, y2 = torch.split(boxes, 1, dim=-1) corners = torch.cat([x1, y1, x2, y1, x1, y2, x2, y2], dim=-1) return corners.reshape(*corners.shape[:-1], 4, 2) @staticmethod def corner2hbox(corners: Tensor) -> Tensor: """Convert box coordinates from corners ((x1, y1), (x2, y1), (x1, y2), (x2, y2)) to (x1, y1, x2, y2). Args: corners (Tensor): Corner tensor with shape of (..., 4, 2). Returns: Tensor: Horizontal box tensor with shape of (..., 4). """ if corners.numel() == 0: return corners.new_zeros((0, 4)) min_xy = corners.min(dim=-2)[0] max_xy = corners.max(dim=-2)[0] return torch.cat([min_xy, max_xy], dim=-1) def rescale_(self, scale_factor: Tuple[float, float]) -> None: """Rescale boxes w.r.t. rescale_factor in-place. Note: Both ``rescale_`` and ``resize_`` will enlarge or shrink boxes w.r.t ``scale_facotr``. The difference is that ``resize_`` only changes the width and the height of boxes, but ``rescale_`` also rescales the box centers simultaneously. Args: scale_factor (Tuple[float, float]): factors for scaling boxes. The length should be 2. """ boxes = self.tensor assert len(scale_factor) == 2 scale_factor = boxes.new_tensor(scale_factor).repeat(2) self.tensor = boxes * scale_factor def resize_(self, scale_factor: Tuple[float, float]) -> None: """Resize the box width and height w.r.t scale_factor in-place. Note: Both ``rescale_`` and ``resize_`` will enlarge or shrink boxes w.r.t ``scale_facotr``. The difference is that ``resize_`` only changes the width and the height of boxes, but ``rescale_`` also rescales the box centers simultaneously. Args: scale_factor (Tuple[float, float]): factors for scaling box shapes. The length should be 2. """ boxes = self.tensor assert len(scale_factor) == 2 ctrs = (boxes[..., 2:] + boxes[..., :2]) / 2 wh = boxes[..., 2:] - boxes[..., :2] scale_factor = boxes.new_tensor(scale_factor) wh = wh * scale_factor xy1 = ctrs - 0.5 * wh xy2 = ctrs + 0.5 * wh self.tensor = torch.cat([xy1, xy2], dim=-1) def is_inside(self, img_shape: Tuple[int, int], all_inside: bool = False, allowed_border: int = 0) -> BoolTensor: """Find boxes inside the image. Args: img_shape (Tuple[int, int]): A tuple of image height and width. all_inside (bool): Whether the boxes are all inside the image or part inside the image. Defaults to False. allowed_border (int): Boxes that extend beyond the image shape boundary by more than ``allowed_border`` are considered "outside" Defaults to 0. Returns: BoolTensor: A BoolTensor indicating whether the box is inside the image. Assuming the original boxes have shape (m, n, 4), the output has shape (m, n). """ img_h, img_w = img_shape boxes = self.tensor if all_inside: return (boxes[:, 0] >= -allowed_border) & \ (boxes[:, 1] >= -allowed_border) & \ (boxes[:, 2] < img_w + allowed_border) & \ (boxes[:, 3] < img_h + allowed_border) else: return (boxes[..., 0] < img_w + allowed_border) & \ (boxes[..., 1] < img_h + allowed_border) & \ (boxes[..., 2] > -allowed_border) & \ (boxes[..., 3] > -allowed_border) def find_inside_points(self, points: Tensor, is_aligned: bool = False) -> BoolTensor: """Find inside box points. Boxes dimension must be 2. Args: points (Tensor): Points coordinates. Has shape of (m, 2). is_aligned (bool): Whether ``points`` has been aligned with boxes or not. If True, the length of boxes and ``points`` should be the same. Defaults to False. Returns: BoolTensor: A BoolTensor indicating whether a point is inside boxes. Assuming the boxes has shape of (n, 4), if ``is_aligned`` is False. The index has shape of (m, n). If ``is_aligned`` is True, m should be equal to n and the index has shape of (m, ). """ boxes = self.tensor assert boxes.dim() == 2, 'boxes dimension must be 2.' if not is_aligned: boxes = boxes[None, :, :] points = points[:, None, :] else: assert boxes.size(0) == points.size(0) x_min, y_min, x_max, y_max = boxes.unbind(dim=-1) return (points[..., 0] >= x_min) & (points[..., 0] <= x_max) & \ (points[..., 1] >= y_min) & (points[..., 1] <= y_max) @staticmethod def overlaps(boxes1: BaseBoxes, boxes2: BaseBoxes, mode: str = 'iou', is_aligned: bool = False, eps: float = 1e-6) -> Tensor: """Calculate overlap between two set of boxes with their types converted to ``HorizontalBoxes``. Args: boxes1 (:obj:`BaseBoxes`): BaseBoxes with shape of (m, box_dim) or empty. boxes2 (:obj:`BaseBoxes`): BaseBoxes with shape of (n, box_dim) or empty. mode (str): "iou" (intersection over union), "iof" (intersection over foreground). Defaults to "iou". is_aligned (bool): If True, then m and n must be equal. Defaults to False. eps (float): A value added to the denominator for numerical stability. Defaults to 1e-6. Returns: Tensor: shape (m, n) if ``is_aligned`` is False else shape (m,) """ boxes1 = boxes1.convert_to('hbox') boxes2 = boxes2.convert_to('hbox') return bbox_overlaps( boxes1.tensor, boxes2.tensor, mode=mode, is_aligned=is_aligned, eps=eps) @staticmethod def from_instance_masks(masks: MaskType) -> 'HorizontalBoxes': """Create horizontal boxes from instance masks. Args: masks (:obj:`BitmapMasks` or :obj:`PolygonMasks`): BitmapMasks or PolygonMasks instance with length of n. Returns: :obj:`HorizontalBoxes`: Converted boxes with shape of (n, 4). """ num_masks = len(masks) boxes = np.zeros((num_masks, 4), dtype=np.float32) if isinstance(masks, BitmapMasks): x_any = masks.masks.any(axis=1) y_any = masks.masks.any(axis=2) for idx in range(num_masks): x = np.where(x_any[idx, :])[0] y = np.where(y_any[idx, :])[0] if len(x) > 0 and len(y) > 0: # use +1 for x_max and y_max so that the right and bottom # boundary of instance masks are fully included by the box boxes[idx, :] = np.array( [x[0], y[0], x[-1] + 1, y[-1] + 1], dtype=np.float32) elif isinstance(masks, PolygonMasks): for idx, poly_per_obj in enumerate(masks.masks): # simply use a number that is big enough for comparison with # coordinates xy_min = np.array([masks.width * 2, masks.height * 2], dtype=np.float32) xy_max = np.zeros(2, dtype=np.float32) for p in poly_per_obj: xy = np.array(p).reshape(-1, 2).astype(np.float32) xy_min = np.minimum(xy_min, np.min(xy, axis=0)) xy_max = np.maximum(xy_max, np.max(xy, axis=0)) boxes[idx, :2] = xy_min boxes[idx, 2:] = xy_max else: raise TypeError( '`masks` must be `BitmapMasks` or `PolygonMasks`, ' f'but got {type(masks)}.') return HorizontalBoxes(boxes) ================================================ FILE: mmdet/structures/bbox/transforms.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional, Sequence, Tuple, Union import numpy as np import torch from torch import Tensor from mmdet.structures.bbox import BaseBoxes def find_inside_bboxes(bboxes: Tensor, img_h: int, img_w: int) -> Tensor: """Find bboxes as long as a part of bboxes is inside the image. Args: bboxes (Tensor): Shape (N, 4). img_h (int): Image height. img_w (int): Image width. Returns: Tensor: Index of the remaining bboxes. """ inside_inds = (bboxes[:, 0] < img_w) & (bboxes[:, 2] > 0) \ & (bboxes[:, 1] < img_h) & (bboxes[:, 3] > 0) return inside_inds def bbox_flip(bboxes: Tensor, img_shape: Tuple[int], direction: str = 'horizontal') -> Tensor: """Flip bboxes horizontally or vertically. Args: bboxes (Tensor): Shape (..., 4*k) img_shape (Tuple[int]): Image shape. direction (str): Flip direction, options are "horizontal", "vertical", "diagonal". Default: "horizontal" Returns: Tensor: Flipped bboxes. """ assert bboxes.shape[-1] % 4 == 0 assert direction in ['horizontal', 'vertical', 'diagonal'] flipped = bboxes.clone() if direction == 'horizontal': flipped[..., 0::4] = img_shape[1] - bboxes[..., 2::4] flipped[..., 2::4] = img_shape[1] - bboxes[..., 0::4] elif direction == 'vertical': flipped[..., 1::4] = img_shape[0] - bboxes[..., 3::4] flipped[..., 3::4] = img_shape[0] - bboxes[..., 1::4] else: flipped[..., 0::4] = img_shape[1] - bboxes[..., 2::4] flipped[..., 1::4] = img_shape[0] - bboxes[..., 3::4] flipped[..., 2::4] = img_shape[1] - bboxes[..., 0::4] flipped[..., 3::4] = img_shape[0] - bboxes[..., 1::4] return flipped def bbox_mapping(bboxes: Tensor, img_shape: Tuple[int], scale_factor: Union[float, Tuple[float]], flip: bool, flip_direction: str = 'horizontal') -> Tensor: """Map bboxes from the original image scale to testing scale.""" new_bboxes = bboxes * bboxes.new_tensor(scale_factor) if flip: new_bboxes = bbox_flip(new_bboxes, img_shape, flip_direction) return new_bboxes def bbox_mapping_back(bboxes: Tensor, img_shape: Tuple[int], scale_factor: Union[float, Tuple[float]], flip: bool, flip_direction: str = 'horizontal') -> Tensor: """Map bboxes from testing scale to original image scale.""" new_bboxes = bbox_flip(bboxes, img_shape, flip_direction) if flip else bboxes new_bboxes = new_bboxes.view(-1, 4) / new_bboxes.new_tensor(scale_factor) return new_bboxes.view(bboxes.shape) def bbox2roi(bbox_list: List[Union[Tensor, BaseBoxes]]) -> Tensor: """Convert a list of bboxes to roi format. Args: bbox_list (List[Union[Tensor, :obj:`BaseBoxes`]): a list of bboxes corresponding to a batch of images. Returns: Tensor: shape (n, box_dim + 1), where ``box_dim`` depends on the different box types. For example, If the box type in ``bbox_list`` is HorizontalBoxes, the output shape is (n, 5). Each row of data indicates [batch_ind, x1, y1, x2, y2]. """ rois_list = [] for img_id, bboxes in enumerate(bbox_list): bboxes = get_box_tensor(bboxes) img_inds = bboxes.new_full((bboxes.size(0), 1), img_id) rois = torch.cat([img_inds, bboxes], dim=-1) rois_list.append(rois) rois = torch.cat(rois_list, 0) return rois def roi2bbox(rois: Tensor) -> List[Tensor]: """Convert rois to bounding box format. Args: rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. Returns: List[Tensor]: Converted boxes of corresponding rois. """ bbox_list = [] img_ids = torch.unique(rois[:, 0].cpu(), sorted=True) for img_id in img_ids: inds = (rois[:, 0] == img_id.item()) bbox = rois[inds, 1:] bbox_list.append(bbox) return bbox_list # TODO remove later def bbox2result(bboxes: Union[Tensor, np.ndarray], labels: Union[Tensor, np.ndarray], num_classes: int) -> List[np.ndarray]: """Convert detection results to a list of numpy arrays. Args: bboxes (Tensor | np.ndarray): shape (n, 5) labels (Tensor | np.ndarray): shape (n, ) num_classes (int): class number, including background class Returns: List(np.ndarray]): bbox results of each class """ if bboxes.shape[0] == 0: return [np.zeros((0, 5), dtype=np.float32) for i in range(num_classes)] else: if isinstance(bboxes, torch.Tensor): bboxes = bboxes.detach().cpu().numpy() labels = labels.detach().cpu().numpy() return [bboxes[labels == i, :] for i in range(num_classes)] def distance2bbox( points: Tensor, distance: Tensor, max_shape: Optional[Union[Sequence[int], Tensor, Sequence[Sequence[int]]]] = None ) -> Tensor: """Decode distance prediction to bounding box. Args: points (Tensor): Shape (B, N, 2) or (N, 2). distance (Tensor): Distance from the given point to 4 boundaries (left, top, right, bottom). Shape (B, N, 4) or (N, 4) max_shape (Union[Sequence[int], Tensor, Sequence[Sequence[int]]], optional): Maximum bounds for boxes, specifies (H, W, C) or (H, W). If priors shape is (B, N, 4), then the max_shape should be a Sequence[Sequence[int]] and the length of max_shape should also be B. Returns: Tensor: Boxes with shape (N, 4) or (B, N, 4) """ x1 = points[..., 0] - distance[..., 0] y1 = points[..., 1] - distance[..., 1] x2 = points[..., 0] + distance[..., 2] y2 = points[..., 1] + distance[..., 3] bboxes = torch.stack([x1, y1, x2, y2], -1) if max_shape is not None: if bboxes.dim() == 2 and not torch.onnx.is_in_onnx_export(): # speed up bboxes[:, 0::2].clamp_(min=0, max=max_shape[1]) bboxes[:, 1::2].clamp_(min=0, max=max_shape[0]) return bboxes # clip bboxes with dynamic `min` and `max` for onnx if torch.onnx.is_in_onnx_export(): # TODO: delete from mmdet.core.export import dynamic_clip_for_onnx x1, y1, x2, y2 = dynamic_clip_for_onnx(x1, y1, x2, y2, max_shape) bboxes = torch.stack([x1, y1, x2, y2], dim=-1) return bboxes if not isinstance(max_shape, torch.Tensor): max_shape = x1.new_tensor(max_shape) max_shape = max_shape[..., :2].type_as(x1) if max_shape.ndim == 2: assert bboxes.ndim == 3 assert max_shape.size(0) == bboxes.size(0) min_xy = x1.new_tensor(0) max_xy = torch.cat([max_shape, max_shape], dim=-1).flip(-1).unsqueeze(-2) bboxes = torch.where(bboxes < min_xy, min_xy, bboxes) bboxes = torch.where(bboxes > max_xy, max_xy, bboxes) return bboxes def bbox2distance(points: Tensor, bbox: Tensor, max_dis: Optional[float] = None, eps: float = 0.1) -> Tensor: """Decode bounding box based on distances. Args: points (Tensor): Shape (n, 2) or (b, n, 2), [x, y]. bbox (Tensor): Shape (n, 4) or (b, n, 4), "xyxy" format max_dis (float, optional): Upper bound of the distance. eps (float): a small value to ensure target < max_dis, instead <= Returns: Tensor: Decoded distances. """ left = points[..., 0] - bbox[..., 0] top = points[..., 1] - bbox[..., 1] right = bbox[..., 2] - points[..., 0] bottom = bbox[..., 3] - points[..., 1] if max_dis is not None: left = left.clamp(min=0, max=max_dis - eps) top = top.clamp(min=0, max=max_dis - eps) right = right.clamp(min=0, max=max_dis - eps) bottom = bottom.clamp(min=0, max=max_dis - eps) return torch.stack([left, top, right, bottom], -1) def bbox_rescale(bboxes: Tensor, scale_factor: float = 1.0) -> Tensor: """Rescale bounding box w.r.t. scale_factor. Args: bboxes (Tensor): Shape (n, 4) for bboxes or (n, 5) for rois scale_factor (float): rescale factor Returns: Tensor: Rescaled bboxes. """ if bboxes.size(1) == 5: bboxes_ = bboxes[:, 1:] inds_ = bboxes[:, 0] else: bboxes_ = bboxes cx = (bboxes_[:, 0] + bboxes_[:, 2]) * 0.5 cy = (bboxes_[:, 1] + bboxes_[:, 3]) * 0.5 w = bboxes_[:, 2] - bboxes_[:, 0] h = bboxes_[:, 3] - bboxes_[:, 1] w = w * scale_factor h = h * scale_factor x1 = cx - 0.5 * w x2 = cx + 0.5 * w y1 = cy - 0.5 * h y2 = cy + 0.5 * h if bboxes.size(1) == 5: rescaled_bboxes = torch.stack([inds_, x1, y1, x2, y2], dim=-1) else: rescaled_bboxes = torch.stack([x1, y1, x2, y2], dim=-1) return rescaled_bboxes def bbox_cxcywh_to_xyxy(bbox: Tensor) -> Tensor: """Convert bbox coordinates from (cx, cy, w, h) to (x1, y1, x2, y2). Args: bbox (Tensor): Shape (n, 4) for bboxes. Returns: Tensor: Converted bboxes. """ cx, cy, w, h = bbox.split((1, 1, 1, 1), dim=-1) bbox_new = [(cx - 0.5 * w), (cy - 0.5 * h), (cx + 0.5 * w), (cy + 0.5 * h)] return torch.cat(bbox_new, dim=-1) def bbox_xyxy_to_cxcywh(bbox: Tensor) -> Tensor: """Convert bbox coordinates from (x1, y1, x2, y2) to (cx, cy, w, h). Args: bbox (Tensor): Shape (n, 4) for bboxes. Returns: Tensor: Converted bboxes. """ x1, y1, x2, y2 = bbox.split((1, 1, 1, 1), dim=-1) bbox_new = [(x1 + x2) / 2, (y1 + y2) / 2, (x2 - x1), (y2 - y1)] return torch.cat(bbox_new, dim=-1) def bbox2corner(bboxes: torch.Tensor) -> torch.Tensor: """Convert bbox coordinates from (x1, y1, x2, y2) to corners ((x1, y1), (x2, y1), (x1, y2), (x2, y2)). Args: bboxes (Tensor): Shape (n, 4) for bboxes. Returns: Tensor: Shape (n*4, 2) for corners. """ x1, y1, x2, y2 = torch.split(bboxes, 1, dim=1) return torch.cat([x1, y1, x2, y1, x1, y2, x2, y2], dim=1).reshape(-1, 2) def corner2bbox(corners: torch.Tensor) -> torch.Tensor: """Convert bbox coordinates from corners ((x1, y1), (x2, y1), (x1, y2), (x2, y2)) to (x1, y1, x2, y2). Args: corners (Tensor): Shape (n*4, 2) for corners. Returns: Tensor: Shape (n, 4) for bboxes. """ corners = corners.reshape(-1, 4, 2) min_xy = corners.min(dim=1)[0] max_xy = corners.max(dim=1)[0] return torch.cat([min_xy, max_xy], dim=1) def bbox_project( bboxes: Union[torch.Tensor, np.ndarray], homography_matrix: Union[torch.Tensor, np.ndarray], img_shape: Optional[Tuple[int, int]] = None ) -> Union[torch.Tensor, np.ndarray]: """Geometric transformation for bbox. Args: bboxes (Union[torch.Tensor, np.ndarray]): Shape (n, 4) for bboxes. homography_matrix (Union[torch.Tensor, np.ndarray]): Shape (3, 3) for geometric transformation. img_shape (Tuple[int, int], optional): Image shape. Defaults to None. Returns: Union[torch.Tensor, np.ndarray]: Converted bboxes. """ bboxes_type = type(bboxes) if bboxes_type is np.ndarray: bboxes = torch.from_numpy(bboxes) if isinstance(homography_matrix, np.ndarray): homography_matrix = torch.from_numpy(homography_matrix) corners = bbox2corner(bboxes) corners = torch.cat( [corners, corners.new_ones(corners.shape[0], 1)], dim=1) corners = torch.matmul(homography_matrix, corners.t()).t() # Convert to homogeneous coordinates by normalization corners = corners[:, :2] / corners[:, 2:3] bboxes = corner2bbox(corners) if img_shape is not None: bboxes[:, 0::2] = bboxes[:, 0::2].clamp(0, img_shape[1]) bboxes[:, 1::2] = bboxes[:, 1::2].clamp(0, img_shape[0]) if bboxes_type is np.ndarray: bboxes = bboxes.numpy() return bboxes def cat_boxes(data_list: List[Union[Tensor, BaseBoxes]], dim: int = 0) -> Union[Tensor, BaseBoxes]: """Concatenate boxes with type of tensor or box type. Args: data_list (List[Union[Tensor, :obj:`BaseBoxes`]]): A list of tensors or box types need to be concatenated. dim (int): The dimension over which the box are concatenated. Defaults to 0. Returns: Union[Tensor, :obj`BaseBoxes`]: Concatenated results. """ if data_list and isinstance(data_list[0], BaseBoxes): return data_list[0].cat(data_list, dim=dim) else: return torch.cat(data_list, dim=dim) def stack_boxes(data_list: List[Union[Tensor, BaseBoxes]], dim: int = 0) -> Union[Tensor, BaseBoxes]: """Stack boxes with type of tensor or box type. Args: data_list (List[Union[Tensor, :obj:`BaseBoxes`]]): A list of tensors or box types need to be stacked. dim (int): The dimension over which the box are stacked. Defaults to 0. Returns: Union[Tensor, :obj`BaseBoxes`]: Stacked results. """ if data_list and isinstance(data_list[0], BaseBoxes): return data_list[0].stack(data_list, dim=dim) else: return torch.stack(data_list, dim=dim) def scale_boxes(boxes: Union[Tensor, BaseBoxes], scale_factor: Tuple[float, float]) -> Union[Tensor, BaseBoxes]: """Scale boxes with type of tensor or box type. Args: boxes (Tensor or :obj:`BaseBoxes`): boxes need to be scaled. Its type can be a tensor or a box type. scale_factor (Tuple[float, float]): factors for scaling boxes. The length should be 2. Returns: Union[Tensor, :obj:`BaseBoxes`]: Scaled boxes. """ if isinstance(boxes, BaseBoxes): boxes.rescale_(scale_factor) return boxes else: # Tensor boxes will be treated as horizontal boxes repeat_num = int(boxes.size(-1) / 2) scale_factor = boxes.new_tensor(scale_factor).repeat((1, repeat_num)) return boxes * scale_factor def get_box_wh(boxes: Union[Tensor, BaseBoxes]) -> Tuple[Tensor, Tensor]: """Get the width and height of boxes with type of tensor or box type. Args: boxes (Tensor or :obj:`BaseBoxes`): boxes with type of tensor or box type. Returns: Tuple[Tensor, Tensor]: the width and height of boxes. """ if isinstance(boxes, BaseBoxes): w = boxes.widths h = boxes.heights else: # Tensor boxes will be treated as horizontal boxes by defaults w = boxes[:, 2] - boxes[:, 0] h = boxes[:, 3] - boxes[:, 1] return w, h def get_box_tensor(boxes: Union[Tensor, BaseBoxes]) -> Tensor: """Get tensor data from box type boxes. Args: boxes (Tensor or BaseBoxes): boxes with type of tensor or box type. If its type is a tensor, the boxes will be directly returned. If its type is a box type, the `boxes.tensor` will be returned. Returns: Tensor: boxes tensor. """ if isinstance(boxes, BaseBoxes): boxes = boxes.tensor return boxes def empty_box_as(boxes: Union[Tensor, BaseBoxes]) -> Union[Tensor, BaseBoxes]: """Generate empty box according to input ``boxes` type and device. Args: boxes (Tensor or :obj:`BaseBoxes`): boxes with type of tensor or box type. Returns: Union[Tensor, BaseBoxes]: Generated empty box. """ if isinstance(boxes, BaseBoxes): return boxes.empty_boxes() else: # Tensor boxes will be treated as horizontal boxes by defaults return boxes.new_zeros(0, 4) ================================================ FILE: mmdet/structures/det_data_sample.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Optional from mmengine.structures import BaseDataElement, InstanceData, PixelData class DetDataSample(BaseDataElement): """A data structure interface of MMDetection. They are used as interfaces between different components. The attributes in ``DetDataSample`` are divided into several parts: - ``proposals``(InstanceData): Region proposals used in two-stage detectors. - ``gt_instances``(InstanceData): Ground truth of instance annotations. - ``pred_instances``(InstanceData): Instances of model predictions. - ``ignored_instances``(InstanceData): Instances to be ignored during training/testing. - ``gt_panoptic_seg``(PixelData): Ground truth of panoptic segmentation. - ``pred_panoptic_seg``(PixelData): Prediction of panoptic segmentation. - ``gt_sem_seg``(PixelData): Ground truth of semantic segmentation. - ``pred_sem_seg``(PixelData): Prediction of semantic segmentation. Examples: >>> import torch >>> import numpy as np >>> from mmengine.structures import InstanceData >>> from mmdet.structures import DetDataSample >>> data_sample = DetDataSample() >>> img_meta = dict(img_shape=(800, 1196, 3), ... pad_shape=(800, 1216, 3)) >>> gt_instances = InstanceData(metainfo=img_meta) >>> gt_instances.bboxes = torch.rand((5, 4)) >>> gt_instances.labels = torch.rand((5,)) >>> data_sample.gt_instances = gt_instances >>> assert 'img_shape' in data_sample.gt_instances.metainfo_keys() >>> len(data_sample.gt_instances) 5 >>> print(data_sample) ) at 0x7f21fb1b9880> >>> pred_instances = InstanceData(metainfo=img_meta) >>> pred_instances.bboxes = torch.rand((5, 4)) >>> pred_instances.scores = torch.rand((5,)) >>> data_sample = DetDataSample(pred_instances=pred_instances) >>> assert 'pred_instances' in data_sample >>> data_sample = DetDataSample() >>> gt_instances_data = dict( ... bboxes=torch.rand(2, 4), ... labels=torch.rand(2), ... masks=np.random.rand(2, 2, 2)) >>> gt_instances = InstanceData(**gt_instances_data) >>> data_sample.gt_instances = gt_instances >>> assert 'gt_instances' in data_sample >>> assert 'masks' in data_sample.gt_instances >>> data_sample = DetDataSample() >>> gt_panoptic_seg_data = dict(panoptic_seg=torch.rand(2, 4)) >>> gt_panoptic_seg = PixelData(**gt_panoptic_seg_data) >>> data_sample.gt_panoptic_seg = gt_panoptic_seg >>> print(data_sample) gt_panoptic_seg: ) at 0x7f66c2bb7280> >>> data_sample = DetDataSample() >>> gt_segm_seg_data = dict(segm_seg=torch.rand(2, 2, 2)) >>> gt_segm_seg = PixelData(**gt_segm_seg_data) >>> data_sample.gt_segm_seg = gt_segm_seg >>> assert 'gt_segm_seg' in data_sample >>> assert 'segm_seg' in data_sample.gt_segm_seg """ @property def proposals(self) -> InstanceData: return self._proposals @proposals.setter def proposals(self, value: InstanceData): self.set_field(value, '_proposals', dtype=InstanceData) @proposals.deleter def proposals(self): del self._proposals @property def gt_instances(self) -> InstanceData: return self._gt_instances @gt_instances.setter def gt_instances(self, value: InstanceData): self.set_field(value, '_gt_instances', dtype=InstanceData) @gt_instances.deleter def gt_instances(self): del self._gt_instances @property def pred_instances(self) -> InstanceData: return self._pred_instances @pred_instances.setter def pred_instances(self, value: InstanceData): self.set_field(value, '_pred_instances', dtype=InstanceData) @pred_instances.deleter def pred_instances(self): del self._pred_instances @property def ignored_instances(self) -> InstanceData: return self._ignored_instances @ignored_instances.setter def ignored_instances(self, value: InstanceData): self.set_field(value, '_ignored_instances', dtype=InstanceData) @ignored_instances.deleter def ignored_instances(self): del self._ignored_instances @property def gt_panoptic_seg(self) -> PixelData: return self._gt_panoptic_seg @gt_panoptic_seg.setter def gt_panoptic_seg(self, value: PixelData): self.set_field(value, '_gt_panoptic_seg', dtype=PixelData) @gt_panoptic_seg.deleter def gt_panoptic_seg(self): del self._gt_panoptic_seg @property def pred_panoptic_seg(self) -> PixelData: return self._pred_panoptic_seg @pred_panoptic_seg.setter def pred_panoptic_seg(self, value: PixelData): self.set_field(value, '_pred_panoptic_seg', dtype=PixelData) @pred_panoptic_seg.deleter def pred_panoptic_seg(self): del self._pred_panoptic_seg @property def gt_sem_seg(self) -> PixelData: return self._gt_sem_seg @gt_sem_seg.setter def gt_sem_seg(self, value: PixelData): self.set_field(value, '_gt_sem_seg', dtype=PixelData) @gt_sem_seg.deleter def gt_sem_seg(self): del self._gt_sem_seg @property def pred_sem_seg(self) -> PixelData: return self._pred_sem_seg @pred_sem_seg.setter def pred_sem_seg(self, value: PixelData): self.set_field(value, '_pred_sem_seg', dtype=PixelData) @pred_sem_seg.deleter def pred_sem_seg(self): del self._pred_sem_seg SampleList = List[DetDataSample] OptSampleList = Optional[SampleList] ================================================ FILE: mmdet/structures/mask/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .mask_target import mask_target from .structures import (BaseInstanceMasks, BitmapMasks, PolygonMasks, bitmap_to_polygon, polygon_to_bitmap) from .utils import encode_mask_results, mask2bbox, split_combined_polys __all__ = [ 'split_combined_polys', 'mask_target', 'BaseInstanceMasks', 'BitmapMasks', 'PolygonMasks', 'encode_mask_results', 'mask2bbox', 'polygon_to_bitmap', 'bitmap_to_polygon' ] ================================================ FILE: mmdet/structures/mask/mask_target.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from torch.nn.modules.utils import _pair def mask_target(pos_proposals_list, pos_assigned_gt_inds_list, gt_masks_list, cfg): """Compute mask target for positive proposals in multiple images. Args: pos_proposals_list (list[Tensor]): Positive proposals in multiple images, each has shape (num_pos, 4). pos_assigned_gt_inds_list (list[Tensor]): Assigned GT indices for each positive proposals, each has shape (num_pos,). gt_masks_list (list[:obj:`BaseInstanceMasks`]): Ground truth masks of each image. cfg (dict): Config dict that specifies the mask size. Returns: Tensor: Mask target of each image, has shape (num_pos, w, h). Example: >>> from mmengine.config import Config >>> import mmdet >>> from mmdet.data_elements.mask import BitmapMasks >>> from mmdet.data_elements.mask.mask_target import * >>> H, W = 17, 18 >>> cfg = Config({'mask_size': (13, 14)}) >>> rng = np.random.RandomState(0) >>> # Positive proposals (tl_x, tl_y, br_x, br_y) for each image >>> pos_proposals_list = [ >>> torch.Tensor([ >>> [ 7.2425, 5.5929, 13.9414, 14.9541], >>> [ 7.3241, 3.6170, 16.3850, 15.3102], >>> ]), >>> torch.Tensor([ >>> [ 4.8448, 6.4010, 7.0314, 9.7681], >>> [ 5.9790, 2.6989, 7.4416, 4.8580], >>> [ 0.0000, 0.0000, 0.1398, 9.8232], >>> ]), >>> ] >>> # Corresponding class index for each proposal for each image >>> pos_assigned_gt_inds_list = [ >>> torch.LongTensor([7, 0]), >>> torch.LongTensor([5, 4, 1]), >>> ] >>> # Ground truth mask for each true object for each image >>> gt_masks_list = [ >>> BitmapMasks(rng.rand(8, H, W), height=H, width=W), >>> BitmapMasks(rng.rand(6, H, W), height=H, width=W), >>> ] >>> mask_targets = mask_target( >>> pos_proposals_list, pos_assigned_gt_inds_list, >>> gt_masks_list, cfg) >>> assert mask_targets.shape == (5,) + cfg['mask_size'] """ cfg_list = [cfg for _ in range(len(pos_proposals_list))] mask_targets = map(mask_target_single, pos_proposals_list, pos_assigned_gt_inds_list, gt_masks_list, cfg_list) mask_targets = list(mask_targets) if len(mask_targets) > 0: mask_targets = torch.cat(mask_targets) return mask_targets def mask_target_single(pos_proposals, pos_assigned_gt_inds, gt_masks, cfg): """Compute mask target for each positive proposal in the image. Args: pos_proposals (Tensor): Positive proposals. pos_assigned_gt_inds (Tensor): Assigned GT inds of positive proposals. gt_masks (:obj:`BaseInstanceMasks`): GT masks in the format of Bitmap or Polygon. cfg (dict): Config dict that indicate the mask size. Returns: Tensor: Mask target of each positive proposals in the image. Example: >>> from mmengine.config import Config >>> import mmdet >>> from mmdet.data_elements.mask import BitmapMasks >>> from mmdet.data_elements.mask.mask_target import * # NOQA >>> H, W = 32, 32 >>> cfg = Config({'mask_size': (7, 11)}) >>> rng = np.random.RandomState(0) >>> # Masks for each ground truth box (relative to the image) >>> gt_masks_data = rng.rand(3, H, W) >>> gt_masks = BitmapMasks(gt_masks_data, height=H, width=W) >>> # Predicted positive boxes in one image >>> pos_proposals = torch.FloatTensor([ >>> [ 16.2, 5.5, 19.9, 20.9], >>> [ 17.3, 13.6, 19.3, 19.3], >>> [ 14.8, 16.4, 17.0, 23.7], >>> [ 0.0, 0.0, 16.0, 16.0], >>> [ 4.0, 0.0, 20.0, 16.0], >>> ]) >>> # For each predicted proposal, its assignment to a gt mask >>> pos_assigned_gt_inds = torch.LongTensor([0, 1, 2, 1, 1]) >>> mask_targets = mask_target_single( >>> pos_proposals, pos_assigned_gt_inds, gt_masks, cfg) >>> assert mask_targets.shape == (5,) + cfg['mask_size'] """ device = pos_proposals.device mask_size = _pair(cfg.mask_size) binarize = not cfg.get('soft_mask_target', False) num_pos = pos_proposals.size(0) if num_pos > 0: proposals_np = pos_proposals.cpu().numpy() maxh, maxw = gt_masks.height, gt_masks.width proposals_np[:, [0, 2]] = np.clip(proposals_np[:, [0, 2]], 0, maxw) proposals_np[:, [1, 3]] = np.clip(proposals_np[:, [1, 3]], 0, maxh) pos_assigned_gt_inds = pos_assigned_gt_inds.cpu().numpy() mask_targets = gt_masks.crop_and_resize( proposals_np, mask_size, device=device, inds=pos_assigned_gt_inds, binarize=binarize).to_ndarray() mask_targets = torch.from_numpy(mask_targets).float().to(device) else: mask_targets = pos_proposals.new_zeros((0, ) + mask_size) return mask_targets ================================================ FILE: mmdet/structures/mask/structures.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import itertools from abc import ABCMeta, abstractmethod from typing import Sequence, Type, TypeVar import cv2 import mmcv import numpy as np import pycocotools.mask as maskUtils import torch from mmcv.ops.roi_align import roi_align T = TypeVar('T') class BaseInstanceMasks(metaclass=ABCMeta): """Base class for instance masks.""" @abstractmethod def rescale(self, scale, interpolation='nearest'): """Rescale masks as large as possible while keeping the aspect ratio. For details can refer to `mmcv.imrescale`. Args: scale (tuple[int]): The maximum size (h, w) of rescaled mask. interpolation (str): Same as :func:`mmcv.imrescale`. Returns: BaseInstanceMasks: The rescaled masks. """ @abstractmethod def resize(self, out_shape, interpolation='nearest'): """Resize masks to the given out_shape. Args: out_shape: Target (h, w) of resized mask. interpolation (str): See :func:`mmcv.imresize`. Returns: BaseInstanceMasks: The resized masks. """ @abstractmethod def flip(self, flip_direction='horizontal'): """Flip masks alone the given direction. Args: flip_direction (str): Either 'horizontal' or 'vertical'. Returns: BaseInstanceMasks: The flipped masks. """ @abstractmethod def pad(self, out_shape, pad_val): """Pad masks to the given size of (h, w). Args: out_shape (tuple[int]): Target (h, w) of padded mask. pad_val (int): The padded value. Returns: BaseInstanceMasks: The padded masks. """ @abstractmethod def crop(self, bbox): """Crop each mask by the given bbox. Args: bbox (ndarray): Bbox in format [x1, y1, x2, y2], shape (4, ). Return: BaseInstanceMasks: The cropped masks. """ @abstractmethod def crop_and_resize(self, bboxes, out_shape, inds, device, interpolation='bilinear', binarize=True): """Crop and resize masks by the given bboxes. This function is mainly used in mask targets computation. It firstly align mask to bboxes by assigned_inds, then crop mask by the assigned bbox and resize to the size of (mask_h, mask_w) Args: bboxes (Tensor): Bboxes in format [x1, y1, x2, y2], shape (N, 4) out_shape (tuple[int]): Target (h, w) of resized mask inds (ndarray): Indexes to assign masks to each bbox, shape (N,) and values should be between [0, num_masks - 1]. device (str): Device of bboxes interpolation (str): See `mmcv.imresize` binarize (bool): if True fractional values are rounded to 0 or 1 after the resize operation. if False and unsupported an error will be raised. Defaults to True. Return: BaseInstanceMasks: the cropped and resized masks. """ @abstractmethod def expand(self, expanded_h, expanded_w, top, left): """see :class:`Expand`.""" @property @abstractmethod def areas(self): """ndarray: areas of each instance.""" @abstractmethod def to_ndarray(self): """Convert masks to the format of ndarray. Return: ndarray: Converted masks in the format of ndarray. """ @abstractmethod def to_tensor(self, dtype, device): """Convert masks to the format of Tensor. Args: dtype (str): Dtype of converted mask. device (torch.device): Device of converted masks. Returns: Tensor: Converted masks in the format of Tensor. """ @abstractmethod def translate(self, out_shape, offset, direction='horizontal', border_value=0, interpolation='bilinear'): """Translate the masks. Args: out_shape (tuple[int]): Shape for output mask, format (h, w). offset (int | float): The offset for translate. direction (str): The translate direction, either "horizontal" or "vertical". border_value (int | float): Border value. Default 0. interpolation (str): Same as :func:`mmcv.imtranslate`. Returns: Translated masks. """ def shear(self, out_shape, magnitude, direction='horizontal', border_value=0, interpolation='bilinear'): """Shear the masks. Args: out_shape (tuple[int]): Shape for output mask, format (h, w). magnitude (int | float): The magnitude used for shear. direction (str): The shear direction, either "horizontal" or "vertical". border_value (int | tuple[int]): Value used in case of a constant border. Default 0. interpolation (str): Same as in :func:`mmcv.imshear`. Returns: ndarray: Sheared masks. """ @abstractmethod def rotate(self, out_shape, angle, center=None, scale=1.0, border_value=0): """Rotate the masks. Args: out_shape (tuple[int]): Shape for output mask, format (h, w). angle (int | float): Rotation angle in degrees. Positive values mean counter-clockwise rotation. center (tuple[float], optional): Center point (w, h) of the rotation in source image. If not specified, the center of the image will be used. scale (int | float): Isotropic scale factor. border_value (int | float): Border value. Default 0 for masks. Returns: Rotated masks. """ def get_bboxes(self, dst_type='hbb'): """Get the certain type boxes from masks. Please refer to ``mmdet.structures.bbox.box_type`` for more details of the box type. Args: dst_type: Destination box type. Returns: :obj:`BaseBoxes`: Certain type boxes. """ from ..bbox import get_box_type _, box_type_cls = get_box_type(dst_type) return box_type_cls.from_instance_masks(self) @classmethod @abstractmethod def cat(cls: Type[T], masks: Sequence[T]) -> T: """Concatenate a sequence of masks into one single mask instance. Args: masks (Sequence[T]): A sequence of mask instances. Returns: T: Concatenated mask instance. """ class BitmapMasks(BaseInstanceMasks): """This class represents masks in the form of bitmaps. Args: masks (ndarray): ndarray of masks in shape (N, H, W), where N is the number of objects. height (int): height of masks width (int): width of masks Example: >>> from mmdet.data_elements.mask.structures import * # NOQA >>> num_masks, H, W = 3, 32, 32 >>> rng = np.random.RandomState(0) >>> masks = (rng.rand(num_masks, H, W) > 0.1).astype(np.int64) >>> self = BitmapMasks(masks, height=H, width=W) >>> # demo crop_and_resize >>> num_boxes = 5 >>> bboxes = np.array([[0, 0, 30, 10.0]] * num_boxes) >>> out_shape = (14, 14) >>> inds = torch.randint(0, len(self), size=(num_boxes,)) >>> device = 'cpu' >>> interpolation = 'bilinear' >>> new = self.crop_and_resize( ... bboxes, out_shape, inds, device, interpolation) >>> assert len(new) == num_boxes >>> assert new.height, new.width == out_shape """ def __init__(self, masks, height, width): self.height = height self.width = width if len(masks) == 0: self.masks = np.empty((0, self.height, self.width), dtype=np.uint8) else: assert isinstance(masks, (list, np.ndarray)) if isinstance(masks, list): assert isinstance(masks[0], np.ndarray) assert masks[0].ndim == 2 # (H, W) else: assert masks.ndim == 3 # (N, H, W) self.masks = np.stack(masks).reshape(-1, height, width) assert self.masks.shape[1] == self.height assert self.masks.shape[2] == self.width def __getitem__(self, index): """Index the BitmapMask. Args: index (int | ndarray): Indices in the format of integer or ndarray. Returns: :obj:`BitmapMasks`: Indexed bitmap masks. """ masks = self.masks[index].reshape(-1, self.height, self.width) return BitmapMasks(masks, self.height, self.width) def __iter__(self): return iter(self.masks) def __repr__(self): s = self.__class__.__name__ + '(' s += f'num_masks={len(self.masks)}, ' s += f'height={self.height}, ' s += f'width={self.width})' return s def __len__(self): """Number of masks.""" return len(self.masks) def rescale(self, scale, interpolation='nearest'): """See :func:`BaseInstanceMasks.rescale`.""" if len(self.masks) == 0: new_w, new_h = mmcv.rescale_size((self.width, self.height), scale) rescaled_masks = np.empty((0, new_h, new_w), dtype=np.uint8) else: rescaled_masks = np.stack([ mmcv.imrescale(mask, scale, interpolation=interpolation) for mask in self.masks ]) height, width = rescaled_masks.shape[1:] return BitmapMasks(rescaled_masks, height, width) def resize(self, out_shape, interpolation='nearest'): """See :func:`BaseInstanceMasks.resize`.""" if len(self.masks) == 0: resized_masks = np.empty((0, *out_shape), dtype=np.uint8) else: resized_masks = np.stack([ mmcv.imresize( mask, out_shape[::-1], interpolation=interpolation) for mask in self.masks ]) return BitmapMasks(resized_masks, *out_shape) def flip(self, flip_direction='horizontal'): """See :func:`BaseInstanceMasks.flip`.""" assert flip_direction in ('horizontal', 'vertical', 'diagonal') if len(self.masks) == 0: flipped_masks = self.masks else: flipped_masks = np.stack([ mmcv.imflip(mask, direction=flip_direction) for mask in self.masks ]) return BitmapMasks(flipped_masks, self.height, self.width) def pad(self, out_shape, pad_val=0): """See :func:`BaseInstanceMasks.pad`.""" if len(self.masks) == 0: padded_masks = np.empty((0, *out_shape), dtype=np.uint8) else: padded_masks = np.stack([ mmcv.impad(mask, shape=out_shape, pad_val=pad_val) for mask in self.masks ]) return BitmapMasks(padded_masks, *out_shape) def crop(self, bbox): """See :func:`BaseInstanceMasks.crop`.""" assert isinstance(bbox, np.ndarray) assert bbox.ndim == 1 # clip the boundary bbox = bbox.copy() bbox[0::2] = np.clip(bbox[0::2], 0, self.width) bbox[1::2] = np.clip(bbox[1::2], 0, self.height) x1, y1, x2, y2 = bbox w = np.maximum(x2 - x1, 1) h = np.maximum(y2 - y1, 1) if len(self.masks) == 0: cropped_masks = np.empty((0, h, w), dtype=np.uint8) else: cropped_masks = self.masks[:, y1:y1 + h, x1:x1 + w] return BitmapMasks(cropped_masks, h, w) def crop_and_resize(self, bboxes, out_shape, inds, device='cpu', interpolation='bilinear', binarize=True): """See :func:`BaseInstanceMasks.crop_and_resize`.""" if len(self.masks) == 0: empty_masks = np.empty((0, *out_shape), dtype=np.uint8) return BitmapMasks(empty_masks, *out_shape) # convert bboxes to tensor if isinstance(bboxes, np.ndarray): bboxes = torch.from_numpy(bboxes).to(device=device) if isinstance(inds, np.ndarray): inds = torch.from_numpy(inds).to(device=device) num_bbox = bboxes.shape[0] fake_inds = torch.arange( num_bbox, device=device).to(dtype=bboxes.dtype)[:, None] rois = torch.cat([fake_inds, bboxes], dim=1) # Nx5 rois = rois.to(device=device) if num_bbox > 0: gt_masks_th = torch.from_numpy(self.masks).to(device).index_select( 0, inds).to(dtype=rois.dtype) targets = roi_align(gt_masks_th[:, None, :, :], rois, out_shape, 1.0, 0, 'avg', True).squeeze(1) if binarize: resized_masks = (targets >= 0.5).cpu().numpy() else: resized_masks = targets.cpu().numpy() else: resized_masks = [] return BitmapMasks(resized_masks, *out_shape) def expand(self, expanded_h, expanded_w, top, left): """See :func:`BaseInstanceMasks.expand`.""" if len(self.masks) == 0: expanded_mask = np.empty((0, expanded_h, expanded_w), dtype=np.uint8) else: expanded_mask = np.zeros((len(self), expanded_h, expanded_w), dtype=np.uint8) expanded_mask[:, top:top + self.height, left:left + self.width] = self.masks return BitmapMasks(expanded_mask, expanded_h, expanded_w) def translate(self, out_shape, offset, direction='horizontal', border_value=0, interpolation='bilinear'): """Translate the BitmapMasks. Args: out_shape (tuple[int]): Shape for output mask, format (h, w). offset (int | float): The offset for translate. direction (str): The translate direction, either "horizontal" or "vertical". border_value (int | float): Border value. Default 0 for masks. interpolation (str): Same as :func:`mmcv.imtranslate`. Returns: BitmapMasks: Translated BitmapMasks. Example: >>> from mmdet.data_elements.mask.structures import BitmapMasks >>> self = BitmapMasks.random(dtype=np.uint8) >>> out_shape = (32, 32) >>> offset = 4 >>> direction = 'horizontal' >>> border_value = 0 >>> interpolation = 'bilinear' >>> # Note, There seem to be issues when: >>> # * the mask dtype is not supported by cv2.AffineWarp >>> new = self.translate(out_shape, offset, direction, >>> border_value, interpolation) >>> assert len(new) == len(self) >>> assert new.height, new.width == out_shape """ if len(self.masks) == 0: translated_masks = np.empty((0, *out_shape), dtype=np.uint8) else: masks = self.masks if masks.shape[-2:] != out_shape: empty_masks = np.zeros((masks.shape[0], *out_shape), dtype=masks.dtype) min_h = min(out_shape[0], masks.shape[1]) min_w = min(out_shape[1], masks.shape[2]) empty_masks[:, :min_h, :min_w] = masks[:, :min_h, :min_w] masks = empty_masks translated_masks = mmcv.imtranslate( masks.transpose((1, 2, 0)), offset, direction, border_value=border_value, interpolation=interpolation) if translated_masks.ndim == 2: translated_masks = translated_masks[:, :, None] translated_masks = translated_masks.transpose( (2, 0, 1)).astype(self.masks.dtype) return BitmapMasks(translated_masks, *out_shape) def shear(self, out_shape, magnitude, direction='horizontal', border_value=0, interpolation='bilinear'): """Shear the BitmapMasks. Args: out_shape (tuple[int]): Shape for output mask, format (h, w). magnitude (int | float): The magnitude used for shear. direction (str): The shear direction, either "horizontal" or "vertical". border_value (int | tuple[int]): Value used in case of a constant border. interpolation (str): Same as in :func:`mmcv.imshear`. Returns: BitmapMasks: The sheared masks. """ if len(self.masks) == 0: sheared_masks = np.empty((0, *out_shape), dtype=np.uint8) else: sheared_masks = mmcv.imshear( self.masks.transpose((1, 2, 0)), magnitude, direction, border_value=border_value, interpolation=interpolation) if sheared_masks.ndim == 2: sheared_masks = sheared_masks[:, :, None] sheared_masks = sheared_masks.transpose( (2, 0, 1)).astype(self.masks.dtype) return BitmapMasks(sheared_masks, *out_shape) def rotate(self, out_shape, angle, center=None, scale=1.0, border_value=0, interpolation='bilinear'): """Rotate the BitmapMasks. Args: out_shape (tuple[int]): Shape for output mask, format (h, w). angle (int | float): Rotation angle in degrees. Positive values mean counter-clockwise rotation. center (tuple[float], optional): Center point (w, h) of the rotation in source image. If not specified, the center of the image will be used. scale (int | float): Isotropic scale factor. border_value (int | float): Border value. Default 0 for masks. interpolation (str): Same as in :func:`mmcv.imrotate`. Returns: BitmapMasks: Rotated BitmapMasks. """ if len(self.masks) == 0: rotated_masks = np.empty((0, *out_shape), dtype=self.masks.dtype) else: rotated_masks = mmcv.imrotate( self.masks.transpose((1, 2, 0)), angle, center=center, scale=scale, border_value=border_value, interpolation=interpolation) if rotated_masks.ndim == 2: # case when only one mask, (h, w) rotated_masks = rotated_masks[:, :, None] # (h, w, 1) rotated_masks = rotated_masks.transpose( (2, 0, 1)).astype(self.masks.dtype) return BitmapMasks(rotated_masks, *out_shape) @property def areas(self): """See :py:attr:`BaseInstanceMasks.areas`.""" return self.masks.sum((1, 2)) def to_ndarray(self): """See :func:`BaseInstanceMasks.to_ndarray`.""" return self.masks def to_tensor(self, dtype, device): """See :func:`BaseInstanceMasks.to_tensor`.""" return torch.tensor(self.masks, dtype=dtype, device=device) @classmethod def random(cls, num_masks=3, height=32, width=32, dtype=np.uint8, rng=None): """Generate random bitmap masks for demo / testing purposes. Example: >>> from mmdet.data_elements.mask.structures import BitmapMasks >>> self = BitmapMasks.random() >>> print('self = {}'.format(self)) self = BitmapMasks(num_masks=3, height=32, width=32) """ from mmdet.utils.util_random import ensure_rng rng = ensure_rng(rng) masks = (rng.rand(num_masks, height, width) > 0.1).astype(dtype) self = cls(masks, height=height, width=width) return self @classmethod def cat(cls: Type[T], masks: Sequence[T]) -> T: """Concatenate a sequence of masks into one single mask instance. Args: masks (Sequence[BitmapMasks]): A sequence of mask instances. Returns: BitmapMasks: Concatenated mask instance. """ assert isinstance(masks, Sequence) if len(masks) == 0: raise ValueError('masks should not be an empty list.') assert all(isinstance(m, cls) for m in masks) mask_array = np.concatenate([m.masks for m in masks], axis=0) return cls(mask_array, *mask_array.shape[1:]) class PolygonMasks(BaseInstanceMasks): """This class represents masks in the form of polygons. Polygons is a list of three levels. The first level of the list corresponds to objects, the second level to the polys that compose the object, the third level to the poly coordinates Args: masks (list[list[ndarray]]): The first level of the list corresponds to objects, the second level to the polys that compose the object, the third level to the poly coordinates height (int): height of masks width (int): width of masks Example: >>> from mmdet.data_elements.mask.structures import * # NOQA >>> masks = [ >>> [ np.array([0, 0, 10, 0, 10, 10., 0, 10, 0, 0]) ] >>> ] >>> height, width = 16, 16 >>> self = PolygonMasks(masks, height, width) >>> # demo translate >>> new = self.translate((16, 16), 4., direction='horizontal') >>> assert np.all(new.masks[0][0][1::2] == masks[0][0][1::2]) >>> assert np.all(new.masks[0][0][0::2] == masks[0][0][0::2] + 4) >>> # demo crop_and_resize >>> num_boxes = 3 >>> bboxes = np.array([[0, 0, 30, 10.0]] * num_boxes) >>> out_shape = (16, 16) >>> inds = torch.randint(0, len(self), size=(num_boxes,)) >>> device = 'cpu' >>> interpolation = 'bilinear' >>> new = self.crop_and_resize( ... bboxes, out_shape, inds, device, interpolation) >>> assert len(new) == num_boxes >>> assert new.height, new.width == out_shape """ def __init__(self, masks, height, width): assert isinstance(masks, list) if len(masks) > 0: assert isinstance(masks[0], list) assert isinstance(masks[0][0], np.ndarray) self.height = height self.width = width self.masks = masks def __getitem__(self, index): """Index the polygon masks. Args: index (ndarray | List): The indices. Returns: :obj:`PolygonMasks`: The indexed polygon masks. """ if isinstance(index, np.ndarray): if index.dtype == bool: index = np.where(index)[0].tolist() else: index = index.tolist() if isinstance(index, list): masks = [self.masks[i] for i in index] else: try: masks = self.masks[index] except Exception: raise ValueError( f'Unsupported input of type {type(index)} for indexing!') if len(masks) and isinstance(masks[0], np.ndarray): masks = [masks] # ensure a list of three levels return PolygonMasks(masks, self.height, self.width) def __iter__(self): return iter(self.masks) def __repr__(self): s = self.__class__.__name__ + '(' s += f'num_masks={len(self.masks)}, ' s += f'height={self.height}, ' s += f'width={self.width})' return s def __len__(self): """Number of masks.""" return len(self.masks) def rescale(self, scale, interpolation=None): """see :func:`BaseInstanceMasks.rescale`""" new_w, new_h = mmcv.rescale_size((self.width, self.height), scale) if len(self.masks) == 0: rescaled_masks = PolygonMasks([], new_h, new_w) else: rescaled_masks = self.resize((new_h, new_w)) return rescaled_masks def resize(self, out_shape, interpolation=None): """see :func:`BaseInstanceMasks.resize`""" if len(self.masks) == 0: resized_masks = PolygonMasks([], *out_shape) else: h_scale = out_shape[0] / self.height w_scale = out_shape[1] / self.width resized_masks = [] for poly_per_obj in self.masks: resized_poly = [] for p in poly_per_obj: p = p.copy() p[0::2] = p[0::2] * w_scale p[1::2] = p[1::2] * h_scale resized_poly.append(p) resized_masks.append(resized_poly) resized_masks = PolygonMasks(resized_masks, *out_shape) return resized_masks def flip(self, flip_direction='horizontal'): """see :func:`BaseInstanceMasks.flip`""" assert flip_direction in ('horizontal', 'vertical', 'diagonal') if len(self.masks) == 0: flipped_masks = PolygonMasks([], self.height, self.width) else: flipped_masks = [] for poly_per_obj in self.masks: flipped_poly_per_obj = [] for p in poly_per_obj: p = p.copy() if flip_direction == 'horizontal': p[0::2] = self.width - p[0::2] elif flip_direction == 'vertical': p[1::2] = self.height - p[1::2] else: p[0::2] = self.width - p[0::2] p[1::2] = self.height - p[1::2] flipped_poly_per_obj.append(p) flipped_masks.append(flipped_poly_per_obj) flipped_masks = PolygonMasks(flipped_masks, self.height, self.width) return flipped_masks def crop(self, bbox): """see :func:`BaseInstanceMasks.crop`""" assert isinstance(bbox, np.ndarray) assert bbox.ndim == 1 # clip the boundary bbox = bbox.copy() bbox[0::2] = np.clip(bbox[0::2], 0, self.width) bbox[1::2] = np.clip(bbox[1::2], 0, self.height) x1, y1, x2, y2 = bbox w = np.maximum(x2 - x1, 1) h = np.maximum(y2 - y1, 1) if len(self.masks) == 0: cropped_masks = PolygonMasks([], h, w) else: cropped_masks = [] for poly_per_obj in self.masks: cropped_poly_per_obj = [] for p in poly_per_obj: # pycocotools will clip the boundary p = p.copy() p[0::2] = p[0::2] - bbox[0] p[1::2] = p[1::2] - bbox[1] cropped_poly_per_obj.append(p) cropped_masks.append(cropped_poly_per_obj) cropped_masks = PolygonMasks(cropped_masks, h, w) return cropped_masks def pad(self, out_shape, pad_val=0): """padding has no effect on polygons`""" return PolygonMasks(self.masks, *out_shape) def expand(self, *args, **kwargs): """TODO: Add expand for polygon""" raise NotImplementedError def crop_and_resize(self, bboxes, out_shape, inds, device='cpu', interpolation='bilinear', binarize=True): """see :func:`BaseInstanceMasks.crop_and_resize`""" out_h, out_w = out_shape if len(self.masks) == 0: return PolygonMasks([], out_h, out_w) if not binarize: raise ValueError('Polygons are always binary, ' 'setting binarize=False is unsupported') resized_masks = [] for i in range(len(bboxes)): mask = self.masks[inds[i]] bbox = bboxes[i, :] x1, y1, x2, y2 = bbox w = np.maximum(x2 - x1, 1) h = np.maximum(y2 - y1, 1) h_scale = out_h / max(h, 0.1) # avoid too large scale w_scale = out_w / max(w, 0.1) resized_mask = [] for p in mask: p = p.copy() # crop # pycocotools will clip the boundary p[0::2] = p[0::2] - bbox[0] p[1::2] = p[1::2] - bbox[1] # resize p[0::2] = p[0::2] * w_scale p[1::2] = p[1::2] * h_scale resized_mask.append(p) resized_masks.append(resized_mask) return PolygonMasks(resized_masks, *out_shape) def translate(self, out_shape, offset, direction='horizontal', border_value=None, interpolation=None): """Translate the PolygonMasks. Example: >>> self = PolygonMasks.random(dtype=np.int64) >>> out_shape = (self.height, self.width) >>> new = self.translate(out_shape, 4., direction='horizontal') >>> assert np.all(new.masks[0][0][1::2] == self.masks[0][0][1::2]) >>> assert np.all(new.masks[0][0][0::2] == self.masks[0][0][0::2] + 4) # noqa: E501 """ assert border_value is None or border_value == 0, \ 'Here border_value is not '\ f'used, and defaultly should be None or 0. got {border_value}.' if len(self.masks) == 0: translated_masks = PolygonMasks([], *out_shape) else: translated_masks = [] for poly_per_obj in self.masks: translated_poly_per_obj = [] for p in poly_per_obj: p = p.copy() if direction == 'horizontal': p[0::2] = np.clip(p[0::2] + offset, 0, out_shape[1]) elif direction == 'vertical': p[1::2] = np.clip(p[1::2] + offset, 0, out_shape[0]) translated_poly_per_obj.append(p) translated_masks.append(translated_poly_per_obj) translated_masks = PolygonMasks(translated_masks, *out_shape) return translated_masks def shear(self, out_shape, magnitude, direction='horizontal', border_value=0, interpolation='bilinear'): """See :func:`BaseInstanceMasks.shear`.""" if len(self.masks) == 0: sheared_masks = PolygonMasks([], *out_shape) else: sheared_masks = [] if direction == 'horizontal': shear_matrix = np.stack([[1, magnitude], [0, 1]]).astype(np.float32) elif direction == 'vertical': shear_matrix = np.stack([[1, 0], [magnitude, 1]]).astype(np.float32) for poly_per_obj in self.masks: sheared_poly = [] for p in poly_per_obj: p = np.stack([p[0::2], p[1::2]], axis=0) # [2, n] new_coords = np.matmul(shear_matrix, p) # [2, n] new_coords[0, :] = np.clip(new_coords[0, :], 0, out_shape[1]) new_coords[1, :] = np.clip(new_coords[1, :], 0, out_shape[0]) sheared_poly.append( new_coords.transpose((1, 0)).reshape(-1)) sheared_masks.append(sheared_poly) sheared_masks = PolygonMasks(sheared_masks, *out_shape) return sheared_masks def rotate(self, out_shape, angle, center=None, scale=1.0, border_value=0, interpolation='bilinear'): """See :func:`BaseInstanceMasks.rotate`.""" if len(self.masks) == 0: rotated_masks = PolygonMasks([], *out_shape) else: rotated_masks = [] rotate_matrix = cv2.getRotationMatrix2D(center, -angle, scale) for poly_per_obj in self.masks: rotated_poly = [] for p in poly_per_obj: p = p.copy() coords = np.stack([p[0::2], p[1::2]], axis=1) # [n, 2] # pad 1 to convert from format [x, y] to homogeneous # coordinates format [x, y, 1] coords = np.concatenate( (coords, np.ones((coords.shape[0], 1), coords.dtype)), axis=1) # [n, 3] rotated_coords = np.matmul( rotate_matrix[None, :, :], coords[:, :, None])[..., 0] # [n, 2, 1] -> [n, 2] rotated_coords[:, 0] = np.clip(rotated_coords[:, 0], 0, out_shape[1]) rotated_coords[:, 1] = np.clip(rotated_coords[:, 1], 0, out_shape[0]) rotated_poly.append(rotated_coords.reshape(-1)) rotated_masks.append(rotated_poly) rotated_masks = PolygonMasks(rotated_masks, *out_shape) return rotated_masks def to_bitmap(self): """convert polygon masks to bitmap masks.""" bitmap_masks = self.to_ndarray() return BitmapMasks(bitmap_masks, self.height, self.width) @property def areas(self): """Compute areas of masks. This func is modified from `detectron2 `_. The function only works with Polygons using the shoelace formula. Return: ndarray: areas of each instance """ # noqa: W501 area = [] for polygons_per_obj in self.masks: area_per_obj = 0 for p in polygons_per_obj: area_per_obj += self._polygon_area(p[0::2], p[1::2]) area.append(area_per_obj) return np.asarray(area) def _polygon_area(self, x, y): """Compute the area of a component of a polygon. Using the shoelace formula: https://stackoverflow.com/questions/24467972/calculate-area-of-polygon-given-x-y-coordinates Args: x (ndarray): x coordinates of the component y (ndarray): y coordinates of the component Return: float: the are of the component """ # noqa: 501 return 0.5 * np.abs( np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))) def to_ndarray(self): """Convert masks to the format of ndarray.""" if len(self.masks) == 0: return np.empty((0, self.height, self.width), dtype=np.uint8) bitmap_masks = [] for poly_per_obj in self.masks: bitmap_masks.append( polygon_to_bitmap(poly_per_obj, self.height, self.width)) return np.stack(bitmap_masks) def to_tensor(self, dtype, device): """See :func:`BaseInstanceMasks.to_tensor`.""" if len(self.masks) == 0: return torch.empty((0, self.height, self.width), dtype=dtype, device=device) ndarray_masks = self.to_ndarray() return torch.tensor(ndarray_masks, dtype=dtype, device=device) @classmethod def random(cls, num_masks=3, height=32, width=32, n_verts=5, dtype=np.float32, rng=None): """Generate random polygon masks for demo / testing purposes. Adapted from [1]_ References: .. [1] https://gitlab.kitware.com/computer-vision/kwimage/-/blob/928cae35ca8/kwimage/structs/polygon.py#L379 # noqa: E501 Example: >>> from mmdet.data_elements.mask.structures import PolygonMasks >>> self = PolygonMasks.random() >>> print('self = {}'.format(self)) """ from mmdet.utils.util_random import ensure_rng rng = ensure_rng(rng) def _gen_polygon(n, irregularity, spikeyness): """Creates the polygon by sampling points on a circle around the centre. Random noise is added by varying the angular spacing between sequential points, and by varying the radial distance of each point from the centre. Based on original code by Mike Ounsworth Args: n (int): number of vertices irregularity (float): [0,1] indicating how much variance there is in the angular spacing of vertices. [0,1] will map to [0, 2pi/numberOfVerts] spikeyness (float): [0,1] indicating how much variance there is in each vertex from the circle of radius aveRadius. [0,1] will map to [0, aveRadius] Returns: a list of vertices, in CCW order. """ from scipy.stats import truncnorm # Generate around the unit circle cx, cy = (0.0, 0.0) radius = 1 tau = np.pi * 2 irregularity = np.clip(irregularity, 0, 1) * 2 * np.pi / n spikeyness = np.clip(spikeyness, 1e-9, 1) # generate n angle steps lower = (tau / n) - irregularity upper = (tau / n) + irregularity angle_steps = rng.uniform(lower, upper, n) # normalize the steps so that point 0 and point n+1 are the same k = angle_steps.sum() / (2 * np.pi) angles = (angle_steps / k).cumsum() + rng.uniform(0, tau) # Convert high and low values to be wrt the standard normal range # https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.truncnorm.html low = 0 high = 2 * radius mean = radius std = spikeyness a = (low - mean) / std b = (high - mean) / std tnorm = truncnorm(a=a, b=b, loc=mean, scale=std) # now generate the points radii = tnorm.rvs(n, random_state=rng) x_pts = cx + radii * np.cos(angles) y_pts = cy + radii * np.sin(angles) points = np.hstack([x_pts[:, None], y_pts[:, None]]) # Scale to 0-1 space points = points - points.min(axis=0) points = points / points.max(axis=0) # Randomly place within 0-1 space points = points * (rng.rand() * .8 + .2) min_pt = points.min(axis=0) max_pt = points.max(axis=0) high = (1 - max_pt) low = (0 - min_pt) offset = (rng.rand(2) * (high - low)) + low points = points + offset return points def _order_vertices(verts): """ References: https://stackoverflow.com/questions/1709283/how-can-i-sort-a-coordinate-list-for-a-rectangle-counterclockwise """ mlat = verts.T[0].sum() / len(verts) mlng = verts.T[1].sum() / len(verts) tau = np.pi * 2 angle = (np.arctan2(mlat - verts.T[0], verts.T[1] - mlng) + tau) % tau sortx = angle.argsort() verts = verts.take(sortx, axis=0) return verts # Generate a random exterior for each requested mask masks = [] for _ in range(num_masks): exterior = _order_vertices(_gen_polygon(n_verts, 0.9, 0.9)) exterior = (exterior * [(width, height)]).astype(dtype) masks.append([exterior.ravel()]) self = cls(masks, height, width) return self @classmethod def cat(cls: Type[T], masks: Sequence[T]) -> T: """Concatenate a sequence of masks into one single mask instance. Args: masks (Sequence[PolygonMasks]): A sequence of mask instances. Returns: PolygonMasks: Concatenated mask instance. """ assert isinstance(masks, Sequence) if len(masks) == 0: raise ValueError('masks should not be an empty list.') assert all(isinstance(m, cls) for m in masks) mask_list = list(itertools.chain(*[m.masks for m in masks])) return cls(mask_list, masks[0].height, masks[0].width) def polygon_to_bitmap(polygons, height, width): """Convert masks from the form of polygons to bitmaps. Args: polygons (list[ndarray]): masks in polygon representation height (int): mask height width (int): mask width Return: ndarray: the converted masks in bitmap representation """ rles = maskUtils.frPyObjects(polygons, height, width) rle = maskUtils.merge(rles) bitmap_mask = maskUtils.decode(rle).astype(bool) return bitmap_mask def bitmap_to_polygon(bitmap): """Convert masks from the form of bitmaps to polygons. Args: bitmap (ndarray): masks in bitmap representation. Return: list[ndarray]: the converted mask in polygon representation. bool: whether the mask has holes. """ bitmap = np.ascontiguousarray(bitmap).astype(np.uint8) # cv2.RETR_CCOMP: retrieves all of the contours and organizes them # into a two-level hierarchy. At the top level, there are external # boundaries of the components. At the second level, there are # boundaries of the holes. If there is another contour inside a hole # of a connected component, it is still put at the top level. # cv2.CHAIN_APPROX_NONE: stores absolutely all the contour points. outs = cv2.findContours(bitmap, cv2.RETR_CCOMP, cv2.CHAIN_APPROX_NONE) contours = outs[-2] hierarchy = outs[-1] if hierarchy is None: return [], False # hierarchy[i]: 4 elements, for the indexes of next, previous, # parent, or nested contours. If there is no corresponding contour, # it will be -1. with_hole = (hierarchy.reshape(-1, 4)[:, 3] >= 0).any() contours = [c.reshape(-1, 2) for c in contours] return contours, with_hole ================================================ FILE: mmdet/structures/mask/utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pycocotools.mask as mask_util import torch from mmengine.utils import slice_list def split_combined_polys(polys, poly_lens, polys_per_mask): """Split the combined 1-D polys into masks. A mask is represented as a list of polys, and a poly is represented as a 1-D array. In dataset, all masks are concatenated into a single 1-D tensor. Here we need to split the tensor into original representations. Args: polys (list): a list (length = image num) of 1-D tensors poly_lens (list): a list (length = image num) of poly length polys_per_mask (list): a list (length = image num) of poly number of each mask Returns: list: a list (length = image num) of list (length = mask num) of \ list (length = poly num) of numpy array. """ mask_polys_list = [] for img_id in range(len(polys)): polys_single = polys[img_id] polys_lens_single = poly_lens[img_id].tolist() polys_per_mask_single = polys_per_mask[img_id].tolist() split_polys = slice_list(polys_single, polys_lens_single) mask_polys = slice_list(split_polys, polys_per_mask_single) mask_polys_list.append(mask_polys) return mask_polys_list # TODO: move this function to more proper place def encode_mask_results(mask_results): """Encode bitmap mask to RLE code. Args: mask_results (list): bitmap mask results. Returns: list | tuple: RLE encoded mask. """ encoded_mask_results = [] for mask in mask_results: encoded_mask_results.append( mask_util.encode( np.array(mask[:, :, np.newaxis], order='F', dtype='uint8'))[0]) # encoded with RLE return encoded_mask_results def mask2bbox(masks): """Obtain tight bounding boxes of binary masks. Args: masks (Tensor): Binary mask of shape (n, h, w). Returns: Tensor: Bboxe with shape (n, 4) of \ positive region in binary mask. """ N = masks.shape[0] bboxes = masks.new_zeros((N, 4), dtype=torch.float32) x_any = torch.any(masks, dim=1) y_any = torch.any(masks, dim=2) for i in range(N): x = torch.where(x_any[i, :])[0] y = torch.where(y_any[i, :])[0] if len(x) > 0 and len(y) > 0: bboxes[i, :] = bboxes.new_tensor( [x[0], y[0], x[-1] + 1, y[-1] + 1]) return bboxes ================================================ FILE: mmdet/testing/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from ._fast_stop_training_hook import FastStopTrainingHook # noqa: F401,F403 from ._utils import (demo_mm_inputs, demo_mm_proposals, demo_mm_sampling_results, get_detector_cfg, get_roi_head_cfg, replace_to_ceph) __all__ = [ 'demo_mm_inputs', 'get_detector_cfg', 'get_roi_head_cfg', 'demo_mm_proposals', 'demo_mm_sampling_results', 'replace_to_ceph' ] ================================================ FILE: mmdet/testing/_fast_stop_training_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmengine.hooks import Hook from mmdet.registry import HOOKS @HOOKS.register_module() class FastStopTrainingHook(Hook): """Set runner's epoch information to the model.""" def __init__(self, by_epoch, save_ckpt=False, stop_iter_or_epoch=5): self.by_epoch = by_epoch self.save_ckpt = save_ckpt self.stop_iter_or_epoch = stop_iter_or_epoch def after_train_iter(self, runner, batch_idx: int, data_batch: None, outputs: None) -> None: if self.save_ckpt and self.by_epoch: # If it is epoch-based and want to save weights, # we must run at least 1 epoch. return if runner.iter >= self.stop_iter_or_epoch: raise RuntimeError('quick exit') def after_train_epoch(self, runner) -> None: if runner.epoch >= self.stop_iter_or_epoch - 1: raise RuntimeError('quick exit') ================================================ FILE: mmdet/testing/_utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from os.path import dirname, exists, join import numpy as np import torch from mmengine.config import Config from mmengine.dataset import pseudo_collate from mmengine.structures import InstanceData, PixelData from ..registry import TASK_UTILS from ..structures import DetDataSample from ..structures.bbox import HorizontalBoxes def _get_config_directory(): """Find the predefined detector config directory.""" try: # Assume we are running in the source mmdetection repo repo_dpath = dirname(dirname(dirname(__file__))) except NameError: # For IPython development when this __file__ is not defined import mmdet repo_dpath = dirname(dirname(mmdet.__file__)) config_dpath = join(repo_dpath, 'configs') if not exists(config_dpath): raise Exception('Cannot find config path') return config_dpath def _get_config_module(fname): """Load a configuration as a python module.""" config_dpath = _get_config_directory() config_fpath = join(config_dpath, fname) config_mod = Config.fromfile(config_fpath) return config_mod def get_detector_cfg(fname): """Grab configs necessary to create a detector. These are deep copied to allow for safe modification of parameters without influencing other tests. """ config = _get_config_module(fname) model = copy.deepcopy(config.model) return model def get_roi_head_cfg(fname): """Grab configs necessary to create a roi_head. These are deep copied to allow for safe modification of parameters without influencing other tests. """ config = _get_config_module(fname) model = copy.deepcopy(config.model) roi_head = model.roi_head train_cfg = None if model.train_cfg is None else model.train_cfg.rcnn test_cfg = None if model.test_cfg is None else model.test_cfg.rcnn roi_head.update(dict(train_cfg=train_cfg, test_cfg=test_cfg)) return roi_head def _rand_bboxes(rng, num_boxes, w, h): cx, cy, bw, bh = rng.rand(num_boxes, 4).T tl_x = ((cx * w) - (w * bw / 2)).clip(0, w) tl_y = ((cy * h) - (h * bh / 2)).clip(0, h) br_x = ((cx * w) + (w * bw / 2)).clip(0, w) br_y = ((cy * h) + (h * bh / 2)).clip(0, h) bboxes = np.vstack([tl_x, tl_y, br_x, br_y]).T return bboxes def _rand_masks(rng, num_boxes, bboxes, img_w, img_h): from mmdet.structures.mask import BitmapMasks masks = np.zeros((num_boxes, img_h, img_w)) for i, bbox in enumerate(bboxes): bbox = bbox.astype(np.int32) mask = (rng.rand(1, bbox[3] - bbox[1], bbox[2] - bbox[0]) > 0.3).astype(np.int64) masks[i:i + 1, bbox[1]:bbox[3], bbox[0]:bbox[2]] = mask return BitmapMasks(masks, height=img_h, width=img_w) def demo_mm_inputs(batch_size=2, image_shapes=(3, 128, 128), num_items=None, num_classes=10, sem_seg_output_strides=1, with_mask=False, with_semantic=False, use_box_type=False, device='cpu'): """Create a superset of inputs needed to run test or train batches. Args: batch_size (int): batch size. Defaults to 2. image_shapes (List[tuple], Optional): image shape. Defaults to (3, 128, 128) num_items (None | List[int]): specifies the number of boxes in each batch item. Default to None. num_classes (int): number of different labels a box might have. Defaults to 10. with_mask (bool): Whether to return mask annotation. Defaults to False. with_semantic (bool): whether to return semantic. Defaults to False. device (str): Destination device type. Defaults to cpu. """ rng = np.random.RandomState(0) if isinstance(image_shapes, list): assert len(image_shapes) == batch_size else: image_shapes = [image_shapes] * batch_size if isinstance(num_items, list): assert len(num_items) == batch_size packed_inputs = [] for idx in range(batch_size): image_shape = image_shapes[idx] c, h, w = image_shape image = rng.randint(0, 255, size=image_shape, dtype=np.uint8) mm_inputs = dict() mm_inputs['inputs'] = torch.from_numpy(image).to(device) img_meta = { 'img_id': idx, 'img_shape': image_shape[1:], 'ori_shape': image_shape[1:], 'filename': '.png', 'scale_factor': np.array([1.1, 1.2]), 'flip': False, 'flip_direction': None, 'border': [1, 1, 1, 1] # Only used by CenterNet } data_sample = DetDataSample() data_sample.set_metainfo(img_meta) # gt_instances gt_instances = InstanceData() if num_items is None: num_boxes = rng.randint(1, 10) else: num_boxes = num_items[idx] bboxes = _rand_bboxes(rng, num_boxes, w, h) labels = rng.randint(1, num_classes, size=num_boxes) # TODO: remove this part when all model adapted with BaseBoxes if use_box_type: gt_instances.bboxes = HorizontalBoxes(bboxes, dtype=torch.float32) else: gt_instances.bboxes = torch.FloatTensor(bboxes) gt_instances.labels = torch.LongTensor(labels) if with_mask: masks = _rand_masks(rng, num_boxes, bboxes, w, h) gt_instances.masks = masks # TODO: waiting for ci to be fixed # masks = np.random.randint(0, 2, (len(bboxes), h, w), dtype=np.uint8) # gt_instances.mask = BitmapMasks(masks, h, w) data_sample.gt_instances = gt_instances # ignore_instances ignore_instances = InstanceData() bboxes = _rand_bboxes(rng, num_boxes, w, h) if use_box_type: ignore_instances.bboxes = HorizontalBoxes( bboxes, dtype=torch.float32) else: ignore_instances.bboxes = torch.FloatTensor(bboxes) data_sample.ignored_instances = ignore_instances # gt_sem_seg if with_semantic: # assume gt_semantic_seg using scale 1/8 of the img gt_semantic_seg = torch.from_numpy( np.random.randint( 0, num_classes, (1, h // sem_seg_output_strides, w // sem_seg_output_strides), dtype=np.uint8)) gt_sem_seg_data = dict(sem_seg=gt_semantic_seg) data_sample.gt_sem_seg = PixelData(**gt_sem_seg_data) mm_inputs['data_samples'] = data_sample.to(device) # TODO: gt_ignore packed_inputs.append(mm_inputs) data = pseudo_collate(packed_inputs) return data def demo_mm_proposals(image_shapes, num_proposals, device='cpu'): """Create a list of fake porposals. Args: image_shapes (list[tuple[int]]): Batch image shapes. num_proposals (int): The number of fake proposals. """ rng = np.random.RandomState(0) results = [] for img_shape in image_shapes: result = InstanceData() w, h = img_shape[1:] proposals = _rand_bboxes(rng, num_proposals, w, h) result.bboxes = torch.from_numpy(proposals).float() result.scores = torch.from_numpy(rng.rand(num_proposals)).float() result.labels = torch.zeros(num_proposals).long() results.append(result.to(device)) return results def demo_mm_sampling_results(proposals_list, batch_gt_instances, batch_gt_instances_ignore=None, assigner_cfg=None, sampler_cfg=None, feats=None): """Create sample results that can be passed to BBoxHead.get_targets.""" assert len(proposals_list) == len(batch_gt_instances) if batch_gt_instances_ignore is None: batch_gt_instances_ignore = [None for _ in batch_gt_instances] else: assert len(batch_gt_instances_ignore) == len(batch_gt_instances) default_assigner_cfg = dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, ignore_iof_thr=-1) assigner_cfg = assigner_cfg if assigner_cfg is not None \ else default_assigner_cfg default_sampler_cfg = dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True) sampler_cfg = sampler_cfg if sampler_cfg is not None \ else default_sampler_cfg bbox_assigner = TASK_UTILS.build(assigner_cfg) bbox_sampler = TASK_UTILS.build(sampler_cfg) sampling_results = [] for i in range(len(batch_gt_instances)): if feats is not None: feats = [lvl_feat[i][None] for lvl_feat in feats] # rename proposals.bboxes to proposals.priors proposals = proposals_list[i] proposals.priors = proposals.pop('bboxes') assign_result = bbox_assigner.assign(proposals, batch_gt_instances[i], batch_gt_instances_ignore[i]) sampling_result = bbox_sampler.sample( assign_result, proposals, batch_gt_instances[i], feats=feats) sampling_results.append(sampling_result) return sampling_results # TODO: Support full ceph def replace_to_ceph(cfg): file_client_args = dict( backend='petrel', path_mapping=dict({ './data/': 's3://openmmlab/datasets/detection/', 'data/': 's3://openmmlab/datasets/detection/' })) # TODO: name is a reserved interface, which will be used later. def _process_pipeline(dataset, name): def replace_img(pipeline): if pipeline['type'] == 'LoadImageFromFile': pipeline['file_client_args'] = file_client_args def replace_ann(pipeline): if pipeline['type'] == 'LoadAnnotations' or pipeline[ 'type'] == 'LoadPanopticAnnotations': pipeline['file_client_args'] = file_client_args if 'pipeline' in dataset: replace_img(dataset.pipeline[0]) replace_ann(dataset.pipeline[1]) if 'dataset' in dataset: # dataset wrapper replace_img(dataset.dataset.pipeline[0]) replace_ann(dataset.dataset.pipeline[1]) else: # dataset wrapper replace_img(dataset.dataset.pipeline[0]) replace_ann(dataset.dataset.pipeline[1]) def _process_evaluator(evaluator, name): if evaluator['type'] == 'CocoPanopticMetric': evaluator['file_client_args'] = file_client_args # half ceph _process_pipeline(cfg.train_dataloader.dataset, cfg.filename) _process_pipeline(cfg.val_dataloader.dataset, cfg.filename) _process_pipeline(cfg.test_dataloader.dataset, cfg.filename) _process_evaluator(cfg.val_evaluator, cfg.filename) _process_evaluator(cfg.test_evaluator, cfg.filename) ================================================ FILE: mmdet/utils/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .collect_env import collect_env from .compat_config import compat_cfg from .dist_utils import (all_reduce_dict, allreduce_grads, reduce_mean, sync_random_seed) from .logger import get_caller_name, log_img_scale from .memory import AvoidCUDAOOM, AvoidOOM from .misc import (find_latest_checkpoint, get_test_pipeline_cfg, update_data_root) from .replace_cfg_vals import replace_cfg_vals from .setup_env import register_all_modules, setup_multi_processes from .split_batch import split_batch from .typing_utils import (ConfigType, InstanceList, MultiConfig, OptConfigType, OptInstanceList, OptMultiConfig, OptPixelList, PixelList, RangeType) __all__ = [ 'collect_env', 'find_latest_checkpoint', 'update_data_root', 'setup_multi_processes', 'get_caller_name', 'log_img_scale', 'compat_cfg', 'split_batch', 'register_all_modules', 'replace_cfg_vals', 'AvoidOOM', 'AvoidCUDAOOM', 'all_reduce_dict', 'allreduce_grads', 'reduce_mean', 'sync_random_seed', 'ConfigType', 'InstanceList', 'MultiConfig', 'OptConfigType', 'OptInstanceList', 'OptMultiConfig', 'OptPixelList', 'PixelList', 'RangeType', 'get_test_pipeline_cfg' ] ================================================ FILE: mmdet/utils/benchmark.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import time from functools import partial from typing import List, Optional, Union import numpy as np import torch import torch.nn as nn from mmcv.cnn import fuse_conv_bn # TODO need update # from mmcv.runner import wrap_fp16_model from mmengine import MMLogger from mmengine.config import Config from mmengine.device import get_max_cuda_memory from mmengine.dist import get_world_size from mmengine.runner import Runner, load_checkpoint from mmengine.utils.dl_utils import set_multi_processing from torch.nn.parallel import DistributedDataParallel from mmdet.registry import DATASETS, MODELS try: import psutil except ImportError: psutil = None def custom_round(value: Union[int, float], factor: Union[int, float], precision: int = 2) -> float: """Custom round function.""" return round(value / factor, precision) gb_round = partial(custom_round, factor=1024**3) def print_log(msg: str, logger: Optional[MMLogger] = None) -> None: """Print a log message.""" if logger is None: print(msg, flush=True) else: logger.info(msg) def print_process_memory(p: psutil.Process, logger: Optional[MMLogger] = None) -> None: """print process memory info.""" mem_used = gb_round(psutil.virtual_memory().used) memory_full_info = p.memory_full_info() uss_mem = gb_round(memory_full_info.uss) pss_mem = gb_round(memory_full_info.pss) for children in p.children(): child_mem_info = children.memory_full_info() uss_mem += gb_round(child_mem_info.uss) pss_mem += gb_round(child_mem_info.pss) process_count = 1 + len(p.children()) print_log( f'(GB) mem_used: {mem_used:.2f} | uss: {uss_mem:.2f} | ' f'pss: {pss_mem:.2f} | total_proc: {process_count}', logger) class BaseBenchmark: """The benchmark base class. The ``run`` method is an external calling interface, and it will call the ``run_once`` method ``repeat_num`` times for benchmarking. Finally, call the ``average_multiple_runs`` method to further process the results of multiple runs. Args: max_iter (int): maximum iterations of benchmark. log_interval (int): interval of logging. num_warmup (int): Number of Warmup. logger (MMLogger, optional): Formatted logger used to record messages. """ def __init__(self, max_iter: int, log_interval: int, num_warmup: int, logger: Optional[MMLogger] = None): self.max_iter = max_iter self.log_interval = log_interval self.num_warmup = num_warmup self.logger = logger def run(self, repeat_num: int = 1) -> dict: """benchmark entry method. Args: repeat_num (int): Number of repeat benchmark. Defaults to 1. """ assert repeat_num >= 1 results = [] for _ in range(repeat_num): results.append(self.run_once()) results = self.average_multiple_runs(results) return results def run_once(self) -> dict: """Executes the benchmark once.""" raise NotImplementedError() def average_multiple_runs(self, results: List[dict]) -> dict: """Average the results of multiple runs.""" raise NotImplementedError() class InferenceBenchmark(BaseBenchmark): """The inference benchmark class. It will be statistical inference FPS, CUDA memory and CPU memory information. Args: cfg (mmengine.Config): config. checkpoint (str): Accept local filepath, URL, ``torchvision://xxx``, ``open-mmlab://xxx``. distributed (bool): distributed testing flag. is_fuse_conv_bn (bool): Whether to fuse conv and bn, this will slightly increase the inference speed. max_iter (int): maximum iterations of benchmark. Defaults to 2000. log_interval (int): interval of logging. Defaults to 50. num_warmup (int): Number of Warmup. Defaults to 5. logger (MMLogger, optional): Formatted logger used to record messages. """ def __init__(self, cfg: Config, checkpoint: str, distributed: bool, is_fuse_conv_bn: bool, max_iter: int = 2000, log_interval: int = 50, num_warmup: int = 5, logger: Optional[MMLogger] = None): super().__init__(max_iter, log_interval, num_warmup, logger) assert get_world_size( ) == 1, 'Inference benchmark does not allow distributed multi-GPU' self.cfg = copy.deepcopy(cfg) self.distributed = distributed if psutil is None: raise ImportError('psutil is not installed, please install it by: ' 'pip install psutil') self._process = psutil.Process() env_cfg = self.cfg.get('env_cfg') if env_cfg.get('cudnn_benchmark'): torch.backends.cudnn.benchmark = True mp_cfg: dict = env_cfg.get('mp_cfg', {}) set_multi_processing(**mp_cfg, distributed=self.distributed) print_log('before build: ', self.logger) print_process_memory(self._process, self.logger) self.cfg.model.pretrained = None self.model = self._init_model(checkpoint, is_fuse_conv_bn) # Because multiple processes will occupy additional CPU resources, # FPS statistics will be more unstable when num_workers is not 0. # It is reasonable to set num_workers to 0. dataloader_cfg = cfg.test_dataloader dataloader_cfg['num_workers'] = 0 dataloader_cfg['batch_size'] = 1 dataloader_cfg['persistent_workers'] = False self.data_loader = Runner.build_dataloader(dataloader_cfg) print_log('after build: ', self.logger) print_process_memory(self._process, self.logger) def _init_model(self, checkpoint: str, is_fuse_conv_bn: bool) -> nn.Module: """Initialize the model.""" model = MODELS.build(self.cfg.model) # TODO need update # fp16_cfg = self.cfg.get('fp16', None) # if fp16_cfg is not None: # wrap_fp16_model(model) load_checkpoint(model, checkpoint, map_location='cpu') if is_fuse_conv_bn: model = fuse_conv_bn(model) model = model.cuda() if self.distributed: model = DistributedDataParallel( model, device_ids=[torch.cuda.current_device()], broadcast_buffers=False, find_unused_parameters=False) model.eval() return model def run_once(self) -> dict: """Executes the benchmark once.""" pure_inf_time = 0 fps = 0 for i, data in enumerate(self.data_loader): if (i + 1) % self.log_interval == 0: print_log('==================================', self.logger) torch.cuda.synchronize() start_time = time.perf_counter() with torch.no_grad(): self.model(data, return_loss=False) torch.cuda.synchronize() elapsed = time.perf_counter() - start_time if i >= self.num_warmup: pure_inf_time += elapsed if (i + 1) % self.log_interval == 0: fps = (i + 1 - self.num_warmup) / pure_inf_time cuda_memory = get_max_cuda_memory() print_log( f'Done image [{i + 1:<3}/{self.max_iter}], ' f'fps: {fps:.1f} img/s, ' f'times per image: {1000 / fps:.1f} ms/img, ' f'cuda memory: {cuda_memory} MB', self.logger) print_process_memory(self._process, self.logger) if (i + 1) == self.max_iter: fps = (i + 1 - self.num_warmup) / pure_inf_time break return {'fps': fps} def average_multiple_runs(self, results: List[dict]) -> dict: """Average the results of multiple runs.""" print_log('============== Done ==================', self.logger) fps_list_ = [round(result['fps'], 1) for result in results] avg_fps_ = sum(fps_list_) / len(fps_list_) outputs = {'avg_fps': avg_fps_, 'fps_list': fps_list_} if len(fps_list_) > 1: times_pre_image_list_ = [ round(1000 / result['fps'], 1) for result in results ] avg_times_pre_image_ = sum(times_pre_image_list_) / len( times_pre_image_list_) print_log( f'Overall fps: {fps_list_}[{avg_fps_:.1f}] img/s, ' 'times per image: ' f'{times_pre_image_list_}[{avg_times_pre_image_:.1f}] ' 'ms/img', self.logger) else: print_log( f'Overall fps: {fps_list_[0]:.1f} img/s, ' f'times per image: {1000 / fps_list_[0]:.1f} ms/img', self.logger) print_log(f'cuda memory: {get_max_cuda_memory()} MB', self.logger) print_process_memory(self._process, self.logger) return outputs class DataLoaderBenchmark(BaseBenchmark): """The dataloader benchmark class. It will be statistical inference FPS and CPU memory information. Args: cfg (mmengine.Config): config. distributed (bool): distributed testing flag. dataset_type (str): benchmark data type, only supports ``train``, ``val`` and ``test``. max_iter (int): maximum iterations of benchmark. Defaults to 2000. log_interval (int): interval of logging. Defaults to 50. num_warmup (int): Number of Warmup. Defaults to 5. logger (MMLogger, optional): Formatted logger used to record messages. """ def __init__(self, cfg: Config, distributed: bool, dataset_type: str, max_iter: int = 2000, log_interval: int = 50, num_warmup: int = 5, logger: Optional[MMLogger] = None): super().__init__(max_iter, log_interval, num_warmup, logger) assert dataset_type in ['train', 'val', 'test'], \ 'dataset_type only supports train,' \ f' val and test, but got {dataset_type}' assert get_world_size( ) == 1, 'Dataloader benchmark does not allow distributed multi-GPU' self.cfg = copy.deepcopy(cfg) self.distributed = distributed if psutil is None: raise ImportError('psutil is not installed, please install it by: ' 'pip install psutil') self._process = psutil.Process() mp_cfg = self.cfg.get('env_cfg', {}).get('mp_cfg') if mp_cfg is not None: set_multi_processing(distributed=self.distributed, **mp_cfg) else: set_multi_processing(distributed=self.distributed) print_log('before build: ', self.logger) print_process_memory(self._process, self.logger) if dataset_type == 'train': self.data_loader = Runner.build_dataloader(cfg.train_dataloader) elif dataset_type == 'test': self.data_loader = Runner.build_dataloader(cfg.test_dataloader) else: self.data_loader = Runner.build_dataloader(cfg.val_dataloader) self.batch_size = self.data_loader.batch_size self.num_workers = self.data_loader.num_workers print_log('after build: ', self.logger) print_process_memory(self._process, self.logger) def run_once(self) -> dict: """Executes the benchmark once.""" pure_inf_time = 0 fps = 0 # benchmark with 2000 image and take the average start_time = time.perf_counter() for i, data in enumerate(self.data_loader): elapsed = time.perf_counter() - start_time if (i + 1) % self.log_interval == 0: print_log('==================================', self.logger) if i >= self.num_warmup: pure_inf_time += elapsed if (i + 1) % self.log_interval == 0: fps = (i + 1 - self.num_warmup) / pure_inf_time print_log( f'Done batch [{i + 1:<3}/{self.max_iter}], ' f'fps: {fps:.1f} batch/s, ' f'times per batch: {1000 / fps:.1f} ms/batch, ' f'batch size: {self.batch_size}, num_workers: ' f'{self.num_workers}', self.logger) print_process_memory(self._process, self.logger) if (i + 1) == self.max_iter: fps = (i + 1 - self.num_warmup) / pure_inf_time break start_time = time.perf_counter() return {'fps': fps} def average_multiple_runs(self, results: List[dict]) -> dict: """Average the results of multiple runs.""" print_log('============== Done ==================', self.logger) fps_list_ = [round(result['fps'], 1) for result in results] avg_fps_ = sum(fps_list_) / len(fps_list_) outputs = {'avg_fps': avg_fps_, 'fps_list': fps_list_} if len(fps_list_) > 1: times_pre_image_list_ = [ round(1000 / result['fps'], 1) for result in results ] avg_times_pre_image_ = sum(times_pre_image_list_) / len( times_pre_image_list_) print_log( f'Overall fps: {fps_list_}[{avg_fps_:.1f}] img/s, ' 'times per batch: ' f'{times_pre_image_list_}[{avg_times_pre_image_:.1f}] ' f'ms/batch, batch size: {self.batch_size}, num_workers: ' f'{self.num_workers}', self.logger) else: print_log( f'Overall fps: {fps_list_[0]:.1f} batch/s, ' f'times per batch: {1000 / fps_list_[0]:.1f} ms/batch, ' f'batch size: {self.batch_size}, num_workers: ' f'{self.num_workers}', self.logger) print_process_memory(self._process, self.logger) return outputs class DatasetBenchmark(BaseBenchmark): """The dataset benchmark class. It will be statistical inference FPS, FPS pre transform and CPU memory information. Args: cfg (mmengine.Config): config. dataset_type (str): benchmark data type, only supports ``train``, ``val`` and ``test``. max_iter (int): maximum iterations of benchmark. Defaults to 2000. log_interval (int): interval of logging. Defaults to 50. num_warmup (int): Number of Warmup. Defaults to 5. logger (MMLogger, optional): Formatted logger used to record messages. """ def __init__(self, cfg: Config, dataset_type: str, max_iter: int = 2000, log_interval: int = 50, num_warmup: int = 5, logger: Optional[MMLogger] = None): super().__init__(max_iter, log_interval, num_warmup, logger) assert dataset_type in ['train', 'val', 'test'], \ 'dataset_type only supports train,' \ f' val and test, but got {dataset_type}' assert get_world_size( ) == 1, 'Dataset benchmark does not allow distributed multi-GPU' self.cfg = copy.deepcopy(cfg) if dataset_type == 'train': dataloader_cfg = copy.deepcopy(cfg.train_dataloader) elif dataset_type == 'test': dataloader_cfg = copy.deepcopy(cfg.test_dataloader) else: dataloader_cfg = copy.deepcopy(cfg.val_dataloader) dataset_cfg = dataloader_cfg.pop('dataset') dataset = DATASETS.build(dataset_cfg) if hasattr(dataset, 'full_init'): dataset.full_init() self.dataset = dataset def run_once(self) -> dict: """Executes the benchmark once.""" pure_inf_time = 0 fps = 0 total_index = list(range(len(self.dataset))) np.random.shuffle(total_index) start_time = time.perf_counter() for i, idx in enumerate(total_index): if (i + 1) % self.log_interval == 0: print_log('==================================', self.logger) get_data_info_start_time = time.perf_counter() data_info = self.dataset.get_data_info(idx) get_data_info_elapsed = time.perf_counter( ) - get_data_info_start_time if (i + 1) % self.log_interval == 0: print_log(f'get_data_info - {get_data_info_elapsed * 1000} ms', self.logger) for t in self.dataset.pipeline.transforms: transform_start_time = time.perf_counter() data_info = t(data_info) transform_elapsed = time.perf_counter() - transform_start_time if (i + 1) % self.log_interval == 0: print_log( f'{t.__class__.__name__} - ' f'{transform_elapsed * 1000} ms', self.logger) if data_info is None: break elapsed = time.perf_counter() - start_time if i >= self.num_warmup: pure_inf_time += elapsed if (i + 1) % self.log_interval == 0: fps = (i + 1 - self.num_warmup) / pure_inf_time print_log( f'Done img [{i + 1:<3}/{self.max_iter}], ' f'fps: {fps:.1f} img/s, ' f'times per img: {1000 / fps:.1f} ms/img', self.logger) if (i + 1) == self.max_iter: fps = (i + 1 - self.num_warmup) / pure_inf_time break start_time = time.perf_counter() return {'fps': fps} def average_multiple_runs(self, results: List[dict]) -> dict: """Average the results of multiple runs.""" print_log('============== Done ==================', self.logger) fps_list_ = [round(result['fps'], 1) for result in results] avg_fps_ = sum(fps_list_) / len(fps_list_) outputs = {'avg_fps': avg_fps_, 'fps_list': fps_list_} if len(fps_list_) > 1: times_pre_image_list_ = [ round(1000 / result['fps'], 1) for result in results ] avg_times_pre_image_ = sum(times_pre_image_list_) / len( times_pre_image_list_) print_log( f'Overall fps: {fps_list_}[{avg_fps_:.1f}] img/s, ' 'times per img: ' f'{times_pre_image_list_}[{avg_times_pre_image_:.1f}] ' 'ms/img', self.logger) else: print_log( f'Overall fps: {fps_list_[0]:.1f} img/s, ' f'times per img: {1000 / fps_list_[0]:.1f} ms/img', self.logger) return outputs ================================================ FILE: mmdet/utils/collect_env.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmengine.utils import get_git_hash from mmengine.utils.dl_utils import collect_env as collect_base_env import mmdet def collect_env(): """Collect the information of the running environments.""" env_info = collect_base_env() env_info['MMDetection'] = mmdet.__version__ + '+' + get_git_hash()[:7] return env_info if __name__ == '__main__': for name, val in collect_env().items(): print(f'{name}: {val}') ================================================ FILE: mmdet/utils/compat_config.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import warnings from mmengine.config import ConfigDict def compat_cfg(cfg): """This function would modify some filed to keep the compatibility of config. For example, it will move some args which will be deprecated to the correct fields. """ cfg = copy.deepcopy(cfg) cfg = compat_imgs_per_gpu(cfg) cfg = compat_loader_args(cfg) cfg = compat_runner_args(cfg) return cfg def compat_runner_args(cfg): if 'runner' not in cfg: cfg.runner = ConfigDict({ 'type': 'EpochBasedRunner', 'max_epochs': cfg.total_epochs }) warnings.warn( 'config is now expected to have a `runner` section, ' 'please set `runner` in your config.', UserWarning) else: if 'total_epochs' in cfg: assert cfg.total_epochs == cfg.runner.max_epochs return cfg def compat_imgs_per_gpu(cfg): cfg = copy.deepcopy(cfg) if 'imgs_per_gpu' in cfg.data: warnings.warn('"imgs_per_gpu" is deprecated in MMDet V2.0. ' 'Please use "samples_per_gpu" instead') if 'samples_per_gpu' in cfg.data: warnings.warn( f'Got "imgs_per_gpu"={cfg.data.imgs_per_gpu} and ' f'"samples_per_gpu"={cfg.data.samples_per_gpu}, "imgs_per_gpu"' f'={cfg.data.imgs_per_gpu} is used in this experiments') else: warnings.warn('Automatically set "samples_per_gpu"="imgs_per_gpu"=' f'{cfg.data.imgs_per_gpu} in this experiments') cfg.data.samples_per_gpu = cfg.data.imgs_per_gpu return cfg def compat_loader_args(cfg): """Deprecated sample_per_gpu in cfg.data.""" cfg = copy.deepcopy(cfg) if 'train_dataloader' not in cfg.data: cfg.data['train_dataloader'] = ConfigDict() if 'val_dataloader' not in cfg.data: cfg.data['val_dataloader'] = ConfigDict() if 'test_dataloader' not in cfg.data: cfg.data['test_dataloader'] = ConfigDict() # special process for train_dataloader if 'samples_per_gpu' in cfg.data: samples_per_gpu = cfg.data.pop('samples_per_gpu') assert 'samples_per_gpu' not in \ cfg.data.train_dataloader, ('`samples_per_gpu` are set ' 'in `data` field and ` ' 'data.train_dataloader` ' 'at the same time. ' 'Please only set it in ' '`data.train_dataloader`. ') cfg.data.train_dataloader['samples_per_gpu'] = samples_per_gpu if 'persistent_workers' in cfg.data: persistent_workers = cfg.data.pop('persistent_workers') assert 'persistent_workers' not in \ cfg.data.train_dataloader, ('`persistent_workers` are set ' 'in `data` field and ` ' 'data.train_dataloader` ' 'at the same time. ' 'Please only set it in ' '`data.train_dataloader`. ') cfg.data.train_dataloader['persistent_workers'] = persistent_workers if 'workers_per_gpu' in cfg.data: workers_per_gpu = cfg.data.pop('workers_per_gpu') cfg.data.train_dataloader['workers_per_gpu'] = workers_per_gpu cfg.data.val_dataloader['workers_per_gpu'] = workers_per_gpu cfg.data.test_dataloader['workers_per_gpu'] = workers_per_gpu # special process for val_dataloader if 'samples_per_gpu' in cfg.data.val: # keep default value of `sample_per_gpu` is 1 assert 'samples_per_gpu' not in \ cfg.data.val_dataloader, ('`samples_per_gpu` are set ' 'in `data.val` field and ` ' 'data.val_dataloader` at ' 'the same time. ' 'Please only set it in ' '`data.val_dataloader`. ') cfg.data.val_dataloader['samples_per_gpu'] = \ cfg.data.val.pop('samples_per_gpu') # special process for val_dataloader # in case the test dataset is concatenated if isinstance(cfg.data.test, dict): if 'samples_per_gpu' in cfg.data.test: assert 'samples_per_gpu' not in \ cfg.data.test_dataloader, ('`samples_per_gpu` are set ' 'in `data.test` field and ` ' 'data.test_dataloader` ' 'at the same time. ' 'Please only set it in ' '`data.test_dataloader`. ') cfg.data.test_dataloader['samples_per_gpu'] = \ cfg.data.test.pop('samples_per_gpu') elif isinstance(cfg.data.test, list): for ds_cfg in cfg.data.test: if 'samples_per_gpu' in ds_cfg: assert 'samples_per_gpu' not in \ cfg.data.test_dataloader, ('`samples_per_gpu` are set ' 'in `data.test` field and ` ' 'data.test_dataloader` at' ' the same time. ' 'Please only set it in ' '`data.test_dataloader`. ') samples_per_gpu = max( [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test]) cfg.data.test_dataloader['samples_per_gpu'] = samples_per_gpu return cfg ================================================ FILE: mmdet/utils/contextmanagers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import asyncio import contextlib import logging import os import time from typing import List import torch logger = logging.getLogger(__name__) DEBUG_COMPLETED_TIME = bool(os.environ.get('DEBUG_COMPLETED_TIME', False)) @contextlib.asynccontextmanager async def completed(trace_name='', name='', sleep_interval=0.05, streams: List[torch.cuda.Stream] = None): """Async context manager that waits for work to complete on given CUDA streams.""" if not torch.cuda.is_available(): yield return stream_before_context_switch = torch.cuda.current_stream() if not streams: streams = [stream_before_context_switch] else: streams = [s if s else stream_before_context_switch for s in streams] end_events = [ torch.cuda.Event(enable_timing=DEBUG_COMPLETED_TIME) for _ in streams ] if DEBUG_COMPLETED_TIME: start = torch.cuda.Event(enable_timing=True) stream_before_context_switch.record_event(start) cpu_start = time.monotonic() logger.debug('%s %s starting, streams: %s', trace_name, name, streams) grad_enabled_before = torch.is_grad_enabled() try: yield finally: current_stream = torch.cuda.current_stream() assert current_stream == stream_before_context_switch if DEBUG_COMPLETED_TIME: cpu_end = time.monotonic() for i, stream in enumerate(streams): event = end_events[i] stream.record_event(event) grad_enabled_after = torch.is_grad_enabled() # observed change of torch.is_grad_enabled() during concurrent run of # async_test_bboxes code assert (grad_enabled_before == grad_enabled_after ), 'Unexpected is_grad_enabled() value change' are_done = [e.query() for e in end_events] logger.debug('%s %s completed: %s streams: %s', trace_name, name, are_done, streams) with torch.cuda.stream(stream_before_context_switch): while not all(are_done): await asyncio.sleep(sleep_interval) are_done = [e.query() for e in end_events] logger.debug( '%s %s completed: %s streams: %s', trace_name, name, are_done, streams, ) current_stream = torch.cuda.current_stream() assert current_stream == stream_before_context_switch if DEBUG_COMPLETED_TIME: cpu_time = (cpu_end - cpu_start) * 1000 stream_times_ms = '' for i, stream in enumerate(streams): elapsed_time = start.elapsed_time(end_events[i]) stream_times_ms += f' {stream} {elapsed_time:.2f} ms' logger.info('%s %s %.2f ms %s', trace_name, name, cpu_time, stream_times_ms) @contextlib.asynccontextmanager async def concurrent(streamqueue: asyncio.Queue, trace_name='concurrent', name='stream'): """Run code concurrently in different streams. :param streamqueue: asyncio.Queue instance. Queue tasks define the pool of streams used for concurrent execution. """ if not torch.cuda.is_available(): yield return initial_stream = torch.cuda.current_stream() with torch.cuda.stream(initial_stream): stream = await streamqueue.get() assert isinstance(stream, torch.cuda.Stream) try: with torch.cuda.stream(stream): logger.debug('%s %s is starting, stream: %s', trace_name, name, stream) yield current = torch.cuda.current_stream() assert current == stream logger.debug('%s %s has finished, stream: %s', trace_name, name, stream) finally: streamqueue.task_done() streamqueue.put_nowait(stream) ================================================ FILE: mmdet/utils/dist_utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import functools import pickle import warnings from collections import OrderedDict import numpy as np import torch import torch.distributed as dist from mmengine.dist import get_dist_info from torch._utils import (_flatten_dense_tensors, _take_tensors, _unflatten_dense_tensors) def _allreduce_coalesced(tensors, world_size, bucket_size_mb=-1): if bucket_size_mb > 0: bucket_size_bytes = bucket_size_mb * 1024 * 1024 buckets = _take_tensors(tensors, bucket_size_bytes) else: buckets = OrderedDict() for tensor in tensors: tp = tensor.type() if tp not in buckets: buckets[tp] = [] buckets[tp].append(tensor) buckets = buckets.values() for bucket in buckets: flat_tensors = _flatten_dense_tensors(bucket) dist.all_reduce(flat_tensors) flat_tensors.div_(world_size) for tensor, synced in zip( bucket, _unflatten_dense_tensors(flat_tensors, bucket)): tensor.copy_(synced) def allreduce_grads(params, coalesce=True, bucket_size_mb=-1): """Allreduce gradients. Args: params (list[torch.Parameters]): List of parameters of a model coalesce (bool, optional): Whether allreduce parameters as a whole. Defaults to True. bucket_size_mb (int, optional): Size of bucket, the unit is MB. Defaults to -1. """ grads = [ param.grad.data for param in params if param.requires_grad and param.grad is not None ] world_size = dist.get_world_size() if coalesce: _allreduce_coalesced(grads, world_size, bucket_size_mb) else: for tensor in grads: dist.all_reduce(tensor.div_(world_size)) def reduce_mean(tensor): """"Obtain the mean of tensor on different GPUs.""" if not (dist.is_available() and dist.is_initialized()): return tensor tensor = tensor.clone() dist.all_reduce(tensor.div_(dist.get_world_size()), op=dist.ReduceOp.SUM) return tensor def obj2tensor(pyobj, device='cuda'): """Serialize picklable python object to tensor.""" storage = torch.ByteStorage.from_buffer(pickle.dumps(pyobj)) return torch.ByteTensor(storage).to(device=device) def tensor2obj(tensor): """Deserialize tensor to picklable python object.""" return pickle.loads(tensor.cpu().numpy().tobytes()) @functools.lru_cache() def _get_global_gloo_group(): """Return a process group based on gloo backend, containing all the ranks The result is cached.""" if dist.get_backend() == 'nccl': return dist.new_group(backend='gloo') else: return dist.group.WORLD def all_reduce_dict(py_dict, op='sum', group=None, to_float=True): """Apply all reduce function for python dict object. The code is modified from https://github.com/Megvii- BaseDetection/YOLOX/blob/main/yolox/utils/allreduce_norm.py. NOTE: make sure that py_dict in different ranks has the same keys and the values should be in the same shape. Currently only supports nccl backend. Args: py_dict (dict): Dict to be applied all reduce op. op (str): Operator, could be 'sum' or 'mean'. Default: 'sum' group (:obj:`torch.distributed.group`, optional): Distributed group, Default: None. to_float (bool): Whether to convert all values of dict to float. Default: True. Returns: OrderedDict: reduced python dict object. """ warnings.warn( 'group` is deprecated. Currently only supports NCCL backend.') _, world_size = get_dist_info() if world_size == 1: return py_dict # all reduce logic across different devices. py_key = list(py_dict.keys()) if not isinstance(py_dict, OrderedDict): py_key_tensor = obj2tensor(py_key) dist.broadcast(py_key_tensor, src=0) py_key = tensor2obj(py_key_tensor) tensor_shapes = [py_dict[k].shape for k in py_key] tensor_numels = [py_dict[k].numel() for k in py_key] if to_float: warnings.warn('Note: the "to_float" is True, you need to ' 'ensure that the behavior is reasonable.') flatten_tensor = torch.cat( [py_dict[k].flatten().float() for k in py_key]) else: flatten_tensor = torch.cat([py_dict[k].flatten() for k in py_key]) dist.all_reduce(flatten_tensor, op=dist.ReduceOp.SUM) if op == 'mean': flatten_tensor /= world_size split_tensors = [ x.reshape(shape) for x, shape in zip( torch.split(flatten_tensor, tensor_numels), tensor_shapes) ] out_dict = {k: v for k, v in zip(py_key, split_tensors)} if isinstance(py_dict, OrderedDict): out_dict = OrderedDict(out_dict) return out_dict def sync_random_seed(seed=None, device='cuda'): """Make sure different ranks share the same seed. All workers must call this function, otherwise it will deadlock. This method is generally used in `DistributedSampler`, because the seed should be identical across all processes in the distributed group. In distributed sampling, different ranks should sample non-overlapped data in the dataset. Therefore, this function is used to make sure that each rank shuffles the data indices in the same order based on the same seed. Then different ranks could use different indices to select non-overlapped data from the same data list. Args: seed (int, Optional): The seed. Default to None. device (str): The device where the seed will be put on. Default to 'cuda'. Returns: int: Seed to be used. """ if seed is None: seed = np.random.randint(2**31) assert isinstance(seed, int) rank, world_size = get_dist_info() if world_size == 1: return seed if rank == 0: random_num = torch.tensor(seed, dtype=torch.int32, device=device) else: random_num = torch.tensor(0, dtype=torch.int32, device=device) dist.broadcast(random_num, src=0) return random_num.item() ================================================ FILE: mmdet/utils/logger.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import inspect from mmengine.logging import print_log def get_caller_name(): """Get name of caller method.""" # this_func_frame = inspect.stack()[0][0] # i.e., get_caller_name # callee_frame = inspect.stack()[1][0] # e.g., log_img_scale caller_frame = inspect.stack()[2][0] # e.g., caller of log_img_scale caller_method = caller_frame.f_code.co_name try: caller_class = caller_frame.f_locals['self'].__class__.__name__ return f'{caller_class}.{caller_method}' except KeyError: # caller is a function return caller_method def log_img_scale(img_scale, shape_order='hw', skip_square=False): """Log image size. Args: img_scale (tuple): Image size to be logged. shape_order (str, optional): The order of image shape. 'hw' for (height, width) and 'wh' for (width, height). Defaults to 'hw'. skip_square (bool, optional): Whether to skip logging for square img_scale. Defaults to False. Returns: bool: Whether to have done logging. """ if shape_order == 'hw': height, width = img_scale elif shape_order == 'wh': width, height = img_scale else: raise ValueError(f'Invalid shape_order {shape_order}.') if skip_square and (height == width): return False caller = get_caller_name() print_log( f'image shape: height={height}, width={width} in {caller}', logger='current') return True ================================================ FILE: mmdet/utils/memory.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings from collections import abc from contextlib import contextmanager from functools import wraps import torch from mmengine.logging import MMLogger def cast_tensor_type(inputs, src_type=None, dst_type=None): """Recursively convert Tensor in inputs from ``src_type`` to ``dst_type``. Args: inputs: Inputs that to be casted. src_type (torch.dtype | torch.device): Source type. src_type (torch.dtype | torch.device): Destination type. Returns: The same type with inputs, but all contained Tensors have been cast. """ assert dst_type is not None if isinstance(inputs, torch.Tensor): if isinstance(dst_type, torch.device): # convert Tensor to dst_device if hasattr(inputs, 'to') and \ hasattr(inputs, 'device') and \ (inputs.device == src_type or src_type is None): return inputs.to(dst_type) else: return inputs else: # convert Tensor to dst_dtype if hasattr(inputs, 'to') and \ hasattr(inputs, 'dtype') and \ (inputs.dtype == src_type or src_type is None): return inputs.to(dst_type) else: return inputs # we need to ensure that the type of inputs to be casted are the same # as the argument `src_type`. elif isinstance(inputs, abc.Mapping): return type(inputs)({ k: cast_tensor_type(v, src_type=src_type, dst_type=dst_type) for k, v in inputs.items() }) elif isinstance(inputs, abc.Iterable): return type(inputs)( cast_tensor_type(item, src_type=src_type, dst_type=dst_type) for item in inputs) # TODO: Currently not supported # elif isinstance(inputs, InstanceData): # for key, value in inputs.items(): # inputs[key] = cast_tensor_type( # value, src_type=src_type, dst_type=dst_type) # return inputs else: return inputs @contextmanager def _ignore_torch_cuda_oom(): """A context which ignores CUDA OOM exception from pytorch. Code is modified from # noqa: E501 """ try: yield except RuntimeError as e: # NOTE: the string may change? if 'CUDA out of memory. ' in str(e): pass else: raise class AvoidOOM: """Try to convert inputs to FP16 and CPU if got a PyTorch's CUDA Out of Memory error. It will do the following steps: 1. First retry after calling `torch.cuda.empty_cache()`. 2. If that still fails, it will then retry by converting inputs to FP16. 3. If that still fails trying to convert inputs to CPUs. In this case, it expects the function to dispatch to CPU implementation. Args: to_cpu (bool): Whether to convert outputs to CPU if get an OOM error. This will slow down the code significantly. Defaults to True. test (bool): Skip `_ignore_torch_cuda_oom` operate that can use lightweight data in unit test, only used in test unit. Defaults to False. Examples: >>> from mmdet.utils.memory import AvoidOOM >>> AvoidCUDAOOM = AvoidOOM() >>> output = AvoidOOM.retry_if_cuda_oom( >>> some_torch_function)(input1, input2) >>> # To use as a decorator >>> # from mmdet.utils import AvoidCUDAOOM >>> @AvoidCUDAOOM.retry_if_cuda_oom >>> def function(*args, **kwargs): >>> return None ``` Note: 1. The output may be on CPU even if inputs are on GPU. Processing on CPU will slow down the code significantly. 2. When converting inputs to CPU, it will only look at each argument and check if it has `.device` and `.to` for conversion. Nested structures of tensors are not supported. 3. Since the function might be called more than once, it has to be stateless. """ def __init__(self, to_cpu=True, test=False): self.to_cpu = to_cpu self.test = test def retry_if_cuda_oom(self, func): """Makes a function retry itself after encountering pytorch's CUDA OOM error. The implementation logic is referred to https://github.com/facebookresearch/detectron2/blob/main/detectron2/utils/memory.py Args: func: a stateless callable that takes tensor-like objects as arguments. Returns: func: a callable which retries `func` if OOM is encountered. """ # noqa: W605 @wraps(func) def wrapped(*args, **kwargs): # raw function if not self.test: with _ignore_torch_cuda_oom(): return func(*args, **kwargs) # Clear cache and retry torch.cuda.empty_cache() with _ignore_torch_cuda_oom(): return func(*args, **kwargs) # get the type and device of first tensor dtype, device = None, None values = args + tuple(kwargs.values()) for value in values: if isinstance(value, torch.Tensor): dtype = value.dtype device = value.device break if dtype is None or device is None: raise ValueError('There is no tensor in the inputs, ' 'cannot get dtype and device.') # Convert to FP16 fp16_args = cast_tensor_type(args, dst_type=torch.half) fp16_kwargs = cast_tensor_type(kwargs, dst_type=torch.half) logger = MMLogger.get_current_instance() logger.warning(f'Attempting to copy inputs of {str(func)} ' 'to FP16 due to CUDA OOM') # get input tensor type, the output type will same as # the first parameter type. with _ignore_torch_cuda_oom(): output = func(*fp16_args, **fp16_kwargs) output = cast_tensor_type( output, src_type=torch.half, dst_type=dtype) if not self.test: return output logger.warning('Using FP16 still meet CUDA OOM') # Try on CPU. This will slow down the code significantly, # therefore print a notice. if self.to_cpu: logger.warning(f'Attempting to copy inputs of {str(func)} ' 'to CPU due to CUDA OOM') cpu_device = torch.empty(0).device cpu_args = cast_tensor_type(args, dst_type=cpu_device) cpu_kwargs = cast_tensor_type(kwargs, dst_type=cpu_device) # convert outputs to GPU with _ignore_torch_cuda_oom(): logger.warning(f'Convert outputs to GPU (device={device})') output = func(*cpu_args, **cpu_kwargs) output = cast_tensor_type( output, src_type=cpu_device, dst_type=device) return output warnings.warn('Cannot convert output to GPU due to CUDA OOM, ' 'the output is now on CPU, which might cause ' 'errors if the output need to interact with GPU ' 'data in subsequent operations') logger.warning('Cannot convert output to GPU due to ' 'CUDA OOM, the output is on CPU now.') return func(*cpu_args, **cpu_kwargs) else: # may still get CUDA OOM error return func(*args, **kwargs) return wrapped # To use AvoidOOM as a decorator AvoidCUDAOOM = AvoidOOM() ================================================ FILE: mmdet/utils/misc.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import glob import os import os.path as osp import warnings from typing import Union from mmengine.config import Config, ConfigDict from mmengine.logging import print_log def find_latest_checkpoint(path, suffix='pth'): """Find the latest checkpoint from the working directory. Args: path(str): The path to find checkpoints. suffix(str): File extension. Defaults to pth. Returns: latest_path(str | None): File path of the latest checkpoint. References: .. [1] https://github.com/microsoft/SoftTeacher /blob/main/ssod/utils/patch.py """ if not osp.exists(path): warnings.warn('The path of checkpoints does not exist.') return None if osp.exists(osp.join(path, f'latest.{suffix}')): return osp.join(path, f'latest.{suffix}') checkpoints = glob.glob(osp.join(path, f'*.{suffix}')) if len(checkpoints) == 0: warnings.warn('There are no checkpoints in the path.') return None latest = -1 latest_path = None for checkpoint in checkpoints: count = int(osp.basename(checkpoint).split('_')[-1].split('.')[0]) if count > latest: latest = count latest_path = checkpoint return latest_path def update_data_root(cfg, logger=None): """Update data root according to env MMDET_DATASETS. If set env MMDET_DATASETS, update cfg.data_root according to MMDET_DATASETS. Otherwise, using cfg.data_root as default. Args: cfg (:obj:`Config`): The model config need to modify logger (logging.Logger | str | None): the way to print msg """ assert isinstance(cfg, Config), \ f'cfg got wrong type: {type(cfg)}, expected mmengine.Config' if 'MMDET_DATASETS' in os.environ: dst_root = os.environ['MMDET_DATASETS'] print_log(f'MMDET_DATASETS has been set to be {dst_root}.' f'Using {dst_root} as data root.') else: return assert isinstance(cfg, Config), \ f'cfg got wrong type: {type(cfg)}, expected mmengine.Config' def update(cfg, src_str, dst_str): for k, v in cfg.items(): if isinstance(v, ConfigDict): update(cfg[k], src_str, dst_str) if isinstance(v, str) and src_str in v: cfg[k] = v.replace(src_str, dst_str) update(cfg.data, cfg.data_root, dst_root) cfg.data_root = dst_root def get_test_pipeline_cfg(cfg: Union[str, ConfigDict]) -> ConfigDict: """Get the test dataset pipeline from entire config. Args: cfg (str or :obj:`ConfigDict`): the entire config. Can be a config file or a ``ConfigDict``. Returns: :obj:`ConfigDict`: the config of test dataset. """ if isinstance(cfg, str): cfg = Config.fromfile(cfg) def _get_test_pipeline_cfg(dataset_cfg): if 'pipeline' in dataset_cfg: return dataset_cfg.pipeline # handle dataset wrapper elif 'dataset' in dataset_cfg: return _get_test_pipeline_cfg(dataset_cfg.dataset) # handle dataset wrappers like ConcatDataset elif 'datasets' in dataset_cfg: return _get_test_pipeline_cfg(dataset_cfg.datasets[0]) raise RuntimeError('Cannot find `pipeline` in `test_dataloader`') return _get_test_pipeline_cfg(cfg.test_dataloader.dataset) ================================================ FILE: mmdet/utils/profiling.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import contextlib import sys import time import torch if sys.version_info >= (3, 7): @contextlib.contextmanager def profile_time(trace_name, name, enabled=True, stream=None, end_stream=None): """Print time spent by CPU and GPU. Useful as a temporary context manager to find sweet spots of code suitable for async implementation. """ if (not enabled) or not torch.cuda.is_available(): yield return stream = stream if stream else torch.cuda.current_stream() end_stream = end_stream if end_stream else stream start = torch.cuda.Event(enable_timing=True) end = torch.cuda.Event(enable_timing=True) stream.record_event(start) try: cpu_start = time.monotonic() yield finally: cpu_end = time.monotonic() end_stream.record_event(end) end.synchronize() cpu_time = (cpu_end - cpu_start) * 1000 gpu_time = start.elapsed_time(end) msg = f'{trace_name} {name} cpu_time {cpu_time:.2f} ms ' msg += f'gpu_time {gpu_time:.2f} ms stream {stream}' print(msg, end_stream) ================================================ FILE: mmdet/utils/replace_cfg_vals.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import re from mmengine.config import Config def replace_cfg_vals(ori_cfg): """Replace the string "${key}" with the corresponding value. Replace the "${key}" with the value of ori_cfg.key in the config. And support replacing the chained ${key}. Such as, replace "${key0.key1}" with the value of cfg.key0.key1. Code is modified from `vars.py < https://github.com/microsoft/SoftTeacher/blob/main/ssod/utils/vars.py>`_ # noqa: E501 Args: ori_cfg (mmengine.config.Config): The origin config with "${key}" generated from a file. Returns: updated_cfg [mmengine.config.Config]: The config with "${key}" replaced by the corresponding value. """ def get_value(cfg, key): for k in key.split('.'): cfg = cfg[k] return cfg def replace_value(cfg): if isinstance(cfg, dict): return {key: replace_value(value) for key, value in cfg.items()} elif isinstance(cfg, list): return [replace_value(item) for item in cfg] elif isinstance(cfg, tuple): return tuple([replace_value(item) for item in cfg]) elif isinstance(cfg, str): # the format of string cfg may be: # 1) "${key}", which will be replaced with cfg.key directly # 2) "xxx${key}xxx" or "xxx${key1}xxx${key2}xxx", # which will be replaced with the string of the cfg.key keys = pattern_key.findall(cfg) values = [get_value(ori_cfg, key[2:-1]) for key in keys] if len(keys) == 1 and keys[0] == cfg: # the format of string cfg is "${key}" cfg = values[0] else: for key, value in zip(keys, values): # the format of string cfg is # "xxx${key}xxx" or "xxx${key1}xxx${key2}xxx" assert not isinstance(value, (dict, list, tuple)), \ f'for the format of string cfg is ' \ f"'xxxxx${key}xxxxx' or 'xxx${key}xxx${key}xxx', " \ f"the type of the value of '${key}' " \ f'can not be dict, list, or tuple' \ f'but you input {type(value)} in {cfg}' cfg = cfg.replace(key, str(value)) return cfg else: return cfg # the pattern of string "${key}" pattern_key = re.compile(r'\$\{[a-zA-Z\d_.]*\}') # the type of ori_cfg._cfg_dict is mmengine.config.ConfigDict updated_cfg = Config( replace_value(ori_cfg._cfg_dict), filename=ori_cfg.filename) # replace the model with model_wrapper if updated_cfg.get('model_wrapper', None) is not None: updated_cfg.model = updated_cfg.model_wrapper updated_cfg.pop('model_wrapper') return updated_cfg ================================================ FILE: mmdet/utils/setup_env.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import datetime import os import platform import warnings import cv2 import torch.multiprocessing as mp from mmengine import DefaultScope def setup_multi_processes(cfg): """Setup multi-processing environment variables.""" # set multi-process start method as `fork` to speed up the training if platform.system() != 'Windows': mp_start_method = cfg.get('mp_start_method', 'fork') current_method = mp.get_start_method(allow_none=True) if current_method is not None and current_method != mp_start_method: warnings.warn( f'Multi-processing start method `{mp_start_method}` is ' f'different from the previous setting `{current_method}`.' f'It will be force set to `{mp_start_method}`. You can change ' f'this behavior by changing `mp_start_method` in your config.') mp.set_start_method(mp_start_method, force=True) # disable opencv multithreading to avoid system being overloaded opencv_num_threads = cfg.get('opencv_num_threads', 0) cv2.setNumThreads(opencv_num_threads) # setup OMP threads # This code is referred from https://github.com/pytorch/pytorch/blob/master/torch/distributed/run.py # noqa workers_per_gpu = cfg.data.get('workers_per_gpu', 1) if 'train_dataloader' in cfg.data: workers_per_gpu = \ max(cfg.data.train_dataloader.get('workers_per_gpu', 1), workers_per_gpu) if 'OMP_NUM_THREADS' not in os.environ and workers_per_gpu > 1: omp_num_threads = 1 warnings.warn( f'Setting OMP_NUM_THREADS environment variable for each process ' f'to be {omp_num_threads} in default, to avoid your system being ' f'overloaded, please further tune the variable for optimal ' f'performance in your application as needed.') os.environ['OMP_NUM_THREADS'] = str(omp_num_threads) # setup MKL threads if 'MKL_NUM_THREADS' not in os.environ and workers_per_gpu > 1: mkl_num_threads = 1 warnings.warn( f'Setting MKL_NUM_THREADS environment variable for each process ' f'to be {mkl_num_threads} in default, to avoid your system being ' f'overloaded, please further tune the variable for optimal ' f'performance in your application as needed.') os.environ['MKL_NUM_THREADS'] = str(mkl_num_threads) def register_all_modules(init_default_scope: bool = True) -> None: """Register all modules in mmdet into the registries. Args: init_default_scope (bool): Whether initialize the mmdet default scope. When `init_default_scope=True`, the global default scope will be set to `mmdet`, and all registries will build modules from mmdet's registry node. To understand more about the registry, please refer to https://github.com/open-mmlab/mmengine/blob/main/docs/en/tutorials/registry.md Defaults to True. """ # noqa import mmdet.datasets # noqa: F401,F403 import mmdet.engine # noqa: F401,F403 import mmdet.evaluation # noqa: F401,F403 import mmdet.models # noqa: F401,F403 import mmdet.visualization # noqa: F401,F403 if init_default_scope: never_created = DefaultScope.get_current_instance() is None \ or not DefaultScope.check_instance_created('mmdet') if never_created: DefaultScope.get_instance('mmdet', scope_name='mmdet') return current_scope = DefaultScope.get_current_instance() if current_scope.scope_name != 'mmdet': warnings.warn('The current default scope ' f'"{current_scope.scope_name}" is not "mmdet", ' '`register_all_modules` will force the current' 'default scope to be "mmdet". If this is not ' 'expected, please set `init_default_scope=False`.') # avoid name conflict new_instance_name = f'mmdet-{datetime.datetime.now()}' DefaultScope.get_instance(new_instance_name, scope_name='mmdet') ================================================ FILE: mmdet/utils/split_batch.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch def split_batch(img, img_metas, kwargs): """Split data_batch by tags. Code is modified from # noqa: E501 Args: img (Tensor): of shape (N, C, H, W) encoding input images. Typically these should be mean centered and std scaled. img_metas (list[dict]): List of image info dict where each dict has: 'img_shape', 'scale_factor', 'flip', and may also contain 'filename', 'ori_shape', 'pad_shape', and 'img_norm_cfg'. For details on the values of these keys, see :class:`mmdet.datasets.pipelines.Collect`. kwargs (dict): Specific to concrete implementation. Returns: data_groups (dict): a dict that data_batch splited by tags, such as 'sup', 'unsup_teacher', and 'unsup_student'. """ # only stack img in the batch def fuse_list(obj_list, obj): return torch.stack(obj_list) if isinstance(obj, torch.Tensor) else obj_list # select data with tag from data_batch def select_group(data_batch, current_tag): group_flag = [tag == current_tag for tag in data_batch['tag']] return { k: fuse_list([vv for vv, gf in zip(v, group_flag) if gf], v) for k, v in data_batch.items() } kwargs.update({'img': img, 'img_metas': img_metas}) kwargs.update({'tag': [meta['tag'] for meta in img_metas]}) tags = list(set(kwargs['tag'])) data_groups = {tag: select_group(kwargs, tag) for tag in tags} for tag, group in data_groups.items(): group.pop('tag') return data_groups ================================================ FILE: mmdet/utils/typing_utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """Collecting some commonly used type hint in mmdetection.""" from typing import List, Optional, Sequence, Tuple, Union from mmengine.config import ConfigDict from mmengine.structures import InstanceData, PixelData # TODO: Need to avoid circular import with assigner and sampler # Type hint of config data ConfigType = Union[ConfigDict, dict] OptConfigType = Optional[ConfigType] # Type hint of one or more config data MultiConfig = Union[ConfigType, List[ConfigType]] OptMultiConfig = Optional[MultiConfig] InstanceList = List[InstanceData] OptInstanceList = Optional[InstanceList] PixelList = List[PixelData] OptPixelList = Optional[PixelList] RangeType = Sequence[Tuple[int, int]] ================================================ FILE: mmdet/utils/util_mixins.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """This module defines the :class:`NiceRepr` mixin class, which defines a ``__repr__`` and ``__str__`` method that only depend on a custom ``__nice__`` method, which you must define. This means you only have to overload one function instead of two. Furthermore, if the object defines a ``__len__`` method, then the ``__nice__`` method defaults to something sensible, otherwise it is treated as abstract and raises ``NotImplementedError``. To use simply have your object inherit from :class:`NiceRepr` (multi-inheritance should be ok). This code was copied from the ubelt library: https://github.com/Erotemic/ubelt Example: >>> # Objects that define __nice__ have a default __str__ and __repr__ >>> class Student(NiceRepr): ... def __init__(self, name): ... self.name = name ... def __nice__(self): ... return self.name >>> s1 = Student('Alice') >>> s2 = Student('Bob') >>> print(f's1 = {s1}') >>> print(f's2 = {s2}') s1 = s2 = Example: >>> # Objects that define __len__ have a default __nice__ >>> class Group(NiceRepr): ... def __init__(self, data): ... self.data = data ... def __len__(self): ... return len(self.data) >>> g = Group([1, 2, 3]) >>> print(f'g = {g}') g = """ import warnings class NiceRepr: """Inherit from this class and define ``__nice__`` to "nicely" print your objects. Defines ``__str__`` and ``__repr__`` in terms of ``__nice__`` function Classes that inherit from :class:`NiceRepr` should redefine ``__nice__``. If the inheriting class has a ``__len__``, method then the default ``__nice__`` method will return its length. Example: >>> class Foo(NiceRepr): ... def __nice__(self): ... return 'info' >>> foo = Foo() >>> assert str(foo) == '' >>> assert repr(foo).startswith('>> class Bar(NiceRepr): ... pass >>> bar = Bar() >>> import pytest >>> with pytest.warns(None) as record: >>> assert 'object at' in str(bar) >>> assert 'object at' in repr(bar) Example: >>> class Baz(NiceRepr): ... def __len__(self): ... return 5 >>> baz = Baz() >>> assert str(baz) == '' """ def __nice__(self): """str: a "nice" summary string describing this module""" if hasattr(self, '__len__'): # It is a common pattern for objects to use __len__ in __nice__ # As a convenience we define a default __nice__ for these objects return str(len(self)) else: # In all other cases force the subclass to overload __nice__ raise NotImplementedError( f'Define the __nice__ method for {self.__class__!r}') def __repr__(self): """str: the string of the module""" try: nice = self.__nice__() classname = self.__class__.__name__ return f'<{classname}({nice}) at {hex(id(self))}>' except NotImplementedError as ex: warnings.warn(str(ex), category=RuntimeWarning) return object.__repr__(self) def __str__(self): """str: the string of the module""" try: classname = self.__class__.__name__ nice = self.__nice__() return f'<{classname}({nice})>' except NotImplementedError as ex: warnings.warn(str(ex), category=RuntimeWarning) return object.__repr__(self) ================================================ FILE: mmdet/utils/util_random.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """Helpers for random number generators.""" import numpy as np def ensure_rng(rng=None): """Coerces input into a random number generator. If the input is None, then a global random state is returned. If the input is a numeric value, then that is used as a seed to construct a random state. Otherwise the input is returned as-is. Adapted from [1]_. Args: rng (int | numpy.random.RandomState | None): if None, then defaults to the global rng. Otherwise this can be an integer or a RandomState class Returns: (numpy.random.RandomState) : rng - a numpy random number generator References: .. [1] https://gitlab.kitware.com/computer-vision/kwarray/blob/master/kwarray/util_random.py#L270 # noqa: E501 """ if rng is None: rng = np.random.mtrand._rand elif isinstance(rng, int): rng = np.random.RandomState(rng) else: rng = rng return rng ================================================ FILE: mmdet/version.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. __version__ = '3.0.0rc6' short_version = __version__ def parse_version_info(version_str): """Parse a version string into a tuple. Args: version_str (str): The version string. Returns: tuple[int | str]: The version info, e.g., "1.3.0" is parsed into (1, 3, 0), and "2.0.0rc1" is parsed into (2, 0, 0, 'rc1'). """ version_info = [] for x in version_str.split('.'): if x.isdigit(): version_info.append(int(x)) elif x.find('rc') != -1: patch_version = x.split('rc') version_info.append(int(patch_version[0])) version_info.append(f'rc{patch_version[1]}') return tuple(version_info) version_info = parse_version_info(__version__) ================================================ FILE: mmdet/visualization/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .local_visualizer import DetLocalVisualizer from .palette import get_palette, jitter_color, palette_val __all__ = ['palette_val', 'get_palette', 'DetLocalVisualizer', 'jitter_color'] ================================================ FILE: mmdet/visualization/local_visualizer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Dict, List, Optional, Tuple, Union import cv2 import mmcv import numpy as np import torch from mmengine.dist import master_only from mmengine.structures import InstanceData, PixelData from mmengine.visualization import Visualizer from ..evaluation import INSTANCE_OFFSET from ..registry import VISUALIZERS from ..structures import DetDataSample from ..structures.mask import BitmapMasks, PolygonMasks, bitmap_to_polygon from .palette import _get_adaptive_scales, get_palette, jitter_color @VISUALIZERS.register_module() class DetLocalVisualizer(Visualizer): """MMDetection Local Visualizer. Args: name (str): Name of the instance. Defaults to 'visualizer'. image (np.ndarray, optional): the origin image to draw. The format should be RGB. Defaults to None. vis_backends (list, optional): Visual backend config list. Defaults to None. save_dir (str, optional): Save file dir for all storage backends. If it is None, the backend storage will not save any data. bbox_color (str, tuple(int), optional): Color of bbox lines. The tuple of color should be in BGR order. Defaults to None. text_color (str, tuple(int), optional): Color of texts. The tuple of color should be in BGR order. Defaults to (200, 200, 200). mask_color (str, tuple(int), optional): Color of masks. The tuple of color should be in BGR order. Defaults to None. line_width (int, float): The linewidth of lines. Defaults to 3. alpha (int, float): The transparency of bboxes or mask. Defaults to 0.8. Examples: >>> import numpy as np >>> import torch >>> from mmengine.structures import InstanceData >>> from mmdet.structures import DetDataSample >>> from mmdet.visualization import DetLocalVisualizer >>> det_local_visualizer = DetLocalVisualizer() >>> image = np.random.randint(0, 256, ... size=(10, 12, 3)).astype('uint8') >>> gt_instances = InstanceData() >>> gt_instances.bboxes = torch.Tensor([[1, 2, 2, 5]]) >>> gt_instances.labels = torch.randint(0, 2, (1,)) >>> gt_det_data_sample = DetDataSample() >>> gt_det_data_sample.gt_instances = gt_instances >>> det_local_visualizer.add_datasample('image', image, ... gt_det_data_sample) >>> det_local_visualizer.add_datasample( ... 'image', image, gt_det_data_sample, ... out_file='out_file.jpg') >>> det_local_visualizer.add_datasample( ... 'image', image, gt_det_data_sample, ... show=True) >>> pred_instances = InstanceData() >>> pred_instances.bboxes = torch.Tensor([[2, 4, 4, 8]]) >>> pred_instances.labels = torch.randint(0, 2, (1,)) >>> pred_det_data_sample = DetDataSample() >>> pred_det_data_sample.pred_instances = pred_instances >>> det_local_visualizer.add_datasample('image', image, ... gt_det_data_sample, ... pred_det_data_sample) """ def __init__(self, name: str = 'visualizer', image: Optional[np.ndarray] = None, vis_backends: Optional[Dict] = None, save_dir: Optional[str] = None, bbox_color: Optional[Union[str, Tuple[int]]] = None, text_color: Optional[Union[str, Tuple[int]]] = (200, 200, 200), mask_color: Optional[Union[str, Tuple[int]]] = None, line_width: Union[int, float] = 3, alpha: float = 0.8) -> None: super().__init__( name=name, image=image, vis_backends=vis_backends, save_dir=save_dir) self.bbox_color = bbox_color self.text_color = text_color self.mask_color = mask_color self.line_width = line_width self.alpha = alpha # Set default value. When calling # `DetLocalVisualizer().dataset_meta=xxx`, # it will override the default value. self.dataset_meta = {} def _draw_instances(self, image: np.ndarray, instances: ['InstanceData'], classes: Optional[List[str]], palette: Optional[List[tuple]]) -> np.ndarray: """Draw instances of GT or prediction. Args: image (np.ndarray): The image to draw. instances (:obj:`InstanceData`): Data structure for instance-level annotations or predictions. classes (List[str], optional): Category information. palette (List[tuple], optional): Palette information corresponding to the category. Returns: np.ndarray: the drawn image which channel is RGB. """ self.set_image(image) if 'bboxes' in instances: bboxes = instances.bboxes labels = instances.labels max_label = int(max(labels) if len(labels) > 0 else 0) text_palette = get_palette(self.text_color, max_label + 1) text_colors = [text_palette[label] for label in labels] bbox_color = palette if self.bbox_color is None \ else self.bbox_color bbox_palette = get_palette(bbox_color, max_label + 1) colors = [bbox_palette[label] for label in labels] self.draw_bboxes( bboxes, edge_colors=colors, alpha=self.alpha, line_widths=self.line_width) positions = bboxes[:, :2] + self.line_width areas = (bboxes[:, 3] - bboxes[:, 1]) * ( bboxes[:, 2] - bboxes[:, 0]) scales = _get_adaptive_scales(areas) for i, (pos, label) in enumerate(zip(positions, labels)): label_text = classes[ label] if classes is not None else f'class {label}' if 'scores' in instances: score = round(float(instances.scores[i]) * 100, 1) label_text += f': {score}' self.draw_texts( label_text, pos, colors=text_colors[i], font_sizes=int(13 * scales[i]), bboxes=[{ 'facecolor': 'black', 'alpha': 0.8, 'pad': 0.7, 'edgecolor': 'none' }]) if 'masks' in instances: labels = instances.labels masks = instances.masks if isinstance(masks, torch.Tensor): masks = masks.numpy() elif isinstance(masks, (PolygonMasks, BitmapMasks)): masks = masks.to_ndarray() masks = masks.astype(bool) max_label = int(max(labels) if len(labels) > 0 else 0) mask_color = palette if self.mask_color is None \ else self.mask_color mask_palette = get_palette(mask_color, max_label + 1) colors = [jitter_color(mask_palette[label]) for label in labels] text_palette = get_palette(self.text_color, max_label + 1) text_colors = [text_palette[label] for label in labels] polygons = [] for i, mask in enumerate(masks): contours, _ = bitmap_to_polygon(mask) polygons.extend(contours) self.draw_polygons(polygons, edge_colors='w', alpha=self.alpha) self.draw_binary_masks(masks, colors=colors, alphas=self.alpha) if len(labels) > 0 and \ ('bboxes' not in instances or instances.bboxes.sum() == 0): # instances.bboxes.sum()==0 represent dummy bboxes. # A typical example of SOLO does not exist bbox branch. areas = [] positions = [] for mask in masks: _, _, stats, centroids = cv2.connectedComponentsWithStats( mask.astype(np.uint8), connectivity=8) if stats.shape[0] > 1: largest_id = np.argmax(stats[1:, -1]) + 1 positions.append(centroids[largest_id]) areas.append(stats[largest_id, -1]) areas = np.stack(areas, axis=0) scales = _get_adaptive_scales(areas) for i, (pos, label) in enumerate(zip(positions, labels)): label_text = classes[ label] if classes is not None else f'class {label}' if 'scores' in instances: score = round(float(instances.scores[i]) * 100, 1) label_text += f': {score}' self.draw_texts( label_text, pos, colors=text_colors[i], font_sizes=int(13 * scales[i]), horizontal_alignments='center', bboxes=[{ 'facecolor': 'black', 'alpha': 0.8, 'pad': 0.7, 'edgecolor': 'none' }]) return self.get_image() def _draw_panoptic_seg(self, image: np.ndarray, panoptic_seg: ['PixelData'], classes: Optional[List[str]]) -> np.ndarray: """Draw panoptic seg of GT or prediction. Args: image (np.ndarray): The image to draw. panoptic_seg (:obj:`PixelData`): Data structure for pixel-level annotations or predictions. classes (List[str], optional): Category information. Returns: np.ndarray: the drawn image which channel is RGB. """ # TODO: Is there a way to bypass？ num_classes = len(classes) panoptic_seg = panoptic_seg.sem_seg[0] ids = np.unique(panoptic_seg)[::-1] legal_indices = ids != num_classes # for VOID label ids = ids[legal_indices] labels = np.array([id % INSTANCE_OFFSET for id in ids], dtype=np.int64) segms = (panoptic_seg[None] == ids[:, None, None]) max_label = int(max(labels) if len(labels) > 0 else 0) mask_palette = get_palette(self.mask_color, max_label + 1) colors = [mask_palette[label] for label in labels] self.set_image(image) # draw segm polygons = [] for i, mask in enumerate(segms): contours, _ = bitmap_to_polygon(mask) polygons.extend(contours) self.draw_polygons(polygons, edge_colors='w', alpha=self.alpha) self.draw_binary_masks(segms, colors=colors, alphas=self.alpha) # draw label areas = [] positions = [] for mask in segms: _, _, stats, centroids = cv2.connectedComponentsWithStats( mask.astype(np.uint8), connectivity=8) max_id = np.argmax(stats[1:, -1]) + 1 positions.append(centroids[max_id]) areas.append(stats[max_id, -1]) areas = np.stack(areas, axis=0) scales = _get_adaptive_scales(areas) text_palette = get_palette(self.text_color, max_label + 1) text_colors = [text_palette[label] for label in labels] for i, (pos, label) in enumerate(zip(positions, labels)): label_text = classes[label] self.draw_texts( label_text, pos, colors=text_colors[i], font_sizes=int(13 * scales[i]), bboxes=[{ 'facecolor': 'black', 'alpha': 0.8, 'pad': 0.7, 'edgecolor': 'none' }], horizontal_alignments='center') return self.get_image() @master_only def add_datasample( self, name: str, image: np.ndarray, data_sample: Optional['DetDataSample'] = None, draw_gt: bool = True, draw_pred: bool = True, show: bool = False, wait_time: float = 0, # TODO: Supported in mmengine's Viusalizer. out_file: Optional[str] = None, pred_score_thr: float = 0.3, step: int = 0) -> None: """Draw datasample and save to all backends. - If GT and prediction are plotted at the same time, they are displayed in a stitched image where the left image is the ground truth and the right image is the prediction. - If ``show`` is True, all storage backends are ignored, and the images will be displayed in a local window. - If ``out_file`` is specified, the drawn image will be saved to ``out_file``. t is usually used when the display is not available. Args: name (str): The image identifier. image (np.ndarray): The image to draw. data_sample (:obj:`DetDataSample`, optional): A data sample that contain annotations and predictions. Defaults to None. draw_gt (bool): Whether to draw GT DetDataSample. Default to True. draw_pred (bool): Whether to draw Prediction DetDataSample. Defaults to True. show (bool): Whether to display the drawn image. Default to False. wait_time (float): The interval of show (s). Defaults to 0. out_file (str): Path to output file. Defaults to None. pred_score_thr (float): The threshold to visualize the bboxes and masks. Defaults to 0.3. step (int): Global step value to record. Defaults to 0. """ image = image.clip(0, 255).astype(np.uint8) classes = self.dataset_meta.get('classes', None) palette = self.dataset_meta.get('palette', None) gt_img_data = None pred_img_data = None if data_sample is not None: data_sample = data_sample.cpu() if draw_gt and data_sample is not None: gt_img_data = image if 'gt_instances' in data_sample: gt_img_data = self._draw_instances(image, data_sample.gt_instances, classes, palette) if 'gt_panoptic_seg' in data_sample: assert classes is not None, 'class information is ' \ 'not provided when ' \ 'visualizing panoptic ' \ 'segmentation results.' gt_img_data = self._draw_panoptic_seg( gt_img_data, data_sample.gt_panoptic_seg, classes) if draw_pred and data_sample is not None: pred_img_data = image if 'pred_instances' in data_sample: pred_instances = data_sample.pred_instances pred_instances = pred_instances[ pred_instances.scores > pred_score_thr] pred_img_data = self._draw_instances(image, pred_instances, classes, palette) if 'pred_panoptic_seg' in data_sample: assert classes is not None, 'class information is ' \ 'not provided when ' \ 'visualizing panoptic ' \ 'segmentation results.' pred_img_data = self._draw_panoptic_seg( pred_img_data, data_sample.pred_panoptic_seg.numpy(), classes) if gt_img_data is not None and pred_img_data is not None: drawn_img = np.concatenate((gt_img_data, pred_img_data), axis=1) elif gt_img_data is not None: drawn_img = gt_img_data elif pred_img_data is not None: drawn_img = pred_img_data else: # Display the original image directly if nothing is drawn. drawn_img = image # It is convenient for users to obtain the drawn image. # For example, the user wants to obtain the drawn image and # save it as a video during video inference. self.set_image(drawn_img) if show: self.show(drawn_img, win_name=name, wait_time=wait_time) if out_file is not None: mmcv.imwrite(drawn_img[..., ::-1], out_file) else: self.add_image(name, drawn_img, step) ================================================ FILE: mmdet/visualization/palette.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Tuple, Union import mmcv import numpy as np from mmengine.utils import is_str def palette_val(palette: List[tuple]) -> List[tuple]: """Convert palette to matplotlib palette. Args: palette (List[tuple]): A list of color tuples. Returns: List[tuple[float]]: A list of RGB matplotlib color tuples. """ new_palette = [] for color in palette: color = [c / 255 for c in color] new_palette.append(tuple(color)) return new_palette def get_palette(palette: Union[List[tuple], str, tuple], num_classes: int) -> List[Tuple[int]]: """Get palette from various inputs. Args: palette (list[tuple] | str | tuple): palette inputs. num_classes (int): the number of classes. Returns: list[tuple[int]]: A list of color tuples. """ assert isinstance(num_classes, int) if isinstance(palette, list): dataset_palette = palette elif isinstance(palette, tuple): dataset_palette = [palette] * num_classes elif palette == 'random' or palette is None: state = np.random.get_state() # random color np.random.seed(42) palette = np.random.randint(0, 256, size=(num_classes, 3)) np.random.set_state(state) dataset_palette = [tuple(c) for c in palette] elif palette == 'coco': from mmdet.datasets import CocoDataset, CocoPanopticDataset dataset_palette = CocoDataset.METAINFO['palette'] if len(dataset_palette) < num_classes: dataset_palette = CocoPanopticDataset.METAINFO['palette'] elif palette == 'citys': from mmdet.datasets import CityscapesDataset dataset_palette = CityscapesDataset.METAINFO['palette'] elif palette == 'voc': from mmdet.datasets import VOCDataset dataset_palette = VOCDataset.METAINFO['palette'] elif is_str(palette): dataset_palette = [mmcv.color_val(palette)[::-1]] * num_classes else: raise TypeError(f'Invalid type for palette: {type(palette)}') assert len(dataset_palette) >= num_classes, \ 'The length of palette should not be less than `num_classes`.' return dataset_palette def _get_adaptive_scales(areas: np.ndarray, min_area: int = 800, max_area: int = 30000) -> np.ndarray: """Get adaptive scales according to areas. The scale range is [0.5, 1.0]. When the area is less than ``min_area``, the scale is 0.5 while the area is larger than ``max_area``, the scale is 1.0. Args: areas (ndarray): The areas of bboxes or masks with the shape of (n, ). min_area (int): Lower bound areas for adaptive scales. Defaults to 800. max_area (int): Upper bound areas for adaptive scales. Defaults to 30000. Returns: ndarray: The adaotive scales with the shape of (n, ). """ scales = 0.5 + (areas - min_area) / (max_area - min_area) scales = np.clip(scales, 0.5, 1.0) return scales def jitter_color(color: tuple) -> tuple: """Randomly jitter the given color in order to better distinguish instances with the same class. Args: color (tuple): The RGB color tuple. Each value is between [0, 255]. Returns: tuple: The jittered color tuple. """ jitter = np.random.rand(3) jitter = (jitter / np.linalg.norm(jitter) - 0.5) * 0.5 * 255 color = np.clip(jitter + color, 0, 255).astype(np.uint8) return tuple(color) ================================================ FILE: model-index.yml ================================================ Import: - configs/atss/metafile.yml - configs/autoassign/metafile.yml - configs/carafe/metafile.yml - configs/cascade_rcnn/metafile.yml - configs/cascade_rpn/metafile.yml - configs/centernet/metafile.yml - configs/centripetalnet/metafile.yml - configs/cornernet/metafile.yml - configs/condinst/metafile.yml - configs/convnext/metafile.yml - configs/dcn/metafile.yml - configs/dcnv2/metafile.yml - configs/ddod/metafile.yml - configs/deformable_detr/metafile.yml - configs/detectors/metafile.yml - configs/detr/metafile.yml - configs/double_heads/metafile.yml - configs/dyhead/metafile.yml - configs/dynamic_rcnn/metafile.yml - configs/efficientnet/metafile.yml - configs/empirical_attention/metafile.yml - configs/faster_rcnn/metafile.yml - configs/fcos/metafile.yml - configs/foveabox/metafile.yml - configs/fpg/metafile.yml - configs/free_anchor/metafile.yml - configs/fsaf/metafile.yml - configs/gcnet/metafile.yml - configs/gfl/metafile.yml - configs/ghm/metafile.yml - configs/gn/metafile.yml - configs/gn+ws/metafile.yml - configs/grid_rcnn/metafile.yml - configs/groie/metafile.yml - configs/guided_anchoring/metafile.yml - configs/hrnet/metafile.yml - configs/htc/metafile.yml - configs/instaboost/metafile.yml - configs/lad/metafile.yml - configs/ld/metafile.yml - configs/libra_rcnn/metafile.yml - configs/mask2former/metafile.yml - configs/mask_rcnn/metafile.yml - configs/maskformer/metafile.yml - configs/ms_rcnn/metafile.yml - configs/nas_fcos/metafile.yml - configs/nas_fpn/metafile.yml - configs/openimages/metafile.yml - configs/paa/metafile.yml - configs/pafpn/metafile.yml - configs/panoptic_fpn/metafile.yml - configs/pvt/metafile.yml - configs/pisa/metafile.yml - configs/point_rend/metafile.yml - configs/queryinst/metafile.yml - configs/rtmdet/metafile.yml - configs/regnet/metafile.yml - configs/reppoints/metafile.yml - configs/res2net/metafile.yml - configs/resnest/metafile.yml - configs/resnet_strikes_back/metafile.yml - configs/retinanet/metafile.yml - configs/rtmdet/metafile.yml - configs/sabl/metafile.yml - configs/scnet/metafile.yml - configs/scratch/metafile.yml - configs/seesaw_loss/metafile.yml - configs/simple_copy_paste/metafile.yml - configs/sparse_rcnn/metafile.yml - configs/solo/metafile.yml - configs/solov2/metafile.yml - configs/ssd/metafile.yml - configs/swin/metafile.yml - configs/tridentnet/metafile.yml - configs/tood/metafile.yml - configs/vfnet/metafile.yml - configs/yolact/metafile.yml - configs/yolo/metafile.yml - configs/yolof/metafile.yml - configs/yolox/metafile.yml ================================================ FILE: projects/ConvNeXt-V2/README.md ================================================ # ConvNeXt-V2 > [ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders](http://arxiv.org/abs/2301.00808) ## Abstract Driven by improved architectures and better representation learning frameworks, the field of visual recognition has enjoyed rapid modernization and performance boost in the early 2020s. For example, modern ConvNets, represented by ConvNeXt \[52\], have demonstrated strong performance in various scenarios. While these models were originally designed for supervised learning with ImageNet labels, they can also potentially benefit from self-supervised learning techniques such as masked autoencoders (MAE) . However, we found that simply combining these two approaches leads to subpar performance. In this paper, we propose a fully convolutional masked autoencoder framework and a new Global Response Normalization (GRN) layer that can be added to the ConvNeXt architecture to enhance inter-channel feature competition. This co-design of self-supervised learning techniques and architectural improvement results in a new model family called ConvNeXt V2, which significantly improves the performance of pure ConvNets on various recognition benchmarks, including ImageNet classification, COCO detection, and ADE20K segmentation. We also provide pre-trained ConvNeXt V2 models of various sizes, ranging from an efficient 3.7Mparameter Atto model with 76.7% top-1 accuracy on Im-ageNet, to a 650M Huge model that achieves a state-of-theart 88.9% accuracy using only public training data.
## Results and models | Method | Backbone | Pretrain | Lr schd | Augmentation | Mem (GB) | box AP | mask AP | Config | Download | | :--------: | :-----------: | :------: | :-----: | :----------: | :------: | :----: | :-----: | :----------------------------------------------------------: | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | Mask R-CNN | ConvNeXt-V2-B | FCMAE | 3x | LSJ | 22.5 | 52.9 | 46.4 | [config](./mask-rcnn_convnext-v2-b_fpn_lsj-3x-fcmae_coco.py) | [model](https://download.openmmlab.com/mmdetection/v3.0/convnextv2/mask-rcnn_convnext-v2-b_fpn_lsj-3x-fcmae_coco/mask-rcnn_convnext-v2-b_fpn_lsj-3x-fcmae_coco_20230113_110947-757ee2dd.pth) \| [log](https://download.openmmlab.com/mmdetection/v3.0/convnextv2/mask-rcnn_convnext-v2-b_fpn_lsj-3x-fcmae_coco/mask-rcnn_convnext-v2-b_fpn_lsj-3x-fcmae_coco_20230113_110947.log.json) | **Note**: - This is a pre-release version of ConvNeXt-V2 object detection. The official finetuning setting of ConvNeXt-V2 has not been released yet. - ConvNeXt backbone needs to install [MMClassification dev-1.x branch](https://github.com/open-mmlab/mmclassification/tree/dev-1.x) first, which has abundant backbones for downstream tasks. ```shell git clone -b dev-1.x https://github.com/open-mmlab/mmclassification.git cd mmclassification pip install -U openmim && mim install -e . ``` ## Citation ```bibtex @article{Woo2023ConvNeXtV2, title={ConvNeXt V2: Co-designing and Scaling ConvNets with Masked Autoencoders}, author={Sanghyun Woo, Shoubhik Debnath, Ronghang Hu, Xinlei Chen, Zhuang Liu, In So Kweon and Saining Xie}, year={2023}, journal={arXiv preprint arXiv:2301.00808}, } ``` ================================================ FILE: projects/ConvNeXt-V2/configs/mask-rcnn_convnext-v2-b_fpn_lsj-3x-fcmae_coco.py ================================================ _base_ = [ 'mmdet::_base_/models/mask-rcnn_r50_fpn.py', 'mmdet::_base_/datasets/coco_instance.py', 'mmdet::_base_/schedules/schedule_1x.py', 'mmdet::_base_/default_runtime.py' ] # please install the mmclassification dev-1.x branch # import mmcls.models to trigger register_module in mmcls custom_imports = dict(imports=['mmcls.models'], allow_failed_imports=False) checkpoint_file = 'https://download.openmmlab.com/mmclassification/v0/convnext-v2/convnext-v2-base_3rdparty-fcmae_in1k_20230104-8a798eaf.pth' # noqa image_size = (1024, 1024) model = dict( backbone=dict( _delete_=True, type='mmcls.ConvNeXt', arch='base', out_indices=[0, 1, 2, 3], # TODO: verify stochastic depth rate {0.1, 0.2, 0.3, 0.4} drop_path_rate=0.4, layer_scale_init_value=0., # disable layer scale when using GRN gap_before_final_norm=False, use_grn=True, # V2 uses GRN init_cfg=dict( type='Pretrained', checkpoint=checkpoint_file, prefix='backbone.')), neck=dict(in_channels=[128, 256, 512, 1024]), test_cfg=dict( rpn=dict(nms=dict(type='nms')), # TODO: does RPN use soft_nms? rcnn=dict(nms=dict(type='soft_nms')))) train_pipeline = [ dict(type='LoadImageFromFile', file_client_args=_base_.file_client_args), dict(type='LoadAnnotations', with_bbox=True, with_mask=True), dict( type='RandomResize', scale=image_size, ratio_range=(0.1, 2.0), keep_ratio=True), dict( type='RandomCrop', crop_type='absolute_range', crop_size=image_size, recompute_bbox=True, allow_negative_crop=True), dict(type='FilterAnnotations', min_gt_bbox_wh=(1e-2, 1e-2)), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] train_dataloader = dict( batch_size=4, # total_batch_size 32 = 8 GPUS x 4 images num_workers=8, dataset=dict(pipeline=train_pipeline)) max_epochs = 36 train_cfg = dict(max_epochs=max_epochs) # learning rate param_scheduler = [ dict( type='LinearLR', start_factor=0.001, by_epoch=False, begin=0, end=1000), dict( type='MultiStepLR', begin=0, end=max_epochs, by_epoch=True, milestones=[27, 33], gamma=0.1) ] # Enable automatic-mixed-precision training with AmpOptimWrapper. optim_wrapper = dict( type='AmpOptimWrapper', constructor='LearningRateDecayOptimizerConstructor', paramwise_cfg={ 'decay_rate': 0.95, 'decay_type': 'layer_wise', # TODO: sweep layer-wise lr decay? 'num_layers': 12 }, optimizer=dict( _delete_=True, type='AdamW', lr=0.0001, betas=(0.9, 0.999), weight_decay=0.05, )) default_hooks = dict(checkpoint=dict(max_keep_ckpts=1)) ================================================ FILE: projects/Detic/README.md ================================================ # Detecting Twenty-thousand Classes using Image-level Supervision ## Description **Detic**: A **Det**ector with **i**mage **c**lasses that can use image-level labels to easily train detectors.

> [**Detecting Twenty-thousand Classes using Image-level Supervision**](http://arxiv.org/abs/2201.02605), > Xingyi Zhou, Rohit Girdhar, Armand Joulin, Philipp Krähenbühl, Ishan Misra, > *ECCV 2022 ([arXiv 2201.02605](http://arxiv.org/abs/2201.02605))* ## Usage ## Installation Detic requires to install CLIP. ```shell pip install git+https://github.com/openai/CLIP.git ``` ### Demo #### Inference with existing dataset vocabulary embeddings First, go to the Detic project folder. ```shell cd projects/Detic ``` Then, download the pre-computed CLIP embeddings from [dataset metainfo](https://github.com/facebookresearch/Detic/tree/main/datasets/metadata) to the `datasets/metadata` folder. The CLIP embeddings will be loaded to the zero-shot classifier during inference. For example, you can download LVIS's class name embeddings with the following command: ```shell wget -P datasets/metadata https://raw.githubusercontent.com/facebookresearch/Detic/main/datasets/metadata/lvis_v1_clip_a%2Bcname.npy ``` You can run demo like this: ```shell python demo.py \ ${IMAGE_PATH} \ ${CONFIG_PATH} \ ${MODEL_PATH} \ --show \ --score-thr 0.5 \ --dataset lvis ``` ![image](https://user-images.githubusercontent.com/12907710/213624759-f0a2ba0c-0f5c-4424-a350-5ba5349e5842.png) ### Inference with custom vocabularies - Detic can detects any class given class names by using CLIP. You can detect custom classes with `--class-name` command: ``` python demo.py \ ${IMAGE_PATH} \ ${CONFIG_PATH} \ ${MODEL_PATH} \ --show \ --score-thr 0.3 \ --class-name headphone webcam paper coffe ``` ![image](https://user-images.githubusercontent.com/12907710/213624637-e9e8a313-9821-4782-a18a-4408c876852b.png) Note that `headphone`, `paper` and `coffe` (typo intended) are not LVIS classes. Despite the misspelled class name, Detic can produce a reasonable detection for `coffe`. ## Results Here we only provide the Detic Swin-B model for the open vocabulary demo. Multi-dataset training and open-vocabulary testing will be supported in the future. To find more variants, please visit the [official model zoo](https://github.com/facebookresearch/Detic/blob/main/docs/MODEL_ZOO.md). | Backbone | Training data | Config | Download | | :------: | :------------------------: | :-------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | Swin-B | ImageNet-21K & LVIS & COCO | [config](./configs/detic_centernet2_swin-b_fpn_4x_lvis-coco-in21k.py) | [model](https://download.openmmlab.com/mmdetection/v3.0/detic/detic_centernet2_swin-b_fpn_4x_lvis-coco-in21k/detic_centernet2_swin-b_fpn_4x_lvis-coco-in21k_20230120-0d301978.pth) | ## Citation If you find Detic is useful in your research or applications, please consider giving a star 🌟 to the [official repository](https://github.com/facebookresearch/Detic) and citing Detic by the following BibTeX entry. ```BibTeX @inproceedings{zhou2022detecting, title={Detecting Twenty-thousand Classes using Image-level Supervision}, author={Zhou, Xingyi and Girdhar, Rohit and Joulin, Armand and Kr{\"a}henb{\"u}hl, Philipp and Misra, Ishan}, booktitle={ECCV}, year={2022} } ``` ## Checklist - [x] Milestone 1: PR-ready, and acceptable to be one of the `projects/`. - [x] Finish the code - [x] Basic docstrings & proper citation - [x] Test-time correctness - [x] A full README - [ ] Milestone 2: Indicates a successful model implementation. - [ ] Training-time correctness - [ ] Milestone 3: Good to be a part of our core package! - [ ] Type hints and docstrings - [ ] Unit tests - [ ] Code polishing - [ ] Metafile.yml - [ ] Move your modules into the core package following the codebase's file hierarchy structure. - [ ] Refactor your modules into the core package following the codebase's file hierarchy structure. ================================================ FILE: projects/Detic/configs/detic_centernet2_swin-b_fpn_4x_lvis-coco-in21k.py ================================================ _base_ = 'mmdet::common/lsj-200e_coco-detection.py' custom_imports = dict( imports=['projects.Detic.detic'], allow_failed_imports=False) image_size = (1024, 1024) batch_augments = [dict(type='BatchFixedSizePad', size=image_size)] cls_layer = dict( type='ZeroShotClassifier', zs_weight_path='rand', zs_weight_dim=512, use_bias=0.0, norm_weight=True, norm_temperature=50.0) reg_layer = [ dict(type='Linear', in_features=1024, out_features=1024), dict(type='ReLU', inplace=True), dict(type='Linear', in_features=1024, out_features=4) ] num_classes = 22047 model = dict( type='CascadeRCNN', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32, batch_augments=batch_augments), backbone=dict( type='SwinTransformer', embed_dims=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=7, mlp_ratio=4, qkv_bias=True, qk_scale=None, drop_rate=0., attn_drop_rate=0., drop_path_rate=0.3, patch_norm=True, out_indices=(1, 2, 3), with_cp=False), neck=dict( type='FPN', in_channels=[256, 512, 1024], out_channels=256, start_level=0, add_extra_convs='on_output', num_outs=5, init_cfg=dict(type='Caffe2Xavier', layer='Conv2d'), relu_before_extra_convs=True), rpn_head=dict( type='CenterNetRPNHead', num_classes=1, in_channels=256, stacked_convs=4, feat_channels=256, strides=[8, 16, 32, 64, 128], conv_bias=True, norm_cfg=dict(type='GN', num_groups=32, requires_grad=True), loss_cls=dict( type='GaussianFocalLoss', pos_weight=0.25, neg_weight=0.75, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), ), roi_head=dict( type='DeticRoIHead', num_stages=3, stage_loss_weights=[1, 0.5, 0.25], bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict( type='RoIAlign', output_size=7, sampling_ratio=0, use_torchvision=True), out_channels=256, featmap_strides=[8, 16, 32], # approximately equal to # canonical_box_size=224, canonical_level=4 in D2 finest_scale=112), bbox_head=[ dict( type='DeticBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=num_classes, cls_predictor_cfg=cls_layer, reg_predictor_cfg=reg_layer, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=True, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)), dict( type='DeticBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=num_classes, cls_predictor_cfg=cls_layer, reg_predictor_cfg=reg_layer, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.05, 0.05, 0.1, 0.1]), reg_class_agnostic=True, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)), dict( type='DeticBBoxHead', in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=num_classes, cls_predictor_cfg=cls_layer, reg_predictor_cfg=reg_layer, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.033, 0.033, 0.067, 0.067]), reg_class_agnostic=True, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0)) ], mask_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=14, sampling_ratio=0), out_channels=256, featmap_strides=[8, 16, 32], # approximately equal to # canonical_box_size=224, canonical_level=4 in D2 finest_scale=112), mask_head=dict( type='FCNMaskHead', num_convs=4, in_channels=256, conv_out_channels=256, class_agnostic=True, num_classes=num_classes, loss_mask=dict( type='CrossEntropyLoss', use_mask=True, loss_weight=1.0))), # model training and testing settings train_cfg=dict( rpn=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=0, pos_weight=-1, debug=False), rpn_proposal=dict( nms_pre=2000, max_per_img=2000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0), rcnn=[ dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.6, neg_iou_thr=0.6, min_pos_iou=0.6, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), mask_size=28, pos_weight=-1, debug=False), dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.7, min_pos_iou=0.7, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), mask_size=28, pos_weight=-1, debug=False), dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.8, neg_iou_thr=0.8, min_pos_iou=0.8, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), mask_size=28, pos_weight=-1, debug=False) ]), test_cfg=dict( rpn=dict( score_thr=0.0001, nms_pre=1000, max_per_img=256, nms=dict(type='nms', iou_threshold=0.9), min_bbox_size=0), rcnn=dict( score_thr=0.02, nms=dict(type='nms', iou_threshold=0.5), max_per_img=300, mask_thr_binary=0.5))) backend = 'pillow' test_pipeline = [ dict( type='LoadImageFromFile', file_client_args=_base_.file_client_args, imdecode_backend=backend), dict(type='Resize', scale=(1333, 800), keep_ratio=True, backend=backend), dict( type='LoadAnnotations', with_bbox=True, with_mask=True, poly2mask=False), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict(batch_size=8, num_workers=4) val_dataloader = dict(dataset=dict(pipeline=test_pipeline)) test_dataloader = val_dataloader # Enable automatic-mixed-precision training with AmpOptimWrapper. optim_wrapper = dict( type='AmpOptimWrapper', optimizer=dict( type='SGD', lr=0.01 * 4, momentum=0.9, weight_decay=0.00004), paramwise_cfg=dict(norm_decay_mult=0.)) param_scheduler = [ dict( type='LinearLR', start_factor=0.00025, by_epoch=False, begin=0, end=4000), dict( type='MultiStepLR', begin=0, end=25, by_epoch=True, milestones=[22, 24], gamma=0.1) ] # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (8 GPUs) x (8 samples per GPU) auto_scale_lr = dict(base_batch_size=64) ================================================ FILE: projects/Detic/demo.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os import urllib from argparse import ArgumentParser import mmcv import torch from mmengine.logging import print_log from mmengine.utils import ProgressBar, scandir from mmdet.apis import inference_detector, init_detector from mmdet.registry import VISUALIZERS from mmdet.utils import register_all_modules IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', '.tiff', '.webp') def get_file_list(source_root: str) -> [list, dict]: """Get file list. Args: source_root (str): image or video source path Return: source_file_path_list (list): A list for all source file. source_type (dict): Source type: file or url or dir. """ is_dir = os.path.isdir(source_root) is_url = source_root.startswith(('http:/', 'https:/')) is_file = os.path.splitext(source_root)[-1].lower() in IMG_EXTENSIONS source_file_path_list = [] if is_dir: # when input source is dir for file in scandir(source_root, IMG_EXTENSIONS, recursive=True): source_file_path_list.append(os.path.join(source_root, file)) elif is_url: # when input source is url filename = os.path.basename( urllib.parse.unquote(source_root).split('?')[0]) file_save_path = os.path.join(os.getcwd(), filename) print(f'Downloading source file to {file_save_path}') torch.hub.download_url_to_file(source_root, file_save_path) source_file_path_list = [file_save_path] elif is_file: # when input source is single image source_file_path_list = [source_root] else: print('Cannot find image file.') source_type = dict(is_dir=is_dir, is_url=is_url, is_file=is_file) return source_file_path_list, source_type def parse_args(): parser = ArgumentParser() parser.add_argument( 'img', help='Image path, include image file, dir and URL.') parser.add_argument('config', help='Config file') parser.add_argument('checkpoint', help='Checkpoint file') parser.add_argument( '--out-dir', default='./output', help='Path to output file') parser.add_argument( '--device', default='cuda:0', help='Device used for inference') parser.add_argument( '--show', action='store_true', help='Show the detection results') parser.add_argument( '--score-thr', type=float, default=0.3, help='Bbox score threshold') parser.add_argument( '--dataset', type=str, help='dataset name to load the text embedding') parser.add_argument( '--class-name', nargs='+', type=str, help='custom class names') args = parser.parse_args() return args def main(): args = parse_args() # register all modules in mmdet into the registries register_all_modules() # build the model from a config file and a checkpoint file model = init_detector(args.config, args.checkpoint, device=args.device) if not os.path.exists(args.out_dir) and not args.show: os.mkdir(args.out_dir) # init visualizer visualizer = VISUALIZERS.build(model.cfg.visualizer) visualizer.dataset_meta = model.dataset_meta # get file list files, source_type = get_file_list(args.img) from detic.utils import (get_class_names, get_text_embeddings, reset_cls_layer_weight) # class name embeddings if args.class_name: dataset_classes = args.class_name elif args.dataset: dataset_classes = get_class_names(args.dataset) embedding = get_text_embeddings( dataset=args.dataset, custom_vocabulary=args.class_name) visualizer.dataset_meta['classes'] = dataset_classes reset_cls_layer_weight(model, embedding) # start detector inference progress_bar = ProgressBar(len(files)) for file in files: result = inference_detector(model, file) img = mmcv.imread(file) img = mmcv.imconvert(img, 'bgr', 'rgb') if source_type['is_dir']: filename = os.path.relpath(file, args.img).replace('/', '_') else: filename = os.path.basename(file) out_file = None if args.show else os.path.join(args.out_dir, filename) progress_bar.update() visualizer.add_datasample( filename, img, data_sample=result, draw_gt=False, show=args.show, wait_time=0, out_file=out_file, pred_score_thr=args.score_thr) if not args.show: print_log( f'\nResults have been saved at {os.path.abspath(args.out_dir)}') if __name__ == '__main__': main() ================================================ FILE: projects/Detic/detic/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .centernet_rpn_head import CenterNetRPNHead from .detic_bbox_head import DeticBBoxHead from .detic_roi_head import DeticRoIHead from .zero_shot_classifier import ZeroShotClassifier __all__ = [ 'CenterNetRPNHead', 'DeticBBoxHead', 'DeticRoIHead', 'ZeroShotClassifier' ] ================================================ FILE: projects/Detic/detic/centernet_rpn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy from typing import List, Sequence, Tuple import torch import torch.nn as nn from mmcv.cnn import Scale from mmengine import ConfigDict from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.dense_heads import CenterNetUpdateHead from mmdet.models.utils import multi_apply from mmdet.registry import MODELS INF = 1000000000 RangeType = Sequence[Tuple[int, int]] @MODELS.register_module(force=True) # avoid bug class CenterNetRPNHead(CenterNetUpdateHead): """CenterNetUpdateHead is an improved version of CenterNet in CenterNet2. Paper link ``_. """ def _init_layers(self) -> None: """Initialize layers of the head.""" self._init_reg_convs() self._init_predictor() def _init_predictor(self) -> None: """Initialize predictor layers of the head.""" self.conv_cls = nn.Conv2d( self.feat_channels, self.num_classes, 3, padding=1) self.conv_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1) def forward(self, x: Tuple[Tensor]) -> Tuple[List[Tensor], List[Tensor]]: """Forward features from the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. Returns: tuple: A tuple of each level outputs. - cls_scores (list[Tensor]): Box scores for each scale level, \ each is a 4D-tensor, the channel number is num_classes. - bbox_preds (list[Tensor]): Box energies / deltas for each \ scale level, each is a 4D-tensor, the channel number is 4. """ res = multi_apply(self.forward_single, x, self.scales, self.strides) return res def forward_single(self, x: Tensor, scale: Scale, stride: int) -> Tuple[Tensor, Tensor]: """Forward features of a single scale level. Args: x (Tensor): FPN feature maps of the specified stride. scale (:obj:`mmcv.cnn.Scale`): Learnable scale module to resize the bbox prediction. stride (int): The corresponding stride for feature maps. Returns: tuple: scores for each class, bbox predictions of input feature maps. """ for m in self.reg_convs: x = m(x) cls_score = self.conv_cls(x) bbox_pred = self.conv_reg(x) # scale the bbox_pred of different level # float to avoid overflow when enabling FP16 bbox_pred = scale(bbox_pred).float() # bbox_pred needed for gradient computation has been modified # by F.relu(bbox_pred) when run with PyTorch 1.10. So replace # F.relu(bbox_pred) with bbox_pred.clamp(min=0) bbox_pred = bbox_pred.clamp(min=0) if not self.training: bbox_pred *= stride return cls_score, bbox_pred # score aligned, box larger def _predict_by_feat_single(self, cls_score_list: List[Tensor], bbox_pred_list: List[Tensor], score_factor_list: List[Tensor], mlvl_priors: List[Tensor], img_meta: dict, cfg: ConfigDict, rescale: bool = False, with_nms: bool = True) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: cls_score_list (list[Tensor]): Box scores from all scale levels of a single image, each item has shape (num_priors * num_classes, H, W). bbox_pred_list (list[Tensor]): Box energies / deltas from all scale levels of a single image, each item has shape (num_priors * 4, H, W). score_factor_list (list[Tensor]): Score factor from all scale levels of a single image, each item has shape (num_priors * 1, H, W). mlvl_priors (list[Tensor]): Each element in the list is the priors of a single level in feature pyramid. In all anchor-based methods, it has shape (num_priors, 4). In all anchor-free methods, it has shape (num_priors, 2) when `with_stride=True`, otherwise it still has shape (num_priors, 4). img_meta (dict): Image meta info. cfg (mmengine.Config): Test / postprocessing configuration, if None, test_cfg would be used. rescale (bool): If True, return boxes in original image space. Defaults to False. with_nms (bool): If True, do nms before return boxes. Defaults to True. Returns: :obj:`InstanceData`: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ cfg = self.test_cfg if cfg is None else cfg cfg = copy.deepcopy(cfg) nms_pre = cfg.get('nms_pre', -1) mlvl_bbox_preds = [] mlvl_valid_priors = [] mlvl_scores = [] mlvl_labels = [] for level_idx, (cls_score, bbox_pred, score_factor, priors) in \ enumerate(zip(cls_score_list, bbox_pred_list, score_factor_list, mlvl_priors)): assert cls_score.size()[-2:] == bbox_pred.size()[-2:] dim = self.bbox_coder.encode_size bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, dim) cls_score = cls_score.permute(1, 2, 0).reshape(-1, self.cls_out_channels) heatmap = cls_score.sigmoid() score_thr = cfg.get('score_thr', 0) candidate_inds = heatmap > score_thr # 0.05 pre_nms_top_n = candidate_inds.sum() # N pre_nms_top_n = pre_nms_top_n.clamp(max=nms_pre) # N heatmap = heatmap[candidate_inds] # n candidate_nonzeros = candidate_inds.nonzero() # n box_loc = candidate_nonzeros[:, 0] # n labels = candidate_nonzeros[:, 1] # n bbox_pred = bbox_pred[box_loc] # n x 4 per_grids = priors[box_loc] # n x 2 if candidate_inds.sum().item() > pre_nms_top_n.item(): heatmap, top_k_indices = \ heatmap.topk(pre_nms_top_n, sorted=False) labels = labels[top_k_indices] bbox_pred = bbox_pred[top_k_indices] per_grids = per_grids[top_k_indices] bboxes = self.bbox_coder.decode(per_grids, bbox_pred) # avoid invalid boxes in RoI heads bboxes[:, 2] = torch.max(bboxes[:, 2], bboxes[:, 0] + 0.01) bboxes[:, 3] = torch.max(bboxes[:, 3], bboxes[:, 1] + 0.01) mlvl_bbox_preds.append(bboxes) mlvl_valid_priors.append(priors) mlvl_scores.append(torch.sqrt(heatmap)) mlvl_labels.append(labels) results = InstanceData() results.bboxes = torch.cat(mlvl_bbox_preds) results.scores = torch.cat(mlvl_scores) results.labels = torch.cat(mlvl_labels) return self._bbox_post_process( results=results, cfg=cfg, rescale=rescale, with_nms=with_nms, img_meta=img_meta) ================================================ FILE: projects/Detic/detic/detic_bbox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Union from mmengine.config import ConfigDict from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.layers import multiclass_nms from mmdet.models.roi_heads.bbox_heads import Shared2FCBBoxHead from mmdet.models.utils import empty_instances from mmdet.registry import MODELS from mmdet.structures.bbox import get_box_tensor, scale_boxes @MODELS.register_module(force=True) # avoid bug class DeticBBoxHead(Shared2FCBBoxHead): def __init__(self, *args, init_cfg: Optional[Union[dict, ConfigDict]] = None, **kwargs) -> None: super().__init__(*args, init_cfg=init_cfg, **kwargs) # reconstruct fc_cls and fc_reg since input channels are changed assert self.with_cls cls_channels = self.num_classes cls_predictor_cfg_ = self.cls_predictor_cfg.copy() cls_predictor_cfg_.update( in_features=self.cls_last_dim, out_features=cls_channels) self.fc_cls = MODELS.build(cls_predictor_cfg_) def _predict_by_feat_single( self, roi: Tensor, cls_score: Tensor, bbox_pred: Tensor, img_meta: dict, rescale: bool = False, rcnn_test_cfg: Optional[ConfigDict] = None) -> InstanceData: """Transform a single image's features extracted from the head into bbox results. Args: roi (Tensor): Boxes to be transformed. Has shape (num_boxes, 5). last dimension 5 arrange as (batch_index, x1, y1, x2, y2). cls_score (Tensor): Box scores, has shape (num_boxes, num_classes + 1). bbox_pred (Tensor): Box energies / deltas. has shape (num_boxes, num_classes * 4). img_meta (dict): image information. rescale (bool): If True, return boxes in original image space. Defaults to False. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of Bbox Head. Defaults to None Returns: :obj:`InstanceData`: Detection results of each image\ Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ results = InstanceData() if roi.shape[0] == 0: return empty_instances([img_meta], roi.device, task_type='bbox', instance_results=[results], box_type=self.predict_box_type, use_box_type=False, num_classes=self.num_classes, score_per_cls=rcnn_test_cfg is None)[0] scores = cls_score img_shape = img_meta['img_shape'] num_rois = roi.size(0) num_classes = 1 if self.reg_class_agnostic else self.num_classes roi = roi.repeat_interleave(num_classes, dim=0) bbox_pred = bbox_pred.view(-1, self.bbox_coder.encode_size) bboxes = self.bbox_coder.decode( roi[..., 1:], bbox_pred, max_shape=img_shape) if rescale and bboxes.size(0) > 0: assert img_meta.get('scale_factor') is not None scale_factor = [1 / s for s in img_meta['scale_factor']] bboxes = scale_boxes(bboxes, scale_factor) # Get the inside tensor when `bboxes` is a box type bboxes = get_box_tensor(bboxes) box_dim = bboxes.size(-1) bboxes = bboxes.view(num_rois, -1) if rcnn_test_cfg is None: # This means that it is aug test. # It needs to return the raw results without nms. results.bboxes = bboxes results.scores = scores else: det_bboxes, det_labels = multiclass_nms( bboxes, scores, rcnn_test_cfg.score_thr, rcnn_test_cfg.nms, rcnn_test_cfg.max_per_img, box_dim=box_dim) results.bboxes = det_bboxes[:, :-1] results.scores = det_bboxes[:, -1] results.labels = det_labels return results ================================================ FILE: projects/Detic/detic/detic_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Sequence, Tuple import torch from mmengine.structures import InstanceData from torch import Tensor from mmdet.models.roi_heads import CascadeRoIHead from mmdet.models.task_modules.samplers import SamplingResult from mmdet.models.test_time_augs import merge_aug_masks from mmdet.models.utils.misc import empty_instances from mmdet.registry import MODELS from mmdet.structures import SampleList from mmdet.structures.bbox import bbox2roi, get_box_tensor from mmdet.utils import ConfigType, InstanceList, MultiConfig @MODELS.register_module(force=True) # avoid bug class DeticRoIHead(CascadeRoIHead): def init_mask_head(self, mask_roi_extractor: MultiConfig, mask_head: MultiConfig) -> None: """Initialize mask head and mask roi extractor. Args: mask_head (dict): Config of mask in mask head. mask_roi_extractor (:obj:`ConfigDict`, dict or list): Config of mask roi extractor. """ self.mask_head = MODELS.build(mask_head) if mask_roi_extractor is not None: self.share_roi_extractor = False self.mask_roi_extractor = MODELS.build(mask_roi_extractor) else: self.share_roi_extractor = True self.mask_roi_extractor = self.bbox_roi_extractor def _refine_roi(self, x: Tuple[Tensor], rois: Tensor, batch_img_metas: List[dict], num_proposals_per_img: Sequence[int], **kwargs) -> tuple: """Multi-stage refinement of RoI. Args: x (tuple[Tensor]): List of multi-level img features. rois (Tensor): shape (n, 5), [batch_ind, x1, y1, x2, y2] batch_img_metas (list[dict]): List of image information. num_proposals_per_img (sequence[int]): number of proposals in each image. Returns: tuple: - rois (Tensor): Refined RoI. - cls_scores (list[Tensor]): Average predicted cls score per image. - bbox_preds (list[Tensor]): Bbox branch predictions for the last stage of per image. """ # "ms" in variable names means multi-stage ms_scores = [] for stage in range(self.num_stages): bbox_results = self._bbox_forward( stage=stage, x=x, rois=rois, **kwargs) # split batch bbox prediction back to each image cls_scores = bbox_results['cls_score'].sigmoid() bbox_preds = bbox_results['bbox_pred'] rois = rois.split(num_proposals_per_img, 0) cls_scores = cls_scores.split(num_proposals_per_img, 0) ms_scores.append(cls_scores) bbox_preds = bbox_preds.split(num_proposals_per_img, 0) if stage < self.num_stages - 1: bbox_head = self.bbox_head[stage] refine_rois_list = [] for i in range(len(batch_img_metas)): if rois[i].shape[0] > 0: bbox_label = cls_scores[i][:, :-1].argmax(dim=1) # Refactor `bbox_head.regress_by_class` to only accept # box tensor without img_idx concatenated. refined_bboxes = bbox_head.regress_by_class( rois[i][:, 1:], bbox_label, bbox_preds[i], batch_img_metas[i]) refined_bboxes = get_box_tensor(refined_bboxes) refined_rois = torch.cat( [rois[i][:, [0]], refined_bboxes], dim=1) refine_rois_list.append(refined_rois) rois = torch.cat(refine_rois_list) # ms_scores aligned # average scores of each image by stages cls_scores = [ sum([score[i] for score in ms_scores]) / float(len(ms_scores)) for i in range(len(batch_img_metas)) ] # aligned return rois, cls_scores, bbox_preds def _bbox_forward(self, stage: int, x: Tuple[Tensor], rois: Tensor) -> dict: """Box head forward function used in both training and testing. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): List of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. Returns: dict[str, Tensor]: Usually returns a dictionary with keys: - `cls_score` (Tensor): Classification scores. - `bbox_pred` (Tensor): Box energies / deltas. - `bbox_feats` (Tensor): Extract bbox RoI features. """ bbox_roi_extractor = self.bbox_roi_extractor[stage] bbox_head = self.bbox_head[stage] bbox_feats = bbox_roi_extractor(x[:bbox_roi_extractor.num_inputs], rois) # do not support caffe_c4 model anymore cls_score, bbox_pred = bbox_head(bbox_feats) bbox_results = dict( cls_score=cls_score, bbox_pred=bbox_pred, bbox_feats=bbox_feats) return bbox_results def predict_bbox(self, x: Tuple[Tensor], batch_img_metas: List[dict], rpn_results_list: InstanceList, rcnn_test_cfg: ConfigType, rescale: bool = False, **kwargs) -> InstanceList: """Perform forward propagation of the bbox head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. rcnn_test_cfg (obj:`ConfigDict`): `test_cfg` of R-CNN. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ proposals = [res.bboxes for res in rpn_results_list] proposal_scores = [res.scores for res in rpn_results_list] num_proposals_per_img = tuple(len(p) for p in proposals) rois = bbox2roi(proposals) if rois.shape[0] == 0: return empty_instances( batch_img_metas, rois.device, task_type='bbox', box_type=self.bbox_head[-1].predict_box_type, num_classes=self.bbox_head[-1].num_classes, score_per_cls=rcnn_test_cfg is None) # rois aligned rois, cls_scores, bbox_preds = self._refine_roi( x=x, rois=rois, batch_img_metas=batch_img_metas, num_proposals_per_img=num_proposals_per_img, **kwargs) # score reweighting in centernet2 cls_scores = [(s * ps[:, None])**0.5 for s, ps in zip(cls_scores, proposal_scores)] cls_scores = [ s * (s == s[:, :-1].max(dim=1)[0][:, None]).float() for s in cls_scores ] # fast_rcnn_inference results_list = self.bbox_head[-1].predict_by_feat( rois=rois, cls_scores=cls_scores, bbox_preds=bbox_preds, batch_img_metas=batch_img_metas, rescale=rescale, rcnn_test_cfg=rcnn_test_cfg) return results_list def _mask_forward(self, x: Tuple[Tensor], rois: Tensor) -> dict: """Mask head forward function used in both training and testing. Args: stage (int): The current stage in Cascade RoI Head. x (tuple[Tensor]): Tuple of multi-level img features. rois (Tensor): RoIs with the shape (n, 5) where the first column indicates batch id of each RoI. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. """ mask_feats = self.mask_roi_extractor( x[:self.mask_roi_extractor.num_inputs], rois) # do not support caffe_c4 model anymore mask_preds = self.mask_head(mask_feats) mask_results = dict(mask_preds=mask_preds) return mask_results def mask_loss(self, x, sampling_results: List[SamplingResult], batch_gt_instances: InstanceList) -> dict: """Run forward function and calculate loss for mask head in training. Args: x (tuple[Tensor]): Tuple of multi-level img features. sampling_results (list["obj:`SamplingResult`]): Sampling results. batch_gt_instances (list[:obj:`InstanceData`]): Batch of gt_instance. It usually includes ``bboxes``, ``labels``, and ``masks`` attributes. Returns: dict: Usually returns a dictionary with keys: - `mask_preds` (Tensor): Mask prediction. - `loss_mask` (dict): A dictionary of mask loss components. """ pos_rois = bbox2roi([res.pos_priors for res in sampling_results]) mask_results = self._mask_forward(x, pos_rois) mask_loss_and_target = self.mask_head.loss_and_target( mask_preds=mask_results['mask_preds'], sampling_results=sampling_results, batch_gt_instances=batch_gt_instances, rcnn_train_cfg=self.train_cfg[-1]) mask_results.update(mask_loss_and_target) return mask_results def loss(self, x: Tuple[Tensor], rpn_results_list: InstanceList, batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection roi on the features of the upstream network. Args: x (tuple[Tensor]): List of multi-level img features. rpn_results_list (list[:obj:`InstanceData`]): List of region proposals. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict[str, Tensor]: A dictionary of loss components """ raise NotImplementedError def predict_mask(self, x: Tuple[Tensor], batch_img_metas: List[dict], results_list: List[InstanceData], rescale: bool = False) -> List[InstanceData]: """Perform forward propagation of the mask head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Feature maps of all scale level. batch_img_metas (list[dict]): List of image information. results_list (list[:obj:`InstanceData`]): Detection results of each image. rescale (bool): If True, return boxes in original image space. Defaults to False. Returns: list[:obj:`InstanceData`]: Detection results of each image after the post process. Each item usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). - masks (Tensor): Has a shape (num_instances, H, W). """ bboxes = [res.bboxes for res in results_list] mask_rois = bbox2roi(bboxes) if mask_rois.shape[0] == 0: results_list = empty_instances( batch_img_metas, mask_rois.device, task_type='mask', instance_results=results_list, mask_thr_binary=self.test_cfg.mask_thr_binary) return results_list num_mask_rois_per_img = [len(res) for res in results_list] aug_masks = [] mask_results = self._mask_forward(x, mask_rois) mask_preds = mask_results['mask_preds'] # split batch mask prediction back to each image mask_preds = mask_preds.split(num_mask_rois_per_img, 0) aug_masks.append([m.sigmoid().detach() for m in mask_preds]) merged_masks = [] for i in range(len(batch_img_metas)): aug_mask = [mask[i] for mask in aug_masks] merged_mask = merge_aug_masks(aug_mask, batch_img_metas[i]) merged_masks.append(merged_mask) results_list = self.mask_head.predict_by_feat( mask_preds=merged_masks, results_list=results_list, batch_img_metas=batch_img_metas, rcnn_test_cfg=self.test_cfg, rescale=rescale, activate_map=True) return results_list ================================================ FILE: projects/Detic/detic/text_encoder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import List, Union import torch import torch.nn as nn class CLIPTextEncoder(nn.Module): def __init__(self, model_name='ViT-B/32'): super().__init__() import clip from clip.simple_tokenizer import SimpleTokenizer self.tokenizer = SimpleTokenizer() pretrained_model, _ = clip.load(model_name, device='cpu') self.clip = pretrained_model @property def device(self): return self.clip.device @property def dtype(self): return self.clip.dtype def tokenize(self, texts: Union[str, List[str]], context_length: int = 77) -> torch.LongTensor: if isinstance(texts, str): texts = [texts] sot_token = self.tokenizer.encoder['<|startoftext|>'] eot_token = self.tokenizer.encoder['<|endoftext|>'] all_tokens = [[sot_token] + self.tokenizer.encode(text) + [eot_token] for text in texts] result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) for i, tokens in enumerate(all_tokens): if len(tokens) > context_length: st = torch.randint(len(tokens) - context_length + 1, (1, ))[0].item() tokens = tokens[st:st + context_length] result[i, :len(tokens)] = torch.tensor(tokens) return result def forward(self, text): text = self.tokenize(text) text_features = self.clip.encode_text(text) return text_features ================================================ FILE: projects/Detic/detic/utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch import torch.nn.functional as F from mmengine.logging import print_log from .text_encoder import CLIPTextEncoder # download from # https://github.com/facebookresearch/Detic/tree/main/datasets/metadata DATASET_EMBEDDINGS = { 'lvis': 'datasets/metadata/lvis_v1_clip_a+cname.npy', 'objects365': 'datasets/metadata/o365_clip_a+cnamefix.npy', 'openimages': 'datasets/metadata/oid_clip_a+cname.npy', 'coco': 'datasets/metadata/coco_clip_a+cname.npy', } def get_text_embeddings(dataset=None, custom_vocabulary=None, prompt_prefix='a '): assert (dataset is None) ^ (custom_vocabulary is None), \ 'Either `dataset` or `custom_vocabulary` should be specified.' if dataset: if dataset in DATASET_EMBEDDINGS: return DATASET_EMBEDDINGS[dataset] else: custom_vocabulary = get_class_names(dataset) text_encoder = CLIPTextEncoder() text_encoder.eval() texts = [prompt_prefix + x for x in custom_vocabulary] print_log( f'Computing text embeddings for {len(custom_vocabulary)} classes.') embeddings = text_encoder(texts).detach().permute(1, 0).contiguous().cpu() return embeddings def get_class_names(dataset): if dataset == 'coco': from mmdet.datasets import CocoDataset class_names = CocoDataset.METAINFO['classes'] elif dataset == 'cityscapes': from mmdet.datasets import CityscapesDataset class_names = CityscapesDataset.METAINFO['classes'] elif dataset == 'voc': from mmdet.datasets import VOCDataset class_names = VOCDataset.METAINFO['classes'] elif dataset == 'openimages': from mmdet.datasets import OpenImagesDataset class_names = OpenImagesDataset.METAINFO['classes'] elif dataset == 'lvis': from mmdet.datasets import LVISV1Dataset class_names = LVISV1Dataset.METAINFO['classes'] else: raise TypeError(f'Invalid type for dataset name: {type(dataset)}') return class_names def reset_cls_layer_weight(model, weight): if type(weight) == str: print_log(f'Resetting cls_layer_weight from file: {weight}') zs_weight = torch.tensor( np.load(weight), dtype=torch.float32).permute(1, 0).contiguous() # D x C else: zs_weight = weight zs_weight = torch.cat( [zs_weight, zs_weight.new_zeros( (zs_weight.shape[0], 1))], dim=1) # D x (C + 1) zs_weight = F.normalize(zs_weight, p=2, dim=0) zs_weight = zs_weight.to('cuda') num_classes = zs_weight.shape[-1] for bbox_head in model.roi_head.bbox_head: bbox_head.num_classes = num_classes del bbox_head.fc_cls.zs_weight bbox_head.fc_cls.zs_weight = zs_weight ================================================ FILE: projects/Detic/detic/zero_shot_classifier.py ================================================ # Copyright (c) Facebook, Inc. and its affiliates. import numpy as np import torch from torch import nn from torch.nn import functional as F from mmdet.registry import MODELS @MODELS.register_module(force=True) # avoid bug class ZeroShotClassifier(nn.Module): def __init__( self, in_features: int, out_features: int, # num_classes zs_weight_path: str, zs_weight_dim: int = 512, use_bias: float = 0.0, norm_weight: bool = True, norm_temperature: float = 50.0, ): super().__init__() num_classes = out_features self.norm_weight = norm_weight self.norm_temperature = norm_temperature self.use_bias = use_bias < 0 if self.use_bias: self.cls_bias = nn.Parameter(torch.ones(1) * use_bias) self.linear = nn.Linear(in_features, zs_weight_dim) if zs_weight_path == 'rand': zs_weight = torch.randn((zs_weight_dim, num_classes)) nn.init.normal_(zs_weight, std=0.01) else: zs_weight = torch.tensor( np.load(zs_weight_path), dtype=torch.float32).permute(1, 0).contiguous() # D x C zs_weight = torch.cat( [zs_weight, zs_weight.new_zeros( (zs_weight_dim, 1))], dim=1) # D x (C + 1) if self.norm_weight: zs_weight = F.normalize(zs_weight, p=2, dim=0) if zs_weight_path == 'rand': self.zs_weight = nn.Parameter(zs_weight) else: self.register_buffer('zs_weight', zs_weight) assert self.zs_weight.shape[1] == num_classes + 1, self.zs_weight.shape def forward(self, x, classifier=None): ''' Inputs: x: B x D' classifier_info: (C', C' x D) ''' x = self.linear(x) if classifier is not None: zs_weight = classifier.permute(1, 0).contiguous() # D x C' zs_weight = F.normalize(zs_weight, p=2, dim=0) \ if self.norm_weight else zs_weight else: zs_weight = self.zs_weight if self.norm_weight: x = self.norm_temperature * F.normalize(x, p=2, dim=1) x = torch.mm(x, zs_weight) if self.use_bias: x = x + self.cls_bias return x ================================================ FILE: projects/DiffusionDet/README.md ================================================ ## Description This is an implementation of [DiffusionDet](https://github.com/ShoufaChen/DiffusionDet) based on [MMDetection](https://github.com/open-mmlab/mmdetection/tree/3.x), [MMCV](https://github.com/open-mmlab/mmcv), and [MMEngine](https://github.com/open-mmlab/mmengine).
## Usage ### Comparison of results 1. Download the [DiffusionDet released model](https://github.com/ShoufaChen/DiffusionDet#models). 2. Convert model from DiffusionDet version to MMDetection version. We give a [sample script](model_converters/diffusiondet_resnet_to_mmdet.py) to convert `DiffusionDet-resnet50` model. Users can download the corresponding models from [here](https://github.com/ShoufaChen/DiffusionDet/releases/download/v0.1/diffdet_coco_res50.pth). ```shell python projects/DiffusionDet/model_converters/diffusiondet_resnet_to_mmdet.py ${DiffusionDet ckpt path} ${MMDetectron ckpt path} ``` 3. Testing the model in MMDetection. ```shell python tools/test.py projects/DiffusionDet/configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py ${CHECKPOINT_PATH} ``` **Note:** During inference time, DiffusionDet will randomly generate noisy boxes, which may affect the AP results. If users want to get the same result every inference time, setting seed is a good way. We give a table to compare the inference results on `ResNet50-500-proposals` between DiffusionDet and MMDetection. | Config | Step | AP | | :---------------------------------------------------------------------------------------------------------------------: | :--: | :-------: | | [DiffusionDet](https://github.com/ShoufaChen/DiffusionDet/blob/main/configs/diffdet.coco.res50.yaml) (released results) | 1 | 45.5 | | [DiffusionDet](https://github.com/ShoufaChen/DiffusionDet/blob/main/configs/diffdet.coco.res50.yaml) (seed=0) | 1 | 45.66 | | [MMDetection](configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py) (seed=0) | 1 | 45.7 | | [MMDetection](configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py) (random seed) | 1 | 45.6~45.8 | | [DiffusionDet](https://github.com/ShoufaChen/DiffusionDet/blob/main/configs/diffdet.coco.res50.yaml) (released results) | 4 | 46.1 | | [DiffusionDet](https://github.com/ShoufaChen/DiffusionDet/blob/main/configs/diffdet.coco.res50.yaml) (seed=0) | 4 | 46.38 | | [MMDetection](configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py) (seed=0) | 4 | 46.4 | | [MMDetection](configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py) (random seed) | 4 | 46.2~46.4 | - `seed=0` means hard set seed before generating random boxes. ```python # hard set seed=0 before generating random boxes seed = 0 random.seed(seed) torch.manual_seed(seed) # torch.cuda.manual_seed(seed) torch.cuda.manual_seed_all(seed) ... noise_bboxes_raw = torch.randn( (self.num_proposals, 4), device=device) ... ``` - `random seed` means do not hard set seed before generating random boxes. ### Training commands In MMDetection's root directory, run the following command to train the model: ```bash python tools/train.py projects/DiffusionDet/configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py ``` For multi-gpu training, run: ```bash python -m torch.distributed.launch --nnodes=1 --node_rank=0 --nproc_per_node=${NUM_GPUS} --master_port=29506 --master_addr="127.0.0.1" tools/train.py projects/DiffusionDet/configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py ``` ### Testing commands In MMDetection's root directory, run the following command to test the model: ```bash # for 1 step inference # test command python tools/test.py projects/DiffusionDet/configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py ${CHECKPOINT_PATH} # for 4 steps inference # test command python tools/test.py projects/DiffusionDet/configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py ${CHECKPOINT_PATH} --cfg-options model.bbox_head.sampling_timesteps=4 ``` **Note:** There is no difference between 1 step or 4 steps (or other multi-step) during training. Users can set different steps during inference through `--cfg-options model.bbox_head.sampling_timesteps=${STEPS}`, but larger `sampling_timesteps` will affect the inference time. ## Results Here we provide the baseline version of DiffusionDet with ResNet50 backbone. To find more variants, please visit the [official model zoo](https://github.com/ShoufaChen/DiffusionDet#models). | Backbone | Style | Lr schd | AP (Step=1) | AP (Step=4) | Config | Download | | :------: | :-----: | :-----: | :---------: | :---------: | :----------------------------------------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | R-50 | PyTorch | 450k | 44.5 | 46.2 | [config](./configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py) | [model](https://download.openmmlab.com/mmdetection/v3.0/diffusiondet/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco_20230215_090925-7d6ed504.pth) \| [log](https://download.openmmlab.com/mmdetection/v3.0/diffusiondet/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco_20230215_090925.log.json) | ## License DiffusionDet is under the [CC-BY-NC 4.0 license](https://github.com/ShoufaChen/DiffusionDet/blob/main/LICENSE). Users should be careful about adopting these features in any commercial matters. ## Citation If you find DiffusionDet is useful in your research or applications, please consider giving a star 🌟 to the [official repository](https://github.com/ShoufaChen/DiffusionDet) and citing DiffusionDet by the following BibTeX entry. ```BibTeX @article{chen2022diffusiondet, title={DiffusionDet: Diffusion Model for Object Detection}, author={Chen, Shoufa and Sun, Peize and Song, Yibing and Luo, Ping}, journal={arXiv preprint arXiv:2211.09788}, year={2022} } ``` ## Checklist - [x] Milestone 1: PR-ready, and acceptable to be one of the `projects/`. - [x] Finish the code - [x] Basic docstrings & proper citation - [x] Test-time correctness - [x] A full README - [x] Milestone 2: Indicates a successful model implementation. - [x] Training-time correctness - [ ] Milestone 3: Good to be a part of our core package! - [ ] Type hints and docstrings - [ ] Unit tests - [ ] Code polishing - [ ] Metafile.yml - [ ] Move your modules into the core package following the codebase's file hierarchy structure. - [ ] Refactor your modules into the core package following the codebase's file hierarchy structure. ================================================ FILE: projects/DiffusionDet/configs/diffusiondet_r50_fpn_500-proposals_1-step_crop-ms-480-800-450k_coco.py ================================================ _base_ = [ 'mmdet::_base_/datasets/coco_detection.py', 'mmdet::_base_/schedules/schedule_1x.py', 'mmdet::_base_/default_runtime.py' ] custom_imports = dict( imports=['projects.DiffusionDet.diffusiondet'], allow_failed_imports=False) # model settings model = dict( type='DiffusionDet', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), neck=dict( type='FPN', in_channels=[256, 512, 1024, 2048], out_channels=256, num_outs=4), bbox_head=dict( type='DynamicDiffusionDetHead', num_classes=80, feat_channels=256, num_proposals=500, num_heads=6, deep_supervision=True, prior_prob=0.01, snr_scale=2.0, sampling_timesteps=1, ddim_sampling_eta=1.0, single_head=dict( type='SingleDiffusionDetHead', num_cls_convs=1, num_reg_convs=3, dim_feedforward=2048, num_heads=8, dropout=0.0, act_cfg=dict(type='ReLU', inplace=True), dynamic_conv=dict(dynamic_dim=64, dynamic_num=2)), roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=2), out_channels=256, featmap_strides=[4, 8, 16, 32]), # criterion criterion=dict( type='DiffusionDetCriterion', num_classes=80, assigner=dict( type='DiffusionDetMatcher', match_costs=[ dict( type='FocalLossCost', alpha=0.25, gamma=2.0, weight=2.0, eps=1e-8), dict(type='BBoxL1Cost', weight=5.0, box_format='xyxy'), dict(type='IoUCost', iou_mode='giou', weight=2.0) ], center_radius=2.5, candidate_topk=5), loss_cls=dict( type='FocalLoss', use_sigmoid=True, alpha=0.25, gamma=2.0, reduction='sum', loss_weight=2.0), loss_bbox=dict(type='L1Loss', reduction='sum', loss_weight=5.0), loss_giou=dict(type='GIoULoss', reduction='sum', loss_weight=2.0))), test_cfg=dict( use_nms=True, score_thr=0.5, min_bbox_size=0, nms=dict(type='nms', iou_threshold=0.5), )) backend = 'pillow' train_pipeline = [ dict( type='LoadImageFromFile', file_client_args=_base_.file_client_args, imdecode_backend=backend), dict(type='LoadAnnotations', with_bbox=True), dict(type='RandomFlip', prob=0.5), dict( type='RandomChoice', transforms=[[ dict( type='RandomChoiceResize', scales=[(480, 1333), (512, 1333), (544, 1333), (576, 1333), (608, 1333), (640, 1333), (672, 1333), (704, 1333), (736, 1333), (768, 1333), (800, 1333)], keep_ratio=True, backend=backend), ], [ dict( type='RandomChoiceResize', scales=[(400, 1333), (500, 1333), (600, 1333)], keep_ratio=True, backend=backend), dict( type='RandomCrop', crop_type='absolute_range', crop_size=(384, 600), allow_negative_crop=True), dict( type='RandomChoiceResize', scales=[(480, 1333), (512, 1333), (544, 1333), (576, 1333), (608, 1333), (640, 1333), (672, 1333), (704, 1333), (736, 1333), (768, 1333), (800, 1333)], keep_ratio=True, backend=backend) ]]), dict(type='PackDetInputs') ] test_pipeline = [ dict( type='LoadImageFromFile', file_client_args=_base_.file_client_args, imdecode_backend=backend), dict(type='Resize', scale=(1333, 800), keep_ratio=True, backend=backend), # If you don't have a gt annotation, delete the pipeline dict(type='LoadAnnotations', with_bbox=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( sampler=dict(type='InfiniteSampler'), dataset=dict( filter_cfg=dict(filter_empty_gt=False, min_size=1e-5), pipeline=train_pipeline)) val_dataloader = dict(dataset=dict(pipeline=test_pipeline)) test_dataloader = val_dataloader # optimizer optim_wrapper = dict( type='OptimWrapper', optimizer=dict( _delete_=True, type='AdamW', lr=0.000025, weight_decay=0.0001), clip_grad=dict(max_norm=1.0, norm_type=2)) train_cfg = dict( _delete_=True, type='IterBasedTrainLoop', max_iters=450000, val_interval=75000) # learning rate param_scheduler = [ dict( type='LinearLR', start_factor=0.01, by_epoch=False, begin=0, end=1000), dict( type='MultiStepLR', begin=0, end=450000, by_epoch=False, milestones=[350000, 420000], gamma=0.1) ] default_hooks = dict( checkpoint=dict(by_epoch=False, interval=75000, max_keep_ckpts=3)) log_processor = dict(by_epoch=False) ================================================ FILE: projects/DiffusionDet/diffusiondet/__init__.py ================================================ from .diffusiondet import DiffusionDet from .head import (DynamicConv, DynamicDiffusionDetHead, SingleDiffusionDetHead, SinusoidalPositionEmbeddings) from .loss import DiffusionDetCriterion, DiffusionDetMatcher __all__ = [ 'DiffusionDet', 'DynamicDiffusionDetHead', 'SingleDiffusionDetHead', 'SinusoidalPositionEmbeddings', 'DynamicConv', 'DiffusionDetCriterion', 'DiffusionDetMatcher' ] ================================================ FILE: projects/DiffusionDet/diffusiondet/diffusiondet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.models import SingleStageDetector from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig @MODELS.register_module() class DiffusionDet(SingleStageDetector): """Implementation of `DiffusionDet <>`_""" def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: projects/DiffusionDet/diffusiondet/head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved # Modified from https://github.com/ShoufaChen/DiffusionDet/blob/main/diffusiondet/detector.py # noqa # Modified from https://github.com/ShoufaChen/DiffusionDet/blob/main/diffusiondet/head.py # noqa # This work is licensed under the CC-BY-NC 4.0 License. # Users should be careful about adopting these features in any commercial matters. # noqa # For more details, please refer to https://github.com/ShoufaChen/DiffusionDet/blob/main/LICENSE # noqa import copy import math import random import warnings from typing import Tuple import torch import torch.nn as nn import torch.nn.functional as F from mmcv.cnn import build_activation_layer from mmcv.ops import batched_nms from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures import SampleList from mmdet.structures.bbox import (bbox2roi, bbox_cxcywh_to_xyxy, bbox_xyxy_to_cxcywh, get_box_wh, scale_boxes) from mmdet.utils import InstanceList _DEFAULT_SCALE_CLAMP = math.log(100000.0 / 16) def cosine_beta_schedule(timesteps, s=0.008): """Cosine schedule as proposed in https://openreview.net/forum?id=-NEXDKk8gZ.""" steps = timesteps + 1 x = torch.linspace(0, timesteps, steps, dtype=torch.float64) alphas_cumprod = torch.cos( ((x / timesteps) + s) / (1 + s) * math.pi * 0.5)**2 alphas_cumprod = alphas_cumprod / alphas_cumprod[0] betas = 1 - (alphas_cumprod[1:] / alphas_cumprod[:-1]) return torch.clip(betas, 0, 0.999) def extract(a, t, x_shape): """extract the appropriate t index for a batch of indices.""" batch_size = t.shape[0] out = a.gather(-1, t) return out.reshape(batch_size, *((1, ) * (len(x_shape) - 1))) class SinusoidalPositionEmbeddings(nn.Module): def __init__(self, dim): super().__init__() self.dim = dim def forward(self, time): device = time.device half_dim = self.dim // 2 embeddings = math.log(10000) / (half_dim - 1) embeddings = torch.exp( torch.arange(half_dim, device=device) * -embeddings) embeddings = time[:, None] * embeddings[None, :] embeddings = torch.cat((embeddings.sin(), embeddings.cos()), dim=-1) return embeddings @MODELS.register_module() class DynamicDiffusionDetHead(nn.Module): def __init__(self, num_classes=80, feat_channels=256, num_proposals=500, num_heads=6, prior_prob=0.01, snr_scale=2.0, timesteps=1000, sampling_timesteps=1, self_condition=False, box_renewal=True, use_ensemble=True, deep_supervision=True, ddim_sampling_eta=1.0, criterion=dict( type='DiffusionDetCriterion', num_classes=80, assigner=dict( type='DiffusionDetMatcher', match_costs=[ dict( type='FocalLossCost', alpha=2.0, gamma=0.25, weight=2.0), dict( type='BBoxL1Cost', weight=5.0, box_format='xyxy'), dict(type='IoUCost', iou_mode='giou', weight=2.0) ], center_radius=2.5, candidate_topk=5), ), single_head=dict( type='DiffusionDetHead', num_cls_convs=1, num_reg_convs=3, dim_feedforward=2048, num_heads=8, dropout=0.0, act_cfg=dict(type='ReLU'), dynamic_conv=dict(dynamic_dim=64, dynamic_num=2)), roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict( type='RoIAlign', output_size=7, sampling_ratio=2), out_channels=256, featmap_strides=[4, 8, 16, 32]), test_cfg=None, **kwargs) -> None: super().__init__() self.roi_extractor = MODELS.build(roi_extractor) self.num_classes = num_classes self.num_classes = num_classes self.feat_channels = feat_channels self.num_proposals = num_proposals self.num_heads = num_heads # Build Diffusion assert isinstance(timesteps, int), 'The type of `timesteps` should ' \ f'be int but got {type(timesteps)}' assert sampling_timesteps <= timesteps self.timesteps = timesteps self.sampling_timesteps = sampling_timesteps self.snr_scale = snr_scale self.ddim_sampling = self.sampling_timesteps < self.timesteps self.ddim_sampling_eta = ddim_sampling_eta self.self_condition = self_condition self.box_renewal = box_renewal self.use_ensemble = use_ensemble self._build_diffusion() # Build assigner assert criterion.get('assigner', None) is not None assigner = TASK_UTILS.build(criterion.get('assigner')) # Init parameters. self.use_focal_loss = assigner.use_focal_loss self.use_fed_loss = assigner.use_fed_loss # build criterion criterion.update(deep_supervision=deep_supervision) self.criterion = TASK_UTILS.build(criterion) # Build Dynamic Head. single_head_ = single_head.copy() single_head_num_classes = single_head_.get('num_classes', None) if single_head_num_classes is None: single_head_.update(num_classes=num_classes) else: if single_head_num_classes != num_classes: warnings.warn( 'The `num_classes` of `DynamicDiffusionDetHead` and ' '`SingleDiffusionDetHead` should be same, changing ' f'`single_head.num_classes` to {num_classes}') single_head_.update(num_classes=num_classes) single_head_feat_channels = single_head_.get('feat_channels', None) if single_head_feat_channels is None: single_head_.update(feat_channels=feat_channels) else: if single_head_feat_channels != feat_channels: warnings.warn( 'The `feat_channels` of `DynamicDiffusionDetHead` and ' '`SingleDiffusionDetHead` should be same, changing ' f'`single_head.feat_channels` to {feat_channels}') single_head_.update(feat_channels=feat_channels) default_pooler_resolution = roi_extractor['roi_layer'].get( 'output_size') assert default_pooler_resolution is not None single_head_pooler_resolution = single_head_.get('pooler_resolution') if single_head_pooler_resolution is None: single_head_.update(pooler_resolution=default_pooler_resolution) else: if single_head_pooler_resolution != default_pooler_resolution: warnings.warn( 'The `pooler_resolution` of `DynamicDiffusionDetHead` ' 'and `SingleDiffusionDetHead` should be same, changing ' f'`single_head.pooler_resolution` to {num_classes}') single_head_.update( pooler_resolution=default_pooler_resolution) single_head_.update( use_focal_loss=self.use_focal_loss, use_fed_loss=self.use_fed_loss) single_head_module = MODELS.build(single_head_) self.num_heads = num_heads self.head_series = nn.ModuleList( [copy.deepcopy(single_head_module) for _ in range(num_heads)]) self.deep_supervision = deep_supervision # Gaussian random feature embedding layer for time time_dim = feat_channels * 4 self.time_mlp = nn.Sequential( SinusoidalPositionEmbeddings(feat_channels), nn.Linear(feat_channels, time_dim), nn.GELU(), nn.Linear(time_dim, time_dim)) self.prior_prob = prior_prob self.test_cfg = test_cfg self.use_nms = self.test_cfg.get('use_nms', True) self._init_weights() def _init_weights(self): # init all parameters. bias_value = -math.log((1 - self.prior_prob) / self.prior_prob) for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) # initialize the bias for focal loss and fed loss. if self.use_focal_loss or self.use_fed_loss: if p.shape[-1] == self.num_classes or \ p.shape[-1] == self.num_classes + 1: nn.init.constant_(p, bias_value) def _build_diffusion(self): betas = cosine_beta_schedule(self.timesteps) alphas = 1. - betas alphas_cumprod = torch.cumprod(alphas, dim=0) alphas_cumprod_prev = F.pad(alphas_cumprod[:-1], (1, 0), value=1.) self.register_buffer('betas', betas) self.register_buffer('alphas_cumprod', alphas_cumprod) self.register_buffer('alphas_cumprod_prev', alphas_cumprod_prev) # calculations for diffusion q(x_t | x_{t-1}) and others self.register_buffer('sqrt_alphas_cumprod', torch.sqrt(alphas_cumprod)) self.register_buffer('sqrt_one_minus_alphas_cumprod', torch.sqrt(1. - alphas_cumprod)) self.register_buffer('log_one_minus_alphas_cumprod', torch.log(1. - alphas_cumprod)) self.register_buffer('sqrt_recip_alphas_cumprod', torch.sqrt(1. / alphas_cumprod)) self.register_buffer('sqrt_recipm1_alphas_cumprod', torch.sqrt(1. / alphas_cumprod - 1)) # calculations for posterior q(x_{t-1} | x_t, x_0) # equal to 1. / (1. / (1. - alpha_cumprod_tm1) + alpha_t / beta_t) posterior_variance = betas * (1. - alphas_cumprod_prev) / ( 1. - alphas_cumprod) self.register_buffer('posterior_variance', posterior_variance) # log calculation clipped because the posterior variance is 0 at # the beginning of the diffusion chain self.register_buffer('posterior_log_variance_clipped', torch.log(posterior_variance.clamp(min=1e-20))) self.register_buffer( 'posterior_mean_coef1', betas * torch.sqrt(alphas_cumprod_prev) / (1. - alphas_cumprod)) self.register_buffer('posterior_mean_coef2', (1. - alphas_cumprod_prev) * torch.sqrt(alphas) / (1. - alphas_cumprod)) def forward(self, features, init_bboxes, init_t, init_features=None): time = self.time_mlp(init_t, ) inter_class_logits = [] inter_pred_bboxes = [] bs = len(features[0]) bboxes = init_bboxes if init_features is not None: init_features = init_features[None].repeat(1, bs, 1) proposal_features = init_features.clone() else: proposal_features = None for head_idx, single_head in enumerate(self.head_series): class_logits, pred_bboxes, proposal_features = single_head( features, bboxes, proposal_features, self.roi_extractor, time) if self.deep_supervision: inter_class_logits.append(class_logits) inter_pred_bboxes.append(pred_bboxes) bboxes = pred_bboxes.detach() if self.deep_supervision: return torch.stack(inter_class_logits), torch.stack( inter_pred_bboxes) else: return class_logits[None, ...], pred_bboxes[None, ...] def loss(self, x: Tuple[Tensor], batch_data_samples: SampleList) -> dict: """Perform forward propagation and loss calculation of the detection head on the features of the upstream network. Args: x (tuple[Tensor]): Features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ prepare_outputs = self.prepare_training_targets(batch_data_samples) (batch_gt_instances, batch_pred_instances, batch_gt_instances_ignore, batch_img_metas) = prepare_outputs batch_diff_bboxes = torch.stack([ pred_instances.diff_bboxes_abs for pred_instances in batch_pred_instances ]) batch_time = torch.stack( [pred_instances.time for pred_instances in batch_pred_instances]) pred_logits, pred_bboxes = self(x, batch_diff_bboxes, batch_time) output = { 'pred_logits': pred_logits[-1], 'pred_boxes': pred_bboxes[-1] } if self.deep_supervision: output['aux_outputs'] = [{ 'pred_logits': a, 'pred_boxes': b } for a, b in zip(pred_logits[:-1], pred_bboxes[:-1])] losses = self.criterion(output, batch_gt_instances, batch_img_metas) return losses def prepare_training_targets(self, batch_data_samples): # hard-setting seed to keep results same (if necessary) # random.seed(0) # torch.manual_seed(0) # torch.cuda.manual_seed_all(0) # torch.backends.cudnn.deterministic = True # torch.backends.cudnn.benchmark = False batch_gt_instances = [] batch_pred_instances = [] batch_gt_instances_ignore = [] batch_img_metas = [] for data_sample in batch_data_samples: img_meta = data_sample.metainfo gt_instances = data_sample.gt_instances gt_bboxes = gt_instances.bboxes h, w = img_meta['img_shape'] image_size = gt_bboxes.new_tensor([w, h, w, h]) norm_gt_bboxes = gt_bboxes / image_size norm_gt_bboxes_cxcywh = bbox_xyxy_to_cxcywh(norm_gt_bboxes) pred_instances = self.prepare_diffusion(norm_gt_bboxes_cxcywh, image_size) gt_instances.set_metainfo(dict(image_size=image_size)) gt_instances.norm_bboxes_cxcywh = norm_gt_bboxes_cxcywh batch_gt_instances.append(gt_instances) batch_pred_instances.append(pred_instances) batch_img_metas.append(data_sample.metainfo) if 'ignored_instances' in data_sample: batch_gt_instances_ignore.append(data_sample.ignored_instances) else: batch_gt_instances_ignore.append(None) return (batch_gt_instances, batch_pred_instances, batch_gt_instances_ignore, batch_img_metas) def prepare_diffusion(self, gt_boxes, image_size): device = gt_boxes.device time = torch.randint( 0, self.timesteps, (1, ), dtype=torch.long, device=device) noise = torch.randn(self.num_proposals, 4, device=device) num_gt = gt_boxes.shape[0] if num_gt < self.num_proposals: # 3 * sigma = 1/2 --> sigma: 1/6 box_placeholder = torch.randn( self.num_proposals - num_gt, 4, device=device) / 6. + 0.5 box_placeholder[:, 2:] = torch.clip( box_placeholder[:, 2:], min=1e-4) x_start = torch.cat((gt_boxes, box_placeholder), dim=0) else: select_mask = [True] * self.num_proposals + \ [False] * (num_gt - self.num_proposals) random.shuffle(select_mask) x_start = gt_boxes[select_mask] x_start = (x_start * 2. - 1.) * self.snr_scale # noise sample x = self.q_sample(x_start=x_start, time=time, noise=noise) x = torch.clamp(x, min=-1 * self.snr_scale, max=self.snr_scale) x = ((x / self.snr_scale) + 1) / 2. diff_bboxes = bbox_cxcywh_to_xyxy(x) # convert to abs bboxes diff_bboxes_abs = diff_bboxes * image_size metainfo = dict(time=time.squeeze(-1)) pred_instances = InstanceData(metainfo=metainfo) pred_instances.diff_bboxes = diff_bboxes pred_instances.diff_bboxes_abs = diff_bboxes_abs pred_instances.noise = noise return pred_instances # forward diffusion def q_sample(self, x_start, time, noise=None): if noise is None: noise = torch.randn_like(x_start) x_start_shape = x_start.shape sqrt_alphas_cumprod_t = extract(self.sqrt_alphas_cumprod, time, x_start_shape) sqrt_one_minus_alphas_cumprod_t = extract( self.sqrt_one_minus_alphas_cumprod, time, x_start_shape) return sqrt_alphas_cumprod_t * x_start + \ sqrt_one_minus_alphas_cumprod_t * noise def predict(self, x: Tuple[Tensor], batch_data_samples: SampleList, rescale: bool = False) -> InstanceList: """Perform forward propagation of the detection head and predict detection results on the features of the upstream network. Args: x (tuple[Tensor]): Multi-level features from the upstream network, each is a 4D-tensor. batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool, optional): Whether to rescale the results. Defaults to False. Returns: list[obj:`InstanceData`]: Detection results of each image after the post process. """ # hard-setting seed to keep results same (if necessary) # seed = 0 # random.seed(seed) # torch.manual_seed(seed) # torch.cuda.manual_seed_all(seed) device = x[-1].device batch_img_metas = [ data_samples.metainfo for data_samples in batch_data_samples ] (time_pairs, batch_noise_bboxes, batch_noise_bboxes_raw, batch_image_size) = self.prepare_testing_targets( batch_img_metas, device) predictions = self.predict_by_feat( x, time_pairs=time_pairs, batch_noise_bboxes=batch_noise_bboxes, batch_noise_bboxes_raw=batch_noise_bboxes_raw, batch_image_size=batch_image_size, device=device, batch_img_metas=batch_img_metas) return predictions def predict_by_feat(self, x, time_pairs, batch_noise_bboxes, batch_noise_bboxes_raw, batch_image_size, device, batch_img_metas=None, cfg=None, rescale=True): batch_size = len(batch_img_metas) cfg = self.test_cfg if cfg is None else cfg cfg = copy.deepcopy(cfg) ensemble_score, ensemble_label, ensemble_coord = [], [], [] for time, time_next in time_pairs: batch_time = torch.full((batch_size, ), time, device=device, dtype=torch.long) # self_condition = x_start if self.self_condition else None pred_logits, pred_bboxes = self(x, batch_noise_bboxes, batch_time) x_start = pred_bboxes[-1] x_start = x_start / batch_image_size[:, None, :] x_start = bbox_xyxy_to_cxcywh(x_start) x_start = (x_start * 2 - 1.) * self.snr_scale x_start = torch.clamp( x_start, min=-1 * self.snr_scale, max=self.snr_scale) pred_noise = self.predict_noise_from_start(batch_noise_bboxes_raw, batch_time, x_start) pred_noise_list, x_start_list = [], [] noise_bboxes_list, num_remain_list = [], [] if self.box_renewal: # filter score_thr = cfg.get('score_thr', 0) for img_id in range(batch_size): score_per_image = pred_logits[-1][img_id] score_per_image = torch.sigmoid(score_per_image) value, _ = torch.max(score_per_image, -1, keepdim=False) keep_idx = value > score_thr num_remain_list.append(torch.sum(keep_idx)) pred_noise_list.append(pred_noise[img_id, keep_idx, :]) x_start_list.append(x_start[img_id, keep_idx, :]) noise_bboxes_list.append(batch_noise_bboxes[img_id, keep_idx, :]) if time_next < 0: # Not same as original DiffusionDet if self.use_ensemble and self.sampling_timesteps > 1: box_pred_per_image, scores_per_image, labels_per_image = \ self.inference( box_cls=pred_logits[-1], box_pred=pred_bboxes[-1], cfg=cfg, device=device) ensemble_score.append(scores_per_image) ensemble_label.append(labels_per_image) ensemble_coord.append(box_pred_per_image) continue alpha = self.alphas_cumprod[time] alpha_next = self.alphas_cumprod[time_next] sigma = self.ddim_sampling_eta * ((1 - alpha / alpha_next) * (1 - alpha_next) / (1 - alpha)).sqrt() c = (1 - alpha_next - sigma**2).sqrt() batch_noise_bboxes_list = [] batch_noise_bboxes_raw_list = [] for idx in range(batch_size): pred_noise = pred_noise_list[idx] x_start = x_start_list[idx] noise_bboxes = noise_bboxes_list[idx] num_remain = num_remain_list[idx] noise = torch.randn_like(noise_bboxes) noise_bboxes = x_start * alpha_next.sqrt() + \ c * pred_noise + sigma * noise if self.box_renewal: # filter # replenish with randn boxes if num_remain < self.num_proposals: noise_bboxes = torch.cat( (noise_bboxes, torch.randn( self.num_proposals - num_remain, 4, device=device)), dim=0) else: select_mask = [True] * self.num_proposals + \ [False] * (num_remain - self.num_proposals) random.shuffle(select_mask) noise_bboxes = noise_bboxes[select_mask] # raw noise boxes batch_noise_bboxes_raw_list.append(noise_bboxes) # resize to xyxy noise_bboxes = torch.clamp( noise_bboxes, min=-1 * self.snr_scale, max=self.snr_scale) noise_bboxes = ((noise_bboxes / self.snr_scale) + 1) / 2 noise_bboxes = bbox_cxcywh_to_xyxy(noise_bboxes) noise_bboxes = noise_bboxes * batch_image_size[idx] batch_noise_bboxes_list.append(noise_bboxes) batch_noise_bboxes = torch.stack(batch_noise_bboxes_list) batch_noise_bboxes_raw = torch.stack(batch_noise_bboxes_raw_list) if self.use_ensemble and self.sampling_timesteps > 1: box_pred_per_image, scores_per_image, labels_per_image = \ self.inference( box_cls=pred_logits[-1], box_pred=pred_bboxes[-1], cfg=cfg, device=device) ensemble_score.append(scores_per_image) ensemble_label.append(labels_per_image) ensemble_coord.append(box_pred_per_image) if self.use_ensemble and self.sampling_timesteps > 1: steps = len(ensemble_score) results_list = [] for idx in range(batch_size): ensemble_score_per_img = [ ensemble_score[i][idx] for i in range(steps) ] ensemble_label_per_img = [ ensemble_label[i][idx] for i in range(steps) ] ensemble_coord_per_img = [ ensemble_coord[i][idx] for i in range(steps) ] scores_per_image = torch.cat(ensemble_score_per_img, dim=0) labels_per_image = torch.cat(ensemble_label_per_img, dim=0) box_pred_per_image = torch.cat(ensemble_coord_per_img, dim=0) if self.use_nms: det_bboxes, keep_idxs = batched_nms( box_pred_per_image, scores_per_image, labels_per_image, cfg.nms) box_pred_per_image = box_pred_per_image[keep_idxs] labels_per_image = labels_per_image[keep_idxs] scores_per_image = det_bboxes[:, -1] results = InstanceData() results.bboxes = box_pred_per_image results.scores = scores_per_image results.labels = labels_per_image results_list.append(results) else: box_cls = pred_logits[-1] box_pred = pred_bboxes[-1] results_list = self.inference(box_cls, box_pred, cfg, device) if rescale: results_list = self.do_results_post_process( results_list, cfg, batch_img_metas=batch_img_metas) return results_list @staticmethod def do_results_post_process(results_list, cfg, batch_img_metas=None): processed_results = [] for results, img_meta in zip(results_list, batch_img_metas): assert img_meta.get('scale_factor') is not None scale_factor = [1 / s for s in img_meta['scale_factor']] results.bboxes = scale_boxes(results.bboxes, scale_factor) # clip w, h h, w = img_meta['ori_shape'] results.bboxes[:, 0::2] = results.bboxes[:, 0::2].clamp( min=0, max=w) results.bboxes[:, 1::2] = results.bboxes[:, 1::2].clamp( min=0, max=h) # filter small size bboxes if cfg.get('min_bbox_size', 0) >= 0: w, h = get_box_wh(results.bboxes) valid_mask = (w > cfg.min_bbox_size) & (h > cfg.min_bbox_size) if not valid_mask.all(): results = results[valid_mask] processed_results.append(results) return processed_results def prepare_testing_targets(self, batch_img_metas, device): # [-1, 0, 1, 2, ..., T-1] when sampling_timesteps == timesteps times = torch.linspace( -1, self.timesteps - 1, steps=self.sampling_timesteps + 1) times = list(reversed(times.int().tolist())) # [(T-1, T-2), (T-2, T-3), ..., (1, 0), (0, -1)] time_pairs = list(zip(times[:-1], times[1:])) noise_bboxes_list = [] noise_bboxes_raw_list = [] image_size_list = [] for img_meta in batch_img_metas: h, w = img_meta['img_shape'] image_size = torch.tensor([w, h, w, h], dtype=torch.float32, device=device) noise_bboxes_raw = torch.randn((self.num_proposals, 4), device=device) noise_bboxes = torch.clamp( noise_bboxes_raw, min=-1 * self.snr_scale, max=self.snr_scale) noise_bboxes = ((noise_bboxes / self.snr_scale) + 1) / 2 noise_bboxes = bbox_cxcywh_to_xyxy(noise_bboxes) noise_bboxes = noise_bboxes * image_size noise_bboxes_raw_list.append(noise_bboxes_raw) noise_bboxes_list.append(noise_bboxes) image_size_list.append(image_size[None]) batch_noise_bboxes = torch.stack(noise_bboxes_list) batch_image_size = torch.cat(image_size_list) batch_noise_bboxes_raw = torch.stack(noise_bboxes_raw_list) return (time_pairs, batch_noise_bboxes, batch_noise_bboxes_raw, batch_image_size) def predict_noise_from_start(self, x_t, t, x0): results = (extract( self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - x0) / \ extract(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape) return results def inference(self, box_cls, box_pred, cfg, device): """ Args: box_cls (Tensor): tensor of shape (batch_size, num_proposals, K). The tensor predicts the classification probability for each proposal. box_pred (Tensor): tensors of shape (batch_size, num_proposals, 4). The tensor predicts 4-vector (x,y,w,h) box regression values for every proposal Returns: results (List[Instances]): a list of #images elements. """ results = [] if self.use_focal_loss or self.use_fed_loss: scores = torch.sigmoid(box_cls) labels = torch.arange( self.num_classes, device=device).unsqueeze(0).repeat(self.num_proposals, 1).flatten(0, 1) box_pred_list = [] scores_list = [] labels_list = [] for i, (scores_per_image, box_pred_per_image) in enumerate(zip(scores, box_pred)): scores_per_image, topk_indices = scores_per_image.flatten( 0, 1).topk( self.num_proposals, sorted=False) labels_per_image = labels[topk_indices] box_pred_per_image = box_pred_per_image.view(-1, 1, 4).repeat( 1, self.num_classes, 1).view(-1, 4) box_pred_per_image = box_pred_per_image[topk_indices] if self.use_ensemble and self.sampling_timesteps > 1: box_pred_list.append(box_pred_per_image) scores_list.append(scores_per_image) labels_list.append(labels_per_image) continue if self.use_nms: det_bboxes, keep_idxs = batched_nms( box_pred_per_image, scores_per_image, labels_per_image, cfg.nms) box_pred_per_image = box_pred_per_image[keep_idxs] labels_per_image = labels_per_image[keep_idxs] # some nms would reweight the score, such as softnms scores_per_image = det_bboxes[:, -1] result = InstanceData() result.bboxes = box_pred_per_image result.scores = scores_per_image result.labels = labels_per_image results.append(result) else: # For each box we assign the best class or the second # best if the best on is `no_object`. scores, labels = F.softmax(box_cls, dim=-1)[:, :, :-1].max(-1) for i, (scores_per_image, labels_per_image, box_pred_per_image) in enumerate( zip(scores, labels, box_pred)): if self.use_ensemble and self.sampling_timesteps > 1: return box_pred_per_image, scores_per_image, \ labels_per_image if self.use_nms: det_bboxes, keep_idxs = batched_nms( box_pred_per_image, scores_per_image, labels_per_image, cfg.nms) box_pred_per_image = box_pred_per_image[keep_idxs] labels_per_image = labels_per_image[keep_idxs] # some nms would reweight the score, such as softnms scores_per_image = det_bboxes[:, -1] result = InstanceData() result.bboxes = box_pred_per_image result.scores = scores_per_image result.labels = labels_per_image results.append(result) if self.use_ensemble and self.sampling_timesteps > 1: return box_pred_list, scores_list, labels_list else: return results @MODELS.register_module() class SingleDiffusionDetHead(nn.Module): def __init__( self, num_classes=80, feat_channels=256, dim_feedforward=2048, num_cls_convs=1, num_reg_convs=3, num_heads=8, dropout=0.0, pooler_resolution=7, scale_clamp=_DEFAULT_SCALE_CLAMP, bbox_weights=(2.0, 2.0, 1.0, 1.0), use_focal_loss=True, use_fed_loss=False, act_cfg=dict(type='ReLU', inplace=True), dynamic_conv=dict(dynamic_dim=64, dynamic_num=2) ) -> None: super().__init__() self.feat_channels = feat_channels # Dynamic self.self_attn = nn.MultiheadAttention( feat_channels, num_heads, dropout=dropout) self.inst_interact = DynamicConv( feat_channels=feat_channels, pooler_resolution=pooler_resolution, dynamic_dim=dynamic_conv['dynamic_dim'], dynamic_num=dynamic_conv['dynamic_num']) self.linear1 = nn.Linear(feat_channels, dim_feedforward) self.dropout = nn.Dropout(dropout) self.linear2 = nn.Linear(dim_feedforward, feat_channels) self.norm1 = nn.LayerNorm(feat_channels) self.norm2 = nn.LayerNorm(feat_channels) self.norm3 = nn.LayerNorm(feat_channels) self.dropout1 = nn.Dropout(dropout) self.dropout2 = nn.Dropout(dropout) self.dropout3 = nn.Dropout(dropout) self.activation = build_activation_layer(act_cfg) # block time mlp self.block_time_mlp = nn.Sequential( nn.SiLU(), nn.Linear(feat_channels * 4, feat_channels * 2)) # cls. cls_module = list() for _ in range(num_cls_convs): cls_module.append(nn.Linear(feat_channels, feat_channels, False)) cls_module.append(nn.LayerNorm(feat_channels)) cls_module.append(nn.ReLU(inplace=True)) self.cls_module = nn.ModuleList(cls_module) # reg. reg_module = list() for _ in range(num_reg_convs): reg_module.append(nn.Linear(feat_channels, feat_channels, False)) reg_module.append(nn.LayerNorm(feat_channels)) reg_module.append(nn.ReLU(inplace=True)) self.reg_module = nn.ModuleList(reg_module) # pred. self.use_focal_loss = use_focal_loss self.use_fed_loss = use_fed_loss if self.use_focal_loss or self.use_fed_loss: self.class_logits = nn.Linear(feat_channels, num_classes) else: self.class_logits = nn.Linear(feat_channels, num_classes + 1) self.bboxes_delta = nn.Linear(feat_channels, 4) self.scale_clamp = scale_clamp self.bbox_weights = bbox_weights def forward(self, features, bboxes, pro_features, pooler, time_emb): """ :param bboxes: (N, num_boxes, 4) :param pro_features: (N, num_boxes, feat_channels) """ N, num_boxes = bboxes.shape[:2] # roi_feature. proposal_boxes = list() for b in range(N): proposal_boxes.append(bboxes[b]) rois = bbox2roi(proposal_boxes) roi_features = pooler(features, rois) if pro_features is None: pro_features = roi_features.view(N, num_boxes, self.feat_channels, -1).mean(-1) roi_features = roi_features.view(N * num_boxes, self.feat_channels, -1).permute(2, 0, 1) # self_att. pro_features = pro_features.view(N, num_boxes, self.feat_channels).permute(1, 0, 2) pro_features2 = self.self_attn( pro_features, pro_features, value=pro_features)[0] pro_features = pro_features + self.dropout1(pro_features2) pro_features = self.norm1(pro_features) # inst_interact. pro_features = pro_features.view( num_boxes, N, self.feat_channels).permute(1, 0, 2).reshape(1, N * num_boxes, self.feat_channels) pro_features2 = self.inst_interact(pro_features, roi_features) pro_features = pro_features + self.dropout2(pro_features2) obj_features = self.norm2(pro_features) # obj_feature. obj_features2 = self.linear2( self.dropout(self.activation(self.linear1(obj_features)))) obj_features = obj_features + self.dropout3(obj_features2) obj_features = self.norm3(obj_features) fc_feature = obj_features.transpose(0, 1).reshape(N * num_boxes, -1) scale_shift = self.block_time_mlp(time_emb) scale_shift = torch.repeat_interleave(scale_shift, num_boxes, dim=0) scale, shift = scale_shift.chunk(2, dim=1) fc_feature = fc_feature * (scale + 1) + shift cls_feature = fc_feature.clone() reg_feature = fc_feature.clone() for cls_layer in self.cls_module: cls_feature = cls_layer(cls_feature) for reg_layer in self.reg_module: reg_feature = reg_layer(reg_feature) class_logits = self.class_logits(cls_feature) bboxes_deltas = self.bboxes_delta(reg_feature) pred_bboxes = self.apply_deltas(bboxes_deltas, bboxes.view(-1, 4)) return (class_logits.view(N, num_boxes, -1), pred_bboxes.view(N, num_boxes, -1), obj_features) def apply_deltas(self, deltas, boxes): """Apply transformation `deltas` (dx, dy, dw, dh) to `boxes`. Args: deltas (Tensor): transformation deltas of shape (N, k*4), where k >= 1. deltas[i] represents k potentially different class-specific box transformations for the single box boxes[i]. boxes (Tensor): boxes to transform, of shape (N, 4) """ boxes = boxes.to(deltas.dtype) widths = boxes[:, 2] - boxes[:, 0] heights = boxes[:, 3] - boxes[:, 1] ctr_x = boxes[:, 0] + 0.5 * widths ctr_y = boxes[:, 1] + 0.5 * heights wx, wy, ww, wh = self.bbox_weights dx = deltas[:, 0::4] / wx dy = deltas[:, 1::4] / wy dw = deltas[:, 2::4] / ww dh = deltas[:, 3::4] / wh # Prevent sending too large values into torch.exp() dw = torch.clamp(dw, max=self.scale_clamp) dh = torch.clamp(dh, max=self.scale_clamp) pred_ctr_x = dx * widths[:, None] + ctr_x[:, None] pred_ctr_y = dy * heights[:, None] + ctr_y[:, None] pred_w = torch.exp(dw) * widths[:, None] pred_h = torch.exp(dh) * heights[:, None] pred_boxes = torch.zeros_like(deltas) pred_boxes[:, 0::4] = pred_ctr_x - 0.5 * pred_w # x1 pred_boxes[:, 1::4] = pred_ctr_y - 0.5 * pred_h # y1 pred_boxes[:, 2::4] = pred_ctr_x + 0.5 * pred_w # x2 pred_boxes[:, 3::4] = pred_ctr_y + 0.5 * pred_h # y2 return pred_boxes class DynamicConv(nn.Module): def __init__(self, feat_channels: int, dynamic_dim: int = 64, dynamic_num: int = 2, pooler_resolution: int = 7) -> None: super().__init__() self.feat_channels = feat_channels self.dynamic_dim = dynamic_dim self.dynamic_num = dynamic_num self.num_params = self.feat_channels * self.dynamic_dim self.dynamic_layer = nn.Linear(self.feat_channels, self.dynamic_num * self.num_params) self.norm1 = nn.LayerNorm(self.dynamic_dim) self.norm2 = nn.LayerNorm(self.feat_channels) self.activation = nn.ReLU(inplace=True) num_output = self.feat_channels * pooler_resolution**2 self.out_layer = nn.Linear(num_output, self.feat_channels) self.norm3 = nn.LayerNorm(self.feat_channels) def forward(self, pro_features: Tensor, roi_features: Tensor) -> Tensor: """Forward function. Args: pro_features: (1, N * num_boxes, self.feat_channels) roi_features: (49, N * num_boxes, self.feat_channels) Returns: """ features = roi_features.permute(1, 0, 2) parameters = self.dynamic_layer(pro_features).permute(1, 0, 2) param1 = parameters[:, :, :self.num_params].view( -1, self.feat_channels, self.dynamic_dim) param2 = parameters[:, :, self.num_params:].view(-1, self.dynamic_dim, self.feat_channels) features = torch.bmm(features, param1) features = self.norm1(features) features = self.activation(features) features = torch.bmm(features, param2) features = self.norm2(features) features = self.activation(features) features = features.flatten(1) features = self.out_layer(features) features = self.norm3(features) features = self.activation(features) return features ================================================ FILE: projects/DiffusionDet/diffusiondet/loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved # Modified from https://github.com/ShoufaChen/DiffusionDet/blob/main/diffusiondet/loss.py # noqa # This work is licensed under the CC-BY-NC 4.0 License. # Users should be careful about adopting these features in any commercial matters. # noqa # For more details, please refer to https://github.com/ShoufaChen/DiffusionDet/blob/main/LICENSE # noqa from typing import List, Tuple, Union import torch import torch.nn as nn from mmengine.config import ConfigDict from mmengine.structures import InstanceData from torch import Tensor from mmdet.registry import MODELS, TASK_UTILS from mmdet.structures.bbox import bbox_cxcywh_to_xyxy, bbox_xyxy_to_cxcywh from mmdet.utils import ConfigType @TASK_UTILS.register_module() class DiffusionDetCriterion(nn.Module): def __init__( self, num_classes, assigner: Union[ConfigDict, nn.Module], deep_supervision=True, loss_cls=dict( type='FocalLoss', use_sigmoid=True, alpha=0.25, gamma=2.0, reduction='sum', loss_weight=2.0), loss_bbox=dict(type='L1Loss', reduction='sum', loss_weight=5.0), loss_giou=dict(type='GIoULoss', reduction='sum', loss_weight=2.0), ): super().__init__() self.num_classes = num_classes if isinstance(assigner, nn.Module): self.assigner = assigner else: self.assigner = TASK_UTILS.build(assigner) self.deep_supervision = deep_supervision self.loss_cls = MODELS.build(loss_cls) self.loss_bbox = MODELS.build(loss_bbox) self.loss_giou = MODELS.build(loss_giou) def forward(self, outputs, batch_gt_instances, batch_img_metas): batch_indices = self.assigner(outputs, batch_gt_instances, batch_img_metas) # Compute all the requested losses loss_cls = self.loss_classification(outputs, batch_gt_instances, batch_indices) loss_bbox, loss_giou = self.loss_boxes(outputs, batch_gt_instances, batch_indices) losses = dict( loss_cls=loss_cls, loss_bbox=loss_bbox, loss_giou=loss_giou) if self.deep_supervision: assert 'aux_outputs' in outputs for i, aux_outputs in enumerate(outputs['aux_outputs']): batch_indices = self.assigner(aux_outputs, batch_gt_instances, batch_img_metas) loss_cls = self.loss_classification(aux_outputs, batch_gt_instances, batch_indices) loss_bbox, loss_giou = self.loss_boxes(aux_outputs, batch_gt_instances, batch_indices) tmp_losses = dict( loss_cls=loss_cls, loss_bbox=loss_bbox, loss_giou=loss_giou) for name, value in tmp_losses.items(): losses[f's.{i}.{name}'] = value return losses def loss_classification(self, outputs, batch_gt_instances, indices): assert 'pred_logits' in outputs src_logits = outputs['pred_logits'] target_classes_list = [ gt.labels[J] for gt, (_, J) in zip(batch_gt_instances, indices) ] target_classes = torch.full( src_logits.shape[:2], self.num_classes, dtype=torch.int64, device=src_logits.device) for idx in range(len(batch_gt_instances)): target_classes[idx, indices[idx][0]] = target_classes_list[idx] src_logits = src_logits.flatten(0, 1) target_classes = target_classes.flatten(0, 1) # comp focal loss. num_instances = max(torch.cat(target_classes_list).shape[0], 1) loss_cls = self.loss_cls( src_logits, target_classes, ) / num_instances return loss_cls def loss_boxes(self, outputs, batch_gt_instances, indices): assert 'pred_boxes' in outputs pred_boxes = outputs['pred_boxes'] target_bboxes_norm_list = [ gt.norm_bboxes_cxcywh[J] for gt, (_, J) in zip(batch_gt_instances, indices) ] target_bboxes_list = [ gt.bboxes[J] for gt, (_, J) in zip(batch_gt_instances, indices) ] pred_bboxes_list = [] pred_bboxes_norm_list = [] for idx in range(len(batch_gt_instances)): pred_bboxes_list.append(pred_boxes[idx, indices[idx][0]]) image_size = batch_gt_instances[idx].image_size pred_bboxes_norm_list.append(pred_boxes[idx, indices[idx][0]] / image_size) pred_boxes_cat = torch.cat(pred_bboxes_list) pred_boxes_norm_cat = torch.cat(pred_bboxes_norm_list) target_bboxes_cat = torch.cat(target_bboxes_list) target_bboxes_norm_cat = torch.cat(target_bboxes_norm_list) if len(pred_boxes_cat) > 0: num_instances = pred_boxes_cat.shape[0] loss_bbox = self.loss_bbox( pred_boxes_norm_cat, bbox_cxcywh_to_xyxy(target_bboxes_norm_cat)) / num_instances loss_giou = self.loss_giou(pred_boxes_cat, target_bboxes_cat) / num_instances else: loss_bbox = pred_boxes.sum() * 0 loss_giou = pred_boxes.sum() * 0 return loss_bbox, loss_giou @TASK_UTILS.register_module() class DiffusionDetMatcher(nn.Module): """This class computes an assignment between the targets and the predictions of the network For efficiency reasons, the targets don't include the no_object. Because of this, in general, there are more predictions than targets. In this case, we do a 1-to-k (dynamic) matching of the best predictions, while the others are un-matched (and thus treated as non-objects). """ def __init__(self, match_costs: Union[List[Union[dict, ConfigDict]], dict, ConfigDict], center_radius: float = 2.5, candidate_topk: int = 5, iou_calculator: ConfigType = dict(type='BboxOverlaps2D'), **kwargs): super().__init__() self.center_radius = center_radius self.candidate_topk = candidate_topk if isinstance(match_costs, dict): match_costs = [match_costs] elif isinstance(match_costs, list): assert len(match_costs) > 0, \ 'match_costs must not be a empty list.' self.use_focal_loss = False self.use_fed_loss = False for _match_cost in match_costs: if _match_cost.get('type') == 'FocalLossCost': self.use_focal_loss = True if _match_cost.get('type') == 'FedLoss': self.use_fed_loss = True raise NotImplementedError self.match_costs = [ TASK_UTILS.build(match_cost) for match_cost in match_costs ] self.iou_calculator = TASK_UTILS.build(iou_calculator) def forward(self, outputs, batch_gt_instances, batch_img_metas): assert 'pred_logits' in outputs and 'pred_boxes' in outputs pred_logits = outputs['pred_logits'] pred_bboxes = outputs['pred_boxes'] batch_size = len(batch_gt_instances) assert batch_size == pred_logits.shape[0] == pred_bboxes.shape[0] batch_indices = [] for i in range(batch_size): pred_instances = InstanceData() pred_instances.bboxes = pred_bboxes[i, ...] pred_instances.scores = pred_logits[i, ...] gt_instances = batch_gt_instances[i] img_meta = batch_img_metas[i] indices = self.single_assigner(pred_instances, gt_instances, img_meta) batch_indices.append(indices) return batch_indices def single_assigner(self, pred_instances, gt_instances, img_meta): with torch.no_grad(): gt_bboxes = gt_instances.bboxes pred_bboxes = pred_instances.bboxes num_gt = gt_bboxes.size(0) if num_gt == 0: # empty object in key frame valid_mask = pred_bboxes.new_zeros((pred_bboxes.shape[0], ), dtype=torch.bool) matched_gt_inds = pred_bboxes.new_zeros((gt_bboxes.shape[0], ), dtype=torch.long) return valid_mask, matched_gt_inds valid_mask, is_in_boxes_and_center = \ self.get_in_gt_and_in_center_info( bbox_xyxy_to_cxcywh(pred_bboxes), bbox_xyxy_to_cxcywh(gt_bboxes) ) cost_list = [] for match_cost in self.match_costs: cost = match_cost( pred_instances=pred_instances, gt_instances=gt_instances, img_meta=img_meta) cost_list.append(cost) pairwise_ious = self.iou_calculator(pred_bboxes, gt_bboxes) cost_list.append((~is_in_boxes_and_center) * 100.0) cost_matrix = torch.stack(cost_list).sum(0) cost_matrix[~valid_mask] = cost_matrix[~valid_mask] + 10000.0 fg_mask_inboxes, matched_gt_inds = \ self.dynamic_k_matching( cost_matrix, pairwise_ious, num_gt) return fg_mask_inboxes, matched_gt_inds def get_in_gt_and_in_center_info( self, pred_bboxes: Tensor, gt_bboxes: Tensor) -> Tuple[Tensor, Tensor]: """Get the information of which prior is in gt bboxes and gt center priors.""" xy_target_gts = bbox_cxcywh_to_xyxy(gt_bboxes) # (x1, y1, x2, y2) pred_bboxes_center_x = pred_bboxes[:, 0].unsqueeze(1) pred_bboxes_center_y = pred_bboxes[:, 1].unsqueeze(1) # whether the center of each anchor is inside a gt box b_l = pred_bboxes_center_x > xy_target_gts[:, 0].unsqueeze(0) b_r = pred_bboxes_center_x < xy_target_gts[:, 2].unsqueeze(0) b_t = pred_bboxes_center_y > xy_target_gts[:, 1].unsqueeze(0) b_b = pred_bboxes_center_y < xy_target_gts[:, 3].unsqueeze(0) # (b_l.long()+b_r.long()+b_t.long()+b_b.long())==4 [300,num_gt] , is_in_boxes = ((b_l.long() + b_r.long() + b_t.long() + b_b.long()) == 4) is_in_boxes_all = is_in_boxes.sum(1) > 0 # [num_query] # in fixed center center_radius = 2.5 # Modified to self-adapted sampling --- the center size depends # on the size of the gt boxes # https://github.com/dulucas/UVO_Challenge/blob/main/Track1/detection/mmdet/core/bbox/assigners/rpn_sim_ota_assigner.py#L212 # noqa b_l = pred_bboxes_center_x > ( gt_bboxes[:, 0] - (center_radius * (xy_target_gts[:, 2] - xy_target_gts[:, 0]))).unsqueeze(0) b_r = pred_bboxes_center_x < ( gt_bboxes[:, 0] + (center_radius * (xy_target_gts[:, 2] - xy_target_gts[:, 0]))).unsqueeze(0) b_t = pred_bboxes_center_y > ( gt_bboxes[:, 1] - (center_radius * (xy_target_gts[:, 3] - xy_target_gts[:, 1]))).unsqueeze(0) b_b = pred_bboxes_center_y < ( gt_bboxes[:, 1] + (center_radius * (xy_target_gts[:, 3] - xy_target_gts[:, 1]))).unsqueeze(0) is_in_centers = ((b_l.long() + b_r.long() + b_t.long() + b_b.long()) == 4) is_in_centers_all = is_in_centers.sum(1) > 0 is_in_boxes_anchor = is_in_boxes_all | is_in_centers_all is_in_boxes_and_center = (is_in_boxes & is_in_centers) return is_in_boxes_anchor, is_in_boxes_and_center def dynamic_k_matching(self, cost: Tensor, pairwise_ious: Tensor, num_gt: int) -> Tuple[Tensor, Tensor]: """Use IoU and matching cost to calculate the dynamic top-k positive targets.""" matching_matrix = torch.zeros_like(cost) # select candidate topk ious for dynamic-k calculation candidate_topk = min(self.candidate_topk, pairwise_ious.size(0)) topk_ious, _ = torch.topk(pairwise_ious, candidate_topk, dim=0) # calculate dynamic k for each gt dynamic_ks = torch.clamp(topk_ious.sum(0).int(), min=1) for gt_idx in range(num_gt): _, pos_idx = torch.topk( cost[:, gt_idx], k=dynamic_ks[gt_idx], largest=False) matching_matrix[:, gt_idx][pos_idx] = 1 del topk_ious, dynamic_ks, pos_idx prior_match_gt_mask = matching_matrix.sum(1) > 1 if prior_match_gt_mask.sum() > 0: _, cost_argmin = torch.min(cost[prior_match_gt_mask, :], dim=1) matching_matrix[prior_match_gt_mask, :] *= 0 matching_matrix[prior_match_gt_mask, cost_argmin] = 1 while (matching_matrix.sum(0) == 0).any(): matched_query_id = matching_matrix.sum(1) > 0 cost[matched_query_id] += 100000.0 unmatch_id = torch.nonzero( matching_matrix.sum(0) == 0, as_tuple=False).squeeze(1) for gt_idx in unmatch_id: pos_idx = torch.argmin(cost[:, gt_idx]) matching_matrix[:, gt_idx][pos_idx] = 1.0 if (matching_matrix.sum(1) > 1).sum() > 0: _, cost_argmin = torch.min(cost[prior_match_gt_mask], dim=1) matching_matrix[prior_match_gt_mask] *= 0 matching_matrix[prior_match_gt_mask, cost_argmin, ] = 1 assert not (matching_matrix.sum(0) == 0).any() # get foreground mask inside box and center prior fg_mask_inboxes = matching_matrix.sum(1) > 0 matched_gt_inds = matching_matrix[fg_mask_inboxes, :].argmax(1) return fg_mask_inboxes, matched_gt_inds ================================================ FILE: projects/DiffusionDet/model_converters/diffusiondet_resnet_to_mmdet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse from collections import OrderedDict import numpy as np import torch from mmengine.fileio import load def convert(src, dst): if src.endswith('pth'): src_model = torch.load(src) else: src_model = load(src) dst_state_dict = OrderedDict() for k, v in src_model['model'].items(): key_name_split = k.split('.') if 'backbone.fpn_lateral' in k: lateral_id = int(key_name_split[-2][-1]) name = f'neck.lateral_convs.{lateral_id - 2}.' \ f'conv.{key_name_split[-1]}' elif 'backbone.fpn_output' in k: lateral_id = int(key_name_split[-2][-1]) name = f'neck.fpn_convs.{lateral_id - 2}.conv.' \ f'{key_name_split[-1]}' elif 'backbone.bottom_up.stem.conv1.norm.' in k: name = f'backbone.bn1.{key_name_split[-1]}' elif 'backbone.bottom_up.stem.conv1.' in k: name = f'backbone.conv1.{key_name_split[-1]}' elif 'backbone.bottom_up.res' in k: # weight_type = key_name_split[-1] res_id = int(key_name_split[2][-1]) - 1 # deal with short cut if 'shortcut' in key_name_split[4]: if 'shortcut' == key_name_split[-2]: name = f'backbone.layer{res_id}.' \ f'{key_name_split[3]}.downsample.0.' \ f'{key_name_split[-1]}' elif 'shortcut' == key_name_split[-3]: name = f'backbone.layer{res_id}.' \ f'{key_name_split[3]}.downsample.1.' \ f'{key_name_split[-1]}' else: print(f'Unvalid key {k}') # deal with conv elif 'conv' in key_name_split[-2]: conv_id = int(key_name_split[-2][-1]) name = f'backbone.layer{res_id}.{key_name_split[3]}' \ f'.conv{conv_id}.{key_name_split[-1]}' # deal with BN elif key_name_split[-2] == 'norm': conv_id = int(key_name_split[-3][-1]) name = f'backbone.layer{res_id}.{key_name_split[3]}.' \ f'bn{conv_id}.{key_name_split[-1]}' else: print(f'{k} is invalid') elif key_name_split[0] == 'head': # d2: head.xxx -> mmdet: bbox_head.xxx name = f'bbox_{k}' else: # some base parameters such as beta will not convert print(f'{k} is not converted!!') continue if not isinstance(v, np.ndarray) and not isinstance(v, torch.Tensor): raise ValueError( 'Unsupported type found in checkpoint! {}: {}'.format( k, type(v))) if not isinstance(v, torch.Tensor): dst_state_dict[name] = torch.from_numpy(v) else: dst_state_dict[name] = v mmdet_model = dict(state_dict=dst_state_dict, meta=dict()) torch.save(mmdet_model, dst) def main(): parser = argparse.ArgumentParser(description='Convert model keys') parser.add_argument('src', help='src detectron model path') parser.add_argument('dst', help='save path') args = parser.parse_args() convert(args.src, args.dst) if __name__ == '__main__': main() ================================================ FILE: projects/EfficientDet/README.md ================================================ # EfficientDet > [**EfficientDet: Scalable and Efficient Object Detection**](https://arxiv.org/pdf/1911.09070.pdf), > Mingxing Tan, Ruoming Pang, Quoc V. Le, > *CVPR 2020* ## Abstract This is an implementation of [EfficientDet](https://github.com/google/automl) based on [MMDetection](https://github.com/open-mmlab/mmdetection/tree/3.x), [MMCV](https://github.com/open-mmlab/mmcv), and [MMEngine](https://github.com/open-mmlab/mmengine).
EfficientDet a new family of object detectors, which consistently achieve much better efficiency than prior art across a wide spectrum of resource constraints. In particular, with single model and single-scale, EfficientDet-D7 achieves stateof-the-art 55.1 AP on COCO test-dev with 77M parameters and 410B FLOP.
BiFPN is a simple yet highly effective weighted bi-directional feature pyramid network, which introduces learnable weights to learn the importance of different input features, while repeatedly applying topdown and bottom-up multi-scale feature fusion.
In contrast to other feature pyramid network, such as FPN, FPN + PAN, NAS-FPN, BiFPN achieves the best accuracy with fewer parameters and FLOPs.
## Usage ### Model conversion Firstly, download EfficientDet [weights](https://github.com/google/automl/tree/master/efficientdet) and unzip, please use the following command ```bash tar -xzvf {EFFICIENTDET_WEIGHT} ``` Then, install tensorflow, please use the following command ```bash pip install tensorflow-gpu==2.6.0 ``` Lastly, convert weights from tensorflow to pytorch, please use the following command ```bash python projects/EfficientDet/convert_tf_to_pt.py --backbone {BACKBONE_NAME} --tensorflow_weight {TENSORFLOW_WEIGHT_PATH} --out_weight {OUT_PATH} ``` ### Testing commands In MMDetection's root directory, run the following command to test the model: ```bash python tools/test.py projects/EfficientDet/configs/efficientdet_effb0_bifpn_8xb16-crop512-300e_coco.py ${CHECKPOINT_PATH} ``` ## Results Based on mmdetection, this project aligns the test accuracy of the [official model](https://github.com/google/automl).
If you want to reproduce the test results, you need to convert model weights first, then run the test command.
The training accuracy will also be aligned with the official in the future | Method | Backbone | Pretrained Model | Training set | Test set | Epoch | Val Box AP | Official AP | | :------------------------------------------------------------------------------: | :-------------: | :--------------: | :------------: | :----------: | :---: | :--------: | :---------: | | [efficientdet-d0](./configs/efficientdet_effb0_bifpn_8xb16-crop512-300e_coco.py) | efficientnet-b0 | ImageNet | COCO2017 Train | COCO2017 Val | 300 | 34.4 | 34.3 | ## Citation ```BibTeX @inproceedings{tan2020efficientdet, title={Efficientdet: Scalable and efficient object detection}, author={Tan, Mingxing and Pang, Ruoming and Le, Quoc V}, booktitle={Proceedings of the IEEE/CVF conference on computer vision and pattern recognition}, pages={10781--10790}, year={2020} } ``` ## Checklist - [x] Milestone 1: PR-ready, and acceptable to be one of the `projects/`. - [x] Finish the code - [x] Basic docstrings & proper citation - [x] Test-time correctness - [x] A full README - [ ] Milestone 2: Indicates a successful model implementation. - [ ] Training-time correctness - [ ] Milestone 3: Good to be a part of our core package! - [ ] Type hints and docstrings - [ ] Unit tests - [ ] Code polishing - [ ] Metafile.yml - [ ] Move your modules into the core package following the codebase's file hierarchy structure. - [ ] Refactor your modules into the core package following the codebase's file hierarchy structure. ================================================ FILE: projects/EfficientDet/configs/efficientdet_effb0_bifpn_16xb8-crop512-300e_coco.py ================================================ _base_ = [ 'mmdet::_base_/datasets/coco_detection.py', 'mmdet::_base_/schedules/schedule_1x.py', 'mmdet::_base_/default_runtime.py' ] custom_imports = dict( imports=['projects.EfficientDet.efficientdet'], allow_failed_imports=False) image_size = 512 dataset_type = 'Coco90Dataset' evalute_type = 'Coco90Metric' batch_augments = [ dict(type='BatchFixedSizePad', size=(image_size, image_size)) ] norm_cfg = dict(type='SyncBN', requires_grad=True, eps=1e-3, momentum=0.01) checkpoint = 'https://download.openmmlab.com/mmclassification/v0/efficientnet/efficientnet-b0_3rdparty_8xb32-aa-advprop_in1k_20220119-26434485.pth' # noqa model = dict( type='EfficientDet', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=image_size, batch_augments=batch_augments), backbone=dict( type='EfficientNet', arch='b0', drop_path_rate=0.2, out_indices=(3, 4, 5), frozen_stages=0, norm_cfg=norm_cfg, norm_eval=False, init_cfg=dict( type='Pretrained', prefix='backbone', checkpoint=checkpoint)), neck=dict( type='BiFPN', num_stages=3, in_channels=[40, 112, 320], out_channels=64, start_level=0, norm_cfg=norm_cfg), bbox_head=dict( type='EfficientDetSepBNHead', num_classes=90, num_ins=5, in_channels=64, feat_channels=64, stacked_convs=3, norm_cfg=norm_cfg, anchor_generator=dict( type='YXYXAnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[1.0, 0.5, 2.0], strides=[8, 16, 32, 64, 128], center_offset=0.5), bbox_coder=dict( type='YXYXDeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=0.11, loss_weight=1.0)), # training and testing settings train_cfg=dict( assigner=dict( type='TransMaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0, ignore_iof_thr=-1), sampler=dict( type='PseudoSampler'), # Focal loss should use PseudoSampler allowed_border=-1, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict( type='soft_nms', iou_threshold=0.3, sigma=0.5, min_score=1e-3, method='gaussian'), max_per_img=100)) # dataset settings train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='LoadAnnotations', with_bbox=True), dict( type='RandomResize', scale=(image_size, image_size), ratio_range=(0.1, 2.0), keep_ratio=True), dict(type='RandomCrop', crop_size=(image_size, image_size)), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}), dict(type='Resize', scale=(image_size, image_size), keep_ratio=True), dict(type='LoadAnnotations', with_bbox=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=16, num_workers=16, dataset=dict(type=dataset_type, pipeline=train_pipeline)) val_dataloader = dict(dataset=dict(type=dataset_type, pipeline=test_pipeline)) test_dataloader = val_dataloader val_evaluator = dict(type=evalute_type) test_evaluator = val_evaluator optim_wrapper = dict( optimizer=dict(lr=0.16), paramwise_cfg=dict(norm_decay_mult=0, bypass_duplicate=True)) # learning policy max_epochs = 300 param_scheduler = [ dict(type='LinearLR', start_factor=0.1, by_epoch=False, begin=0, end=917), dict( type='CosineAnnealingLR', eta_min=0.0016, begin=1, T_max=284, end=285, by_epoch=True, convert_to_iter_based=True) ] train_cfg = dict(max_epochs=max_epochs, val_interval=1) vis_backends = [ dict(type='LocalVisBackend'), dict(type='TensorboardVisBackend') ] visualizer = dict( type='DetLocalVisualizer', vis_backends=vis_backends, name='visualizer') default_hooks = dict(checkpoint=dict(type='CheckpointHook', interval=15)) # cudnn_benchmark=True can accelerate fix-size training env_cfg = dict(cudnn_benchmark=True) # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (8 GPUs) x (32 samples per GPU) auto_scale_lr = dict(base_batch_size=128) ================================================ FILE: projects/EfficientDet/convert_tf_to_pt.py ================================================ import argparse import numpy as np import torch from tensorflow.python.training import py_checkpoint_reader torch.set_printoptions(precision=20) def tf2pth(v): if v.ndim == 4: return np.ascontiguousarray(v.transpose(3, 2, 0, 1)) elif v.ndim == 2: return np.ascontiguousarray(v.transpose()) return v def convert_key(model_name, bifpn_repeats, weights): p6_w1 = [ torch.tensor([-1e4, -1e4], dtype=torch.float64) for _ in range(bifpn_repeats) ] p5_w1 = [ torch.tensor([-1e4, -1e4], dtype=torch.float64) for _ in range(bifpn_repeats) ] p4_w1 = [ torch.tensor([-1e4, -1e4], dtype=torch.float64) for _ in range(bifpn_repeats) ] p3_w1 = [ torch.tensor([-1e4, -1e4], dtype=torch.float64) for _ in range(bifpn_repeats) ] p4_w2 = [ torch.tensor([-1e4, -1e4, -1e4], dtype=torch.float64) for _ in range(bifpn_repeats) ] p5_w2 = [ torch.tensor([-1e4, -1e4, -1e4], dtype=torch.float64) for _ in range(bifpn_repeats) ] p6_w2 = [ torch.tensor([-1e4, -1e4, -1e4], dtype=torch.float64) for _ in range(bifpn_repeats) ] p7_w2 = [ torch.tensor([-1e4, -1e4], dtype=torch.float64) for _ in range(bifpn_repeats) ] idx2key = { 0: '1.0', 1: '2.0', 2: '2.1', 3: '3.0', 4: '3.1', 5: '4.0', 6: '4.1', 7: '4.2', 8: '4.3', 9: '4.4', 10: '4.5', 11: '5.0', 12: '5.1', 13: '5.2', 14: '5.3', 15: '5.4' } m = dict() for k, v in weights.items(): if 'Exponential' in k or 'global_step' in k: continue seg = k.split('/') if len(seg) == 1: continue if seg[2] == 'depthwise_conv2d': v = v.transpose(1, 0) if seg[0] == model_name: if seg[1] == 'stem': prefix = 'backbone.layers.0' mapping = { 'conv2d/kernel': 'conv.weight', 'tpu_batch_normalization/beta': 'bn.bias', 'tpu_batch_normalization/gamma': 'bn.weight', 'tpu_batch_normalization/moving_mean': 'bn.running_mean', 'tpu_batch_normalization/moving_variance': 'bn.running_var', } suffix = mapping['/'.join(seg[2:])] m[prefix + '.' + suffix] = v elif seg[1].startswith('blocks_'): idx = int(seg[1][7:]) prefix = '.'.join(['backbone', 'layers', idx2key[idx]]) base_mapping = { 'depthwise_conv2d/depthwise_kernel': 'depthwise_conv.conv.weight', 'se/conv2d/kernel': 'se.conv1.conv.weight', 'se/conv2d/bias': 'se.conv1.conv.bias', 'se/conv2d_1/kernel': 'se.conv2.conv.weight', 'se/conv2d_1/bias': 'se.conv2.conv.bias' } if idx == 0: mapping = { 'conv2d/kernel': 'linear_conv.conv.weight', 'tpu_batch_normalization/beta': 'depthwise_conv.bn.bias', 'tpu_batch_normalization/gamma': 'depthwise_conv.bn.weight', 'tpu_batch_normalization/moving_mean': 'depthwise_conv.bn.running_mean', 'tpu_batch_normalization/moving_variance': 'depthwise_conv.bn.running_var', 'tpu_batch_normalization_1/beta': 'linear_conv.bn.bias', 'tpu_batch_normalization_1/gamma': 'linear_conv.bn.weight', 'tpu_batch_normalization_1/moving_mean': 'linear_conv.bn.running_mean', 'tpu_batch_normalization_1/moving_variance': 'linear_conv.bn.running_var', } else: mapping = { 'depthwise_conv2d/depthwise_kernel': 'depthwise_conv.conv.weight', 'conv2d/kernel': 'expand_conv.conv.weight', 'conv2d_1/kernel': 'linear_conv.conv.weight', 'tpu_batch_normalization/beta': 'expand_conv.bn.bias', 'tpu_batch_normalization/gamma': 'expand_conv.bn.weight', 'tpu_batch_normalization/moving_mean': 'expand_conv.bn.running_mean', 'tpu_batch_normalization/moving_variance': 'expand_conv.bn.running_var', 'tpu_batch_normalization_1/beta': 'depthwise_conv.bn.bias', 'tpu_batch_normalization_1/gamma': 'depthwise_conv.bn.weight', 'tpu_batch_normalization_1/moving_mean': 'depthwise_conv.bn.running_mean', 'tpu_batch_normalization_1/moving_variance': 'depthwise_conv.bn.running_var', 'tpu_batch_normalization_2/beta': 'linear_conv.bn.bias', 'tpu_batch_normalization_2/gamma': 'linear_conv.bn.weight', 'tpu_batch_normalization_2/moving_mean': 'linear_conv.bn.running_mean', 'tpu_batch_normalization_2/moving_variance': 'linear_conv.bn.running_var', } mapping.update(base_mapping) suffix = mapping['/'.join(seg[2:])] m[prefix + '.' + suffix] = v elif seg[0] == 'resample_p6': prefix = 'neck.bifpn.0.p5_to_p6.0' mapping = { 'conv2d/kernel': 'down_conv.conv.weight', 'conv2d/bias': 'down_conv.conv.bias', 'bn/beta': 'bn.bias', 'bn/gamma': 'bn.weight', 'bn/moving_mean': 'bn.running_mean', 'bn/moving_variance': 'bn.running_var', } suffix = mapping['/'.join(seg[1:])] m[prefix + '.' + suffix] = v elif seg[0] == 'fpn_cells': fpn_idx = int(seg[1][5:]) prefix = '.'.join(['neck', 'bifpn', str(fpn_idx)]) fnode_id = int(seg[2][5]) if fnode_id == 0: mapping = { 'op_after_combine5/conv/depthwise_kernel': 'conv6_up.depthwise_conv.conv.weight', 'op_after_combine5/conv/pointwise_kernel': 'conv6_up.pointwise_conv.conv.weight', 'op_after_combine5/conv/bias': 'conv6_up.pointwise_conv.conv.bias', 'op_after_combine5/bn/beta': 'conv6_up.bn.bias', 'op_after_combine5/bn/gamma': 'conv6_up.bn.weight', 'op_after_combine5/bn/moving_mean': 'conv6_up.bn.running_mean', 'op_after_combine5/bn/moving_variance': 'conv6_up.bn.running_var', } if seg[3] != 'WSM' and seg[3] != 'WSM_1': suffix = mapping['/'.join(seg[3:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif seg[3] == 'WSM': p6_w1[fpn_idx][0] = v elif seg[3] == 'WSM_1': p6_w1[fpn_idx][1] = v if torch.min(p6_w1[fpn_idx]) > -1e4: m[prefix + '.p6_w1'] = p6_w1[fpn_idx] elif fnode_id == 1: base_mapping = { 'op_after_combine6/conv/depthwise_kernel': 'conv5_up.depthwise_conv.conv.weight', 'op_after_combine6/conv/pointwise_kernel': 'conv5_up.pointwise_conv.conv.weight', 'op_after_combine6/conv/bias': 'conv5_up.pointwise_conv.conv.bias', 'op_after_combine6/bn/beta': 'conv5_up.bn.bias', 'op_after_combine6/bn/gamma': 'conv5_up.bn.weight', 'op_after_combine6/bn/moving_mean': 'conv5_up.bn.running_mean', 'op_after_combine6/bn/moving_variance': 'conv5_up.bn.running_var', } if fpn_idx == 0: mapping = { 'resample_0_2_6/conv2d/kernel': 'p5_down_channel.down_conv.conv.weight', 'resample_0_2_6/conv2d/bias': 'p5_down_channel.down_conv.conv.bias', 'resample_0_2_6/bn/beta': 'p5_down_channel.bn.bias', 'resample_0_2_6/bn/gamma': 'p5_down_channel.bn.weight', 'resample_0_2_6/bn/moving_mean': 'p5_down_channel.bn.running_mean', 'resample_0_2_6/bn/moving_variance': 'p5_down_channel.bn.running_var', } base_mapping.update(mapping) if seg[3] != 'WSM' and seg[3] != 'WSM_1': suffix = base_mapping['/'.join(seg[3:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif seg[3] == 'WSM': p5_w1[fpn_idx][0] = v elif seg[3] == 'WSM_1': p5_w1[fpn_idx][1] = v if torch.min(p5_w1[fpn_idx]) > -1e4: m[prefix + '.p5_w1'] = p5_w1[fpn_idx] elif fnode_id == 2: base_mapping = { 'op_after_combine7/conv/depthwise_kernel': 'conv4_up.depthwise_conv.conv.weight', 'op_after_combine7/conv/pointwise_kernel': 'conv4_up.pointwise_conv.conv.weight', 'op_after_combine7/conv/bias': 'conv4_up.pointwise_conv.conv.bias', 'op_after_combine7/bn/beta': 'conv4_up.bn.bias', 'op_after_combine7/bn/gamma': 'conv4_up.bn.weight', 'op_after_combine7/bn/moving_mean': 'conv4_up.bn.running_mean', 'op_after_combine7/bn/moving_variance': 'conv4_up.bn.running_var', } if fpn_idx == 0: mapping = { 'resample_0_1_7/conv2d/kernel': 'p4_down_channel.down_conv.conv.weight', 'resample_0_1_7/conv2d/bias': 'p4_down_channel.down_conv.conv.bias', 'resample_0_1_7/bn/beta': 'p4_down_channel.bn.bias', 'resample_0_1_7/bn/gamma': 'p4_down_channel.bn.weight', 'resample_0_1_7/bn/moving_mean': 'p4_down_channel.bn.running_mean', 'resample_0_1_7/bn/moving_variance': 'p4_down_channel.bn.running_var', } base_mapping.update(mapping) if seg[3] != 'WSM' and seg[3] != 'WSM_1': suffix = base_mapping['/'.join(seg[3:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif seg[3] == 'WSM': p4_w1[fpn_idx][0] = v elif seg[3] == 'WSM_1': p4_w1[fpn_idx][1] = v if torch.min(p4_w1[fpn_idx]) > -1e4: m[prefix + '.p4_w1'] = p4_w1[fpn_idx] elif fnode_id == 3: base_mapping = { 'op_after_combine8/conv/depthwise_kernel': 'conv3_up.depthwise_conv.conv.weight', 'op_after_combine8/conv/pointwise_kernel': 'conv3_up.pointwise_conv.conv.weight', 'op_after_combine8/conv/bias': 'conv3_up.pointwise_conv.conv.bias', 'op_after_combine8/bn/beta': 'conv3_up.bn.bias', 'op_after_combine8/bn/gamma': 'conv3_up.bn.weight', 'op_after_combine8/bn/moving_mean': 'conv3_up.bn.running_mean', 'op_after_combine8/bn/moving_variance': 'conv3_up.bn.running_var', } if fpn_idx == 0: mapping = { 'resample_0_0_8/conv2d/kernel': 'p3_down_channel.down_conv.conv.weight', 'resample_0_0_8/conv2d/bias': 'p3_down_channel.down_conv.conv.bias', 'resample_0_0_8/bn/beta': 'p3_down_channel.bn.bias', 'resample_0_0_8/bn/gamma': 'p3_down_channel.bn.weight', 'resample_0_0_8/bn/moving_mean': 'p3_down_channel.bn.running_mean', 'resample_0_0_8/bn/moving_variance': 'p3_down_channel.bn.running_var', } base_mapping.update(mapping) if seg[3] != 'WSM' and seg[3] != 'WSM_1': suffix = base_mapping['/'.join(seg[3:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif seg[3] == 'WSM': p3_w1[fpn_idx][0] = v elif seg[3] == 'WSM_1': p3_w1[fpn_idx][1] = v if torch.min(p3_w1[fpn_idx]) > -1e4: m[prefix + '.p3_w1'] = p3_w1[fpn_idx] elif fnode_id == 4: base_mapping = { 'op_after_combine9/conv/depthwise_kernel': 'conv4_down.depthwise_conv.conv.weight', 'op_after_combine9/conv/pointwise_kernel': 'conv4_down.pointwise_conv.conv.weight', 'op_after_combine9/conv/bias': 'conv4_down.pointwise_conv.conv.bias', 'op_after_combine9/bn/beta': 'conv4_down.bn.bias', 'op_after_combine9/bn/gamma': 'conv4_down.bn.weight', 'op_after_combine9/bn/moving_mean': 'conv4_down.bn.running_mean', 'op_after_combine9/bn/moving_variance': 'conv4_down.bn.running_var', } if fpn_idx == 0: mapping = { 'resample_0_1_9/conv2d/kernel': 'p4_level_connection.down_conv.conv.weight', 'resample_0_1_9/conv2d/bias': 'p4_level_connection.down_conv.conv.bias', 'resample_0_1_9/bn/beta': 'p4_level_connection.bn.bias', 'resample_0_1_9/bn/gamma': 'p4_level_connection.bn.weight', 'resample_0_1_9/bn/moving_mean': 'p4_level_connection.bn.running_mean', 'resample_0_1_9/bn/moving_variance': 'p4_level_connection.bn.running_var', } base_mapping.update(mapping) if seg[3] != 'WSM' and seg[3] != 'WSM_1' and seg[3] != 'WSM_2': suffix = base_mapping['/'.join(seg[3:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif seg[3] == 'WSM': p4_w2[fpn_idx][0] = v elif seg[3] == 'WSM_1': p4_w2[fpn_idx][1] = v elif seg[3] == 'WSM_2': p4_w2[fpn_idx][2] = v if torch.min(p4_w2[fpn_idx]) > -1e4: m[prefix + '.p4_w2'] = p4_w2[fpn_idx] elif fnode_id == 5: base_mapping = { 'op_after_combine10/conv/depthwise_kernel': 'conv5_down.depthwise_conv.conv.weight', 'op_after_combine10/conv/pointwise_kernel': 'conv5_down.pointwise_conv.conv.weight', 'op_after_combine10/conv/bias': 'conv5_down.pointwise_conv.conv.bias', 'op_after_combine10/bn/beta': 'conv5_down.bn.bias', 'op_after_combine10/bn/gamma': 'conv5_down.bn.weight', 'op_after_combine10/bn/moving_mean': 'conv5_down.bn.running_mean', 'op_after_combine10/bn/moving_variance': 'conv5_down.bn.running_var', } if fpn_idx == 0: mapping = { 'resample_0_2_10/conv2d/kernel': 'p5_level_connection.down_conv.conv.weight', 'resample_0_2_10/conv2d/bias': 'p5_level_connection.down_conv.conv.bias', 'resample_0_2_10/bn/beta': 'p5_level_connection.bn.bias', 'resample_0_2_10/bn/gamma': 'p5_level_connection.bn.weight', 'resample_0_2_10/bn/moving_mean': 'p5_level_connection.bn.running_mean', 'resample_0_2_10/bn/moving_variance': 'p5_level_connection.bn.running_var', } base_mapping.update(mapping) if seg[3] != 'WSM' and seg[3] != 'WSM_1' and seg[3] != 'WSM_2': suffix = base_mapping['/'.join(seg[3:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif seg[3] == 'WSM': p5_w2[fpn_idx][0] = v elif seg[3] == 'WSM_1': p5_w2[fpn_idx][1] = v elif seg[3] == 'WSM_2': p5_w2[fpn_idx][2] = v if torch.min(p5_w2[fpn_idx]) > -1e4: m[prefix + '.p5_w2'] = p5_w2[fpn_idx] elif fnode_id == 6: base_mapping = { 'op_after_combine11/conv/depthwise_kernel': 'conv6_down.depthwise_conv.conv.weight', 'op_after_combine11/conv/pointwise_kernel': 'conv6_down.pointwise_conv.conv.weight', 'op_after_combine11/conv/bias': 'conv6_down.pointwise_conv.conv.bias', 'op_after_combine11/bn/beta': 'conv6_down.bn.bias', 'op_after_combine11/bn/gamma': 'conv6_down.bn.weight', 'op_after_combine11/bn/moving_mean': 'conv6_down.bn.running_mean', 'op_after_combine11/bn/moving_variance': 'conv6_down.bn.running_var', } if seg[3] != 'WSM' and seg[3] != 'WSM_1' and seg[3] != 'WSM_2': suffix = base_mapping['/'.join(seg[3:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif seg[3] == 'WSM': p6_w2[fpn_idx][0] = v elif seg[3] == 'WSM_1': p6_w2[fpn_idx][1] = v elif seg[3] == 'WSM_2': p6_w2[fpn_idx][2] = v if torch.min(p6_w2[fpn_idx]) > -1e4: m[prefix + '.p6_w2'] = p6_w2[fpn_idx] elif fnode_id == 7: base_mapping = { 'op_after_combine12/conv/depthwise_kernel': 'conv7_down.depthwise_conv.conv.weight', 'op_after_combine12/conv/pointwise_kernel': 'conv7_down.pointwise_conv.conv.weight', 'op_after_combine12/conv/bias': 'conv7_down.pointwise_conv.conv.bias', 'op_after_combine12/bn/beta': 'conv7_down.bn.bias', 'op_after_combine12/bn/gamma': 'conv7_down.bn.weight', 'op_after_combine12/bn/moving_mean': 'conv7_down.bn.running_mean', 'op_after_combine12/bn/moving_variance': 'conv7_down.bn.running_var', } if seg[3] != 'WSM' and seg[3] != 'WSM_1' and seg[3] != 'WSM_2': suffix = base_mapping['/'.join(seg[3:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif seg[3] == 'WSM': p7_w2[fpn_idx][0] = v elif seg[3] == 'WSM_1': p7_w2[fpn_idx][1] = v if torch.min(p7_w2[fpn_idx]) > -1e4: m[prefix + '.p7_w2'] = p7_w2[fpn_idx] elif seg[0] == 'box_net': if 'box-predict' in seg[1]: prefix = '.'.join(['bbox_head', 'reg_header']) base_mapping = { 'depthwise_kernel': 'depthwise_conv.conv.weight', 'pointwise_kernel': 'pointwise_conv.conv.weight', 'bias': 'pointwise_conv.conv.bias' } suffix = base_mapping['/'.join(seg[2:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif 'bn' in seg[1]: bbox_conv_idx = int(seg[1][4]) bbox_bn_idx = int(seg[1][9]) - 3 prefix = '.'.join([ 'bbox_head', 'reg_bn_list', str(bbox_conv_idx), str(bbox_bn_idx) ]) base_mapping = { 'beta': 'bias', 'gamma': 'weight', 'moving_mean': 'running_mean', 'moving_variance': 'running_var' } suffix = base_mapping['/'.join(seg[2:])] m[prefix + '.' + suffix] = v else: bbox_conv_idx = int(seg[1][4]) prefix = '.'.join( ['bbox_head', 'reg_conv_list', str(bbox_conv_idx)]) base_mapping = { 'depthwise_kernel': 'depthwise_conv.conv.weight', 'pointwise_kernel': 'pointwise_conv.conv.weight', 'bias': 'pointwise_conv.conv.bias' } suffix = base_mapping['/'.join(seg[2:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif seg[0] == 'class_net': if 'class-predict' in seg[1]: prefix = '.'.join(['bbox_head', 'cls_header']) base_mapping = { 'depthwise_kernel': 'depthwise_conv.conv.weight', 'pointwise_kernel': 'pointwise_conv.conv.weight', 'bias': 'pointwise_conv.conv.bias' } suffix = base_mapping['/'.join(seg[2:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v elif 'bn' in seg[1]: cls_conv_idx = int(seg[1][6]) cls_bn_idx = int(seg[1][11]) - 3 prefix = '.'.join([ 'bbox_head', 'cls_bn_list', str(cls_conv_idx), str(cls_bn_idx) ]) base_mapping = { 'beta': 'bias', 'gamma': 'weight', 'moving_mean': 'running_mean', 'moving_variance': 'running_var' } suffix = base_mapping['/'.join(seg[2:])] m[prefix + '.' + suffix] = v else: cls_conv_idx = int(seg[1][6]) prefix = '.'.join( ['bbox_head', 'cls_conv_list', str(cls_conv_idx)]) base_mapping = { 'depthwise_kernel': 'depthwise_conv.conv.weight', 'pointwise_kernel': 'pointwise_conv.conv.weight', 'bias': 'pointwise_conv.conv.bias' } suffix = base_mapping['/'.join(seg[2:])] if 'depthwise_conv' in suffix: v = v.transpose(1, 0) m[prefix + '.' + suffix] = v return m def parse_args(): parser = argparse.ArgumentParser( description='convert efficientdet weight from tensorflow to pytorch') parser.add_argument( '--backbone', type=str, help='efficientnet model name, like efficientnet-b0') parser.add_argument( '--tensorflow_weight', type=str, help='efficientdet tensorflow weight name, like efficientdet-d0/model') parser.add_argument( '--out_weight', type=str, help='efficientdet pytorch weight name like demo.pth') args = parser.parse_args() return args def main(): args = parse_args() model_name = args.backbone ori_weight_name = args.tensorflow_weight out_name = args.out_weight repeat_map = { 0: 3, 1: 4, 2: 5, 3: 6, 4: 7, 5: 7, 6: 8, 7: 8, } reader = py_checkpoint_reader.NewCheckpointReader(ori_weight_name) weights = { n: torch.as_tensor(tf2pth(reader.get_tensor(n))) for (n, _) in reader.get_variable_to_shape_map().items() } print(weights.keys()) bifpn_repeats = repeat_map[int(model_name[14])] out = convert_key(model_name, bifpn_repeats, weights) result = {'state_dict': out} torch.save(result, out_name) if __name__ == '__main__': main() ================================================ FILE: projects/EfficientDet/efficientdet/__init__.py ================================================ from .anchor_generator import YXYXAnchorGenerator from .bifpn import BiFPN from .coco_90class import Coco90Dataset from .coco_90metric import Coco90Metric from .efficientdet import EfficientDet from .efficientdet_head import EfficientDetSepBNHead from .trans_max_iou_assigner import TransMaxIoUAssigner from .yxyx_bbox_coder import YXYXDeltaXYWHBBoxCoder __all__ = [ 'EfficientDet', 'BiFPN', 'EfficientDetSepBNHead', 'YXYXAnchorGenerator', 'YXYXDeltaXYWHBBoxCoder', 'Coco90Dataset', 'Coco90Metric', 'TransMaxIoUAssigner' ] ================================================ FILE: projects/EfficientDet/efficientdet/anchor_generator.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional, Tuple, Union import torch from torch import Tensor from mmdet.models.task_modules.prior_generators.anchor_generator import \ AnchorGenerator from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import HorizontalBoxes DeviceType = Union[str, torch.device] @TASK_UTILS.register_module() class YXYXAnchorGenerator(AnchorGenerator): def gen_single_level_base_anchors(self, base_size: Union[int, float], scales: Tensor, ratios: Tensor, center: Optional[Tuple[float]] = None) \ -> Tensor: """Generate base anchors of a single level. Args: base_size (int | float): Basic size of an anchor. scales (torch.Tensor): Scales of the anchor. ratios (torch.Tensor): The ratio between the height and width of anchors in a single level. center (tuple[float], optional): The center of the base anchor related to a single feature grid. Defaults to None. Returns: torch.Tensor: Anchors in a single-level feature maps. """ w = base_size h = base_size if center is None: x_center = self.center_offset * w y_center = self.center_offset * h else: x_center, y_center = center h_ratios = torch.sqrt(ratios) w_ratios = 1 / h_ratios if self.scale_major: ws = (w * scales[:, None] * w_ratios[None, :]).view(-1) hs = (h * scales[:, None] * h_ratios[None, :]).view(-1) else: ws = (w * scales[:, None] * w_ratios[None, :]).view(-1) hs = (h * scales[:, None] * h_ratios[None, :]).view(-1) # use float anchor and the anchor's center is aligned with the # pixel center base_anchors = [ y_center - 0.5 * hs, x_center - 0.5 * ws, y_center + 0.5 * hs, x_center + 0.5 * ws, ] base_anchors = torch.stack(base_anchors, dim=-1) return base_anchors def single_level_grid_priors(self, featmap_size: Tuple[int, int], level_idx: int, dtype: torch.dtype = torch.float32, device: DeviceType = 'cuda') -> Tensor: """Generate grid anchors of a single level. Note: This function is usually called by method ``self.grid_priors``. Args: featmap_size (tuple[int, int]): Size of the feature maps. level_idx (int): The index of corresponding feature map level. dtype (obj:`torch.dtype`): Date type of points.Defaults to ``torch.float32``. device (str | torch.device): The device the tensor will be put on. Defaults to 'cuda'. Returns: torch.Tensor: Anchors in the overall feature maps. """ base_anchors = self.base_anchors[level_idx].to(device).to(dtype) feat_h, feat_w = featmap_size stride_w, stride_h = self.strides[level_idx] # First create Range with the default dtype, than convert to # target `dtype` for onnx exporting. shift_x = torch.arange(0, feat_w, device=device).to(dtype) * stride_w shift_y = torch.arange(0, feat_h, device=device).to(dtype) * stride_h shift_xx, shift_yy = self._meshgrid(shift_x, shift_y) shifts = torch.stack([shift_yy, shift_xx, shift_yy, shift_xx], dim=-1) # first feat_w elements correspond to the first row of shifts # add A anchors (1, A, 4) to K shifts (K, 1, 4) to get # shifted anchors (K, A, 4), reshape to (K*A, 4) all_anchors = base_anchors[None, :, :] + shifts[:, None, :] all_anchors = all_anchors.view(-1, 4) # first A rows correspond to A anchors of (0, 0) in feature map, # then (0, 1), (0, 2), ... if self.use_box_type: all_anchors = HorizontalBoxes(all_anchors) return all_anchors ================================================ FILE: projects/EfficientDet/efficientdet/api_wrappers/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .coco_api import COCO, COCOeval, COCOPanoptic __all__ = ['COCO', 'COCOeval', 'COCOPanoptic'] ================================================ FILE: projects/EfficientDet/efficientdet/api_wrappers/coco_api.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # This file add snake case alias for coco api import warnings from collections import defaultdict from typing import List, Optional, Union import pycocotools from pycocotools.coco import COCO as _COCO from pycocotools.cocoeval import COCOeval as _COCOeval class COCO(_COCO): """This class is almost the same as official pycocotools package. It implements some snake case function aliases. So that the COCO class has the same interface as LVIS class. """ def __init__(self, annotation_file=None): if getattr(pycocotools, '__version__', '0') >= '12.0.2': warnings.warn( 'mmpycocotools is deprecated. Please install official pycocotools by "pip install pycocotools"', # noqa: E501 UserWarning) super().__init__(annotation_file=annotation_file) self.img_ann_map = self.imgToAnns self.cat_img_map = self.catToImgs def get_ann_ids(self, img_ids=[], cat_ids=[], area_rng=[], iscrowd=None): return self.getAnnIds(img_ids, cat_ids, area_rng, iscrowd) def get_cat_ids(self, cat_names=[], sup_names=[], cat_ids=[]): # return self.getCatIds(cat_names, sup_names, cat_ids) cat_ids_coco = self.getCatIds(cat_names, sup_names, cat_ids) if None in cat_names: index = [i for i, v in enumerate(cat_names) if v is not None] cat_ids = list(range(len(cat_names))) for i in range(len(index)): cat_ids[index[i]] = cat_ids_coco[i] return cat_ids else: return cat_ids_coco def get_img_ids(self, img_ids=[], cat_ids=[]): return self.getImgIds(img_ids, cat_ids) def load_anns(self, ids): return self.loadAnns(ids) def load_cats(self, ids): return self.loadCats(ids) def load_imgs(self, ids): return self.loadImgs(ids) # just for the ease of import COCOeval = _COCOeval class COCOPanoptic(COCO): """This wrapper is for loading the panoptic style annotation file. The format is shown in the CocoPanopticDataset class. Args: annotation_file (str, optional): Path of annotation file. Defaults to None. """ def __init__(self, annotation_file: Optional[str] = None) -> None: super(COCOPanoptic, self).__init__(annotation_file) def createIndex(self) -> None: """Create index.""" # create index print('creating index...') # anns stores 'segment_id -> annotation' anns, cats, imgs = {}, {}, {} img_to_anns, cat_to_imgs = defaultdict(list), defaultdict(list) if 'annotations' in self.dataset: for ann in self.dataset['annotations']: for seg_ann in ann['segments_info']: # to match with instance.json seg_ann['image_id'] = ann['image_id'] img_to_anns[ann['image_id']].append(seg_ann) # segment_id is not unique in coco dataset orz... # annotations from different images but # may have same segment_id if seg_ann['id'] in anns.keys(): anns[seg_ann['id']].append(seg_ann) else: anns[seg_ann['id']] = [seg_ann] # filter out annotations from other images img_to_anns_ = defaultdict(list) for k, v in img_to_anns.items(): img_to_anns_[k] = [x for x in v if x['image_id'] == k] img_to_anns = img_to_anns_ if 'images' in self.dataset: for img_info in self.dataset['images']: img_info['segm_file'] = img_info['file_name'].replace( 'jpg', 'png') imgs[img_info['id']] = img_info if 'categories' in self.dataset: for cat in self.dataset['categories']: cats[cat['id']] = cat if 'annotations' in self.dataset and 'categories' in self.dataset: for ann in self.dataset['annotations']: for seg_ann in ann['segments_info']: cat_to_imgs[seg_ann['category_id']].append(ann['image_id']) print('index created!') self.anns = anns self.imgToAnns = img_to_anns self.catToImgs = cat_to_imgs self.imgs = imgs self.cats = cats def load_anns(self, ids: Union[List[int], int] = []) -> Optional[List[dict]]: """Load anns with the specified ids. ``self.anns`` is a list of annotation lists instead of a list of annotations. Args: ids (Union[List[int], int]): Integer ids specifying anns. Returns: anns (List[dict], optional): Loaded ann objects. """ anns = [] if hasattr(ids, '__iter__') and hasattr(ids, '__len__'): # self.anns is a list of annotation lists instead of # a list of annotations for id in ids: anns += self.anns[id] return anns elif type(ids) == int: return self.anns[ids] ================================================ FILE: projects/EfficientDet/efficientdet/bifpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. # Modified from https://github.com/zylo117/Yet-Another-EfficientDet-Pytorch from typing import List import torch import torch.nn as nn from mmcv.cnn.bricks import Swish from mmengine.model import BaseModule from mmdet.registry import MODELS from mmdet.utils import MultiConfig, OptConfigType from .utils import (DepthWiseConvBlock, DownChannelBlock, MaxPool2dSamePadding, MemoryEfficientSwish) class BiFPNStage(nn.Module): ''' in_channels: List[int], input dim for P3, P4, P5 out_channels: int, output dim for P2 - P7 first_time: int, whether is the first bifpnstage num_outs: int, BiFPN need feature maps num use_swish: whether use MemoryEfficientSwish norm_cfg: (:obj:`ConfigDict` or dict, optional): Config dict for normalization layer. epsilon: float, hyperparameter in fusion features ''' def __init__(self, in_channels: List[int], out_channels: int, first_time: bool = False, apply_bn_for_resampling: bool = True, conv_bn_act_pattern: bool = False, use_meswish: bool = True, norm_cfg: OptConfigType = dict( type='BN', momentum=1e-2, eps=1e-3), epsilon: float = 1e-4) -> None: super().__init__() assert isinstance(in_channels, list) self.in_channels = in_channels self.out_channels = out_channels self.first_time = first_time self.apply_bn_for_resampling = apply_bn_for_resampling self.conv_bn_act_pattern = conv_bn_act_pattern self.use_meswish = use_meswish self.norm_cfg = norm_cfg self.epsilon = epsilon if self.first_time: self.p5_down_channel = DownChannelBlock( self.in_channels[-1], self.out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.p4_down_channel = DownChannelBlock( self.in_channels[-2], self.out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.p3_down_channel = DownChannelBlock( self.in_channels[-3], self.out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.p5_to_p6 = nn.Sequential( DownChannelBlock( self.in_channels[-1], self.out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg), MaxPool2dSamePadding(3, 2)) self.p6_to_p7 = MaxPool2dSamePadding(3, 2) self.p4_level_connection = DownChannelBlock( self.in_channels[-2], self.out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.p5_level_connection = DownChannelBlock( self.in_channels[-1], self.out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.p6_upsample = nn.Upsample(scale_factor=2, mode='nearest') self.p5_upsample = nn.Upsample(scale_factor=2, mode='nearest') self.p4_upsample = nn.Upsample(scale_factor=2, mode='nearest') self.p3_upsample = nn.Upsample(scale_factor=2, mode='nearest') # bottom to up: feature map down_sample module self.p4_down_sample = MaxPool2dSamePadding(3, 2) self.p5_down_sample = MaxPool2dSamePadding(3, 2) self.p6_down_sample = MaxPool2dSamePadding(3, 2) self.p7_down_sample = MaxPool2dSamePadding(3, 2) # Fuse Conv Layers self.conv6_up = DepthWiseConvBlock( out_channels, out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.conv5_up = DepthWiseConvBlock( out_channels, out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.conv4_up = DepthWiseConvBlock( out_channels, out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.conv3_up = DepthWiseConvBlock( out_channels, out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.conv4_down = DepthWiseConvBlock( out_channels, out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.conv5_down = DepthWiseConvBlock( out_channels, out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.conv6_down = DepthWiseConvBlock( out_channels, out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) self.conv7_down = DepthWiseConvBlock( out_channels, out_channels, apply_norm=self.apply_bn_for_resampling, conv_bn_act_pattern=self.conv_bn_act_pattern, norm_cfg=norm_cfg) # weights self.p6_w1 = nn.Parameter( torch.ones(2, dtype=torch.float32), requires_grad=True) self.p6_w1_relu = nn.ReLU() self.p5_w1 = nn.Parameter( torch.ones(2, dtype=torch.float32), requires_grad=True) self.p5_w1_relu = nn.ReLU() self.p4_w1 = nn.Parameter( torch.ones(2, dtype=torch.float32), requires_grad=True) self.p4_w1_relu = nn.ReLU() self.p3_w1 = nn.Parameter( torch.ones(2, dtype=torch.float32), requires_grad=True) self.p3_w1_relu = nn.ReLU() self.p4_w2 = nn.Parameter( torch.ones(3, dtype=torch.float32), requires_grad=True) self.p4_w2_relu = nn.ReLU() self.p5_w2 = nn.Parameter( torch.ones(3, dtype=torch.float32), requires_grad=True) self.p5_w2_relu = nn.ReLU() self.p6_w2 = nn.Parameter( torch.ones(3, dtype=torch.float32), requires_grad=True) self.p6_w2_relu = nn.ReLU() self.p7_w2 = nn.Parameter( torch.ones(2, dtype=torch.float32), requires_grad=True) self.p7_w2_relu = nn.ReLU() self.swish = MemoryEfficientSwish() if use_meswish else Swish() def combine(self, x): if not self.conv_bn_act_pattern: x = self.swish(x) return x def forward(self, x): if self.first_time: p3, p4, p5 = x # build feature map P6 p6_in = self.p5_to_p6(p5) # build feature map P7 p7_in = self.p6_to_p7(p6_in) p3_in = self.p3_down_channel(p3) p4_in = self.p4_down_channel(p4) p5_in = self.p5_down_channel(p5) else: p3_in, p4_in, p5_in, p6_in, p7_in = x # Weights for P6_0 and P7_0 to P6_1 p6_w1 = self.p6_w1_relu(self.p6_w1) weight = p6_w1 / (torch.sum(p6_w1, dim=0) + self.epsilon) # Connections for P6_0 and P7_0 to P6_1 respectively p6_up = self.conv6_up( self.combine(weight[0] * p6_in + weight[1] * self.p6_upsample(p7_in))) # Weights for P5_0 and P6_1 to P5_1 p5_w1 = self.p5_w1_relu(self.p5_w1) weight = p5_w1 / (torch.sum(p5_w1, dim=0) + self.epsilon) # Connections for P5_0 and P6_1 to P5_1 respectively p5_up = self.conv5_up( self.combine(weight[0] * p5_in + weight[1] * self.p5_upsample(p6_up))) # Weights for P4_0 and P5_1 to P4_1 p4_w1 = self.p4_w1_relu(self.p4_w1) weight = p4_w1 / (torch.sum(p4_w1, dim=0) + self.epsilon) # Connections for P4_0 and P5_1 to P4_1 respectively p4_up = self.conv4_up( self.combine(weight[0] * p4_in + weight[1] * self.p4_upsample(p5_up))) # Weights for P3_0 and P4_1 to P3_2 p3_w1 = self.p3_w1_relu(self.p3_w1) weight = p3_w1 / (torch.sum(p3_w1, dim=0) + self.epsilon) # Connections for P3_0 and P4_1 to P3_2 respectively p3_out = self.conv3_up( self.combine(weight[0] * p3_in + weight[1] * self.p3_upsample(p4_up))) if self.first_time: p4_in = self.p4_level_connection(p4) p5_in = self.p5_level_connection(p5) # Weights for P4_0, P4_1 and P3_2 to P4_2 p4_w2 = self.p4_w2_relu(self.p4_w2) weight = p4_w2 / (torch.sum(p4_w2, dim=0) + self.epsilon) # Connections for P4_0, P4_1 and P3_2 to P4_2 respectively p4_out = self.conv4_down( self.combine(weight[0] * p4_in + weight[1] * p4_up + weight[2] * self.p4_down_sample(p3_out))) # Weights for P5_0, P5_1 and P4_2 to P5_2 p5_w2 = self.p5_w2_relu(self.p5_w2) weight = p5_w2 / (torch.sum(p5_w2, dim=0) + self.epsilon) # Connections for P5_0, P5_1 and P4_2 to P5_2 respectively p5_out = self.conv5_down( self.combine(weight[0] * p5_in + weight[1] * p5_up + weight[2] * self.p5_down_sample(p4_out))) # Weights for P6_0, P6_1 and P5_2 to P6_2 p6_w2 = self.p6_w2_relu(self.p6_w2) weight = p6_w2 / (torch.sum(p6_w2, dim=0) + self.epsilon) # Connections for P6_0, P6_1 and P5_2 to P6_2 respectively p6_out = self.conv6_down( self.combine(weight[0] * p6_in + weight[1] * p6_up + weight[2] * self.p6_down_sample(p5_out))) # Weights for P7_0 and P6_2 to P7_2 p7_w2 = self.p7_w2_relu(self.p7_w2) weight = p7_w2 / (torch.sum(p7_w2, dim=0) + self.epsilon) # Connections for P7_0 and P6_2 to P7_2 p7_out = self.conv7_down( self.combine(weight[0] * p7_in + weight[1] * self.p7_down_sample(p6_out))) return p3_out, p4_out, p5_out, p6_out, p7_out @MODELS.register_module() class BiFPN(BaseModule): ''' num_stages: int, bifpn number of repeats in_channels: List[int], input dim for P3, P4, P5 out_channels: int, output dim for P2 - P7 start_level: int, Index of input features in backbone epsilon: float, hyperparameter in fusion features apply_bn_for_resampling: bool, whether use bn after resampling conv_bn_act_pattern: bool, whether use conv_bn_act_pattern use_swish: whether use MemoryEfficientSwish norm_cfg: (:obj:`ConfigDict` or dict, optional): Config dict for normalization layer. init_cfg: MultiConfig: init method ''' def __init__(self, num_stages: int, in_channels: List[int], out_channels: int, start_level: int = 0, epsilon: float = 1e-4, apply_bn_for_resampling: bool = True, conv_bn_act_pattern: bool = False, use_meswish: bool = True, norm_cfg: OptConfigType = dict( type='BN', momentum=1e-2, eps=1e-3), init_cfg: MultiConfig = None) -> None: super().__init__(init_cfg=init_cfg) self.start_level = start_level self.bifpn = nn.Sequential(*[ BiFPNStage( in_channels=in_channels, out_channels=out_channels, first_time=True if _ == 0 else False, apply_bn_for_resampling=apply_bn_for_resampling, conv_bn_act_pattern=conv_bn_act_pattern, use_meswish=use_meswish, norm_cfg=norm_cfg, epsilon=epsilon) for _ in range(num_stages) ]) def forward(self, x): x = x[self.start_level:] x = self.bifpn(x) return x ================================================ FILE: projects/EfficientDet/efficientdet/coco_90class.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import os.path as osp from typing import List, Union from mmdet.datasets.base_det_dataset import BaseDetDataset from mmdet.registry import DATASETS from .api_wrappers import COCO @DATASETS.register_module() class Coco90Dataset(BaseDetDataset): """Dataset for COCO.""" METAINFO = { 'classes': ('person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', 'truck', 'boat', 'traffic light', 'fire hydrant', None, 'stop sign', 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', 'elephant', 'bear', 'zebra', 'giraffe', None, 'backpack', 'umbrella', None, None, 'handbag', 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', 'bottle', None, 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', None, 'dining table', None, None, 'toilet', None, 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', None, 'book', 'clock', 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush'), # palette is a list of color tuples, which is used for visualization. 'palette': [(220, 20, 60), (119, 11, 32), (0, 0, 142), (0, 0, 230), (106, 0, 228), (0, 60, 100), (0, 80, 100), (0, 0, 70), (0, 0, 192), (250, 170, 30), (100, 170, 30), None, (220, 220, 0), (175, 116, 175), (250, 0, 30), (165, 42, 42), (255, 77, 255), (0, 226, 252), (182, 182, 255), (0, 82, 0), (120, 166, 157), (110, 76, 0), (174, 57, 255), (199, 100, 0), (72, 0, 118), None, (255, 179, 240), (0, 125, 92), None, None, (209, 0, 151), (188, 208, 182), (0, 220, 176), (255, 99, 164), (92, 0, 73), (133, 129, 255), (78, 180, 255), (0, 228, 0), (174, 255, 243), (45, 89, 255), (134, 134, 103), (145, 148, 174), (255, 208, 186), (197, 226, 255), None, (171, 134, 1), (109, 63, 54), (207, 138, 255), (151, 0, 95), (9, 80, 61), (84, 105, 51), (74, 65, 105), (166, 196, 102), (208, 195, 210), (255, 109, 65), (0, 143, 149), (179, 0, 194), (209, 99, 106), (5, 121, 0), (227, 255, 205), (147, 186, 208), (153, 69, 1), (3, 95, 161), (163, 255, 0), (119, 0, 170), None, (0, 182, 199), None, None, (0, 165, 120), None, (183, 130, 88), (95, 32, 0), (130, 114, 135), (110, 129, 133), (166, 74, 118), (219, 142, 185), (79, 210, 114), (178, 90, 62), (65, 70, 15), (127, 167, 115), (59, 105, 106), None, (142, 108, 45), (196, 172, 0), (95, 54, 80), (128, 76, 255), (201, 57, 1), (246, 0, 122), (191, 162, 208)] } COCOAPI = COCO # ann_id is unique in coco dataset. ANN_ID_UNIQUE = True def load_data_list(self) -> List[dict]: """Load annotations from an annotation file named as ``self.ann_file`` Returns: List[dict]: A list of annotation. """ # noqa: E501 with self.file_client.get_local_path(self.ann_file) as local_path: self.coco = self.COCOAPI(local_path) # The order of returned `cat_ids` will not # change with the order of the `classes` self.cat_ids = self.coco.get_cat_ids( cat_names=self.metainfo['classes']) self.cat2label = {cat_id: i for i, cat_id in enumerate(self.cat_ids)} self.cat_img_map = copy.deepcopy(self.coco.cat_img_map) img_ids = self.coco.get_img_ids() data_list = [] total_ann_ids = [] for img_id in img_ids: raw_img_info = self.coco.load_imgs([img_id])[0] raw_img_info['img_id'] = img_id ann_ids = self.coco.get_ann_ids(img_ids=[img_id]) raw_ann_info = self.coco.load_anns(ann_ids) total_ann_ids.extend(ann_ids) parsed_data_info = self.parse_data_info({ 'raw_ann_info': raw_ann_info, 'raw_img_info': raw_img_info }) data_list.append(parsed_data_info) if self.ANN_ID_UNIQUE: assert len(set(total_ann_ids)) == len( total_ann_ids ), f"Annotation ids in '{self.ann_file}' are not unique!" del self.coco return data_list def parse_data_info(self, raw_data_info: dict) -> Union[dict, List[dict]]: """Parse raw annotation to target format. Args: raw_data_info (dict): Raw data information load from ``ann_file`` Returns: Union[dict, List[dict]]: Parsed annotation. """ img_info = raw_data_info['raw_img_info'] ann_info = raw_data_info['raw_ann_info'] data_info = {} # TODO: need to change data_prefix['img'] to data_prefix['img_path'] img_path = osp.join(self.data_prefix['img'], img_info['file_name']) if self.data_prefix.get('seg', None): seg_map_path = osp.join( self.data_prefix['seg'], img_info['file_name'].rsplit('.', 1)[0] + self.seg_map_suffix) else: seg_map_path = None data_info['img_path'] = img_path data_info['img_id'] = img_info['img_id'] data_info['seg_map_path'] = seg_map_path data_info['height'] = img_info['height'] data_info['width'] = img_info['width'] instances = [] for i, ann in enumerate(ann_info): instance = {} if ann.get('ignore', False): continue x1, y1, w, h = ann['bbox'] inter_w = max(0, min(x1 + w, img_info['width']) - max(x1, 0)) inter_h = max(0, min(y1 + h, img_info['height']) - max(y1, 0)) if inter_w * inter_h == 0: continue if ann['area'] <= 0 or w < 1 or h < 1: continue if ann['category_id'] not in self.cat_ids: continue bbox = [x1, y1, x1 + w, y1 + h] if ann.get('iscrowd', False): instance['ignore_flag'] = 1 else: instance['ignore_flag'] = 0 instance['bbox'] = bbox instance['bbox_label'] = self.cat2label[ann['category_id']] if ann.get('segmentation', None): instance['mask'] = ann['segmentation'] instances.append(instance) data_info['instances'] = instances return data_info def filter_data(self) -> List[dict]: """Filter annotations according to filter_cfg. Returns: List[dict]: Filtered results. """ if self.test_mode: return self.data_list if self.filter_cfg is None: return self.data_list filter_empty_gt = self.filter_cfg.get('filter_empty_gt', False) min_size = self.filter_cfg.get('min_size', 0) # obtain images that contain annotation ids_with_ann = set(data_info['img_id'] for data_info in self.data_list) # obtain images that contain annotations of the required categories ids_in_cat = set() for i, class_id in enumerate(self.cat_ids): ids_in_cat |= set(self.cat_img_map[class_id]) # merge the image id sets of the two conditions and use the merged set # to filter out images if self.filter_empty_gt=True ids_in_cat &= ids_with_ann valid_data_infos = [] for i, data_info in enumerate(self.data_list): img_id = data_info['img_id'] width = data_info['width'] height = data_info['height'] if filter_empty_gt and img_id not in ids_in_cat: continue if min(width, height) >= min_size: valid_data_infos.append(data_info) return valid_data_infos ================================================ FILE: projects/EfficientDet/efficientdet/coco_90metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import datetime import itertools import os.path as osp import tempfile from collections import OrderedDict from typing import Dict, List, Optional, Sequence, Union import numpy as np from mmengine.evaluator import BaseMetric from mmengine.fileio import FileClient, dump, load from mmengine.logging import MMLogger from terminaltables import AsciiTable from mmdet.evaluation.functional import eval_recalls from mmdet.registry import METRICS from mmdet.structures.mask import encode_mask_results from .api_wrappers import COCO, COCOeval @METRICS.register_module() class Coco90Metric(BaseMetric): """COCO evaluation metric. Evaluate AR, AP, and mAP for detection tasks including proposal/box detection and instance segmentation. Please refer to https://cocodataset.org/#detection-eval for more details. Args: ann_file (str, optional): Path to the coco format annotation file. If not specified, ground truth annotations from the dataset will be converted to coco format. Defaults to None. metric (str | List[str]): Metrics to be evaluated. Valid metrics include 'bbox', 'segm', 'proposal', and 'proposal_fast'. Defaults to 'bbox'. classwise (bool): Whether to evaluate the metric class-wise. Defaults to False. proposal_nums (Sequence[int]): Numbers of proposals to be evaluated. Defaults to (100, 300, 1000). iou_thrs (float | List[float], optional): IoU threshold to compute AP and AR. If not specified, IoUs from 0.5 to 0.95 will be used. Defaults to None. metric_items (List[str], optional): Metric result names to be recorded in the evaluation result. Defaults to None. format_only (bool): Format the output results without perform evaluation. It is useful when you want to format the result to a specific format and submit it to the test server. Defaults to False. outfile_prefix (str, optional): The prefix of json files. It includes the file path and the prefix of filename, e.g., "a/b/prefix". If not specified, a temp file will be created. Defaults to None. file_client_args (dict): Arguments to instantiate a FileClient. See :class:`mmengine.fileio.FileClient` for details. Defaults to ``dict(backend='disk')``. collect_device (str): Device name used for collecting results from different ranks during distributed training. Must be 'cpu' or 'gpu'. Defaults to 'cpu'. prefix (str, optional): The prefix that will be added in the metric names to disambiguate homonymous metrics of different evaluators. If prefix is not provided in the argument, self.default_prefix will be used instead. Defaults to None. """ default_prefix: Optional[str] = 'coco' def __init__(self, ann_file: Optional[str] = None, metric: Union[str, List[str]] = 'bbox', classwise: bool = False, proposal_nums: Sequence[int] = (100, 300, 1000), iou_thrs: Optional[Union[float, Sequence[float]]] = None, metric_items: Optional[Sequence[str]] = None, format_only: bool = False, outfile_prefix: Optional[str] = None, file_client_args: dict = dict(backend='disk'), collect_device: str = 'cpu', prefix: Optional[str] = None) -> None: super().__init__(collect_device=collect_device, prefix=prefix) # coco evaluation metrics self.metrics = metric if isinstance(metric, list) else [metric] allowed_metrics = ['bbox', 'segm', 'proposal', 'proposal_fast'] for metric in self.metrics: if metric not in allowed_metrics: raise KeyError( "metric should be one of 'bbox', 'segm', 'proposal', " f"'proposal_fast', but got {metric}.") # do class wise evaluation, default False self.classwise = classwise # proposal_nums used to compute recall or precision. self.proposal_nums = list(proposal_nums) # iou_thrs used to compute recall or precision. if iou_thrs is None: iou_thrs = np.linspace( .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) self.iou_thrs = iou_thrs self.metric_items = metric_items self.format_only = format_only if self.format_only: assert outfile_prefix is not None, 'outfile_prefix must be not' 'None when format_only is True, otherwise the result files will' 'be saved to a temp directory which will be cleaned up at the end.' self.outfile_prefix = outfile_prefix self.file_client_args = file_client_args self.file_client = FileClient(**file_client_args) # if ann_file is not specified, # initialize coco api with the converted dataset if ann_file is not None: with self.file_client.get_local_path(ann_file) as local_path: self._coco_api = COCO(local_path) else: self._coco_api = None # handle dataset lazy init self.cat_ids = None self.img_ids = None def fast_eval_recall(self, results: List[dict], proposal_nums: Sequence[int], iou_thrs: Sequence[float], logger: Optional[MMLogger] = None) -> np.ndarray: """Evaluate proposal recall with COCO's fast_eval_recall. Args: results (List[dict]): Results of the dataset. proposal_nums (Sequence[int]): Proposal numbers used for evaluation. iou_thrs (Sequence[float]): IoU thresholds used for evaluation. logger (MMLogger, optional): Logger used for logging the recall summary. Returns: np.ndarray: Averaged recall results. """ gt_bboxes = [] pred_bboxes = [result['bboxes'] for result in results] for i in range(len(self.img_ids)): ann_ids = self._coco_api.get_ann_ids(img_ids=self.img_ids[i]) ann_info = self._coco_api.load_anns(ann_ids) if len(ann_info) == 0: gt_bboxes.append(np.zeros((0, 4))) continue bboxes = [] for ann in ann_info: if ann.get('ignore', False) or ann['iscrowd']: continue x1, y1, w, h = ann['bbox'] bboxes.append([x1, y1, x1 + w, y1 + h]) bboxes = np.array(bboxes, dtype=np.float32) if bboxes.shape[0] == 0: bboxes = np.zeros((0, 4)) gt_bboxes.append(bboxes) recalls = eval_recalls( gt_bboxes, pred_bboxes, proposal_nums, iou_thrs, logger=logger) ar = recalls.mean(axis=1) return ar def xyxy2xywh(self, bbox: np.ndarray) -> list: """Convert ``xyxy`` style bounding boxes to ``xywh`` style for COCO evaluation. Args: bbox (numpy.ndarray): The bounding boxes, shape (4, ), in ``xyxy`` order. Returns: list[float]: The converted bounding boxes, in ``xywh`` order. """ _bbox: List = bbox.tolist() return [ _bbox[0], _bbox[1], _bbox[2] - _bbox[0], _bbox[3] - _bbox[1], ] def results2json(self, results: Sequence[dict], outfile_prefix: str) -> dict: """Dump the detection results to a COCO style json file. There are 3 types of results: proposals, bbox predictions, mask predictions, and they have different data types. This method will automatically recognize the type, and dump them to json files. Args: results (Sequence[dict]): Testing results of the dataset. outfile_prefix (str): The filename prefix of the json files. If the prefix is "somepath/xxx", the json files will be named "somepath/xxx.bbox.json", "somepath/xxx.segm.json", "somepath/xxx.proposal.json". Returns: dict: Possible keys are "bbox", "segm", "proposal", and values are corresponding filenames. """ bbox_json_results = [] segm_json_results = [] if 'masks' in results[0] else None for idx, result in enumerate(results): image_id = result.get('img_id', idx) labels = result['labels'] bboxes = result['bboxes'] scores = result['scores'] # bbox results for i, label in enumerate(labels): data = dict() data['image_id'] = image_id data['bbox'] = self.xyxy2xywh(bboxes[i]) data['score'] = float(scores[i]) data['category_id'] = self.cat_ids[label] bbox_json_results.append(data) if segm_json_results is None: continue # segm results masks = result['masks'] mask_scores = result.get('mask_scores', scores) for i, label in enumerate(labels): data = dict() data['image_id'] = image_id data['bbox'] = self.xyxy2xywh(bboxes[i]) data['score'] = float(mask_scores[i]) data['category_id'] = self.cat_ids[label] if isinstance(masks[i]['counts'], bytes): masks[i]['counts'] = masks[i]['counts'].decode() data['segmentation'] = masks[i] segm_json_results.append(data) result_files = dict() result_files['bbox'] = f'{outfile_prefix}.bbox.json' result_files['proposal'] = f'{outfile_prefix}.bbox.json' dump(bbox_json_results, result_files['bbox']) if segm_json_results is not None: result_files['segm'] = f'{outfile_prefix}.segm.json' dump(segm_json_results, result_files['segm']) return result_files def gt_to_coco_json(self, gt_dicts: Sequence[dict], outfile_prefix: str) -> str: """Convert ground truth to coco format json file. Args: gt_dicts (Sequence[dict]): Ground truth of the dataset. outfile_prefix (str): The filename prefix of the json files. If the prefix is "somepath/xxx", the json file will be named "somepath/xxx.gt.json". Returns: str: The filename of the json file. """ categories = [ dict(id=id, name=name) for id, name in enumerate(self.dataset_meta['classes']) ] image_infos = [] annotations = [] for idx, gt_dict in enumerate(gt_dicts): img_id = gt_dict.get('img_id', idx) image_info = dict( id=img_id, width=gt_dict['width'], height=gt_dict['height'], file_name='') image_infos.append(image_info) for ann in gt_dict['anns']: label = ann['bbox_label'] bbox = ann['bbox'] coco_bbox = [ bbox[0], bbox[1], bbox[2] - bbox[0], bbox[3] - bbox[1], ] annotation = dict( id=len(annotations) + 1, # coco api requires id starts with 1 image_id=img_id, bbox=coco_bbox, iscrowd=ann.get('ignore_flag', 0), category_id=int(label), area=coco_bbox[2] * coco_bbox[3]) if ann.get('mask', None): mask = ann['mask'] # area = mask_util.area(mask) if isinstance(mask, dict) and isinstance( mask['counts'], bytes): mask['counts'] = mask['counts'].decode() annotation['segmentation'] = mask # annotation['area'] = float(area) annotations.append(annotation) info = dict( date_created=str(datetime.datetime.now()), description='Coco json file converted by mmdet CocoMetric.') coco_json = dict( info=info, images=image_infos, categories=categories, licenses=None, ) if len(annotations) > 0: coco_json['annotations'] = annotations converted_json_path = f'{outfile_prefix}.gt.json' dump(coco_json, converted_json_path) return converted_json_path # TODO: data_batch is no longer needed, consider adjusting the # parameter position def process(self, data_batch: dict, data_samples: Sequence[dict]) -> None: """Process one batch of data samples and predictions. The processed results should be stored in ``self.results``, which will be used to compute the metrics when all batches have been processed. Args: data_batch (dict): A batch of data from the dataloader. data_samples (Sequence[dict]): A batch of data samples that contain annotations and predictions. """ for data_sample in data_samples: result = dict() pred = data_sample['pred_instances'] result['img_id'] = data_sample['img_id'] result['bboxes'] = pred['bboxes'].cpu().numpy() result['scores'] = pred['scores'].cpu().numpy() result['labels'] = pred['labels'].cpu().numpy() # encode mask to RLE if 'masks' in pred: result['masks'] = encode_mask_results( pred['masks'].detach().cpu().numpy()) # some detectors use different scores for bbox and mask if 'mask_scores' in pred: result['mask_scores'] = pred['mask_scores'].cpu().numpy() # parse gt gt = dict() gt['width'] = data_sample['ori_shape'][1] gt['height'] = data_sample['ori_shape'][0] gt['img_id'] = data_sample['img_id'] if self._coco_api is None: # TODO: Need to refactor to support LoadAnnotations assert 'instances' in data_sample, \ 'ground truth is required for evaluation when ' \ '`ann_file` is not provided' gt['anns'] = data_sample['instances'] # add converted result to the results list self.results.append((gt, result)) def compute_metrics(self, results: list) -> Dict[str, float]: """Compute the metrics from processed results. Args: results (list): The processed results of each batch. Returns: Dict[str, float]: The computed metrics. The keys are the names of the metrics, and the values are corresponding results. """ logger: MMLogger = MMLogger.get_current_instance() # split gt and prediction list gts, preds = zip(*results) tmp_dir = None if self.outfile_prefix is None: tmp_dir = tempfile.TemporaryDirectory() outfile_prefix = osp.join(tmp_dir.name, 'results') else: outfile_prefix = self.outfile_prefix if self._coco_api is None: # use converted gt json file to initialize coco api logger.info('Converting ground truth to coco format...') coco_json_path = self.gt_to_coco_json( gt_dicts=gts, outfile_prefix=outfile_prefix) self._coco_api = COCO(coco_json_path) # handle lazy init if self.cat_ids is None: self.cat_ids = self._coco_api.get_cat_ids( cat_names=self.dataset_meta['classes']) if self.img_ids is None: self.img_ids = self._coco_api.get_img_ids() # convert predictions to coco format and dump to json file result_files = self.results2json(preds, outfile_prefix) eval_results = OrderedDict() if self.format_only: logger.info('results are saved in ' f'{osp.dirname(outfile_prefix)}') return eval_results for metric in self.metrics: logger.info(f'Evaluating {metric}...') # TODO: May refactor fast_eval_recall to an independent metric? # fast eval recall if metric == 'proposal_fast': ar = self.fast_eval_recall( preds, self.proposal_nums, self.iou_thrs, logger=logger) log_msg = [] for i, num in enumerate(self.proposal_nums): eval_results[f'AR@{num}'] = ar[i] log_msg.append(f'\nAR@{num}\t{ar[i]:.4f}') log_msg = ''.join(log_msg) logger.info(log_msg) continue # evaluate proposal, bbox and segm iou_type = 'bbox' if metric == 'proposal' else metric if metric not in result_files: raise KeyError(f'{metric} is not in results') try: predictions = load(result_files[metric]) if iou_type == 'segm': # Refer to https://github.com/cocodataset/cocoapi/blob/master/PythonAPI/pycocotools/coco.py#L331 # noqa # When evaluating mask AP, if the results contain bbox, # cocoapi will use the box area instead of the mask area # for calculating the instance area. Though the overall AP # is not affected, this leads to different # small/medium/large mask AP results. for x in predictions: x.pop('bbox') coco_dt = self._coco_api.loadRes(predictions) except IndexError: logger.error( 'The testing results of the whole dataset is empty.') break coco_eval = COCOeval(self._coco_api, coco_dt, iou_type) coco_eval.params.catIds = self.cat_ids coco_eval.params.imgIds = self.img_ids coco_eval.params.maxDets = list(self.proposal_nums) coco_eval.params.iouThrs = self.iou_thrs # mapping of cocoEval.stats coco_metric_names = { 'mAP': 0, 'mAP_50': 1, 'mAP_75': 2, 'mAP_s': 3, 'mAP_m': 4, 'mAP_l': 5, 'AR@100': 6, 'AR@300': 7, 'AR@1000': 8, 'AR_s@1000': 9, 'AR_m@1000': 10, 'AR_l@1000': 11 } metric_items = self.metric_items if metric_items is not None: for metric_item in metric_items: if metric_item not in coco_metric_names: raise KeyError( f'metric item "{metric_item}" is not supported') if metric == 'proposal': coco_eval.params.useCats = 0 coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() if metric_items is None: metric_items = [ 'AR@100', 'AR@300', 'AR@1000', 'AR_s@1000', 'AR_m@1000', 'AR_l@1000' ] for item in metric_items: val = float( f'{coco_eval.stats[coco_metric_names[item]]:.3f}') eval_results[item] = val else: coco_eval.evaluate() coco_eval.accumulate() coco_eval.summarize() if self.classwise: # Compute per-category AP # Compute per-category AP # from https://github.com/facebookresearch/detectron2/ precisions = coco_eval.eval['precision'] # precision: (iou, recall, cls, area range, max dets) assert len(self.cat_ids) == precisions.shape[2] results_per_category = [] for idx, cat_id in enumerate(self.cat_ids): # area range index 0: all area ranges # max dets index -1: typically 100 per image nm = self._coco_api.loadCats(cat_id)[0] precision = precisions[:, :, idx, 0, -1] precision = precision[precision > -1] if precision.size: ap = np.mean(precision) else: ap = float('nan') results_per_category.append( (f'{nm["name"]}', f'{round(ap, 3)}')) eval_results[f'{nm["name"]}_precision'] = round(ap, 3) num_columns = min(6, len(results_per_category) * 2) results_flatten = list( itertools.chain(*results_per_category)) headers = ['category', 'AP'] * (num_columns // 2) results_2d = itertools.zip_longest(*[ results_flatten[i::num_columns] for i in range(num_columns) ]) table_data = [headers] table_data += [result for result in results_2d] table = AsciiTable(table_data) logger.info('\n' + table.table) if metric_items is None: metric_items = [ 'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l' ] for metric_item in metric_items: key = f'{metric}_{metric_item}' val = coco_eval.stats[coco_metric_names[metric_item]] eval_results[key] = float(f'{round(val, 3)}') ap = coco_eval.stats[:6] logger.info(f'{metric}_mAP_copypaste: {ap[0]:.3f} ' f'{ap[1]:.3f} {ap[2]:.3f} {ap[3]:.3f} ' f'{ap[4]:.3f} {ap[5]:.3f}') if tmp_dir is not None: tmp_dir.cleanup() return eval_results ================================================ FILE: projects/EfficientDet/efficientdet/efficientdet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.detectors.single_stage import SingleStageDetector from mmdet.registry import MODELS from mmdet.utils import ConfigType, OptConfigType, OptMultiConfig @MODELS.register_module() class EfficientDet(SingleStageDetector): def __init__(self, backbone: ConfigType, neck: ConfigType, bbox_head: ConfigType, train_cfg: OptConfigType = None, test_cfg: OptConfigType = None, data_preprocessor: OptConfigType = None, init_cfg: OptMultiConfig = None) -> None: super().__init__( backbone=backbone, neck=neck, bbox_head=bbox_head, train_cfg=train_cfg, test_cfg=test_cfg, data_preprocessor=data_preprocessor, init_cfg=init_cfg) ================================================ FILE: projects/EfficientDet/efficientdet/efficientdet_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Tuple import torch.nn as nn from mmcv.cnn.bricks import build_norm_layer from mmengine.model import bias_init_with_prob from torch import Tensor from mmdet.models.dense_heads.anchor_head import AnchorHead from mmdet.registry import MODELS from mmdet.utils import OptConfigType, OptMultiConfig from .utils import DepthWiseConvBlock, MemoryEfficientSwish @MODELS.register_module() class EfficientDetSepBNHead(AnchorHead): """EfficientDetHead with separate BN. num_classes (int): Number of categories excluding the background category. in_channels (int): Number of channels in the input feature map. feat_channels (int): Number of hidden channels. stacked_convs (int): Number of repetitions of conv norm_cfg (dict): Config dict for normalization layer. anchor_generator (dict): Config dict for anchor generator bbox_coder (dict): Config of bounding box coder. loss_cls (dict): Config of classification loss. loss_bbox (dict): Config of localization loss. train_cfg (dict): Training config of anchor head. test_cfg (dict): Testing config of anchor head. init_cfg (dict or list[dict], optional): Initialization config dict. """ def __init__(self, num_classes: int, num_ins: int, in_channels: int, feat_channels: int, stacked_convs: int = 3, norm_cfg: OptConfigType = dict( type='BN', momentum=1e-2, eps=1e-3), init_cfg: OptMultiConfig = None, **kwargs) -> None: self.num_ins = num_ins self.stacked_convs = stacked_convs self.norm_cfg = norm_cfg super().__init__( num_classes=num_classes, in_channels=in_channels, feat_channels=feat_channels, init_cfg=init_cfg, **kwargs) def _init_layers(self) -> None: """Initialize layers of the head.""" self.reg_conv_list = nn.ModuleList() self.cls_conv_list = nn.ModuleList() for i in range(self.stacked_convs): channels = self.in_channels if i == 0 else self.feat_channels self.reg_conv_list.append( DepthWiseConvBlock( channels, self.feat_channels, apply_norm=False)) self.cls_conv_list.append( DepthWiseConvBlock( channels, self.feat_channels, apply_norm=False)) self.reg_bn_list = nn.ModuleList([ nn.ModuleList([ build_norm_layer( self.norm_cfg, num_features=self.feat_channels)[1] for j in range(self.num_ins) ]) for i in range(self.stacked_convs) ]) self.cls_bn_list = nn.ModuleList([ nn.ModuleList([ build_norm_layer( self.norm_cfg, num_features=self.feat_channels)[1] for j in range(self.num_ins) ]) for i in range(self.stacked_convs) ]) self.cls_header = DepthWiseConvBlock( self.in_channels, self.num_base_priors * self.cls_out_channels, apply_norm=False) self.reg_header = DepthWiseConvBlock( self.in_channels, self.num_base_priors * 4, apply_norm=False) self.swish = MemoryEfficientSwish() def init_weights(self) -> None: """Initialize weights of the head.""" for m in self.reg_conv_list: nn.init.constant_(m.pointwise_conv.conv.bias, 0.0) for m in self.cls_conv_list: nn.init.constant_(m.pointwise_conv.conv.bias, 0.0) bias_cls = bias_init_with_prob(0.01) nn.init.constant_(self.cls_header.pointwise_conv.conv.bias, bias_cls) nn.init.constant_(self.reg_header.pointwise_conv.conv.bias, 0.0) def forward_single_bbox(self, feat: Tensor, level_id: int, i: int) -> Tensor: conv_op = self.reg_conv_list[i] bn = self.reg_bn_list[i][level_id] feat = conv_op(feat) feat = bn(feat) feat = self.swish(feat) return feat def forward_single_cls(self, feat: Tensor, level_id: int, i: int) -> Tensor: conv_op = self.cls_conv_list[i] bn = self.cls_bn_list[i][level_id] feat = conv_op(feat) feat = bn(feat) feat = self.swish(feat) return feat def forward(self, feats: Tuple[Tensor]) -> tuple: cls_scores = [] bbox_preds = [] for level_id in range(self.num_ins): feat = feats[level_id] for i in range(self.stacked_convs): feat = self.forward_single_bbox(feat, level_id, i) bbox_pred = self.reg_header(feat) bbox_preds.append(bbox_pred) for level_id in range(self.num_ins): feat = feats[level_id] for i in range(self.stacked_convs): feat = self.forward_single_cls(feat, level_id, i) cls_score = self.cls_header(feat) cls_scores.append(cls_score) return cls_scores, bbox_preds ================================================ FILE: projects/EfficientDet/efficientdet/trans_max_iou_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from typing import Optional import torch from mmengine.structures import InstanceData from mmdet.models.task_modules.assigners.assign_result import AssignResult from mmdet.models.task_modules.assigners.max_iou_assigner import MaxIoUAssigner from mmdet.registry import TASK_UTILS @TASK_UTILS.register_module() class TransMaxIoUAssigner(MaxIoUAssigner): def assign(self, pred_instances: InstanceData, gt_instances: InstanceData, gt_instances_ignore: Optional[InstanceData] = None, **kwargs) -> AssignResult: """Assign gt to bboxes. This method assign a gt bbox to every bbox (proposal/anchor), each bbox will be assigned with -1, or a semi-positive number. -1 means negative sample, semi-positive number is the index (0-based) of assigned gt. The assignment is done in following steps, the order matters. 1. assign every bbox to the background 2. assign proposals whose iou with all gts < neg_iou_thr to 0 3. for each bbox, if the iou with its nearest gt >= pos_iou_thr, assign it to that bbox 4. for each gt bbox, assign its nearest proposals (may be more than one) to itself Args: pred_instances (:obj:`InstanceData`): Instances of model predictions. It includes ``priors``, and the priors can be anchors or points, or the bboxes predicted by the previous stage, has shape (n, 4). The bboxes predicted by the current model or stage will be named ``bboxes``, ``labels``, and ``scores``, the same as the ``InstanceData`` in other places. gt_instances (:obj:`InstanceData`): Ground truth of instance annotations. It usually includes ``bboxes``, with shape (k, 4), and ``labels``, with shape (k, ). gt_instances_ignore (:obj:`InstanceData`, optional): Instances to be ignored during training. It includes ``bboxes`` attribute data that is ignored during training and testing. Defaults to None. Returns: :obj:`AssignResult`: The assign result. Example: >>> from mmengine.structures import InstanceData >>> self = MaxIoUAssigner(0.5, 0.5) >>> pred_instances = InstanceData() >>> pred_instances.priors = torch.Tensor([[0, 0, 10, 10], ... [10, 10, 20, 20]]) >>> gt_instances = InstanceData() >>> gt_instances.bboxes = torch.Tensor([[0, 0, 10, 9]]) >>> gt_instances.labels = torch.Tensor([0]) >>> assign_result = self.assign(pred_instances, gt_instances) >>> expected_gt_inds = torch.LongTensor([1, 0]) >>> assert torch.all(assign_result.gt_inds == expected_gt_inds) """ gt_bboxes = gt_instances.bboxes priors = pred_instances.priors gt_labels = gt_instances.labels if gt_instances_ignore is not None: gt_bboxes_ignore = gt_instances_ignore.bboxes else: gt_bboxes_ignore = None assign_on_cpu = True if (self.gpu_assign_thr > 0) and ( gt_bboxes.shape[0] > self.gpu_assign_thr) else False # compute overlap and assign gt on CPU when number of GT is large if assign_on_cpu: device = priors.device priors = priors.cpu() gt_bboxes = gt_bboxes.cpu() gt_labels = gt_labels.cpu() if gt_bboxes_ignore is not None: gt_bboxes_ignore = gt_bboxes_ignore.cpu() trans_priors = torch.cat([ priors[..., 1].view(-1, 1), priors[..., 0].view(-1, 1), priors[..., 3].view(-1, 1), priors[..., 2].view(-1, 1) ], dim=-1) overlaps = self.iou_calculator(gt_bboxes, trans_priors) if (self.ignore_iof_thr > 0 and gt_bboxes_ignore is not None and gt_bboxes_ignore.numel() > 0 and trans_priors.numel() > 0): if self.ignore_wrt_candidates: ignore_overlaps = self.iou_calculator( trans_priors, gt_bboxes_ignore, mode='iof') ignore_max_overlaps, _ = ignore_overlaps.max(dim=1) else: ignore_overlaps = self.iou_calculator( gt_bboxes_ignore, trans_priors, mode='iof') ignore_max_overlaps, _ = ignore_overlaps.max(dim=0) overlaps[:, ignore_max_overlaps > self.ignore_iof_thr] = -1 assign_result = self.assign_wrt_overlaps(overlaps, gt_labels) if assign_on_cpu: assign_result.gt_inds = assign_result.gt_inds.to(device) assign_result.max_overlaps = assign_result.max_overlaps.to(device) if assign_result.labels is not None: assign_result.labels = assign_result.labels.to(device) return assign_result ================================================ FILE: projects/EfficientDet/efficientdet/utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import math from typing import Tuple, Union import torch import torch.nn as nn from mmcv.cnn.bricks import Swish, build_norm_layer from torch.nn import functional as F from mmdet.utils import OptConfigType class SwishImplementation(torch.autograd.Function): @staticmethod def forward(ctx, i): result = i * torch.sigmoid(i) ctx.save_for_backward(i) return result @staticmethod def backward(ctx, grad_output): i = ctx.saved_variables[0] sigmoid_i = torch.sigmoid(i) return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i))) class MemoryEfficientSwish(nn.Module): def forward(self, x): return SwishImplementation.apply(x) class Conv2dSamePadding(nn.Module): def __init__(self, in_channels: int, out_channels: int, kernel_size: Union[int, Tuple[int, int]], stride: Union[int, Tuple[int, int]] = 1, groups: int = 1, bias: bool = True): super().__init__() self.conv = nn.Conv2d( in_channels, out_channels, kernel_size, stride=stride, bias=bias, groups=groups) self.stride = self.conv.stride self.kernel_size = self.conv.kernel_size def forward(self, x): h, w = x.shape[-2:] extra_h = (math.ceil(w / self.stride[1]) - 1) * self.stride[1] - w + self.kernel_size[1] extra_v = (math.ceil(h / self.stride[0]) - 1) * self.stride[0] - h + self.kernel_size[0] left = extra_h // 2 right = extra_h - left top = extra_v // 2 bottom = extra_v - top x = F.pad(x, [left, right, top, bottom]) x = self.conv(x) return x class MaxPool2dSamePadding(nn.Module): def __init__(self, kernel_size: Union[int, Tuple[int, int]] = 3, stride: Union[int, Tuple[int, int]] = 2, **kwargs): super().__init__() self.pool = nn.MaxPool2d(kernel_size, stride, **kwargs) self.stride = self.pool.stride self.kernel_size = self.pool.kernel_size if isinstance(self.stride, int): self.stride = [self.stride] * 2 if isinstance(self.kernel_size, int): self.kernel_size = [self.kernel_size] * 2 def forward(self, x): h, w = x.shape[-2:] extra_h = (math.ceil(w / self.stride[1]) - 1) * self.stride[1] - w + self.kernel_size[1] extra_v = (math.ceil(h / self.stride[0]) - 1) * self.stride[0] - h + self.kernel_size[0] left = extra_h // 2 right = extra_h - left top = extra_v // 2 bottom = extra_v - top x = F.pad(x, [left, right, top, bottom]) x = self.pool(x) return x class DepthWiseConvBlock(nn.Module): def __init__( self, in_channels: int, out_channels: int, apply_norm: bool = True, conv_bn_act_pattern: bool = False, use_meswish: bool = True, norm_cfg: OptConfigType = dict(type='BN', momentum=1e-2, eps=1e-3) ) -> None: super(DepthWiseConvBlock, self).__init__() self.depthwise_conv = Conv2dSamePadding( in_channels, in_channels, kernel_size=3, stride=1, groups=in_channels, bias=False) self.pointwise_conv = Conv2dSamePadding( in_channels, out_channels, kernel_size=1, stride=1) self.apply_norm = apply_norm if self.apply_norm: self.bn = build_norm_layer(norm_cfg, num_features=out_channels)[1] self.apply_activation = conv_bn_act_pattern if self.apply_activation: self.swish = MemoryEfficientSwish() if use_meswish else Swish() def forward(self, x): x = self.depthwise_conv(x) x = self.pointwise_conv(x) if self.apply_norm: x = self.bn(x) if self.apply_activation: x = self.swish(x) return x class DownChannelBlock(nn.Module): def __init__( self, in_channels: int, out_channels: int, apply_norm: bool = True, conv_bn_act_pattern: bool = False, use_meswish: bool = True, norm_cfg: OptConfigType = dict(type='BN', momentum=1e-2, eps=1e-3) ) -> None: super(DownChannelBlock, self).__init__() self.down_conv = Conv2dSamePadding(in_channels, out_channels, 1) self.apply_norm = apply_norm if self.apply_norm: self.bn = build_norm_layer(norm_cfg, num_features=out_channels)[1] self.apply_activation = conv_bn_act_pattern if self.apply_activation: self.swish = MemoryEfficientSwish() if use_meswish else Swish() def forward(self, x): x = self.down_conv(x) if self.apply_norm: x = self.bn(x) if self.apply_activation: x = self.swish(x) return x ================================================ FILE: projects/EfficientDet/efficientdet/yxyx_bbox_coder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import warnings import numpy as np import torch from mmdet.models.task_modules.coders.delta_xywh_bbox_coder import \ DeltaXYWHBBoxCoder from mmdet.registry import TASK_UTILS from mmdet.structures.bbox import HorizontalBoxes, get_box_tensor @TASK_UTILS.register_module() class YXYXDeltaXYWHBBoxCoder(DeltaXYWHBBoxCoder): def encode(self, bboxes, gt_bboxes): """Get box regression transformation deltas that can be used to transform the ``bboxes`` into the ``gt_bboxes``. Args: bboxes (torch.Tensor or :obj:`BaseBoxes`): Source boxes, e.g., object proposals. gt_bboxes (torch.Tensor or :obj:`BaseBoxes`): Target of the transformation, e.g., ground-truth boxes. Returns: torch.Tensor: Box transformation deltas """ bboxes = get_box_tensor(bboxes) gt_bboxes = get_box_tensor(gt_bboxes) assert bboxes.size(0) == gt_bboxes.size(0) assert bboxes.size(-1) == gt_bboxes.size(-1) == 4 encoded_bboxes = YXbbox2delta(bboxes, gt_bboxes, self.means, self.stds) return encoded_bboxes def decode(self, bboxes, pred_bboxes, max_shape=None, wh_ratio_clip=16 / 1000): """Apply transformation `pred_bboxes` to `boxes`. Args: bboxes (torch.Tensor or :obj:`BaseBoxes`): Basic boxes. Shape (B, N, 4) or (N, 4) pred_bboxes (Tensor): Encoded offsets with respect to each roi. Has shape (B, N, num_classes * 4) or (B, N, 4) or (N, num_classes * 4) or (N, 4). Note N = num_anchors * W * H when rois is a grid of anchors.Offset encoding follows [1]_. max_shape (Sequence[int] or torch.Tensor or Sequence[ Sequence[int]],optional): Maximum bounds for boxes, specifies (H, W, C) or (H, W). If bboxes shape is (B, N, 4), then the max_shape should be a Sequence[Sequence[int]] and the length of max_shape should also be B. wh_ratio_clip (float, optional): The allowed ratio between width and height. Returns: Union[torch.Tensor, :obj:`BaseBoxes`]: Decoded boxes. """ bboxes = get_box_tensor(bboxes) assert pred_bboxes.size(0) == bboxes.size(0) if pred_bboxes.ndim == 3: assert pred_bboxes.size(1) == bboxes.size(1) if pred_bboxes.ndim == 2 and not torch.onnx.is_in_onnx_export(): # single image decode decoded_bboxes = YXdelta2bbox(bboxes, pred_bboxes, self.means, self.stds, max_shape, wh_ratio_clip, self.clip_border, self.add_ctr_clamp, self.ctr_clamp) else: if pred_bboxes.ndim == 3 and not torch.onnx.is_in_onnx_export(): warnings.warn( 'DeprecationWarning: onnx_delta2bbox is deprecated ' 'in the case of batch decoding and non-ONNX, ' 'please use “delta2bbox” instead. In order to improve ' 'the decoding speed, the batch function will no ' 'longer be supported. ') decoded_bboxes = YXonnx_delta2bbox(bboxes, pred_bboxes, self.means, self.stds, max_shape, wh_ratio_clip, self.clip_border, self.add_ctr_clamp, self.ctr_clamp) if self.use_box_type: assert decoded_bboxes.size(-1) == 4, \ ('Cannot warp decoded boxes with box type when decoded boxes' 'have shape of (N, num_classes * 4)') decoded_bboxes = HorizontalBoxes(decoded_bboxes) return decoded_bboxes def YXdelta2bbox(rois, deltas, means=(0., 0., 0., 0.), stds=(1., 1., 1., 1.), max_shape=None, hw_ratio_clip=1000 / 16, clip_border=True, add_ctr_clamp=False, ctr_clamp=32): """Apply deltas to shift/scale base boxes. Typically the rois are anchor or proposed bounding boxes and the deltas are network outputs used to shift/scale those boxes. This is the inverse function of :func:`bbox2delta`. Args: rois (Tensor): Boxes to be transformed. Has shape (N, 4). deltas (Tensor): Encoded offsets relative to each roi. Has shape (N, num_classes * 4) or (N, 4). Note N = num_base_anchors * W * H, when rois is a grid of anchors. Offset encoding follows [1]_. means (Sequence[float]): Denormalizing means for delta coordinates. Default (0., 0., 0., 0.). stds (Sequence[float]): Denormalizing standard deviation for delta coordinates. Default (1., 1., 1., 1.). max_shape (tuple[int, int]): Maximum bounds for boxes, specifies (H, W). Default None. wh_ratio_clip (float): Maximum aspect ratio for boxes. Default 16 / 1000. clip_border (bool, optional): Whether clip the objects outside the border of the image. Default True. add_ctr_clamp (bool): Whether to add center clamp. When set to True, the center of the prediction bounding box will be clamped to avoid being too far away from the center of the anchor. Only used by YOLOF. Default False. ctr_clamp (int): the maximum pixel shift to clamp. Only used by YOLOF. Default 32. Returns: Tensor: Boxes with shape (N, num_classes * 4) or (N, 4), where 4 represent tl_x, tl_y, br_x, br_y. References: .. [1] https://arxiv.org/abs/1311.2524 Example: >>> rois = torch.Tensor([[ 0., 0., 1., 1.], >>> [ 0., 0., 1., 1.], >>> [ 0., 0., 1., 1.], >>> [ 5., 5., 5., 5.]]) >>> deltas = torch.Tensor([[ 0., 0., 0., 0.], >>> [ 1., 1., 1., 1.], >>> [ 0., 0., 2., -1.], >>> [ 0.7, -1.9, -0.5, 0.3]]) >>> delta2bbox(rois, deltas, max_shape=(32, 32, 3)) tensor([[0.0000, 0.0000, 1.0000, 1.0000], [0.1409, 0.1409, 2.8591, 2.8591], [0.0000, 0.3161, 4.1945, 0.6839], [5.0000, 5.0000, 5.0000, 5.0000]]) """ num_bboxes, num_classes = deltas.size(0), deltas.size(1) // 4 if num_bboxes == 0: return deltas deltas = deltas.reshape(-1, 4) means = deltas.new_tensor(means).view(1, -1) stds = deltas.new_tensor(stds).view(1, -1) denorm_deltas = deltas * stds + means dyx = denorm_deltas[:, :2] dhw = denorm_deltas[:, 2:] # Compute width/height of each roi rois_ = rois.repeat(1, num_classes).reshape(-1, 4) pyx = ((rois_[:, :2] + rois_[:, 2:]) * 0.5) phw = (rois_[:, 2:] - rois_[:, :2]) dyx_hw = phw * dyx max_ratio = np.abs(np.log(hw_ratio_clip)) if add_ctr_clamp: dyx_hw = torch.clamp(dyx_hw, max=ctr_clamp, min=-ctr_clamp) dhw = torch.clamp(dhw, max=max_ratio) else: dhw = dhw.clamp(min=-max_ratio, max=max_ratio) gyx = pyx + dyx_hw ghw = phw * dhw.exp() y1x1 = gyx - (ghw * 0.5) y2x2 = gyx + (ghw * 0.5) ymin, xmin = y1x1[:, 0].reshape(-1, 1), y1x1[:, 1].reshape(-1, 1) ymax, xmax = y2x2[:, 0].reshape(-1, 1), y2x2[:, 1].reshape(-1, 1) bboxes = torch.cat([xmin, ymin, xmax, ymax], dim=-1) if clip_border and max_shape is not None: bboxes[..., 0::2].clamp_(min=0, max=max_shape[1]) bboxes[..., 1::2].clamp_(min=0, max=max_shape[0]) bboxes = bboxes.reshape(num_bboxes, -1) return bboxes def YXbbox2delta(proposals, gt, means=(0., 0., 0., 0.), stds=(1., 1., 1., 1.)): """Compute deltas of proposals w.r.t. gt. We usually compute the deltas of x, y, w, h of proposals w.r.t ground truth bboxes to get regression target. This is the inverse function of :func:`delta2bbox`. Args: proposals (Tensor): Boxes to be transformed, shape (N, ..., 4) gt (Tensor): Gt bboxes to be used as base, shape (N, ..., 4) means (Sequence[float]): Denormalizing means for delta coordinates stds (Sequence[float]): Denormalizing standard deviation for delta coordinates Returns: Tensor: deltas with shape (N, 4), where columns represent dx, dy, dw, dh. """ assert proposals.size() == gt.size() proposals = proposals.float() gt = gt.float() py = (proposals[..., 0] + proposals[..., 2]) * 0.5 px = (proposals[..., 1] + proposals[..., 3]) * 0.5 ph = proposals[..., 2] - proposals[..., 0] pw = proposals[..., 3] - proposals[..., 1] gx = (gt[..., 0] + gt[..., 2]) * 0.5 gy = (gt[..., 1] + gt[..., 3]) * 0.5 gw = gt[..., 2] - gt[..., 0] gh = gt[..., 3] - gt[..., 1] dx = (gx - px) / pw dy = (gy - py) / ph dw = torch.log(gw / pw) dh = torch.log(gh / ph) deltas = torch.stack([dy, dx, dh, dw], dim=-1) means = deltas.new_tensor(means).unsqueeze(0) stds = deltas.new_tensor(stds).unsqueeze(0) deltas = deltas.sub_(means).div_(stds) return deltas def YXonnx_delta2bbox(rois, deltas, means=(0., 0., 0., 0.), stds=(1., 1., 1., 1.), max_shape=None, wh_ratio_clip=16 / 1000, clip_border=True, add_ctr_clamp=False, ctr_clamp=32): """Apply deltas to shift/scale base boxes. Typically the rois are anchor or proposed bounding boxes and the deltas are network outputs used to shift/scale those boxes. This is the inverse function of :func:`bbox2delta`. Args: rois (Tensor): Boxes to be transformed. Has shape (N, 4) or (B, N, 4) deltas (Tensor): Encoded offsets with respect to each roi. Has shape (B, N, num_classes * 4) or (B, N, 4) or (N, num_classes * 4) or (N, 4). Note N = num_anchors * W * H when rois is a grid of anchors.Offset encoding follows [1]_. means (Sequence[float]): Denormalizing means for delta coordinates. Default (0., 0., 0., 0.). stds (Sequence[float]): Denormalizing standard deviation for delta coordinates. Default (1., 1., 1., 1.). max_shape (Sequence[int] or torch.Tensor or Sequence[ Sequence[int]],optional): Maximum bounds for boxes, specifies (H, W, C) or (H, W). If rois shape is (B, N, 4), then the max_shape should be a Sequence[Sequence[int]] and the length of max_shape should also be B. Default None. wh_ratio_clip (float): Maximum aspect ratio for boxes. Default 16 / 1000. clip_border (bool, optional): Whether clip the objects outside the border of the image. Default True. add_ctr_clamp (bool): Whether to add center clamp, when added, the predicted box is clamped is its center is too far away from the original anchor's center. Only used by YOLOF. Default False. ctr_clamp (int): the maximum pixel shift to clamp. Only used by YOLOF. Default 32. Returns: Tensor: Boxes with shape (B, N, num_classes * 4) or (B, N, 4) or (N, num_classes * 4) or (N, 4), where 4 represent tl_x, tl_y, br_x, br_y. References: .. [1] https://arxiv.org/abs/1311.2524 Example: >>> rois = torch.Tensor([[ 0., 0., 1., 1.], >>> [ 0., 0., 1., 1.], >>> [ 0., 0., 1., 1.], >>> [ 5., 5., 5., 5.]]) >>> deltas = torch.Tensor([[ 0., 0., 0., 0.], >>> [ 1., 1., 1., 1.], >>> [ 0., 0., 2., -1.], >>> [ 0.7, -1.9, -0.5, 0.3]]) >>> delta2bbox(rois, deltas, max_shape=(32, 32, 3)) tensor([[0.0000, 0.0000, 1.0000, 1.0000], [0.1409, 0.1409, 2.8591, 2.8591], [0.0000, 0.3161, 4.1945, 0.6839], [5.0000, 5.0000, 5.0000, 5.0000]]) """ means = deltas.new_tensor(means).view(1, -1).repeat(1, deltas.size(-1) // 4) stds = deltas.new_tensor(stds).view(1, -1).repeat(1, deltas.size(-1) // 4) denorm_deltas = deltas * stds + means dy = denorm_deltas[..., 0::4] dx = denorm_deltas[..., 1::4] dh = denorm_deltas[..., 2::4] dw = denorm_deltas[..., 3::4] y1, x1 = rois[..., 0], rois[..., 1] y2, x2 = rois[..., 2], rois[..., 3] # Compute center of each roi px = ((x1 + x2) * 0.5).unsqueeze(-1).expand_as(dx) py = ((y1 + y2) * 0.5).unsqueeze(-1).expand_as(dy) # Compute width/height of each roi pw = (x2 - x1).unsqueeze(-1).expand_as(dw) ph = (y2 - y1).unsqueeze(-1).expand_as(dh) dx_width = pw * dx dy_height = ph * dy max_ratio = np.abs(np.log(wh_ratio_clip)) if add_ctr_clamp: dx_width = torch.clamp(dx_width, max=ctr_clamp, min=-ctr_clamp) dy_height = torch.clamp(dy_height, max=ctr_clamp, min=-ctr_clamp) dw = torch.clamp(dw, max=max_ratio) dh = torch.clamp(dh, max=max_ratio) else: dw = dw.clamp(min=-max_ratio, max=max_ratio) dh = dh.clamp(min=-max_ratio, max=max_ratio) # Use exp(network energy) to enlarge/shrink each roi gw = pw * dw.exp() gh = ph * dh.exp() # Use network energy to shift the center of each roi gx = px + dx_width gy = py + dy_height # Convert center-xy/width/height to top-left, bottom-right x1 = gx - gw * 0.5 y1 = gy - gh * 0.5 x2 = gx + gw * 0.5 y2 = gy + gh * 0.5 bboxes = torch.stack([x1, y1, x2, y2], dim=-1).view(deltas.size()) if clip_border and max_shape is not None: # clip bboxes with dynamic `min` and `max` for onnx if torch.onnx.is_in_onnx_export(): from mmdet.core.export import dynamic_clip_for_onnx x1, y1, x2, y2 = dynamic_clip_for_onnx(x1, y1, x2, y2, max_shape) bboxes = torch.stack([x1, y1, x2, y2], dim=-1).view(deltas.size()) return bboxes if not isinstance(max_shape, torch.Tensor): max_shape = x1.new_tensor(max_shape) max_shape = max_shape[..., :2].type_as(x1) if max_shape.ndim == 2: assert bboxes.ndim == 3 assert max_shape.size(0) == bboxes.size(0) min_xy = x1.new_tensor(0) max_xy = torch.cat( [max_shape] * (deltas.size(-1) // 2), dim=-1).flip(-1).unsqueeze(-2) bboxes = torch.where(bboxes < min_xy, min_xy, bboxes) bboxes = torch.where(bboxes > max_xy, max_xy, bboxes) return bboxes ================================================ FILE: projects/SparseInst/README.md ================================================


Tianheng Cheng, Xinggang Wang†, Shaoyu Chen, Wenqiang Zhang, Qian Zhang, Chang Huang, Zhaoxiang Zhang, Wenyu Liu
(†: corresponding author)
## Description This is an implementation of [SparseInst](https://github.com/hustvl/SparseInst) based on [MMDetection](https://github.com/open-mmlab/mmdetection/tree/3.x), [MMCV](https://github.com/open-mmlab/mmcv), and [MMEngine](https://github.com/open-mmlab/mmengine). **SparseInst** is a conceptually novel, efficient, and fully convolutional framework for real-time instance segmentation. In contrast to region boxes or anchors (centers), SparseInst adopts a sparse set of **instance activation maps** as object representation, to highlight informative regions for each foreground objects. Then it obtains the instance-level features by aggregating features according to the highlighted regions for recognition and segmentation. The bipartite matching compels the instance activation maps to predict objects in a one-to-one style, thus avoiding non-maximum suppression (NMS) in post-processing. Owing to the simple yet effective designs with instance activation maps, SparseInst has extremely fast inference speed and achieves **40 FPS** and **37.9 AP** on COCO (NVIDIA 2080Ti), significantly outperforms the counter parts in terms of speed and accuracy.
## Usage ### Training commands In MMDetection's root directory, run the following command to train the model: ```bash python tools/train.py projects/SparseInst/configs/sparseinst_r50_iam_8xb8-ms-270k_coco.py ``` For multi-gpu training, run: ```bash python -m torch.distributed.launch --nnodes=1 --node_rank=0 --nproc_per_node=${NUM_GPUS} --master_port=29506 --master_addr="127.0.0.1" tools/train.py projects/SparseInst/configs/sparseinst_r50_iam_8xb8-ms-270k_coco.py ``` ### Testing commands In MMDetection's root directory, run the following command to test the model: ```bash python tools/test.py projects/SparseInst/configs/sparseinst_r50_iam_8xb8-ms-270k_coco.py ${CHECKPOINT_PATH} ``` ## Results Here we provide the baseline version of SparseInst with ResNet50 backbone. To find more variants, please visit the [official model zoo](https://github.com/hustvl/SparseInst#models). | Backbone | Style | Lr schd | Mem (GB) | FPS | mask AP val2017 | Config | Download | | :------: | :-----: | :-----: | :------: | :--: | :-------------: | :---------------------------------------------------------: | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------: | | R-50 | PyTorch | 270k | 8.7 | 44.3 | 32.9 | [config](./configs/sparseinst_r50_iam_8xb8-ms-270k_coco.py) | [model](https://download.openmmlab.com/mmdetection/v3.0/sparseinst/sparseinst_r50_iam_8xb8-ms-270k_coco/sparseinst_r50_iam_8xb8-ms-270k_coco_20221111_181051-72c711cd.pth) \| [log](https://download.openmmlab.com/mmdetection/v3.0/sparseinst/sparseinst_r50_iam_8xb8-ms-270k_coco/sparseinst_r50_iam_8xb8-ms-270k_coco_20221111_181051.json) | ## Citation If you find SparseInst is useful in your research or applications, please consider giving a star 🌟 to the [official repository](https://github.com/hustvl/SparseInst) and citing SparseInst by the following BibTeX entry. ```BibTeX @inproceedings{Cheng2022SparseInst, title = {Sparse Instance Activation for Real-Time Instance Segmentation}, author = {Cheng, Tianheng and Wang, Xinggang and Chen, Shaoyu and Zhang, Wenqiang and Zhang, Qian and Huang, Chang and Zhang, Zhaoxiang and Liu, Wenyu}, booktitle = {Proc. IEEE Conf. Computer Vision and Pattern Recognition (CVPR)}, year = {2022} } ``` ## Checklist - [x] Milestone 1: PR-ready, and acceptable to be one of the `projects/`. - [x] Finish the code - [x] Basic docstrings & proper citation - [x] Test-time correctness - [x] A full README - [x] Milestone 2: Indicates a successful model implementation. - [x] Training-time correctness - [ ] Milestone 3: Good to be a part of our core package! - [ ] Type hints and docstrings - [ ] Unit tests - [ ] Code polishing - [ ] Metafile.yml - [ ] Move your modules into the core package following the codebase's file hierarchy structure. - [ ] Refactor your modules into the core package following the codebase's file hierarchy structure. ================================================ FILE: projects/SparseInst/configs/sparseinst_r50_iam_8xb8-ms-270k_coco.py ================================================ _base_ = [ 'mmdet::_base_/datasets/coco_instance.py', 'mmdet::_base_/schedules/schedule_1x.py', 'mmdet::_base_/default_runtime.py' ] custom_imports = dict( imports=['projects.SparseInst.sparseinst'], allow_failed_imports=False) model = dict( type='SparseInst', data_preprocessor=dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_mask=True, pad_size_divisor=32), backbone=dict( type='ResNet', depth=50, num_stages=4, out_indices=(1, 2, 3), frozen_stages=0, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, style='pytorch', init_cfg=dict(type='Pretrained', checkpoint='torchvision://resnet50')), encoder=dict( type='InstanceContextEncoder', in_channels=[512, 1024, 2048], out_channels=256), decoder=dict( type='BaseIAMDecoder', in_channels=256 + 2, num_classes=80, ins_dim=256, ins_conv=4, mask_dim=256, mask_conv=4, kernel_dim=128, scale_factor=2.0, output_iam=False, num_masks=100), criterion=dict( type='SparseInstCriterion', num_classes=80, assigner=dict(type='SparseInstMatcher', alpha=0.8, beta=0.2), loss_cls=dict( type='FocalLoss', use_sigmoid=True, alpha=0.25, gamma=2.0, reduction='sum', loss_weight=2.0), loss_obj=dict( type='CrossEntropyLoss', use_sigmoid=True, reduction='mean', loss_weight=1.0), loss_mask=dict( type='CrossEntropyLoss', use_sigmoid=True, reduction='mean', loss_weight=5.0), loss_dice=dict( type='DiceLoss', use_sigmoid=True, reduction='sum', eps=5e-5, loss_weight=2.0), ), test_cfg=dict(score_thr=0.005, mask_thr_binary=0.45)) backend = 'pillow' train_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}, imdecode_backend=backend), dict( type='LoadAnnotations', with_bbox=True, with_mask=True, poly2mask=False), dict( type='RandomChoiceResize', scales=[(416, 853), (448, 853), (480, 853), (512, 853), (544, 853), (576, 853), (608, 853), (640, 853)], keep_ratio=True, backend=backend), dict(type='RandomFlip', prob=0.5), dict(type='PackDetInputs') ] test_pipeline = [ dict( type='LoadImageFromFile', file_client_args={{_base_.file_client_args}}, imdecode_backend=backend), dict(type='Resize', scale=(640, 853), keep_ratio=True, backend=backend), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ] train_dataloader = dict( batch_size=8, num_workers=8, sampler=dict(type='InfiniteSampler'), dataset=dict(pipeline=train_pipeline)) test_dataloader = dict(dataset=dict(pipeline=test_pipeline)) val_dataloader = test_dataloader val_evaluator = dict(metric='segm') test_evaluator = val_evaluator # optimizer optim_wrapper = dict( type='OptimWrapper', optimizer=dict(_delete_=True, type='AdamW', lr=0.00005, weight_decay=0.05)) train_cfg = dict( _delete_=True, type='IterBasedTrainLoop', max_iters=270000, val_interval=10000) # learning rate param_scheduler = [ dict( type='MultiStepLR', begin=0, end=270000, by_epoch=False, milestones=[210000, 250000], gamma=0.1) ] default_hooks = dict( checkpoint=dict(by_epoch=False, interval=10000, max_keep_ckpts=3)) log_processor = dict(by_epoch=False) # NOTE: `auto_scale_lr` is for automatically scaling LR, # USER SHOULD NOT CHANGE ITS VALUES. # base_batch_size = (8 GPUs) x (8 samples per GPU) auto_scale_lr = dict(base_batch_size=64, enable=True) ================================================ FILE: projects/SparseInst/sparseinst/__init__.py ================================================ from .decoder import BaseIAMDecoder, GroupIAMDecoder, GroupIAMSoftDecoder from .encoder import PyramidPoolingModule from .loss import SparseInstCriterion, SparseInstMatcher from .sparseinst import SparseInst __all__ = [ 'BaseIAMDecoder', 'GroupIAMDecoder', 'GroupIAMSoftDecoder', 'PyramidPoolingModule', 'SparseInstCriterion', 'SparseInstMatcher', 'SparseInst' ] ================================================ FILE: projects/SparseInst/sparseinst/decoder.py ================================================ # Copyright (c) Tianheng Cheng and its affiliates. All Rights Reserved import math import torch import torch.nn as nn import torch.nn.functional as F from mmengine.model.weight_init import caffe2_xavier_init, kaiming_init from torch.nn import init from mmdet.registry import MODELS def _make_stack_3x3_convs(num_convs, in_channels, out_channels, act_cfg=dict(type='ReLU', inplace=True)): convs = [] for _ in range(num_convs): convs.append(nn.Conv2d(in_channels, out_channels, 3, padding=1)) convs.append(MODELS.build(act_cfg)) in_channels = out_channels return nn.Sequential(*convs) class InstanceBranch(nn.Module): def __init__(self, in_channels, dim=256, num_convs=4, num_masks=100, num_classes=80, kernel_dim=128, act_cfg=dict(type='ReLU', inplace=True)): super().__init__() num_masks = num_masks self.num_classes = num_classes self.inst_convs = _make_stack_3x3_convs(num_convs, in_channels, dim, act_cfg) # iam prediction, a simple conv self.iam_conv = nn.Conv2d(dim, num_masks, 3, padding=1) # outputs self.cls_score = nn.Linear(dim, self.num_classes) self.mask_kernel = nn.Linear(dim, kernel_dim) self.objectness = nn.Linear(dim, 1) self.prior_prob = 0.01 self._init_weights() def _init_weights(self): for m in self.inst_convs.modules(): if isinstance(m, nn.Conv2d): kaiming_init(m) bias_value = -math.log((1 - self.prior_prob) / self.prior_prob) for module in [self.iam_conv, self.cls_score]: init.constant_(module.bias, bias_value) init.normal_(self.iam_conv.weight, std=0.01) init.normal_(self.cls_score.weight, std=0.01) init.normal_(self.mask_kernel.weight, std=0.01) init.constant_(self.mask_kernel.bias, 0.0) def forward(self, features): # instance features (x4 convs) features = self.inst_convs(features) # predict instance activation maps iam = self.iam_conv(features) iam_prob = iam.sigmoid() B, N = iam_prob.shape[:2] C = features.size(1) # BxNxHxW -> BxNx(HW) iam_prob = iam_prob.view(B, N, -1) normalizer = iam_prob.sum(-1).clamp(min=1e-6) iam_prob = iam_prob / normalizer[:, :, None] # aggregate features: BxCxHxW -> Bx(HW)xC inst_features = torch.bmm(iam_prob, features.view(B, C, -1).permute(0, 2, 1)) # predict classification & segmentation kernel & objectness pred_logits = self.cls_score(inst_features) pred_kernel = self.mask_kernel(inst_features) pred_scores = self.objectness(inst_features) return pred_logits, pred_kernel, pred_scores, iam class MaskBranch(nn.Module): def __init__(self, in_channels, dim=256, num_convs=4, kernel_dim=128, act_cfg=dict(type='ReLU', inplace=True)): super().__init__() self.mask_convs = _make_stack_3x3_convs(num_convs, in_channels, dim, act_cfg) self.projection = nn.Conv2d(dim, kernel_dim, kernel_size=1) self._init_weights() def _init_weights(self): for m in self.mask_convs.modules(): if isinstance(m, nn.Conv2d): kaiming_init(m) kaiming_init(self.projection) def forward(self, features): # mask features (x4 convs) features = self.mask_convs(features) return self.projection(features) @MODELS.register_module() class BaseIAMDecoder(nn.Module): def __init__(self, in_channels, num_classes, ins_dim=256, ins_conv=4, mask_dim=256, mask_conv=4, kernel_dim=128, scale_factor=2.0, output_iam=False, num_masks=100, act_cfg=dict(type='ReLU', inplace=True)): super().__init__() # add 2 for coordinates in_channels = in_channels # ENCODER.NUM_CHANNELS + 2 self.scale_factor = scale_factor self.output_iam = output_iam self.inst_branch = InstanceBranch( in_channels, dim=ins_dim, num_convs=ins_conv, num_masks=num_masks, num_classes=num_classes, kernel_dim=kernel_dim, act_cfg=act_cfg) self.mask_branch = MaskBranch( in_channels, dim=mask_dim, num_convs=mask_conv, kernel_dim=kernel_dim, act_cfg=act_cfg) @torch.no_grad() def compute_coordinates_linspace(self, x): # linspace is not supported in ONNX h, w = x.size(2), x.size(3) y_loc = torch.linspace(-1, 1, h, device=x.device) x_loc = torch.linspace(-1, 1, w, device=x.device) y_loc, x_loc = torch.meshgrid(y_loc, x_loc) y_loc = y_loc.expand([x.shape[0], 1, -1, -1]) x_loc = x_loc.expand([x.shape[0], 1, -1, -1]) locations = torch.cat([x_loc, y_loc], 1) return locations.to(x) @torch.no_grad() def compute_coordinates(self, x): h, w = x.size(2), x.size(3) y_loc = -1.0 + 2.0 * torch.arange(h, device=x.device) / (h - 1) x_loc = -1.0 + 2.0 * torch.arange(w, device=x.device) / (w - 1) y_loc, x_loc = torch.meshgrid(y_loc, x_loc) y_loc = y_loc.expand([x.shape[0], 1, -1, -1]) x_loc = x_loc.expand([x.shape[0], 1, -1, -1]) locations = torch.cat([x_loc, y_loc], 1) return locations.to(x) def forward(self, features): coord_features = self.compute_coordinates(features) features = torch.cat([coord_features, features], dim=1) pred_logits, pred_kernel, pred_scores, iam = self.inst_branch(features) mask_features = self.mask_branch(features) N = pred_kernel.shape[1] # mask_features: BxCxHxW B, C, H, W = mask_features.shape pred_masks = torch.bmm(pred_kernel, mask_features.view(B, C, H * W)).view(B, N, H, W) pred_masks = F.interpolate( pred_masks, scale_factor=self.scale_factor, mode='bilinear', align_corners=False) output = { 'pred_logits': pred_logits, 'pred_masks': pred_masks, 'pred_scores': pred_scores, } if self.output_iam: iam = F.interpolate( iam, scale_factor=self.scale_factor, mode='bilinear', align_corners=False) output['pred_iam'] = iam return output class GroupInstanceBranch(nn.Module): def __init__(self, in_channels, num_groups=4, dim=256, num_convs=4, num_masks=100, num_classes=80, kernel_dim=128, act_cfg=dict(type='ReLU', inplace=True)): super().__init__() self.num_groups = num_groups self.num_classes = num_classes self.inst_convs = _make_stack_3x3_convs( num_convs, in_channels, dim, act_cfg=act_cfg) # iam prediction, a group conv expand_dim = dim * self.num_groups self.iam_conv = nn.Conv2d( dim, num_masks * self.num_groups, 3, padding=1, groups=self.num_groups) # outputs self.fc = nn.Linear(expand_dim, expand_dim) self.cls_score = nn.Linear(expand_dim, self.num_classes) self.mask_kernel = nn.Linear(expand_dim, kernel_dim) self.objectness = nn.Linear(expand_dim, 1) self.prior_prob = 0.01 self._init_weights() def _init_weights(self): for m in self.inst_convs.modules(): if isinstance(m, nn.Conv2d): kaiming_init(m) bias_value = -math.log((1 - self.prior_prob) / self.prior_prob) for module in [self.iam_conv, self.cls_score]: init.constant_(module.bias, bias_value) init.normal_(self.iam_conv.weight, std=0.01) init.normal_(self.cls_score.weight, std=0.01) init.normal_(self.mask_kernel.weight, std=0.01) init.constant_(self.mask_kernel.bias, 0.0) caffe2_xavier_init(self.fc) def forward(self, features): # instance features (x4 convs) features = self.inst_convs(features) # predict instance activation maps iam = self.iam_conv(features) iam_prob = iam.sigmoid() B, N = iam_prob.shape[:2] C = features.size(1) # BxNxHxW -> BxNx(HW) iam_prob = iam_prob.view(B, N, -1) normalizer = iam_prob.sum(-1).clamp(min=1e-6) iam_prob = iam_prob / normalizer[:, :, None] # aggregate features: BxCxHxW -> Bx(HW)xC inst_features = torch.bmm(iam_prob, features.view(B, C, -1).permute(0, 2, 1)) inst_features = inst_features.reshape(B, 4, N // self.num_groups, -1).transpose(1, 2).reshape( B, N // self.num_groups, -1) inst_features = F.relu_(self.fc(inst_features)) # predict classification & segmentation kernel & objectness pred_logits = self.cls_score(inst_features) pred_kernel = self.mask_kernel(inst_features) pred_scores = self.objectness(inst_features) return pred_logits, pred_kernel, pred_scores, iam @MODELS.register_module() class GroupIAMDecoder(BaseIAMDecoder): def __init__(self, in_channels, num_classes, num_groups=4, ins_dim=256, ins_conv=4, mask_dim=256, mask_conv=4, kernel_dim=128, scale_factor=2.0, output_iam=False, num_masks=100, act_cfg=dict(type='ReLU', inplace=True)): super().__init__( in_channels=in_channels, num_classes=num_classes, ins_dim=ins_dim, ins_conv=ins_conv, mask_dim=mask_dim, mask_conv=mask_conv, kernel_dim=kernel_dim, scale_factor=scale_factor, output_iam=output_iam, num_masks=num_masks, act_cfg=act_cfg) self.inst_branch = GroupInstanceBranch( in_channels, num_groups=num_groups, dim=ins_dim, num_convs=ins_conv, num_masks=num_masks, num_classes=num_classes, kernel_dim=kernel_dim, act_cfg=act_cfg) class GroupInstanceSoftBranch(GroupInstanceBranch): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.softmax_bias = nn.Parameter(torch.ones([ 1, ])) def forward(self, features): # instance features (x4 convs) features = self.inst_convs(features) # predict instance activation maps iam = self.iam_conv(features) B, N = iam.shape[:2] C = features.size(1) # BxNxHxW -> BxNx(HW) iam_prob = F.softmax(iam.view(B, N, -1) + self.softmax_bias, dim=-1) # aggregate features: BxCxHxW -> Bx(HW)xC inst_features = torch.bmm(iam_prob, features.view(B, C, -1).permute(0, 2, 1)) inst_features = inst_features.reshape(B, self.num_groups, N // self.num_groups, -1).transpose(1, 2).reshape( B, N // self.num_groups, -1) inst_features = F.relu_(self.fc(inst_features)) # predict classification & segmentation kernel & objectness pred_logits = self.cls_score(inst_features) pred_kernel = self.mask_kernel(inst_features) pred_scores = self.objectness(inst_features) return pred_logits, pred_kernel, pred_scores, iam @MODELS.register_module() class GroupIAMSoftDecoder(BaseIAMDecoder): def __init__(self, in_channels, num_classes, num_groups=4, ins_dim=256, ins_conv=4, mask_dim=256, mask_conv=4, kernel_dim=128, scale_factor=2.0, output_iam=False, num_masks=100, act_cfg=dict(type='ReLU', inplace=True)): super().__init__( in_channels=in_channels, num_classes=num_classes, ins_dim=ins_dim, ins_conv=ins_conv, mask_dim=mask_dim, mask_conv=mask_conv, kernel_dim=kernel_dim, scale_factor=scale_factor, output_iam=output_iam, num_masks=num_masks, act_cfg=act_cfg) self.inst_branch = GroupInstanceSoftBranch( in_channels, num_groups=num_groups, dim=ins_dim, num_convs=ins_conv, num_masks=num_masks, num_classes=num_classes, kernel_dim=kernel_dim, act_cfg=act_cfg) ================================================ FILE: projects/SparseInst/sparseinst/encoder.py ================================================ # Copyright (c) Tianheng Cheng and its affiliates. All Rights Reserved import torch import torch.nn as nn import torch.nn.functional as F from mmengine.model.weight_init import caffe2_xavier_init, kaiming_init from mmdet.registry import MODELS class PyramidPoolingModule(nn.Module): def __init__(self, in_channels, channels=512, sizes=(1, 2, 3, 6), act_cfg=dict(type='ReLU')): super().__init__() self.stages = [] self.stages = nn.ModuleList( [self._make_stage(in_channels, channels, size) for size in sizes]) self.bottleneck = nn.Conv2d(in_channels + len(sizes) * channels, in_channels, 1) self.act = MODELS.build(act_cfg) def _make_stage(self, features, out_features, size): prior = nn.AdaptiveAvgPool2d(output_size=(size, size)) conv = nn.Conv2d(features, out_features, 1) return nn.Sequential(prior, conv) def forward(self, feats): h, w = feats.size(2), feats.size(3) priors = [ F.interpolate( input=self.act(stage(feats)), size=(h, w), mode='bilinear', align_corners=False) for stage in self.stages ] + [feats] out = self.act(self.bottleneck(torch.cat(priors, 1))) return out @MODELS.register_module() class InstanceContextEncoder(nn.Module): """ Instance Context Encoder 1. construct feature pyramids from ResNet 2. enlarge receptive fields (ppm) 3. multi-scale fusion """ def __init__(self, in_channels, out_channels=256, with_ppm=True, act_cfg=dict(type='ReLU')): super().__init__() self.num_channels = out_channels self.in_channels = in_channels self.with_ppm = with_ppm fpn_laterals = [] fpn_outputs = [] for in_channel in reversed(self.in_channels): lateral_conv = nn.Conv2d(in_channel, self.num_channels, 1) output_conv = nn.Conv2d( self.num_channels, self.num_channels, 3, padding=1) caffe2_xavier_init(lateral_conv) caffe2_xavier_init(output_conv) fpn_laterals.append(lateral_conv) fpn_outputs.append(output_conv) self.fpn_laterals = nn.ModuleList(fpn_laterals) self.fpn_outputs = nn.ModuleList(fpn_outputs) # ppm if self.with_ppm: self.ppm = PyramidPoolingModule( self.num_channels, self.num_channels // 4, act_cfg=act_cfg) # final fusion self.fusion = nn.Conv2d(self.num_channels * 3, self.num_channels, 1) kaiming_init(self.fusion) def forward(self, features): features = features[::-1] prev_features = self.fpn_laterals[0](features[0]) if self.with_ppm: prev_features = self.ppm(prev_features) outputs = [self.fpn_outputs[0](prev_features)] for feature, lat_conv, output_conv in zip(features[1:], self.fpn_laterals[1:], self.fpn_outputs[1:]): lat_features = lat_conv(feature) top_down_features = F.interpolate( prev_features, scale_factor=2.0, mode='nearest') prev_features = lat_features + top_down_features outputs.insert(0, output_conv(prev_features)) size = outputs[0].shape[2:] features = [outputs[0]] + [ F.interpolate(x, size, mode='bilinear', align_corners=False) for x in outputs[1:] ] features = self.fusion(torch.cat(features, dim=1)) return features ================================================ FILE: projects/SparseInst/sparseinst/loss.py ================================================ # Copyright (c) Tianheng Cheng and its affiliates. All Rights Reserved import torch import torch.nn as nn import torch.nn.functional as F from scipy.optimize import linear_sum_assignment from torch.cuda.amp import autocast from mmdet.registry import MODELS, TASK_UTILS from mmdet.utils import reduce_mean def compute_mask_iou(inputs, targets): inputs = inputs.sigmoid() # thresholding binarized_inputs = (inputs >= 0.4).float() targets = (targets > 0.5).float() intersection = (binarized_inputs * targets).sum(-1) union = targets.sum(-1) + binarized_inputs.sum(-1) - intersection score = intersection / (union + 1e-6) return score def dice_score(inputs, targets): inputs = inputs.sigmoid() numerator = 2 * torch.matmul(inputs, targets.t()) denominator = (inputs * inputs).sum(-1)[:, None] + (targets * targets).sum(-1) score = numerator / (denominator + 1e-4) return score @MODELS.register_module() class SparseInstCriterion(nn.Module): """This part is partially derivated from: https://github.com/facebookresearch/detr/blob/main/models/detr.py. """ def __init__( self, num_classes, assigner, loss_cls=dict( type='FocalLoss', use_sigmoid=True, alpha=0.25, gamma=2.0, reduction='sum', loss_weight=2.0), loss_obj=dict( type='CrossEntropyLoss', use_sigmoid=True, reduction='mean', loss_weight=1.0), loss_mask=dict( type='CrossEntropyLoss', use_sigmoid=True, reduction='mean', loss_weight=5.0), loss_dice=dict( type='DiceLoss', use_sigmoid=True, reduction='sum', eps=5e-5, loss_weight=2.0), ): super().__init__() self.matcher = TASK_UTILS.build(assigner) self.num_classes = num_classes self.loss_cls = MODELS.build(loss_cls) self.loss_obj = MODELS.build(loss_obj) self.loss_mask = MODELS.build(loss_mask) self.loss_dice = MODELS.build(loss_dice) def _get_src_permutation_idx(self, indices): # permute predictions following indices batch_idx = torch.cat( [torch.full_like(src, i) for i, (src, _) in enumerate(indices)]) src_idx = torch.cat([src for (src, _) in indices]) return batch_idx, src_idx def _get_tgt_permutation_idx(self, indices): # permute targets following indices batch_idx = torch.cat( [torch.full_like(tgt, i) for i, (_, tgt) in enumerate(indices)]) tgt_idx = torch.cat([tgt for (_, tgt) in indices]) return batch_idx, tgt_idx def loss_classification(self, outputs, batch_gt_instances, indices, num_instances): assert 'pred_logits' in outputs src_logits = outputs['pred_logits'] idx = self._get_src_permutation_idx(indices) target_classes_o = torch.cat( [gt.labels[J] for gt, (_, J) in zip(batch_gt_instances, indices)]) target_classes = torch.full( src_logits.shape[:2], self.num_classes, dtype=torch.int64, device=src_logits.device) target_classes[idx] = target_classes_o src_logits = src_logits.flatten(0, 1) target_classes = target_classes.flatten(0, 1) # comp focal loss. class_loss = self.loss_cls( src_logits, target_classes, ) / num_instances return class_loss def loss_masks_with_iou_objectness(self, outputs, batch_gt_instances, indices, num_instances): src_idx = self._get_src_permutation_idx(indices) tgt_idx = self._get_tgt_permutation_idx(indices) # Bx100xHxW assert 'pred_masks' in outputs assert 'pred_scores' in outputs src_iou_scores = outputs['pred_scores'] src_masks = outputs['pred_masks'] with torch.no_grad(): target_masks = torch.cat([ gt.masks.to_tensor( dtype=src_masks.dtype, device=src_masks.device) for gt in batch_gt_instances ]) num_masks = [len(gt.masks) for gt in batch_gt_instances] target_masks = target_masks.to(src_masks) if len(target_masks) == 0: loss_dice = src_masks.sum() * 0.0 loss_mask = src_masks.sum() * 0.0 loss_objectness = src_iou_scores.sum() * 0.0 return loss_objectness, loss_dice, loss_mask src_masks = src_masks[src_idx] target_masks = F.interpolate( target_masks[:, None], size=src_masks.shape[-2:], mode='bilinear', align_corners=False).squeeze(1) src_masks = src_masks.flatten(1) # FIXME: tgt_idx mix_tgt_idx = torch.zeros_like(tgt_idx[1]) cum_sum = 0 for num_mask in num_masks: mix_tgt_idx[cum_sum:cum_sum + num_mask] = cum_sum cum_sum += num_mask mix_tgt_idx += tgt_idx[1] target_masks = target_masks[mix_tgt_idx].flatten(1) with torch.no_grad(): ious = compute_mask_iou(src_masks, target_masks) tgt_iou_scores = ious src_iou_scores = src_iou_scores[src_idx] tgt_iou_scores = tgt_iou_scores.flatten(0) src_iou_scores = src_iou_scores.flatten(0) loss_objectness = self.loss_obj(src_iou_scores, tgt_iou_scores) loss_dice = self.loss_dice(src_masks, target_masks) / num_instances loss_mask = self.loss_mask(src_masks, target_masks) return loss_objectness, loss_dice, loss_mask def forward(self, outputs, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore): # Retrieve the matching between the outputs of # the last layer and the targets indices = self.matcher(outputs, batch_gt_instances) # Compute the average number of target boxes # across all nodes, for normalization purposes num_instances = sum(gt.labels.shape[0] for gt in batch_gt_instances) num_instances = torch.as_tensor([num_instances], dtype=torch.float, device=next(iter( outputs.values())).device) num_instances = reduce_mean(num_instances).clamp_(min=1).item() # Compute all the requested losses loss_cls = self.loss_classification(outputs, batch_gt_instances, indices, num_instances) loss_obj, loss_dice, loss_mask = self.loss_masks_with_iou_objectness( outputs, batch_gt_instances, indices, num_instances) return dict( loss_cls=loss_cls, loss_obj=loss_obj, loss_dice=loss_dice, loss_mask=loss_mask) @TASK_UTILS.register_module() class SparseInstMatcher(nn.Module): def __init__(self, alpha=0.8, beta=0.2): super().__init__() self.alpha = alpha self.beta = beta self.mask_score = dice_score def forward(self, outputs, batch_gt_instances): with torch.no_grad(): B, N, H, W = outputs['pred_masks'].shape pred_masks = outputs['pred_masks'] pred_logits = outputs['pred_logits'].sigmoid() device = pred_masks.device tgt_ids = torch.cat([gt.labels for gt in batch_gt_instances]) if tgt_ids.shape[0] == 0: return [(torch.as_tensor([]).to(pred_logits), torch.as_tensor([]).to(pred_logits))] * B tgt_masks = torch.cat([ gt.masks.to_tensor(dtype=pred_masks.dtype, device=device) for gt in batch_gt_instances ]) tgt_masks = F.interpolate( tgt_masks[:, None], size=pred_masks.shape[-2:], mode='bilinear', align_corners=False).squeeze(1) pred_masks = pred_masks.view(B * N, -1) tgt_masks = tgt_masks.flatten(1) with autocast(enabled=False): pred_masks = pred_masks.float() tgt_masks = tgt_masks.float() pred_logits = pred_logits.float() mask_score = self.mask_score(pred_masks, tgt_masks) # Nx(Number of gts) matching_prob = pred_logits.view(B * N, -1)[:, tgt_ids] C = (mask_score**self.alpha) * (matching_prob**self.beta) C = C.view(B, N, -1).cpu() # hungarian matching sizes = [len(gt.masks) for gt in batch_gt_instances] indices = [ linear_sum_assignment(c[i], maximize=True) for i, c in enumerate(C.split(sizes, -1)) ] indices = [(torch.as_tensor(i, dtype=torch.int64), torch.as_tensor(j, dtype=torch.int64)) for i, j in indices] return indices ================================================ FILE: projects/SparseInst/sparseinst/sparseinst.py ================================================ # Copyright (c) Tianheng Cheng and its affiliates. All Rights Reserved from typing import List, Tuple, Union import torch import torch.nn.functional as F from mmengine.structures import InstanceData from torch import Tensor from mmdet.models import BaseDetector from mmdet.models.utils import unpack_gt_instances from mmdet.registry import MODELS from mmdet.structures import OptSampleList, SampleList from mmdet.utils import ConfigType, OptConfigType @torch.jit.script def rescoring_mask(scores, mask_pred, masks): mask_pred_ = mask_pred.float() return scores * ((masks * mask_pred_).sum([1, 2]) / (mask_pred_.sum([1, 2]) + 1e-6)) @MODELS.register_module() class SparseInst(BaseDetector): """Implementation of `SparseInst `_ Args: data_preprocessor (:obj:`ConfigDict` or dict, optional): Config of :class:`DetDataPreprocessor` to process the input data. Defaults to None. backbone (:obj:`ConfigDict` or dict): The backbone module. encoder (:obj:`ConfigDict` or dict): The encoder module. decoder (:obj:`ConfigDict` or dict): The decoder module. criterion (:obj:`ConfigDict` or dict, optional): The training matcher and losses. Defaults to None. test_cfg (:obj:`ConfigDict` or dict, optional): The testing config of SparseInst. Defaults to None. init_cfg (:obj:`ConfigDict` or dict, optional): the config to control the initialization. Defaults to None. """ def __init__(self, data_preprocessor: ConfigType, backbone: ConfigType, encoder: ConfigType, decoder: ConfigType, criterion: OptConfigType = None, test_cfg: OptConfigType = None, init_cfg: OptConfigType = None): super().__init__( data_preprocessor=data_preprocessor, init_cfg=init_cfg) # backbone self.backbone = MODELS.build(backbone) # encoder & decoder self.encoder = MODELS.build(encoder) self.decoder = MODELS.build(decoder) # matcher & loss (matcher is built in loss) self.criterion = MODELS.build(criterion) # inference self.cls_threshold = test_cfg.score_thr self.mask_threshold = test_cfg.mask_thr_binary def _forward( self, batch_inputs: Tensor, batch_data_samples: OptSampleList = None) -> Tuple[List[Tensor]]: """Network forward process. Usually includes backbone, neck and head forward without any post-processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). Returns: tuple[list]: A tuple of features from ``bbox_head`` forward. """ x = self.backbone(batch_inputs) x = self.encoder(x) results = self.decoder(x) return results def predict(self, batch_inputs: Tensor, batch_data_samples: SampleList, rescale: bool = True) -> SampleList: """Predict results from a batch of inputs and data samples with post- processing. Args: batch_inputs (Tensor): Inputs with shape (N, C, H, W). batch_data_samples (List[:obj:`DetDataSample`]): The Data Samples. It usually includes information such as `gt_instance`, `gt_panoptic_seg` and `gt_sem_seg`. rescale (bool): Whether to rescale the results. Defaults to True. Returns: list[:obj:`DetDataSample`]: Detection results of the input images. Each DetDataSample usually contain 'pred_instances'. And the ``pred_instances`` usually contains following keys. - scores (Tensor): Classification scores, has a shape (num_instance, ) - labels (Tensor): Labels of bboxes, has a shape (num_instances, ). - bboxes (Tensor): Has a shape (num_instances, 4), the last dimension 4 arrange as (x1, y1, x2, y2). """ max_shape = batch_inputs.shape[-2:] output = self._forward(batch_inputs) pred_scores = output['pred_logits'].sigmoid() pred_masks = output['pred_masks'].sigmoid() pred_objectness = output['pred_scores'].sigmoid() pred_scores = torch.sqrt(pred_scores * pred_objectness) results_list = [] for batch_idx, (scores_per_image, mask_pred_per_image, datasample) in enumerate( zip(pred_scores, pred_masks, batch_data_samples)): result = InstanceData() # max/argmax scores, labels = scores_per_image.max(dim=-1) # cls threshold keep = scores > self.cls_threshold scores = scores[keep] labels = labels[keep] mask_pred_per_image = mask_pred_per_image[keep] if scores.size(0) == 0: result.scores = scores result.labels = labels results_list.append(result) continue img_meta = datasample.metainfo # rescoring mask using maskness scores = rescoring_mask(scores, mask_pred_per_image > self.mask_threshold, mask_pred_per_image) h, w = img_meta['img_shape'][:2] mask_pred_per_image = F.interpolate( mask_pred_per_image.unsqueeze(1), size=max_shape, mode='bilinear', align_corners=False)[:, :, :h, :w] if rescale: ori_h, ori_w = img_meta['ori_shape'][:2] mask_pred_per_image = F.interpolate( mask_pred_per_image, size=(ori_h, ori_w), mode='bilinear', align_corners=False).squeeze(1) mask_pred = mask_pred_per_image > self.mask_threshold result.masks = mask_pred result.scores = scores result.labels = labels # create an empty bbox in InstanceData to avoid bugs when # calculating metrics. result.bboxes = result.scores.new_zeros(len(scores), 4) results_list.append(result) batch_data_samples = self.add_pred_to_datasample( batch_data_samples, results_list) return batch_data_samples def loss(self, batch_inputs: Tensor, batch_data_samples: SampleList) -> Union[dict, list]: """Calculate losses from a batch of inputs and data samples. Args: batch_inputs (Tensor): Input images of shape (N, C, H, W). These should usually be mean centered and std scaled. batch_data_samples (list[:obj:`DetDataSample`]): The batch data samples. It usually includes information such as `gt_instance` or `gt_panoptic_seg` or `gt_sem_seg`. Returns: dict: A dictionary of loss components. """ outs = self._forward(batch_inputs) (batch_gt_instances, batch_gt_instances_ignore, batch_img_metas) = unpack_gt_instances(batch_data_samples) losses = self.criterion(outs, batch_gt_instances, batch_img_metas, batch_gt_instances_ignore) return losses def extract_feat(self, batch_inputs: Tensor) -> Tuple[Tensor]: """Extract features. Args: batch_inputs (Tensor): Image tensor with shape (N, C, H ,W). Returns: tuple[Tensor]: Multi-level features that may have different resolutions. """ x = self.backbone(batch_inputs) x = self.encoder(x) return x ================================================ FILE: projects/example_project/README.md ================================================ # Dummy ResNet Wrapper This is an example README for community `projects/`. We have provided detailed explanations for each field in the form of html comments, which are visible when you read the source of this README file. If you wish to submit your project to our main repository, then all the fields in this README are mandatory for others to understand what you have achieved in this implementation. For more details, read our [contribution guide](https://mmdetection.readthedocs.io/en/3.x/notes/contribution_guide.html) or approach us in [Discussions](https://github.com/open-mmlab/mmdetection/discussions). ## Description This project implements a dummy ResNet wrapper, which literally does nothing new but prints "hello world" during initialization. ## Usage ### Training commands In MMDetection's root directory, run the following command to train the model: ```bash python tools/train.py projects/example_project/configs/faster-rcnn_dummy-resnet_fpn_1x_coco.py ``` For multi-gpu training, run: ```bash python -m torch.distributed.launch --nnodes=1 --node_rank=0 --nproc_per_node=${NUM_GPUS} --master_port=29506 --master_addr="127.0.0.1" tools/train.py projects/example_project/configs/faster-rcnn_dummy-resnet_fpn_1x_coco.py ``` ### Testing commands In MMDetection's root directory, run the following command to test the model: ```bash python tools/test.py projects/example_project/configs/faster-rcnn_dummy-resnet_fpn_1x_coco.py ${CHECKPOINT_PATH} ``` ## Results | Method | Backbone | Pretrained Model | Training set | Test set | #epoch | box AP | Download | | :-------------------------------------------------------------------: | :---------: | :--------------: | :------------: | :----------: | :----: | :----: | :----------------------: | | [Faster R-CNN dummy](configs/faster-rcnn_dummy-resnet_fpn_1x_coco.py) | DummyResNet | - | COCO2017 Train | COCO2017 Val | 12 | 0.8853 | [model](<>) \| [log](<>) | ## Citation ```latex @article{Ren_2017, title={Faster R-CNN: Towards Real-Time Object Detection with Region Proposal Networks}, journal={IEEE Transactions on Pattern Analysis and Machine Intelligence}, publisher={Institute of Electrical and Electronics Engineers (IEEE)}, author={Ren, Shaoqing and He, Kaiming and Girshick, Ross and Sun, Jian}, year={2017}, month={Jun}, } ``` ## Checklist - [ ] Milestone 1: PR-ready, and acceptable to be one of the `projects/`. - [ ] Finish the code - [ ] Basic docstrings & proper citation - [ ] Test-time correctness - [ ] A full README - [ ] Milestone 2: Indicates a successful model implementation. - [ ] Training-time correctness - [ ] Milestone 3: Good to be a part of our core package! - [ ] Type hints and docstrings - [ ] Unit tests - [ ] Code polishing - [ ] Metafile.yml - [ ] Move your modules into the core package following the codebase's file hierarchy structure. - [ ] Refactor your modules into the core package following the codebase's file hierarchy structure. ================================================ FILE: projects/example_project/configs/faster-rcnn_dummy-resnet_fpn_1x_coco.py ================================================ _base_ = ['../../../configs/faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py'] custom_imports = dict(imports=['projects.example_project.dummy']) _base_.model.backbone.type = 'DummyResNet' ================================================ FILE: projects/example_project/dummy/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .dummy_resnet import DummyResNet __all__ = ['DummyResNet'] ================================================ FILE: projects/example_project/dummy/dummy_resnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from mmdet.models.backbones import ResNet from mmdet.registry import MODELS @MODELS.register_module() class DummyResNet(ResNet): """Implements a dummy ResNet wrapper for demonstration purpose. Args: **kwargs: All the arguments are passed to the parent class. """ def __init__(self, **kwargs) -> None: print('Hello world!') super().__init__(**kwargs) ================================================ FILE: pytest.ini ================================================ [pytest] addopts = --xdoctest --xdoctest-style=auto norecursedirs = .git ignore build __pycache__ data docker docs .eggs filterwarnings= default ignore:.*No cfgstr given in Cacher constructor or call.*:Warning ignore:.*Define the __nice__ method for.*:Warning ================================================ FILE: requirements/albu.txt ================================================ albumentations>=0.3.2 --no-binary qudida,albumentations ================================================ FILE: requirements/build.txt ================================================ # These must be installed before building mmdetection cython numpy ================================================ FILE: requirements/docs.txt ================================================ docutils==0.16.0 myst-parser -e git+https://github.com/open-mmlab/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme sphinx==4.0.2 sphinx-copybutton sphinx_markdown_tables sphinx_rtd_theme==0.5.2 ================================================ FILE: requirements/mminstall.txt ================================================ mmcv>=2.0.0rc4,<2.1.0 mmengine>=0.4.0,<1.0.0 ================================================ FILE: requirements/optional.txt ================================================ cityscapesscripts imagecorruptions scikit-learn ================================================ FILE: requirements/readthedocs.txt ================================================ mmcv>=2.0.0rc1,<2.1.0 mmengine>=0.1.0,<1.0.0 scipy torch torchvision ================================================ FILE: requirements/runtime.txt ================================================ matplotlib numpy pycocotools scipy six terminaltables ================================================ FILE: requirements/tests.txt ================================================ asynctest cityscapesscripts codecov flake8 imagecorruptions instaboostfast interrogate isort==4.3.21 # Note: used for kwarray.group_items, this may be ported to mmcv in the future. kwarray memory_profiler -e git+https://github.com/open-mmlab/mmtracking@dev-1.x#egg=mmtrack onnx==1.7.0 onnxruntime>=1.8.0 parameterized protobuf<=3.20.1 psutil pytest ubelt xdoctest>=0.10.0 yapf ================================================ FILE: requirements.txt ================================================ -r requirements/build.txt -r requirements/optional.txt -r requirements/runtime.txt ================================================ FILE: setup.cfg ================================================ [isort] line_length = 79 multi_line_output = 0 extra_standard_library = setuptools known_first_party = mmdet known_third_party = PIL,asynctest,cityscapesscripts,cv2,gather_models,matplotlib,mmcv,mmengine,numpy,onnx,onnxruntime,pycocotools,parameterized,pytest,pytorch_sphinx_theme,requests,scipy,seaborn,six,terminaltables,torch,ts,yaml no_lines_before = STDLIB,LOCALFOLDER default_section = THIRDPARTY [yapf] BASED_ON_STYLE = pep8 BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = true SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN = true # ignore-words-list needs to be lowercase format. For example, if we want to # ignore word "BA", then we need to append "ba" to ignore-words-list rather # than "BA" [codespell] skip = *.ipynb quiet-level = 3 ignore-words-list = patten,nd,ty,mot,hist,formating,winn,gool,datas,wan,confids,TOOD,tood,ba,warmup,nam,DOTA,dota ================================================ FILE: setup.py ================================================ #!/usr/bin/env python # Copyright (c) OpenMMLab. All rights reserved. import os import os.path as osp import platform import shutil import sys import warnings from setuptools import find_packages, setup import torch from torch.utils.cpp_extension import (BuildExtension, CppExtension, CUDAExtension) def readme(): with open('README.md', encoding='utf-8') as f: content = f.read() return content version_file = 'mmdet/version.py' def get_version(): with open(version_file, 'r') as f: exec(compile(f.read(), version_file, 'exec')) return locals()['__version__'] def make_cuda_ext(name, module, sources, sources_cuda=[]): define_macros = [] extra_compile_args = {'cxx': []} if torch.cuda.is_available() or os.getenv('FORCE_CUDA', '0') == '1': define_macros += [('WITH_CUDA', None)] extension = CUDAExtension extra_compile_args['nvcc'] = [ '-D__CUDA_NO_HALF_OPERATORS__', '-D__CUDA_NO_HALF_CONVERSIONS__', '-D__CUDA_NO_HALF2_OPERATORS__', ] sources += sources_cuda else: print(f'Compiling {name} without CUDA') extension = CppExtension return extension( name=f'{module}.{name}', sources=[os.path.join(*module.split('.'), p) for p in sources], define_macros=define_macros, extra_compile_args=extra_compile_args) def parse_requirements(fname='requirements.txt', with_version=True): """Parse the package dependencies listed in a requirements file but strips specific versioning information. Args: fname (str): path to requirements file with_version (bool, default=False): if True include version specs Returns: List[str]: list of requirements items CommandLine: python -c "import setup; print(setup.parse_requirements())" """ import re import sys from os.path import exists require_fpath = fname def parse_line(line): """Parse information from a line in a requirements text file.""" if line.startswith('-r '): # Allow specifying requirements in other files target = line.split(' ')[1] for info in parse_require_file(target): yield info else: info = {'line': line} if line.startswith('-e '): info['package'] = line.split('#egg=')[1] elif '@git+' in line: info['package'] = line else: # Remove versioning from the package pat = '(' + '|'.join(['>=', '==', '>']) + ')' parts = re.split(pat, line, maxsplit=1) parts = [p.strip() for p in parts] info['package'] = parts[0] if len(parts) > 1: op, rest = parts[1:] if ';' in rest: # Handle platform specific dependencies # http://setuptools.readthedocs.io/en/latest/setuptools.html#declaring-platform-specific-dependencies version, platform_deps = map(str.strip, rest.split(';')) info['platform_deps'] = platform_deps else: version = rest # NOQA info['version'] = (op, version) yield info def parse_require_file(fpath): with open(fpath, 'r') as f: for line in f.readlines(): line = line.strip() if line and not line.startswith('#'): for info in parse_line(line): yield info def gen_packages_items(): if exists(require_fpath): for info in parse_require_file(require_fpath): parts = [info['package']] if with_version and 'version' in info: parts.extend(info['version']) if not sys.version.startswith('3.4'): # apparently package_deps are broken in 3.4 platform_deps = info.get('platform_deps') if platform_deps is not None: parts.append(';' + platform_deps) item = ''.join(parts) yield item packages = list(gen_packages_items()) return packages def add_mim_extension(): """Add extra files that are required to support MIM into the package. These files will be added by creating a symlink to the originals if the package is installed in `editable` mode (e.g. pip install -e .), or by copying from the originals otherwise. """ # parse installment mode if 'develop' in sys.argv: # installed by `pip install -e .` if platform.system() == 'Windows': # set `copy` mode here since symlink fails on Windows. mode = 'copy' else: mode = 'symlink' elif 'sdist' in sys.argv or 'bdist_wheel' in sys.argv: # installed by `pip install .` # or create source distribution by `python setup.py sdist` mode = 'copy' else: return filenames = ['tools', 'configs', 'demo', 'model-index.yml'] repo_path = osp.dirname(__file__) mim_path = osp.join(repo_path, 'mmdet', '.mim') os.makedirs(mim_path, exist_ok=True) for filename in filenames: if osp.exists(filename): src_path = osp.join(repo_path, filename) tar_path = osp.join(mim_path, filename) if osp.isfile(tar_path) or osp.islink(tar_path): os.remove(tar_path) elif osp.isdir(tar_path): shutil.rmtree(tar_path) if mode == 'symlink': src_relpath = osp.relpath(src_path, osp.dirname(tar_path)) os.symlink(src_relpath, tar_path) elif mode == 'copy': if osp.isfile(src_path): shutil.copyfile(src_path, tar_path) elif osp.isdir(src_path): shutil.copytree(src_path, tar_path) else: warnings.warn(f'Cannot copy file {src_path}.') else: raise ValueError(f'Invalid mode {mode}') if __name__ == '__main__': add_mim_extension() setup( name='mmdet', version=get_version(), description='OpenMMLab Detection Toolbox and Benchmark', long_description=readme(), long_description_content_type='text/markdown', author='MMDetection Contributors', author_email='openmmlab@gmail.com', keywords='computer vision, object detection', url='https://github.com/open-mmlab/mmdetection', packages=find_packages(exclude=('configs', 'tools', 'demo')), include_package_data=True, classifiers=[ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', ], license='Apache License 2.0', install_requires=parse_requirements('requirements/runtime.txt'), extras_require={ 'all': parse_requirements('requirements.txt'), 'tests': parse_requirements('requirements/tests.txt'), 'build': parse_requirements('requirements/build.txt'), 'optional': parse_requirements('requirements/optional.txt'), 'mim': parse_requirements('requirements/mminstall.txt'), }, ext_modules=[], cmdclass={'build_ext': BuildExtension}, zip_safe=False) ================================================ FILE: tests/test_apis/test_det_inferencer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp import tempfile from unittest import TestCase, mock from unittest.mock import Mock, patch import mmcv import mmengine import numpy as np import torch from mmengine.structures import InstanceData from mmengine.utils import is_list_of from parameterized import parameterized from mmdet.apis import DetInferencer from mmdet.evaluation.functional import get_classes from mmdet.structures import DetDataSample class TestDetInferencer(TestCase): @mock.patch('mmengine.infer.infer._load_checkpoint', return_value=None) def test_init(self, mock): # init from metafile DetInferencer('rtmdet-t') # init from cfg DetInferencer('configs/yolox/yolox_tiny_8xb8-300e_coco.py') def assert_predictions_equal(self, preds1, preds2): for pred1, pred2 in zip(preds1, preds2): if 'bboxes' in pred1: self.assertTrue( np.allclose(pred1['bboxes'], pred2['bboxes'], 0.1)) if 'scores' in pred1: self.assertTrue( np.allclose(pred1['scores'], pred2['scores'], 0.1)) if 'labels' in pred1: self.assertTrue(np.allclose(pred1['labels'], pred2['labels'])) if 'panoptic_seg_path' in pred1: self.assertTrue( pred1['panoptic_seg_path'] == pred2['panoptic_seg_path']) @parameterized.expand([ 'rtmdet-t', 'mask-rcnn_r50_fpn_1x_coco', 'panoptic_fpn_r50_fpn_1x_coco' ]) def test_call(self, model): # single img img_path = 'tests/data/color.jpg' mock_load = Mock(return_value=None) with patch('mmengine.infer.infer._load_checkpoint', mock_load): inferencer = DetInferencer(model) # In the case of not loading the pretrained weight, the category # defaults to COCO 80, so it needs to be replaced. if model == 'panoptic_fpn_r50_fpn_1x_coco': inferencer.visualizer.dataset_meta = { 'classes': get_classes('coco_panoptic'), 'palette': 'random' } res_path = inferencer(img_path, return_vis=True) # ndarray img = mmcv.imread(img_path) res_ndarray = inferencer(img, return_vis=True) self.assert_predictions_equal(res_path['predictions'], res_ndarray['predictions']) self.assertIn('visualization', res_path) self.assertIn('visualization', res_ndarray) # multiple images img_paths = ['tests/data/color.jpg', 'tests/data/gray.jpg'] res_path = inferencer(img_paths, return_vis=True) # list of ndarray imgs = [mmcv.imread(p) for p in img_paths] res_ndarray = inferencer(imgs, return_vis=True) self.assert_predictions_equal(res_path['predictions'], res_ndarray['predictions']) self.assertIn('visualization', res_path) self.assertIn('visualization', res_ndarray) # img dir, test different batch sizes img_dir = 'tests/data/VOCdevkit/VOC2007/JPEGImages/' res_bs1 = inferencer(img_dir, batch_size=1, return_vis=True) res_bs3 = inferencer(img_dir, batch_size=3, return_vis=True) self.assert_predictions_equal(res_bs1['predictions'], res_bs3['predictions']) # There is a jitter operation when the mask is drawn, # so it cannot be asserted. if model == 'rtmdet-t': for res_bs1_vis, res_bs3_vis in zip(res_bs1['visualization'], res_bs3['visualization']): self.assertTrue(np.allclose(res_bs1_vis, res_bs3_vis)) @parameterized.expand([ 'rtmdet-t', 'mask-rcnn_r50_fpn_1x_coco', 'panoptic_fpn_r50_fpn_1x_coco' ]) def test_visualize(self, model): img_paths = ['tests/data/color.jpg', 'tests/data/gray.jpg'] mock_load = Mock(return_value=None) with patch('mmengine.infer.infer._load_checkpoint', mock_load): inferencer = DetInferencer(model) # In the case of not loading the pretrained weight, the category # defaults to COCO 80, so it needs to be replaced. if model == 'panoptic_fpn_r50_fpn_1x_coco': inferencer.visualizer.dataset_meta = { 'classes': get_classes('coco_panoptic'), 'palette': 'random' } with tempfile.TemporaryDirectory() as tmp_dir: inferencer(img_paths, out_dir=tmp_dir) for img_dir in ['color.jpg', 'gray.jpg']: self.assertTrue(osp.exists(osp.join(tmp_dir, 'vis', img_dir))) @parameterized.expand([ 'rtmdet-t', 'mask-rcnn_r50_fpn_1x_coco', 'panoptic_fpn_r50_fpn_1x_coco' ]) def test_postprocess(self, model): # return_datasample img_path = 'tests/data/color.jpg' mock_load = Mock(return_value=None) with patch('mmengine.infer.infer._load_checkpoint', mock_load): inferencer = DetInferencer(model) # In the case of not loading the pretrained weight, the category # defaults to COCO 80, so it needs to be replaced. if model == 'panoptic_fpn_r50_fpn_1x_coco': inferencer.visualizer.dataset_meta = { 'classes': get_classes('coco_panoptic'), 'palette': 'random' } res = inferencer(img_path, return_datasample=True) self.assertTrue(is_list_of(res['predictions'], DetDataSample)) with tempfile.TemporaryDirectory() as tmp_dir: res = inferencer(img_path, out_dir=tmp_dir, no_save_pred=False) dumped_res = mmengine.load( osp.join(tmp_dir, 'preds', 'color.json')) self.assertEqual(res['predictions'][0], dumped_res) @mock.patch('mmengine.infer.infer._load_checkpoint', return_value=None) def test_pred2dict(self, mock): data_sample = DetDataSample() data_sample.pred_instances = InstanceData() data_sample.pred_instances.bboxes = np.array([[0, 0, 1, 1]]) data_sample.pred_instances.labels = np.array([0]) data_sample.pred_instances.scores = torch.FloatTensor([0.9]) res = DetInferencer('rtmdet-t').pred2dict(data_sample) self.assertListAlmostEqual(res['bboxes'], [[0, 0, 1, 1]]) self.assertListAlmostEqual(res['labels'], [0]) self.assertListAlmostEqual(res['scores'], [0.9]) def assertListAlmostEqual(self, list1, list2, places=7): for i in range(len(list1)): if isinstance(list1[i], list): self.assertListAlmostEqual(list1[i], list2[i], places=places) else: self.assertAlmostEqual(list1[i], list2[i], places=places) ================================================ FILE: tests/test_apis/test_inference.py ================================================ import os from pathlib import Path import numpy as np import pytest import torch from mmdet.apis import inference_detector, init_detector from mmdet.structures import DetDataSample from mmdet.utils import register_all_modules # TODO: Waiting to fix multiple call error bug register_all_modules() @pytest.mark.parametrize('config,devices', [('configs/retinanet/retinanet_r18_fpn_1x_coco.py', ('cpu', 'cuda'))]) def test_init_detector(config, devices): assert all([device in ['cpu', 'cuda'] for device in devices]) project_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) project_dir = os.path.join(project_dir, '..') config_file = os.path.join(project_dir, config) # test init_detector with config_file: str and cfg_options cfg_options = dict( model=dict( backbone=dict( depth=18, init_cfg=dict( type='Pretrained', checkpoint='torchvision://resnet18')))) for device in devices: if device == 'cuda' and not torch.cuda.is_available(): pytest.skip('test requires GPU and torch+cuda') model = init_detector( config_file, device=device, cfg_options=cfg_options) # test init_detector with :obj:`Path` config_path_object = Path(config_file) model = init_detector(config_path_object, device=device) # test init_detector with undesirable type with pytest.raises(TypeError): config_list = [config_file] model = init_detector(config_list) # noqa: F841 @pytest.mark.parametrize('config,devices', [('configs/retinanet/retinanet_r18_fpn_1x_coco.py', ('cpu', 'cuda'))]) def test_inference_detector(config, devices): assert all([device in ['cpu', 'cuda'] for device in devices]) project_dir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) project_dir = os.path.join(project_dir, '..') config_file = os.path.join(project_dir, config) # test init_detector with config_file: str and cfg_options rng = np.random.RandomState(0) img1 = rng.randint(0, 255, (100, 100, 3), dtype=np.uint8) img2 = rng.randint(0, 255, (100, 100, 3), dtype=np.uint8) for device in devices: if device == 'cuda' and not torch.cuda.is_available(): pytest.skip('test requires GPU and torch+cuda') model = init_detector(config_file, device=device) result = inference_detector(model, img1) assert isinstance(result, DetDataSample) result = inference_detector(model, [img1, img2]) assert isinstance(result, list) and len(result) == 2 ================================================ FILE: tests/test_datasets/test_cityscapes.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os import unittest from mmengine.fileio import dump from mmdet.datasets import CityscapesDataset class TestCityscapesDataset(unittest.TestCase): def setUp(self) -> None: image1 = { 'file_name': 'munster/munster_000102_000019_leftImg8bit.png', 'height': 1024, 'width': 2048, 'segm_file': 'munster/munster_000102_000019_gtFine_labelIds.png', 'id': 0 } image2 = { 'file_name': 'munster/munster_000157_000019_leftImg8bit.png', 'height': 1024, 'width': 2048, 'segm_file': 'munster/munster_000157_000019_gtFine_labelIds.png', 'id': 1 } image3 = { 'file_name': 'munster/munster_000139_000019_leftImg8bit.png', 'height': 1024, 'width': 2048, 'segm_file': 'munster/munster_000139_000019_gtFine_labelIds.png', 'id': 2 } image4 = { 'file_name': 'munster/munster_000034_000019_leftImg8bit.png', 'height': 31, 'width': 15, 'segm_file': 'munster/munster_000034_000019_gtFine_labelIds.png', 'id': 3 } images = [image1, image2, image3, image4] categories = [{ 'id': 24, 'name': 'person' }, { 'id': 25, 'name': 'rider' }, { 'id': 26, 'name': 'car' }] annotations = [ { 'iscrowd': 0, 'category_id': 24, 'bbox': [379.0, 435.0, 52.0, 124.0], 'area': 2595, 'segmentation': { 'size': [1024, 2048], 'counts': 'xxx' }, 'image_id': 0, 'id': 0 }, { 'iscrowd': 0, 'category_id': 25, 'bbox': [379.0, 435.0, 52.0, 124.0], 'area': -1, 'segmentation': { 'size': [1024, 2048], 'counts': 'xxx' }, 'image_id': 0, 'id': 1 }, { 'iscrowd': 0, 'category_id': 26, 'bbox': [379.0, 435.0, -1, 124.0], 'area': 2, 'segmentation': { 'size': [1024, 2048], 'counts': 'xxx' }, 'image_id': 0, 'id': 2 }, { 'iscrowd': 0, 'category_id': 24, 'bbox': [379.0, 435.0, 52.0, -1], 'area': 2, 'segmentation': { 'size': [1024, 2048], 'counts': 'xxx' }, 'image_id': 0, 'id': 3 }, { 'iscrowd': 0, 'category_id': 1, 'bbox': [379.0, 435.0, 52.0, 124.0], 'area': 2595, 'segmentation': { 'size': [1024, 2048], 'counts': 'xxx' }, 'image_id': 0, 'id': 4 }, { 'iscrowd': 1, 'category_id': 26, 'bbox': [379.0, 435.0, 52.0, 124.0], 'area': 2595, 'segmentation': { 'size': [1024, 2048], 'counts': 'xxx' }, 'image_id': 1, 'id': 5 }, { 'iscrowd': 0, 'category_id': 26, 'bbox': [379.0, 435.0, 10, 2], 'area': 2595, 'segmentation': { 'size': [1024, 2048], 'counts': 'xxx' }, 'image_id': 3, 'id': 6 }, ] fake_json = { 'images': images, 'annotations': annotations, 'categories': categories } self.json_name = 'cityscapes.json' dump(fake_json, self.json_name) self.metainfo = dict(classes=('person', 'rider', 'car')) def tearDown(self): os.remove(self.json_name) def test_cityscapes_dataset(self): dataset = CityscapesDataset( ann_file=self.json_name, data_prefix=dict(img='imgs'), metainfo=self.metainfo, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[]) self.assertEqual(dataset.metainfo['classes'], self.metainfo['classes']) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 1) self.assertEqual(len(dataset.load_data_list()), 4) dataset = CityscapesDataset( ann_file=self.json_name, data_prefix=dict(img='imgs'), metainfo=self.metainfo, test_mode=True, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[]) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 4) self.assertEqual(len(dataset.load_data_list()), 4) def test_cityscapes_dataset_without_filter_cfg(self): dataset = CityscapesDataset( ann_file=self.json_name, data_prefix=dict(img='imgs'), metainfo=self.metainfo, filter_cfg=None, pipeline=[]) self.assertEqual(dataset.metainfo['classes'], self.metainfo['classes']) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 4) self.assertEqual(len(dataset.load_data_list()), 4) dataset = CityscapesDataset( ann_file=self.json_name, data_prefix=dict(img='imgs'), metainfo=self.metainfo, test_mode=True, filter_cfg=None, pipeline=[]) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 4) self.assertEqual(len(dataset.load_data_list()), 4) ================================================ FILE: tests/test_datasets/test_coco.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from mmdet.datasets import CocoDataset class TestCocoDataset(unittest.TestCase): def test_coco_dataset(self): # test CocoDataset metainfo = dict(classes=('bus', 'car'), task_name='new_task') dataset = CocoDataset( data_prefix=dict(img='imgs'), ann_file='tests/data/coco_sample.json', metainfo=metainfo, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[], serialize_data=False, lazy_init=False) self.assertEqual(dataset.metainfo['classes'], ('bus', 'car')) self.assertEqual(dataset.metainfo['task_name'], 'new_task') self.assertListEqual(dataset.get_cat_ids(0), [0, 1]) def test_coco_dataset_without_filter_cfg(self): # test CocoDataset without filter_cfg dataset = CocoDataset( data_prefix=dict(img='imgs'), ann_file='tests/data/coco_sample.json', pipeline=[]) self.assertEqual(len(dataset), 4) # test with test_mode = True dataset = CocoDataset( data_prefix=dict(img='imgs'), ann_file='tests/data/coco_sample.json', test_mode=True, pipeline=[]) self.assertEqual(len(dataset), 4) def test_coco_annotation_ids_unique(self): # test annotation ids not unique error metainfo = dict(classes=('car', ), task_name='new_task') with self.assertRaisesRegex(AssertionError, 'are not unique!'): CocoDataset( data_prefix=dict(img='imgs'), ann_file='tests/data/coco_wrong_format_sample.json', metainfo=metainfo, pipeline=[]) ================================================ FILE: tests/test_datasets/test_coco_api_wrapper.py ================================================ import os.path as osp import tempfile import unittest from mmengine.fileio import dump from mmdet.datasets.api_wrappers import COCOPanoptic class TestCOCOPanoptic(unittest.TestCase): def setUp(self): self.tmp_dir = tempfile.TemporaryDirectory() def tearDown(self): self.tmp_dir.cleanup() def test_create_index(self): ann_json = {'test': ['test', 'createIndex']} annotation_file = osp.join(self.tmp_dir.name, 'createIndex.json') dump(ann_json, annotation_file) COCOPanoptic(annotation_file) def test_load_anns(self): categories = [{ 'id': 0, 'name': 'person', 'supercategory': 'person', 'isthing': 1 }] images = [{ 'id': 0, 'width': 80, 'height': 60, 'file_name': 'fake_name1.jpg', }] annotations = [{ 'segments_info': [ { 'id': 1, 'category_id': 0, 'area': 400, 'bbox': [10, 10, 10, 40], 'iscrowd': 0 }, ], 'file_name': 'fake_name1.png', 'image_id': 0 }] ann_json = { 'images': images, 'annotations': annotations, 'categories': categories, } annotation_file = osp.join(self.tmp_dir.name, 'load_anns.json') dump(ann_json, annotation_file) api = COCOPanoptic(annotation_file) api.load_anns(1) self.assertIsNone(api.load_anns(0.1)) ================================================ FILE: tests/test_datasets/test_coco_panoptic.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os import unittest from mmengine.fileio import dump from mmdet.datasets import CocoPanopticDataset class TestCocoPanopticDataset(unittest.TestCase): def setUp(self): image1 = { 'id': 0, 'width': 640, 'height': 640, 'file_name': 'fake_name1.jpg', } image2 = { 'id': 1, 'width': 640, 'height': 800, 'file_name': 'fake_name2.jpg', } image3 = { 'id': 2, 'width': 31, 'height': 40, 'file_name': 'fake_name3.jpg', } image4 = { 'id': 3, 'width': 400, 'height': 400, 'file_name': 'fake_name4.jpg', } images = [image1, image2, image3, image4] annotations = [ { 'segments_info': [{ 'id': 1, 'category_id': 0, 'area': 400, 'bbox': [50, 60, 20, 20], 'iscrowd': 0 }, { 'id': 2, 'category_id': 1, 'area': 900, 'bbox': [100, 120, 30, 30], 'iscrowd': 0 }, { 'id': 3, 'category_id': 2, 'iscrowd': 0, 'bbox': [1, 189, 612, 285], 'area': 70036 }], 'file_name': 'fake_name1.jpg', 'image_id': 0 }, { 'segments_info': [ { # Different to instance style json, there # are duplicate ids in panoptic style json 'id': 1, 'category_id': 0, 'area': 400, 'bbox': [50, 60, 20, 20], 'iscrowd': 0 }, { 'id': 4, 'category_id': 1, 'area': 900, 'bbox': [100, 120, 30, 30], 'iscrowd': 1 }, { 'id': 5, 'category_id': 2, 'iscrowd': 0, 'bbox': [100, 200, 200, 300], 'area': 66666 }, { 'id': 6, 'category_id': 0, 'iscrowd': 0, 'bbox': [1, 189, -10, 285], 'area': -2 }, { 'id': 10, 'category_id': 0, 'iscrowd': 0, 'bbox': [1, 189, 10, -285], 'area': 100 } ], 'file_name': 'fake_name2.jpg', 'image_id': 1 }, { 'segments_info': [{ 'id': 7, 'category_id': 0, 'area': 25, 'bbox': [0, 0, 5, 5], 'iscrowd': 0 }], 'file_name': 'fake_name3.jpg', 'image_id': 2 }, { 'segments_info': [{ 'id': 8, 'category_id': 0, 'area': 25, 'bbox': [0, 0, 400, 400], 'iscrowd': 1 }], 'file_name': 'fake_name4.jpg', 'image_id': 3 } ] categories = [{ 'id': 0, 'name': 'car', 'supercategory': 'car', 'isthing': 1 }, { 'id': 1, 'name': 'person', 'supercategory': 'person', 'isthing': 1 }, { 'id': 2, 'name': 'wall', 'supercategory': 'wall', 'isthing': 0 }] fake_json = { 'images': images, 'annotations': annotations, 'categories': categories } self.json_name = 'coco_panoptic.json' dump(fake_json, self.json_name) self.metainfo = dict( classes=('person', 'car', 'wall'), thing_classes=('person', 'car'), stuff_classes=('wall', )) def tearDown(self): os.remove(self.json_name) def test_coco_panoptic_dataset(self): dataset = CocoPanopticDataset( data_root='./', ann_file=self.json_name, data_prefix=dict(img='imgs', seg='seg'), metainfo=self.metainfo, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[]) self.assertEqual(dataset.metainfo['classes'], self.metainfo['classes']) self.assertEqual(dataset.metainfo['thing_classes'], self.metainfo['thing_classes']) self.assertEqual(dataset.metainfo['stuff_classes'], self.metainfo['stuff_classes']) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 2) self.assertEqual(len(dataset.load_data_list()), 4) # test mode dataset = CocoPanopticDataset( data_root='./', ann_file=self.json_name, data_prefix=dict(img='imgs', seg='seg'), metainfo=self.metainfo, test_mode=True, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[]) self.assertEqual(dataset.metainfo['classes'], self.metainfo['classes']) self.assertEqual(dataset.metainfo['thing_classes'], self.metainfo['thing_classes']) self.assertEqual(dataset.metainfo['stuff_classes'], self.metainfo['stuff_classes']) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 4) self.assertEqual(len(dataset.load_data_list()), 4) def test_coco_panoptic_dataset_without_filter_cfg(self): dataset = CocoPanopticDataset( data_root='./', ann_file=self.json_name, data_prefix=dict(img='imgs', seg='seg'), metainfo=self.metainfo, filter_cfg=None, pipeline=[]) self.assertEqual(dataset.metainfo['classes'], self.metainfo['classes']) self.assertEqual(dataset.metainfo['thing_classes'], self.metainfo['thing_classes']) self.assertEqual(dataset.metainfo['stuff_classes'], self.metainfo['stuff_classes']) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 4) self.assertEqual(len(dataset.load_data_list()), 4) # test mode dataset = CocoPanopticDataset( data_root='./', ann_file=self.json_name, data_prefix=dict(img='imgs', seg='seg'), metainfo=self.metainfo, filter_cfg=None, test_mode=True, pipeline=[]) self.assertEqual(dataset.metainfo['classes'], self.metainfo['classes']) self.assertEqual(dataset.metainfo['thing_classes'], self.metainfo['thing_classes']) self.assertEqual(dataset.metainfo['stuff_classes'], self.metainfo['stuff_classes']) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 4) self.assertEqual(len(dataset.load_data_list()), 4) ================================================ FILE: tests/test_datasets/test_crowdhuman.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from mmdet.datasets import CrowdHumanDataset class TestCrowdHumanDataset(unittest.TestCase): def test_crowdhuman_init(self): dataset = CrowdHumanDataset( data_root='tests/data/crowdhuman_dataset/', ann_file='test_annotation_train.odgt', data_prefix=dict(img='Images/'), pipeline=[]) self.assertEqual(len(dataset), 1) self.assertEqual(dataset.metainfo['classes'], ('person', )) ================================================ FILE: tests/test_datasets/test_lvis.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os import unittest from mmengine.fileio import dump from mmdet.datasets import LVISV1Dataset, LVISV05Dataset try: import lvis except ImportError: lvis = None class TestLVISDataset(unittest.TestCase): def setUp(self) -> None: image1 = { # ``coco_url`` for v1 only. 'coco_url': 'http://images.cocodataset.org/train2017/0.jpg', # ``file_name`` for v0.5 only. 'file_name': '0.jpg', 'height': 1024, 'width': 2048, 'neg_category_ids': [], 'not_exhaustive_category_ids': [], 'id': 0 } image2 = { 'coco_url': 'http://images.cocodataset.org/train2017/1.jpg', 'file_name': '1.jpg', 'height': 1024, 'width': 2048, 'neg_category_ids': [], 'not_exhaustive_category_ids': [], 'id': 1 } image3 = { 'coco_url': 'http://images.cocodataset.org/train2017/2.jpg', 'file_name': '2.jpg', 'height': 1024, 'width': 2048, 'neg_category_ids': [], 'not_exhaustive_category_ids': [], 'id': 2 } image4 = { 'coco_url': 'http://images.cocodataset.org/train2017/3.jpg', 'file_name': '3.jpg', 'height': 31, 'width': 15, 'neg_category_ids': [], 'not_exhaustive_category_ids': [], 'id': 3 } images = [image1, image2, image3, image4] categories = [{ 'id': 1, 'name': 'aerosol_can', 'frequency': 'c', 'image_count': 64 }, { 'id': 2, 'name': 'air_conditioner', 'frequency': 'f', 'image_count': 364 }, { 'id': 3, 'name': 'airplane', 'frequency': 'f', 'image_count': 1911 }] annotations = [ { 'category_id': 1, 'bbox': [379.0, 435.0, 52.0, 124.0], 'area': 2595, 'segmentation': [[0.0, 0.0]], 'image_id': 0, 'id': 0 }, { 'category_id': 2, 'bbox': [379.0, 435.0, 52.0, 124.0], 'area': -1, 'segmentation': [[0.0, 0.0]], 'image_id': 0, 'id': 1 }, { 'category_id': 3, 'bbox': [379.0, 435.0, -1, 124.0], 'area': 2, 'segmentation': [[0.0, 0.0]], 'image_id': 0, 'id': 2 }, { 'category_id': 1, 'bbox': [379.0, 435.0, 52.0, -1], 'area': 2, 'segmentation': [[0.0, 0.0]], 'image_id': 0, 'id': 3 }, { 'category_id': 1, 'bbox': [379.0, 435.0, 52.0, 124.0], 'area': 2595, 'segmentation': [[0.0, 0.0]], 'image_id': 0, 'id': 4 }, { 'category_id': 3, 'bbox': [379.0, 435.0, 52.0, 124.0], 'area': 2595, 'segmentation': [[0.0, 0.0]], 'image_id': 1, 'id': 5 }, { 'category_id': 3, 'bbox': [379.0, 435.0, 10, 2], 'area': 2595, 'segmentation': [[0.0, 0.0]], 'image_id': 3, 'id': 6 }, ] fake_json = { 'images': images, 'annotations': annotations, 'categories': categories } self.json_name = 'lvis.json' dump(fake_json, self.json_name) self.metainfo = dict( classes=('aerosol_can', 'air_conditioner', 'airplane')) def tearDown(self): os.remove(self.json_name) @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_lvis05_dataset(self): dataset = LVISV05Dataset( ann_file=self.json_name, data_prefix=dict(img='imgs'), metainfo=self.metainfo, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[]) self.assertEqual(dataset.metainfo['classes'], self.metainfo['classes']) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 2) self.assertEqual(len(dataset.load_data_list()), 4) dataset = LVISV05Dataset( ann_file=self.json_name, data_prefix=dict(img='imgs'), metainfo=self.metainfo, test_mode=True, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[]) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 4) self.assertEqual(len(dataset.load_data_list()), 4) @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_lvis1_dataset(self): dataset = LVISV1Dataset( ann_file=self.json_name, data_prefix=dict(img='imgs'), metainfo=self.metainfo, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[]) self.assertEqual(dataset.metainfo['classes'], self.metainfo['classes']) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 2) self.assertEqual(len(dataset.load_data_list()), 4) dataset = LVISV1Dataset( ann_file=self.json_name, data_prefix=dict(img='imgs'), metainfo=self.metainfo, test_mode=True, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[]) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 4) self.assertEqual(len(dataset.load_data_list()), 4) @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_lvis1_dataset_without_filter_cfg(self): dataset = LVISV1Dataset( ann_file=self.json_name, data_prefix=dict(img='imgs'), metainfo=self.metainfo, filter_cfg=None, pipeline=[]) self.assertEqual(dataset.metainfo['classes'], self.metainfo['classes']) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 4) self.assertEqual(len(dataset.load_data_list()), 4) dataset = LVISV1Dataset( ann_file=self.json_name, data_prefix=dict(img='imgs'), metainfo=self.metainfo, test_mode=True, filter_cfg=None, pipeline=[]) dataset.full_init() # filter images of small size and images # with all illegal annotations self.assertEqual(len(dataset), 4) self.assertEqual(len(dataset.load_data_list()), 4) ================================================ FILE: tests/test_datasets/test_objects365.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from mmdet.datasets import Objects365V1Dataset, Objects365V2Dataset class TestObjects365V1Dataset(unittest.TestCase): def test_obj365v1_dataset(self): # test Objects365V1Dataset metainfo = dict(classes=('bus', 'car'), task_name='new_task') dataset = Objects365V1Dataset( data_prefix=dict(img='imgs'), ann_file='tests/data/coco_sample.json', metainfo=metainfo, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[], serialize_data=False, lazy_init=False) self.assertEqual(dataset.metainfo['classes'], ('bus', 'car')) self.assertEqual(dataset.metainfo['task_name'], 'new_task') self.assertListEqual(dataset.get_cat_ids(0), [0, 1]) self.assertEqual(dataset.cat_ids, [1, 2]) def test_obj365v1_with_unsorted_annotation(self): # test Objects365V1Dataset with unsorted annotations metainfo = dict(classes=('bus', 'car'), task_name='new_task') dataset = Objects365V1Dataset( data_prefix=dict(img='imgs'), ann_file='tests/data/Objects365/unsorted_obj365_sample.json', metainfo=metainfo, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[], serialize_data=False, lazy_init=False) self.assertEqual(dataset.metainfo['classes'], ('bus', 'car')) self.assertEqual(dataset.metainfo['task_name'], 'new_task') # sort the unsorted annotations self.assertListEqual(dataset.get_cat_ids(0), [0, 1]) self.assertEqual(dataset.cat_ids, [1, 2]) def test_obj365v1_annotation_ids_unique(self): # test annotation ids not unique error metainfo = dict(classes=('car', ), task_name='new_task') with self.assertRaisesRegex(AssertionError, 'are not unique!'): Objects365V1Dataset( data_prefix=dict(img='imgs'), ann_file='tests/data/coco_wrong_format_sample.json', metainfo=metainfo, pipeline=[]) class TestObjects365V2Dataset(unittest.TestCase): def test_obj365v2_dataset(self): # test Objects365V2Dataset metainfo = dict(classes=('bus', 'car'), task_name='new_task') dataset = Objects365V2Dataset( data_prefix=dict(img='imgs'), ann_file='tests/data/coco_sample.json', metainfo=metainfo, filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[], serialize_data=False, lazy_init=False) self.assertEqual(dataset.metainfo['classes'], ('bus', 'car')) self.assertEqual(dataset.metainfo['task_name'], 'new_task') self.assertListEqual(dataset.get_cat_ids(0), [0, 1]) self.assertEqual(dataset.cat_ids, [1, 2]) def test_obj365v1_annotation_ids_unique(self): # test annotation ids not unique error metainfo = dict(classes=('car', ), task_name='new_task') with self.assertRaisesRegex(AssertionError, 'are not unique!'): Objects365V2Dataset( data_prefix=dict(img='imgs'), ann_file='tests/data/coco_wrong_format_sample.json', metainfo=metainfo, pipeline=[]) ================================================ FILE: tests/test_datasets/test_openimages.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from mmdet.datasets import OpenImagesChallengeDataset, OpenImagesDataset class TestOpenImagesDataset(unittest.TestCase): def test_init(self): dataset = OpenImagesDataset( data_root='tests/data/OpenImages/', ann_file='annotations/oidv6-train-annotations-bbox.csv', data_prefix=dict(img='OpenImages/train/'), label_file='annotations/class-descriptions-boxable.csv', hierarchy_file='annotations/bbox_labels_600_hierarchy.json', meta_file='annotations/image-metas.pkl', pipeline=[]) dataset.full_init() self.assertEqual(len(dataset), 1) self.assertEqual(dataset.metainfo['classes'], ['Airplane']) class TestOpenImagesChallengeDataset(unittest.TestCase): def test_init(self): dataset = OpenImagesChallengeDataset( data_root='tests/data/OpenImages/', ann_file='challenge2019/challenge-2019-train-detection-bbox.txt', data_prefix=dict(img='OpenImages/train/'), label_file='challenge2019/cls-label-description.csv', hierarchy_file='challenge2019/class_label_tree.np', meta_file='annotations/image-metas.pkl', pipeline=[]) dataset.full_init() self.assertEqual(len(dataset), 1) self.assertEqual(dataset.metainfo['classes'], ['Airplane']) ================================================ FILE: tests/test_datasets/test_pascal_voc.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from mmdet.datasets import VOCDataset class TestVOCDataset(unittest.TestCase): def test_voc2007_init(self): dataset = VOCDataset( data_root='tests/data/VOCdevkit/', ann_file='VOC2007/ImageSets/Main/trainval.txt', data_prefix=dict(sub_data_root='VOC2007/'), filter_cfg=dict( filter_empty_gt=True, min_size=32, bbox_min_size=32), pipeline=[]) dataset.full_init() self.assertEqual(len(dataset), 1) data_list = dataset.load_data_list() self.assertEqual(len(data_list), 1) self.assertEqual(len(data_list[0]['instances']), 2) self.assertEqual(dataset.get_cat_ids(0), [11, 14]) def test_voc2012_init(self): dataset = VOCDataset( data_root='tests/data/VOCdevkit/', ann_file='VOC2012/ImageSets/Main/trainval.txt', data_prefix=dict(sub_data_root='VOC2012/'), filter_cfg=dict(filter_empty_gt=True, min_size=32), pipeline=[]) dataset.full_init() self.assertEqual(len(dataset), 1) data_list = dataset.load_data_list() self.assertEqual(len(data_list), 1) self.assertEqual(len(data_list[0]['instances']), 1) self.assertEqual(dataset.get_cat_ids(0), [18]) ================================================ FILE: tests/test_datasets/test_samplers/test_batch_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase from unittest.mock import patch import numpy as np from mmengine.dataset import DefaultSampler from torch.utils.data import Dataset from mmdet.datasets.samplers import AspectRatioBatchSampler class DummyDataset(Dataset): def __init__(self, length): self.length = length self.shapes = np.random.random((length, 2)) def __len__(self): return self.length def __getitem__(self, idx): return self.shapes[idx] def get_data_info(self, idx): return dict(width=self.shapes[idx][0], height=self.shapes[idx][1]) class TestAspectRatioBatchSampler(TestCase): @patch('mmengine.dist.get_dist_info', return_value=(0, 1)) def setUp(self, mock): self.length = 100 self.dataset = DummyDataset(self.length) self.sampler = DefaultSampler(self.dataset, shuffle=False) def test_invalid_inputs(self): with self.assertRaisesRegex( ValueError, 'batch_size should be a positive integer value'): AspectRatioBatchSampler(self.sampler, batch_size=-1) with self.assertRaisesRegex( TypeError, 'sampler should be an instance of ``Sampler``'): AspectRatioBatchSampler(None, batch_size=1) def test_divisible_batch(self): batch_size = 5 batch_sampler = AspectRatioBatchSampler( self.sampler, batch_size=batch_size, drop_last=True) self.assertEqual(len(batch_sampler), self.length // batch_size) for batch_idxs in batch_sampler: self.assertEqual(len(batch_idxs), batch_size) batch = [self.dataset[idx] for idx in batch_idxs] flag = batch[0][0] < batch[0][1] for i in range(1, batch_size): self.assertEqual(batch[i][0] < batch[i][1], flag) def test_indivisible_batch(self): batch_size = 7 batch_sampler = AspectRatioBatchSampler( self.sampler, batch_size=batch_size, drop_last=False) all_batch_idxs = list(batch_sampler) self.assertEqual( len(batch_sampler), (self.length + batch_size - 1) // batch_size) self.assertEqual( len(all_batch_idxs), (self.length + batch_size - 1) // batch_size) batch_sampler = AspectRatioBatchSampler( self.sampler, batch_size=batch_size, drop_last=True) all_batch_idxs = list(batch_sampler) self.assertEqual(len(batch_sampler), self.length // batch_size) self.assertEqual(len(all_batch_idxs), self.length // batch_size) # the last batch may not have the same aspect ratio for batch_idxs in all_batch_idxs[:-1]: self.assertEqual(len(batch_idxs), batch_size) batch = [self.dataset[idx] for idx in batch_idxs] flag = batch[0][0] < batch[0][1] for i in range(1, batch_size): self.assertEqual(batch[i][0] < batch[i][1], flag) ================================================ FILE: tests/test_datasets/test_samplers/test_multi_source_sampler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import bisect from unittest import TestCase from unittest.mock import patch import numpy as np from torch.utils.data import ConcatDataset, Dataset from mmdet.datasets.samplers import GroupMultiSourceSampler, MultiSourceSampler class DummyDataset(Dataset): def __init__(self, length, flag): self.length = length self.flag = flag self.shapes = np.random.random((length, 2)) def __len__(self): return self.length def __getitem__(self, idx): return self.shapes[idx] def get_data_info(self, idx): return dict( width=self.shapes[idx][0], height=self.shapes[idx][1], flag=self.flag) class DummyConcatDataset(ConcatDataset): def _get_ori_dataset_idx(self, idx): dataset_idx = bisect.bisect_right(self.cumulative_sizes, idx) sample_idx = idx if dataset_idx == 0 else idx - self.cumulative_sizes[ dataset_idx - 1] return dataset_idx, sample_idx def get_data_info(self, idx: int): dataset_idx, sample_idx = self._get_ori_dataset_idx(idx) return self.datasets[dataset_idx].get_data_info(sample_idx) class TestMultiSourceSampler(TestCase): @patch('mmengine.dist.get_dist_info', return_value=(7, 8)) def setUp(self, mock): self.length_a = 100 self.dataset_a = DummyDataset(self.length_a, flag='a') self.length_b = 1000 self.dataset_b = DummyDataset(self.length_b, flag='b') self.dataset = DummyConcatDataset([self.dataset_a, self.dataset_b]) def test_multi_source_sampler(self): # test dataset is not ConcatDataset with self.assertRaises(AssertionError): MultiSourceSampler( self.dataset_a, batch_size=5, source_ratio=[1, 4]) # test invalid batch_size with self.assertRaises(AssertionError): MultiSourceSampler( self.dataset_a, batch_size=-5, source_ratio=[1, 4]) # test source_ratio longer then dataset with self.assertRaises(AssertionError): MultiSourceSampler( self.dataset, batch_size=5, source_ratio=[1, 2, 4]) sampler = MultiSourceSampler( self.dataset, batch_size=5, source_ratio=[1, 4]) sampler = iter(sampler) flags = [] for i in range(100): idx = next(sampler) flags.append(self.dataset.get_data_info(idx)['flag']) flags_gt = ['a', 'b', 'b', 'b', 'b'] * 20 self.assertEqual(flags, flags_gt) class TestGroupMultiSourceSampler(TestCase): @patch('mmengine.dist.get_dist_info', return_value=(7, 8)) def setUp(self, mock): self.length_a = 100 self.dataset_a = DummyDataset(self.length_a, flag='a') self.length_b = 1000 self.dataset_b = DummyDataset(self.length_b, flag='b') self.dataset = DummyConcatDataset([self.dataset_a, self.dataset_b]) def test_group_multi_source_sampler(self): sampler = GroupMultiSourceSampler( self.dataset, batch_size=5, source_ratio=[1, 4]) sampler = iter(sampler) flags = [] groups = [] for i in range(100): idx = next(sampler) data_info = self.dataset.get_data_info(idx) flags.append(data_info['flag']) group = 0 if data_info['width'] < data_info['height'] else 1 groups.append(group) flags_gt = ['a', 'b', 'b', 'b', 'b'] * 20 self.assertEqual(flags, flags_gt) groups = set( [sum(x) for x in (groups[k:k + 5] for k in range(0, 100, 5))]) groups_gt = set([0, 5]) self.assertEqual(groups, groups_gt) ================================================ FILE: tests/test_datasets/test_transforms/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .utils import construct_toy_data, create_full_masks, create_random_bboxes __all__ = ['create_random_bboxes', 'create_full_masks', 'construct_toy_data'] ================================================ FILE: tests/test_datasets/test_transforms/test_augment_wrappers.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import unittest from mmdet.datasets.transforms import (AutoAugment, AutoContrast, Brightness, Color, Contrast, Equalize, Invert, Posterize, RandAugment, Rotate, Sharpness, ShearX, ShearY, Solarize, SolarizeAdd, TranslateX, TranslateY) from mmdet.utils import register_all_modules from .utils import check_result_same, construct_toy_data register_all_modules() class TestAutoAugment(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map', 'homography_matrix') self.results_mask = construct_toy_data(poly2mask=True) self.img_fill_val = (104, 116, 124) self.seg_ignore_label = 255 def test_autoaugment(self): # test AutoAugment equipped with Shear policies = [[ dict(type='ShearX', prob=1.0, level=3, reversal_prob=0.0), dict(type='ShearY', prob=1.0, level=7, reversal_prob=1.0) ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_shearx = ShearX(prob=1.0, level=3, reversal_prob=0.0) transform_sheary = ShearY(prob=1.0, level=7, reversal_prob=1.0) results_sheared = transform_sheary( transform_shearx(copy.deepcopy(self.results_mask))) check_result_same(results_sheared, results_auto, self.check_keys) # test AutoAugment equipped with Rotate policies = [[ dict(type='Rotate', prob=1.0, level=10, reversal_prob=0.0), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_rotate = Rotate(prob=1.0, level=10, reversal_prob=0.0) results_rotated = transform_rotate(copy.deepcopy(self.results_mask)) check_result_same(results_rotated, results_auto, self.check_keys) # test AutoAugment equipped with Translate policies = [[ dict( type='TranslateX', prob=1.0, level=10, max_mag=1.0, reversal_prob=0.0), dict( type='TranslateY', prob=1.0, level=10, max_mag=1.0, reversal_prob=1.0) ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_translatex = TranslateX( prob=1.0, level=10, max_mag=1.0, reversal_prob=0.0) transform_translatey = TranslateY( prob=1.0, level=10, max_mag=1.0, reversal_prob=1.0) results_translated = transform_translatey( transform_translatex(copy.deepcopy(self.results_mask))) check_result_same(results_translated, results_auto, self.check_keys) # test AutoAugment equipped with Brightness policies = [[ dict(type='Brightness', prob=1.0, level=3), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_brightness = Brightness(prob=1.0, level=3) results_brightness = transform_brightness( copy.deepcopy(self.results_mask)) check_result_same(results_brightness, results_auto, self.check_keys) # test AutoAugment equipped with Color policies = [[ dict(type='Color', prob=1.0, level=3), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_color = Color(prob=1.0, level=3) results_colored = transform_color(copy.deepcopy(self.results_mask)) check_result_same(results_colored, results_auto, self.check_keys) # test AutoAugment equipped with Contrast policies = [[ dict(type='Contrast', prob=1.0, level=3), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_contrast = Contrast(prob=1.0, level=3) results_contrasted = transform_contrast( copy.deepcopy(self.results_mask)) check_result_same(results_contrasted, results_auto, self.check_keys) # test AutoAugment equipped with Sharpness policies = [[ dict(type='Sharpness', prob=1.0, level=3), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_sharpness = Sharpness(prob=1.0, level=3) results_sharpness = transform_sharpness( copy.deepcopy(self.results_mask)) check_result_same(results_sharpness, results_auto, self.check_keys) # test AutoAugment equipped with Solarize policies = [[ dict(type='Solarize', prob=1.0, level=3), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_solarize = Solarize(prob=1.0, level=3) results_solarized = transform_solarize( copy.deepcopy(self.results_mask)) check_result_same(results_solarized, results_auto, self.check_keys) # test AutoAugment equipped with SolarizeAdd policies = [[ dict(type='SolarizeAdd', prob=1.0, level=3), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_solarizeadd = SolarizeAdd(prob=1.0, level=3) results_solarizeadded = transform_solarizeadd( copy.deepcopy(self.results_mask)) check_result_same(results_solarizeadded, results_auto, self.check_keys) # test AutoAugment equipped with Posterize policies = [[ dict(type='Posterize', prob=1.0, level=3), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_posterize = Posterize(prob=1.0, level=3) results_posterized = transform_posterize( copy.deepcopy(self.results_mask)) check_result_same(results_posterized, results_auto, self.check_keys) # test AutoAugment equipped with Equalize policies = [[ dict(type='Equalize', prob=1.0), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_equalize = Equalize(prob=1.0) results_equalized = transform_equalize( copy.deepcopy(self.results_mask)) check_result_same(results_equalized, results_auto, self.check_keys) # test AutoAugment equipped with AutoContrast policies = [[ dict(type='AutoContrast', prob=1.0), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_autocontrast = AutoContrast(prob=1.0) results_autocontrast = transform_autocontrast( copy.deepcopy(self.results_mask)) check_result_same(results_autocontrast, results_auto, self.check_keys) # test AutoAugment equipped with Invert policies = [[ dict(type='Invert', prob=1.0), ]] transform_auto = AutoAugment(policies=policies) results_auto = transform_auto(copy.deepcopy(self.results_mask)) transform_invert = Invert(prob=1.0) results_inverted = transform_invert(copy.deepcopy(self.results_mask)) check_result_same(results_inverted, results_auto, self.check_keys) # test AutoAugment equipped with default policies transform_auto = AutoAugment() transform_auto(copy.deepcopy(self.results_mask)) def test_repr(self): policies = [[ dict(type='Rotate', prob=1.0, level=10, reversal_prob=0.0), dict(type='Invert', prob=1.0), ]] transform = AutoAugment(policies=policies) self.assertEqual( repr(transform), ('AutoAugment(' 'policies=[[' "{'type': 'Rotate', 'prob': 1.0, " "'level': 10, 'reversal_prob': 0.0}, " "{'type': 'Invert', 'prob': 1.0}]], " 'prob=None)')) class TestRandAugment(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map', 'homography_matrix') self.results_mask = construct_toy_data(poly2mask=True) self.img_fill_val = (104, 116, 124) self.seg_ignore_label = 255 def test_randaugment(self): # test RandAugment equipped with Rotate aug_space = [[ dict(type='Rotate', prob=1.0, level=10, reversal_prob=0.0) ]] transform_rand = RandAugment(aug_space=aug_space, aug_num=1) results_rand = transform_rand(copy.deepcopy(self.results_mask)) transform_rotate = Rotate(prob=1.0, level=10, reversal_prob=0.0) results_rotated = transform_rotate(copy.deepcopy(self.results_mask)) check_result_same(results_rotated, results_rand, self.check_keys) # test RandAugment equipped with default augmentation space transform_rand = RandAugment() transform_rand(copy.deepcopy(self.results_mask)) def test_repr(self): aug_space = [ [dict(type='Rotate')], [dict(type='Invert')], ] transform = RandAugment(aug_space=aug_space) self.assertEqual( repr(transform), ('RandAugment(' 'aug_space=[' "[{'type': 'Rotate'}], " "[{'type': 'Invert'}]], " 'aug_num=2, ' 'prob=None)')) ================================================ FILE: tests/test_datasets/test_transforms/test_colorspace.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import unittest from mmdet.datasets.transforms import (AutoContrast, Brightness, Color, ColorTransform, Contrast, Equalize, Invert, Posterize, Sharpness, Solarize, SolarizeAdd) from .utils import check_result_same, construct_toy_data class TestColorTransform(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_colortransform(self): # test assertion for invalid value of level with self.assertRaises(AssertionError): transform = ColorTransform(level=-1) # test assertion for invalid prob with self.assertRaises(AssertionError): transform = ColorTransform(level=1, prob=-0.5) # test case when no translation is called (prob=0) transform = ColorTransform(prob=0.0, level=10) results_wo_color = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_color, self.check_keys) def test_repr(self): transform = ColorTransform(level=10, prob=1.) self.assertEqual( repr(transform), ('ColorTransform(prob=1.0, ' 'level=10, ' 'min_mag=0.1, ' 'max_mag=1.9)')) class TestColor(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_color(self): # test case when level=5 (without color aug) transform = Color(prob=1.0, level=5) results_wo_color = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_color, self.check_keys) # test case when level=0 transform = Color(prob=1.0, level=0) transform(copy.deepcopy(self.results_mask)) # test case when level=10 transform = Color(prob=1.0, level=10) transform(copy.deepcopy(self.results_mask)) def test_repr(self): transform = Color(level=10, prob=1.) self.assertEqual( repr(transform), ('Color(prob=1.0, ' 'level=10, ' 'min_mag=0.1, ' 'max_mag=1.9)')) class TestBrightness(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_brightness(self): # test case when level=5 (without Brightness aug) transform = Brightness(level=5, prob=1.0) results_wo_brightness = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_brightness, self.check_keys) # test case when level=0 transform = Brightness(prob=1.0, level=0) transform(copy.deepcopy(self.results_mask)) # test case when level=10 transform = Brightness(prob=1.0, level=10) transform(copy.deepcopy(self.results_mask)) def test_repr(self): transform = Brightness(prob=1.0, level=10) self.assertEqual( repr(transform), ('Brightness(prob=1.0, ' 'level=10, ' 'min_mag=0.1, ' 'max_mag=1.9)')) class TestContrast(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_contrast(self): # test case when level=5 (without Contrast aug) transform = Contrast(prob=1.0, level=5) results_wo_contrast = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_contrast, self.check_keys) # test case when level=0 transform = Contrast(prob=1.0, level=0) transform(copy.deepcopy(self.results_mask)) # test case when level=10 transform = Contrast(prob=1.0, level=10) transform(copy.deepcopy(self.results_mask)) def test_repr(self): transform = Contrast(level=10, prob=1.) self.assertEqual( repr(transform), ('Contrast(prob=1.0, ' 'level=10, ' 'min_mag=0.1, ' 'max_mag=1.9)')) class TestSharpness(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_sharpness(self): # test case when level=5 (without Sharpness aug) transform = Sharpness(prob=1.0, level=5) results_wo_sharpness = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_sharpness, self.check_keys) # test case when level=0 transform = Sharpness(prob=1.0, level=0) transform(copy.deepcopy(self.results_mask)) # test case when level=10 transform = Sharpness(prob=1.0, level=10) transform(copy.deepcopy(self.results_mask)) def test_repr(self): transform = Sharpness(level=10, prob=1.) self.assertEqual( repr(transform), ('Sharpness(prob=1.0, ' 'level=10, ' 'min_mag=0.1, ' 'max_mag=1.9)')) class TestSolarize(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_solarize(self): # test case when level=10 (without Solarize aug) transform = Solarize(prob=1.0, level=10) results_wo_solarize = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_solarize, self.check_keys) # test case when level=0 transform = Solarize(prob=1.0, level=0) transform(copy.deepcopy(self.results_mask)) def test_repr(self): transform = Solarize(level=10, prob=1.) self.assertEqual( repr(transform), ('Solarize(prob=1.0, ' 'level=10, ' 'min_mag=0.0, ' 'max_mag=256.0)')) class TestSolarizeAdd(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_solarize(self): # test case when level=0 (without Solarize aug) transform = SolarizeAdd(prob=1.0, level=0) results_wo_solarizeadd = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_solarizeadd, self.check_keys) # test case when level=10 transform = SolarizeAdd(prob=1.0, level=10) transform(copy.deepcopy(self.results_mask)) def test_repr(self): transform = SolarizeAdd(level=10, prob=1.) self.assertEqual( repr(transform), ('SolarizeAdd(prob=1.0, ' 'level=10, ' 'min_mag=0.0, ' 'max_mag=110.0)')) class TestPosterize(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_posterize(self): # test case when level=10 (without Posterize aug) transform = Posterize(prob=1.0, level=10, max_mag=8.0) results_wo_posterize = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_posterize, self.check_keys) # test case when level=0 transform = Posterize(prob=1.0, level=0) transform(copy.deepcopy(self.results_mask)) def test_repr(self): transform = Posterize(level=10, prob=1.) self.assertEqual( repr(transform), ('Posterize(prob=1.0, ' 'level=10, ' 'min_mag=0.0, ' 'max_mag=4.0)')) class TestEqualize(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_equalize(self): # test case when no translation is called (prob=0) transform = Equalize(prob=0.0) results_wo_equalize = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_equalize, self.check_keys) # test case when translation is called transform = Equalize(prob=1.0) transform(copy.deepcopy(self.results_mask)) def test_repr(self): transform = Equalize(prob=1.0) self.assertEqual( repr(transform), ('Equalize(prob=1.0, ' 'level=None, ' 'min_mag=0.1, ' 'max_mag=1.9)')) class TestAutoContrast(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_autocontrast(self): # test case when no translation is called (prob=0) transform = AutoContrast(prob=0.0) results_wo_autocontrast = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_autocontrast, self.check_keys) # test case when translation is called transform = AutoContrast(prob=1.0) transform(copy.deepcopy(self.results_mask)) def test_repr(self): transform = AutoContrast(prob=1.0) self.assertEqual( repr(transform), ('AutoContrast(prob=1.0, ' 'level=None, ' 'min_mag=0.1, ' 'max_mag=1.9)')) class TestInvert(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) def test_invert(self): # test case when no translation is called (prob=0) transform = Invert(prob=0.0) results_wo_invert = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_invert, self.check_keys) # test case when translation is called transform = Invert(prob=1.0) transform(copy.deepcopy(self.results_mask)) def test_repr(self): transform = Invert(prob=1.0) self.assertEqual( repr(transform), ('Invert(prob=1.0, ' 'level=None, ' 'min_mag=0.1, ' 'max_mag=1.9)')) ================================================ FILE: tests/test_datasets/test_transforms/test_formatting.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import os.path as osp import unittest import numpy as np import torch from mmengine.structures import InstanceData, PixelData from mmdet.datasets.transforms import PackDetInputs from mmdet.structures import DetDataSample from mmdet.structures.mask import BitmapMasks class TestPackDetInputs(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ data_prefix = osp.join(osp.dirname(__file__), '../../data') img_path = osp.join(data_prefix, 'color.jpg') rng = np.random.RandomState(0) self.results1 = { 'img_id': 1, 'img_path': img_path, 'ori_shape': (300, 400), 'img_shape': (600, 800), 'scale_factor': 2.0, 'flip': False, 'img': rng.rand(300, 400), 'gt_seg_map': rng.rand(300, 400), 'gt_masks': BitmapMasks(rng.rand(3, 300, 400), height=300, width=400), 'gt_bboxes_labels': rng.rand(3, ), 'gt_ignore_flags': np.array([0, 0, 1], dtype=bool), 'proposals': rng.rand(2, 4), 'proposals_scores': rng.rand(2, ) } self.results2 = { 'img_id': 1, 'img_path': img_path, 'ori_shape': (300, 400), 'img_shape': (600, 800), 'scale_factor': 2.0, 'flip': False, 'img': rng.rand(300, 400), 'gt_seg_map': rng.rand(300, 400), 'gt_masks': BitmapMasks(rng.rand(3, 300, 400), height=300, width=400), 'gt_bboxes_labels': rng.rand(3, ), 'proposals': rng.rand(2, 4), 'proposals_scores': rng.rand(2, ) } self.meta_keys = ('img_id', 'img_path', 'ori_shape', 'scale_factor', 'flip') def test_transform(self): transform = PackDetInputs(meta_keys=self.meta_keys) results = transform(copy.deepcopy(self.results1)) self.assertIn('data_samples', results) self.assertIsInstance(results['data_samples'], DetDataSample) self.assertIsInstance(results['data_samples'].gt_instances, InstanceData) self.assertIsInstance(results['data_samples'].ignored_instances, InstanceData) self.assertEqual(len(results['data_samples'].gt_instances), 2) self.assertEqual(len(results['data_samples'].ignored_instances), 1) self.assertIsInstance(results['data_samples'].gt_sem_seg, PixelData) self.assertIsInstance(results['data_samples'].proposals, InstanceData) self.assertEqual(len(results['data_samples'].proposals), 2) self.assertIsInstance(results['data_samples'].proposals.bboxes, torch.Tensor) self.assertIsInstance(results['data_samples'].proposals.scores, torch.Tensor) def test_transform_without_ignore(self): transform = PackDetInputs(meta_keys=self.meta_keys) results = transform(copy.deepcopy(self.results2)) self.assertIn('data_samples', results) self.assertIsInstance(results['data_samples'], DetDataSample) self.assertIsInstance(results['data_samples'].gt_instances, InstanceData) self.assertIsInstance(results['data_samples'].ignored_instances, InstanceData) self.assertEqual(len(results['data_samples'].gt_instances), 3) self.assertEqual(len(results['data_samples'].ignored_instances), 0) self.assertIsInstance(results['data_samples'].gt_sem_seg, PixelData) self.assertIsInstance(results['data_samples'].proposals, InstanceData) self.assertEqual(len(results['data_samples'].proposals), 2) self.assertIsInstance(results['data_samples'].proposals.bboxes, torch.Tensor) self.assertIsInstance(results['data_samples'].proposals.scores, torch.Tensor) def test_repr(self): transform = PackDetInputs(meta_keys=self.meta_keys) self.assertEqual( repr(transform), f'PackDetInputs(meta_keys={self.meta_keys})') ================================================ FILE: tests/test_datasets/test_transforms/test_geometric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import unittest import numpy as np from mmdet.datasets.transforms import (GeomTransform, Rotate, ShearX, ShearY, TranslateX, TranslateY) from mmdet.structures.bbox import HorizontalBoxes from mmdet.structures.mask import BitmapMasks, PolygonMasks from .utils import check_result_same, construct_toy_data class TestGeomTransform(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) self.img_border_value = (104, 116, 124) self.seg_ignore_label = 255 def test_geomtransform(self): # test assertion for invalid prob with self.assertRaises(AssertionError): transform = GeomTransform( prob=-0.5, level=1, min_mag=0.0, max_mag=1.0) # test assertion for invalid value of level with self.assertRaises(AssertionError): transform = GeomTransform( prob=0.5, level=-1, min_mag=0.0, max_mag=1.0) # test assertion for invalid value of min_mag and max_mag with self.assertRaises(AssertionError): transform = ShearX(prob=0.5, level=2, min_mag=1.0, max_mag=0.0) # test assertion for the num of elements in tuple img_border_value with self.assertRaises(AssertionError): transform = GeomTransform( prob=0.5, level=1, min_mag=0.0, max_mag=1.0, img_border_value=(128, 128, 128, 128)) # test ValueError for invalid type of img_border_value with self.assertRaises(ValueError): transform = GeomTransform( prob=0.5, level=1, min_mag=0.0, max_mag=1.0, img_border_value=[128, 128, 128]) # test assertion for invalid value of img_border_value with self.assertRaises(AssertionError): transform = GeomTransform( prob=0.5, level=1, min_mag=0.0, max_mag=1.0, img_border_value=(128, -1, 256)) # test case when no aug (prob=0) transform = GeomTransform( prob=0., level=10, min_mag=0.0, max_mag=1.0, img_border_value=self.img_border_value) results_wo_aug = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_aug, self.check_keys) def test_repr(self): transform = GeomTransform( prob=0.5, level=5, min_mag=0.0, max_mag=1.0, ) self.assertEqual( repr(transform), ('GeomTransform(prob=0.5, ' 'level=5, ' 'min_mag=0.0, ' 'max_mag=1.0, ' 'reversal_prob=0.5, ' 'img_border_value=(128.0, 128.0, 128.0), ' 'mask_border_value=0, ' 'seg_ignore_label=255, ' 'interpolation=bilinear)')) class TestShearX(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) self.results_poly = construct_toy_data(poly2mask=False) self.results_mask_boxtype = construct_toy_data( poly2mask=True, use_box_type=True) self.img_border_value = (104, 116, 124) self.seg_ignore_label = 255 def test_shearx(self): # test assertion for invalid value of min_mag with self.assertRaises(AssertionError): transform = ShearX(prob=0.5, level=2, min_mag=-30.) # test assertion for invalid value of max_mag with self.assertRaises(AssertionError): transform = ShearX(prob=0.5, level=2, max_mag=100.) # test case when no shear horizontally (level=0) transform = ShearX( prob=1.0, level=0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label, ) results_wo_shearx = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_shearx, self.check_keys) # test shear horizontally, magnitude=-1 transform = ShearX( prob=1.0, level=10, max_mag=45., reversal_prob=1.0, img_border_value=self.img_border_value) results_sheared = transform(copy.deepcopy(self.results_mask)) results_gt = copy.deepcopy(self.results_mask) img_gt = np.array([[1, 2, 3, 4], [0, 5, 6, 7], [0, 0, 9, 10]], dtype=np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) img_gt[1, 0, :] = np.array(self.img_border_value) img_gt[2, 0, :] = np.array(self.img_border_value) img_gt[2, 1, :] = np.array(self.img_border_value) results_gt['img'] = img_gt results_gt['gt_bboxes'] = np.array([[1, 0, 4, 2]], dtype=np.float32) results_gt['gt_bboxes_labels'] = np.array([13], dtype=np.int64) gt_masks = np.array([[0, 1, 0, 0], [0, 0, 1, 1], [0, 0, 0, 1]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[255, 13, 255, 255], [255, 255, 13, 13], [255, 255, 255, 13]], dtype=self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_sheared, self.check_keys) # test PolygonMasks with shear horizontally, magnitude=1 results_sheared = transform(copy.deepcopy(self.results_poly)) gt_masks = [[np.array([3, 2, 1, 0, 3, 1], dtype=np.float32)]] results_gt['gt_masks'] = PolygonMasks(gt_masks, 3, 4) check_result_same(results_gt, results_sheared, self.check_keys) def test_shearx_use_box_type(self): # test case when no shear horizontally (level=0) transform = ShearX( prob=1.0, level=0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label, ) results_wo_shearx = transform(copy.deepcopy(self.results_mask_boxtype)) check_result_same(self.results_mask_boxtype, results_wo_shearx, self.check_keys) # test shear horizontally, magnitude=-1 transform = ShearX( prob=1.0, level=10, max_mag=45., reversal_prob=1.0, img_border_value=self.img_border_value) results_sheared = transform(copy.deepcopy(self.results_mask_boxtype)) results_gt = copy.deepcopy(self.results_mask_boxtype) img_gt = np.array([[1, 2, 3, 4], [0, 5, 6, 7], [0, 0, 9, 10]], dtype=np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) img_gt[1, 0, :] = np.array(self.img_border_value) img_gt[2, 0, :] = np.array(self.img_border_value) img_gt[2, 1, :] = np.array(self.img_border_value) results_gt['img'] = img_gt results_gt['gt_bboxes'] = HorizontalBoxes( np.array([[1, 0, 4, 2]], dtype=np.float32)) results_gt['gt_bboxes_labels'] = np.array([13], dtype=np.int64) gt_masks = np.array([[0, 1, 0, 0], [0, 0, 1, 1], [0, 0, 0, 1]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[255, 13, 255, 255], [255, 255, 13, 13], [255, 255, 255, 13]], dtype=self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_sheared, self.check_keys) def test_repr(self): transform = ShearX(prob=0.5, level=10) self.assertEqual( repr(transform), ('ShearX(prob=0.5, ' 'level=10, ' 'min_mag=0.0, ' 'max_mag=30.0, ' 'reversal_prob=0.5, ' 'img_border_value=(128.0, 128.0, 128.0), ' 'mask_border_value=0, ' 'seg_ignore_label=255, ' 'interpolation=bilinear)')) class TestShearY(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) self.results_poly = construct_toy_data(poly2mask=False) self.results_mask_boxtype = construct_toy_data( poly2mask=True, use_box_type=True) self.img_border_value = (104, 116, 124) self.seg_ignore_label = 255 def test_sheary(self): # test assertion for invalid value of min_mag with self.assertRaises(AssertionError): transform = ShearY(prob=0.5, level=2, min_mag=-30.) # test assertion for invalid value of max_mag with self.assertRaises(AssertionError): transform = ShearY(prob=0.5, level=2, max_mag=100.) # test case when no shear vertically (level=0) transform = ShearY( prob=1.0, level=0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label, ) results_wo_sheary = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_sheary, self.check_keys) # test shear vertically, magnitude=1 transform = ShearY(prob=1., level=10, max_mag=45., reversal_prob=0.) results_sheared = transform(copy.deepcopy(self.results_mask)) results_gt = copy.deepcopy(self.results_mask) img_gt = np.array( [[1, 6, 11, 128], [5, 10, 128, 128], [9, 128, 128, 128]], dtype=np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) results_gt['img'] = img_gt results_gt['gt_bboxes'] = np.array([[1, 0, 2, 1]], dtype=np.float32) results_gt['gt_bboxes_labels'] = np.array([13], dtype=np.int64) gt_masks = np.array([[0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[255, 13, 255, 255], [255, 13, 255, 255], [255, 255, 255, 255]], dtype=self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_sheared, self.check_keys) # test PolygonMasks with shear vertically, magnitude=-1 results_sheared = transform(copy.deepcopy(self.results_poly)) gt_masks = [[np.array([1, 1, 1, 0, 2, 0], dtype=np.float32)]] results_gt['gt_masks'] = PolygonMasks(gt_masks, 3, 4) check_result_same(results_gt, results_sheared, self.check_keys) def test_sheary_use_box_type(self): # test case when no shear vertically (level=0) transform = ShearY( prob=1.0, level=0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label, ) results_wo_sheary = transform(copy.deepcopy(self.results_mask_boxtype)) check_result_same(self.results_mask_boxtype, results_wo_sheary, self.check_keys) # test shear vertically, magnitude=1 transform = ShearY(prob=1., level=10, max_mag=45., reversal_prob=0.) results_sheared = transform(copy.deepcopy(self.results_mask_boxtype)) results_gt = copy.deepcopy(self.results_mask_boxtype) img_gt = np.array( [[1, 6, 11, 128], [5, 10, 128, 128], [9, 128, 128, 128]], dtype=np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) results_gt['img'] = img_gt results_gt['gt_bboxes'] = HorizontalBoxes( np.array([[1, 0, 2, 1]], dtype=np.float32)) results_gt['gt_bboxes_labels'] = np.array([13], dtype=np.int64) gt_masks = np.array([[0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 0, 0]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[255, 13, 255, 255], [255, 13, 255, 255], [255, 255, 255, 255]], dtype=self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_sheared, self.check_keys) def test_repr(self): transform = ShearY(prob=0.5, level=10) self.assertEqual( repr(transform), ('ShearY(prob=0.5, ' 'level=10, ' 'min_mag=0.0, ' 'max_mag=30.0, ' 'reversal_prob=0.5, ' 'img_border_value=(128.0, 128.0, 128.0), ' 'mask_border_value=0, ' 'seg_ignore_label=255, ' 'interpolation=bilinear)')) class TestRotate(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) self.results_poly = construct_toy_data(poly2mask=False) self.results_mask_boxtype = construct_toy_data( poly2mask=True, use_box_type=True) self.img_border_value = (104, 116, 124) self.seg_ignore_label = 255 def test_rotate(self): # test assertion for invalid value of min_mag with self.assertRaises(AssertionError): transform = ShearY(prob=0.5, level=2, min_mag=-90.0) # test assertion for invalid value of max_mag with self.assertRaises(AssertionError): transform = ShearY(prob=0.5, level=2, max_mag=270.0) # test case when no rotate aug (level=0) transform = Rotate( prob=1., level=0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label, ) results_wo_rotate = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_rotate, self.check_keys) # test clockwise rotation with angle 90 transform = Rotate( prob=1., level=10, max_mag=90.0, # set reversal_prob to 1 for clockwise rotation reversal_prob=1., ) results_rotated = transform(copy.deepcopy(self.results_mask)) # The image, masks, and semantic segmentation map # will be bilinearly interpolated. img_gt = np.array([[69, 8, 4, 65], [69, 9, 5, 65], [70, 10, 6, 66]]).astype(np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) results_gt = copy.deepcopy(self.results_mask) results_gt['img'] = img_gt results_gt['gt_bboxes'] = np.array([[0.5, 0.5, 2.5, 1.5]], dtype=np.float32) gt_masks = np.array([[0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[255, 13, 13, 13], [255, 255, 13, 255], [255, 255, 255, 255]]).astype(self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_rotated, self.check_keys) # test clockwise rotation with angle 90, PolygonMasks results_rotated = transform(copy.deepcopy(self.results_poly)) gt_masks = [[np.array([0, 1, 0, 1, 0, 2], dtype=np.float)]] results_gt['gt_masks'] = PolygonMasks(gt_masks, 3, 4) check_result_same(results_gt, results_rotated, self.check_keys) # test counter-clockwise rotation with angle 90 transform = Rotate( prob=1.0, level=10, max_mag=90.0, # set reversal_prob to 0 for counter-clockwise rotation reversal_prob=0.0, ) results_rotated = transform(copy.deepcopy(self.results_mask)) # The image, masks, and semantic segmentation map # will be bilinearly interpolated. img_gt = np.array([[66, 6, 10, 70], [65, 5, 9, 69], [65, 4, 8, 69]]).astype(np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) results_gt = copy.deepcopy(self.results_mask) results_gt['img'] = img_gt results_gt['gt_bboxes'] = np.array([[0.5, 0.5, 2.5, 1.5]], dtype=np.float32) gt_masks = np.array([[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[255, 255, 255, 255], [255, 13, 255, 255], [13, 13, 13, 255]]).astype(self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_rotated, self.check_keys) # test counter-clockwise rotation with angle 90, PolygonMasks results_rotated = transform(copy.deepcopy(self.results_poly)) gt_masks = [[np.array([2, 0, 0, 0, 1, 0], dtype=np.float)]] results_gt['gt_masks'] = PolygonMasks(gt_masks, 3, 4) check_result_same(results_gt, results_rotated, self.check_keys) def test_rotate_use_box_type(self): # test case when no rotate aug (level=0) transform = Rotate( prob=1., level=0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label, ) results_wo_rotate = transform(copy.deepcopy(self.results_mask_boxtype)) check_result_same(self.results_mask_boxtype, results_wo_rotate, self.check_keys) # test clockwise rotation with angle 90 transform = Rotate( prob=1., level=10, max_mag=90.0, # set reversal_prob to 1 for clockwise rotation reversal_prob=1., ) results_rotated = transform(copy.deepcopy(self.results_mask_boxtype)) # The image, masks, and semantic segmentation map # will be bilinearly interpolated. img_gt = np.array([[69, 8, 4, 65], [69, 9, 5, 65], [70, 10, 6, 66]]).astype(np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) results_gt = copy.deepcopy(self.results_mask_boxtype) results_gt['img'] = img_gt results_gt['gt_bboxes'] = HorizontalBoxes( np.array([[0.5, 0.5, 2.5, 1.5]], dtype=np.float32)) gt_masks = np.array([[0, 1, 1, 0], [0, 1, 1, 0], [0, 0, 0, 0]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[255, 13, 13, 13], [255, 255, 13, 255], [255, 255, 255, 255]]).astype(self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_rotated, self.check_keys) # test counter-clockwise rotation with angle 90 transform = Rotate( prob=1.0, level=10, max_mag=90.0, # set reversal_prob to 0 for counter-clockwise rotation reversal_prob=0.0, ) results_rotated = transform(copy.deepcopy(self.results_mask_boxtype)) # The image, masks, and semantic segmentation map # will be bilinearly interpolated. img_gt = np.array([[66, 6, 10, 70], [65, 5, 9, 69], [65, 4, 8, 69]]).astype(np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) results_gt = copy.deepcopy(self.results_mask_boxtype) results_gt['img'] = img_gt results_gt['gt_bboxes'] = HorizontalBoxes( np.array([[0.5, 0.5, 2.5, 1.5]], dtype=np.float32)) gt_masks = np.array([[0, 0, 0, 0], [0, 1, 1, 0], [0, 1, 1, 0]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[255, 255, 255, 255], [255, 13, 255, 255], [13, 13, 13, 255]]).astype(self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_rotated, self.check_keys) def test_repr(self): transform = Rotate(prob=0.5, level=5) self.assertEqual( repr(transform), ('Rotate(prob=0.5, ' 'level=5, ' 'min_mag=0.0, ' 'max_mag=30.0, ' 'reversal_prob=0.5, ' 'img_border_value=(128.0, 128.0, 128.0), ' 'mask_border_value=0, ' 'seg_ignore_label=255, ' 'interpolation=bilinear)')) class TestTranslateX(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) self.results_poly = construct_toy_data(poly2mask=False) self.results_mask_boxtype = construct_toy_data( poly2mask=True, use_box_type=True) self.img_border_value = (104, 116, 124) self.seg_ignore_label = 255 def test_translatex(self): # test assertion for invalid value of min_mag with self.assertRaises(AssertionError): transform = TranslateX(prob=0.5, level=2, min_mag=-1.) # test assertion for invalid value of max_mag with self.assertRaises(AssertionError): transform = TranslateX(prob=0.5, level=2, max_mag=1.1) # test case when level=0 (without translate aug) transform = TranslateX( prob=1.0, level=0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label) results_wo_translatex = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_translatex, self.check_keys) # test translate horizontally, magnitude=-1 transform = TranslateX( prob=1.0, level=10, max_mag=0.3, reversal_prob=0.0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label) results_translated = transform(copy.deepcopy(self.results_mask)) img_gt = np.array([[2, 3, 4, 0], [6, 7, 8, 0], [10, 11, 12, 0]]).astype(np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) img_gt[:, 3, :] = np.array(self.img_border_value) results_gt = copy.deepcopy(self.results_mask) results_gt['img'] = img_gt results_gt['gt_bboxes'] = np.array([[0, 0, 1, 2]], dtype=np.float32) gt_masks = np.array([[1, 0, 0, 0], [1, 1, 0, 0], [1, 0, 0, 0]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[13, 255, 255, 255], [13, 13, 255, 255], [13, 255, 255, 255]]).astype(self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_translated, self.check_keys) # test PolygonMasks with translate horizontally. results_translated = transform(copy.deepcopy(self.results_poly)) gt_masks = [[np.array([0, 2, 0, 0, 1, 1], dtype=np.float32)]] results_gt['gt_masks'] = PolygonMasks(gt_masks, 3, 4) check_result_same(results_gt, results_translated, self.check_keys) def test_translatex_use_box_type(self): # test case when level=0 (without translate aug) transform = TranslateX( prob=1.0, level=0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label) results_wo_translatex = transform( copy.deepcopy(self.results_mask_boxtype)) check_result_same(self.results_mask_boxtype, results_wo_translatex, self.check_keys) # test translate horizontally, magnitude=-1 transform = TranslateX( prob=1.0, level=10, max_mag=0.3, reversal_prob=0.0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label) results_translated = transform( copy.deepcopy(self.results_mask_boxtype)) img_gt = np.array([[2, 3, 4, 0], [6, 7, 8, 0], [10, 11, 12, 0]]).astype(np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) img_gt[:, 3, :] = np.array(self.img_border_value) results_gt = copy.deepcopy(self.results_mask) results_gt['img'] = img_gt results_gt['gt_bboxes'] = HorizontalBoxes( np.array([[0, 0, 1, 2]], dtype=np.float32)) gt_masks = np.array([[1, 0, 0, 0], [1, 1, 0, 0], [1, 0, 0, 0]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[13, 255, 255, 255], [13, 13, 255, 255], [13, 255, 255, 255]]).astype(self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_translated, self.check_keys) def test_repr(self): transform = TranslateX(prob=0.5, level=5) self.assertEqual( repr(transform), ('TranslateX(prob=0.5, ' 'level=5, ' 'min_mag=0.0, ' 'max_mag=0.1, ' 'reversal_prob=0.5, ' 'img_border_value=(128.0, 128.0, 128.0), ' 'mask_border_value=0, ' 'seg_ignore_label=255, ' 'interpolation=bilinear)')) class TestTranslateY(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.check_keys = ('img', 'gt_bboxes', 'gt_bboxes_labels', 'gt_masks', 'gt_ignore_flags', 'gt_seg_map') self.results_mask = construct_toy_data(poly2mask=True) self.results_poly = construct_toy_data(poly2mask=False) self.results_mask_boxtype = construct_toy_data( poly2mask=True, use_box_type=True) self.img_border_value = (104, 116, 124) self.seg_ignore_label = 255 def test_translatey(self): # test assertion for invalid value of min_mag with self.assertRaises(AssertionError): transform = TranslateY(prob=0.5, level=2, min_mag=-1.0) # test assertion for invalid value of max_mag with self.assertRaises(AssertionError): transform = TranslateY(prob=0.5, level=2, max_mag=1.1) # test case when level=0 (without translate aug) transform = TranslateY( prob=1.0, level=0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label) results_wo_translatey = transform(copy.deepcopy(self.results_mask)) check_result_same(self.results_mask, results_wo_translatey, self.check_keys) # test translate vertically, magnitude=-1 transform = TranslateY( prob=1.0, level=10, max_mag=0.4, reversal_prob=0.0, seg_ignore_label=self.seg_ignore_label) results_translated = transform(copy.deepcopy(self.results_mask)) img_gt = np.array([[5, 6, 7, 8], [9, 10, 11, 12], [128, 128, 128, 128]]).astype(np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) results_gt = copy.deepcopy(self.results_mask) results_gt['img'] = img_gt results_gt['gt_bboxes'] = np.array([[1, 0, 2, 1]], dtype=np.float32) gt_masks = np.array([[0, 1, 1, 0], [0, 1, 0, 0], [0, 0, 0, 0]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[255, 13, 13, 255], [255, 13, 255, 255], [255, 255, 255, 255]]).astype(self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_translated, self.check_keys) # test PolygonMasks with translate vertically. results_translated = transform(copy.deepcopy(self.results_poly)) gt_masks = [[np.array([1, 1, 1, 0, 2, 0], dtype=np.float32)]] results_gt['gt_masks'] = PolygonMasks(gt_masks, 3, 4) check_result_same(results_gt, results_translated, self.check_keys) def test_translatey_use_box_type(self): # test case when level=0 (without translate aug) transform = TranslateY( prob=1.0, level=0, img_border_value=self.img_border_value, seg_ignore_label=self.seg_ignore_label) results_wo_translatey = transform( copy.deepcopy(self.results_mask_boxtype)) check_result_same(self.results_mask_boxtype, results_wo_translatey, self.check_keys) # test translate vertically, magnitude=-1 transform = TranslateY( prob=1.0, level=10, max_mag=0.4, reversal_prob=0.0, seg_ignore_label=self.seg_ignore_label) results_translated = transform( copy.deepcopy(self.results_mask_boxtype)) img_gt = np.array([[5, 6, 7, 8], [9, 10, 11, 12], [128, 128, 128, 128]]).astype(np.uint8) img_gt = np.stack([img_gt, img_gt, img_gt], axis=-1) results_gt = copy.deepcopy(self.results_mask_boxtype) results_gt['img'] = img_gt results_gt['gt_bboxes'] = HorizontalBoxes( np.array([[1, 0, 2, 1]], dtype=np.float32)) gt_masks = np.array([[0, 1, 1, 0], [0, 1, 0, 0], [0, 0, 0, 0]], dtype=np.uint8)[None, :, :] results_gt['gt_masks'] = BitmapMasks(gt_masks, 3, 4) results_gt['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results_gt['gt_seg_map'] = np.array( [[255, 13, 13, 255], [255, 13, 255, 255], [255, 255, 255, 255]]).astype(self.results_mask['gt_seg_map'].dtype) check_result_same(results_gt, results_translated, self.check_keys) def test_repr(self): transform = TranslateX(prob=0.5, level=5) self.assertEqual( repr(transform), ('TranslateX(prob=0.5, ' 'level=5, ' 'min_mag=0.0, ' 'max_mag=0.1, ' 'reversal_prob=0.5, ' 'img_border_value=(128.0, 128.0, 128.0), ' 'mask_border_value=0, ' 'seg_ignore_label=255, ' 'interpolation=bilinear)')) ================================================ FILE: tests/test_datasets/test_transforms/test_instaboost.py ================================================ import os.path as osp import unittest import numpy as np from mmdet.registry import TRANSFORMS from mmdet.utils import register_all_modules register_all_modules() class TestInstaboost(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ img_path = osp.join(osp.dirname(__file__), '../../data/gray.jpg') self.results = { 'img_path': img_path, 'img_shape': (300, 400), 'instances': [{ 'bbox': [0, 0, 10, 20], 'bbox_label': 1, 'mask': [[0, 0, 0, 20, 10, 20, 10, 0]], 'ignore_flag': 0 }, { 'bbox': [10, 10, 110, 120], 'bbox_label': 2, 'mask': [[10, 10, 110, 10, 110, 120, 110, 10]], 'ignore_flag': 0 }, { 'bbox': [50, 50, 60, 80], 'bbox_label': 2, 'mask': [[50, 50, 60, 50, 60, 80, 50, 80]], 'ignore_flag': 1 }] } def test_transform(self): load = TRANSFORMS.build(dict(type='LoadImageFromFile')) instaboost_transform = TRANSFORMS.build(dict(type='InstaBoost')) # Execute transforms results = load(self.results) results = instaboost_transform(results) self.assertEqual(results['img'].dtype, np.uint8) self.assertIn('instances', results) def test_repr(self): instaboost_transform = TRANSFORMS.build(dict(type='InstaBoost')) self.assertEqual( repr(instaboost_transform), 'InstaBoost(aug_ratio=0.5)') ================================================ FILE: tests/test_datasets/test_transforms/test_loading.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import os import os.path as osp import sys import unittest from unittest.mock import MagicMock, Mock, patch import mmcv import numpy as np from mmdet.datasets.transforms import (FilterAnnotations, LoadAnnotations, LoadEmptyAnnotations, LoadImageFromNDArray, LoadMultiChannelImageFromFiles, LoadProposals) from mmdet.evaluation import INSTANCE_OFFSET from mmdet.structures.mask import BitmapMasks, PolygonMasks try: import panopticapi except ImportError: panopticapi = None class TestLoadAnnotations(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ data_prefix = osp.join(osp.dirname(__file__), '../../data') seg_map = osp.join(data_prefix, 'gray.jpg') self.results = { 'ori_shape': (300, 400), 'seg_map_path': seg_map, 'instances': [{ 'bbox': [0, 0, 10, 20], 'bbox_label': 1, 'mask': [[0, 0, 0, 20, 10, 20, 10, 0]], 'ignore_flag': 0 }, { 'bbox': [10, 10, 110, 120], 'bbox_label': 2, 'mask': [[10, 10, 110, 10, 110, 120, 110, 10]], 'ignore_flag': 0 }, { 'bbox': [50, 50, 60, 80], 'bbox_label': 2, 'mask': [[50, 50, 60, 50, 60, 80, 50, 80]], 'ignore_flag': 1 }] } def test_load_bboxes(self): transform = LoadAnnotations( with_bbox=True, with_label=False, with_seg=False, with_mask=False, box_type=None) results = transform(copy.deepcopy(self.results)) self.assertIn('gt_bboxes', results) self.assertTrue((results['gt_bboxes'] == np.array([[0, 0, 10, 20], [10, 10, 110, 120], [50, 50, 60, 80]])).all()) self.assertEqual(results['gt_bboxes'].dtype, np.float32) self.assertTrue((results['gt_ignore_flags'] == np.array([0, 0, 1])).all()) self.assertEqual(results['gt_ignore_flags'].dtype, bool) def test_load_labels(self): transform = LoadAnnotations( with_bbox=False, with_label=True, with_seg=False, with_mask=False, ) results = transform(copy.deepcopy(self.results)) self.assertIn('gt_bboxes_labels', results) self.assertTrue((results['gt_bboxes_labels'] == np.array([1, 2, 2])).all()) self.assertEqual(results['gt_bboxes_labels'].dtype, np.int64) def test_load_mask(self): transform = LoadAnnotations( with_bbox=False, with_label=False, with_seg=False, with_mask=True, poly2mask=False) results = transform(copy.deepcopy(self.results)) self.assertIn('gt_masks', results) self.assertEqual(len(results['gt_masks']), 3) self.assertIsInstance(results['gt_masks'], PolygonMasks) def test_load_mask_poly2mask(self): transform = LoadAnnotations( with_bbox=False, with_label=False, with_seg=False, with_mask=True, poly2mask=True) results = transform(copy.deepcopy(self.results)) self.assertIn('gt_masks', results) self.assertEqual(len(results['gt_masks']), 3) self.assertIsInstance(results['gt_masks'], BitmapMasks) def test_repr(self): transform = LoadAnnotations( with_bbox=True, with_label=False, with_seg=False, with_mask=False, ) self.assertEqual( repr(transform), ('LoadAnnotations(with_bbox=True, ' 'with_label=False, with_mask=False, ' 'with_seg=False, poly2mask=True, ' "imdecode_backend='cv2', " 'file_client_args=None)')) class TestFilterAnnotations(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ rng = np.random.RandomState(0) self.results = { 'img': np.random.random((224, 224, 3)), 'img_shape': (224, 224), 'gt_bboxes_labels': np.array([1, 2, 3], dtype=np.int64), 'gt_bboxes': np.array([[10, 10, 20, 20], [20, 20, 40, 40], [40, 40, 80, 80]]), 'gt_ignore_flags': np.array([0, 0, 1], dtype=np.bool8), 'gt_masks': BitmapMasks(rng.rand(3, 224, 224), height=224, width=224), } def test_transform(self): # test keep_empty = True transform = FilterAnnotations( min_gt_bbox_wh=(50, 50), keep_empty=True, ) results = transform(copy.deepcopy(self.results)) self.assertIsNone(results) # test keep_empty = False transform = FilterAnnotations( min_gt_bbox_wh=(50, 50), keep_empty=False, ) results = transform(copy.deepcopy(self.results)) self.assertTrue(isinstance(results, dict)) # test filter annotations transform = FilterAnnotations(min_gt_bbox_wh=(15, 15), ) results = transform(copy.deepcopy(self.results)) self.assertIsInstance(results, dict) self.assertTrue((results['gt_bboxes_labels'] == np.array([2, 3])).all()) self.assertTrue((results['gt_bboxes'] == np.array([[20, 20, 40, 40], [40, 40, 80, 80]])).all()) self.assertEqual(len(results['gt_masks']), 2) self.assertEqual(len(results['gt_ignore_flags']), 2) def test_repr(self): transform = FilterAnnotations( min_gt_bbox_wh=(1, 1), keep_empty=False, ) self.assertEqual( repr(transform), ('FilterAnnotations(min_gt_bbox_wh=(1, 1), ' 'keep_empty=False)')) class TestLoadPanopticAnnotations(unittest.TestCase): def setUp(self): seg_map = np.zeros((10, 10), dtype=np.int32) seg_map[:5, :10] = 1 + 10 * INSTANCE_OFFSET seg_map[5:10, :5] = 4 + 11 * INSTANCE_OFFSET seg_map[5:10, 5:10] = 6 + 0 * INSTANCE_OFFSET rgb_seg_map = np.zeros((10, 10, 3), dtype=np.uint8) rgb_seg_map[:, :, 0] = seg_map / (256 * 256) rgb_seg_map[:, :, 1] = seg_map % (256 * 256) / 256 rgb_seg_map[:, :, 2] = seg_map % 256 self.seg_map_path = './1.png' mmcv.imwrite(rgb_seg_map, self.seg_map_path) self.seg_map = seg_map self.rgb_seg_map = rgb_seg_map self.results = { 'ori_shape': (10, 10), 'instances': [{ 'bbox': [0, 0, 10, 5], 'bbox_label': 0, 'ignore_flag': 0, }, { 'bbox': [0, 5, 5, 10], 'bbox_label': 1, 'ignore_flag': 1, }], 'segments_info': [ { 'id': 1 + 10 * INSTANCE_OFFSET, 'category': 0, 'is_thing': True, }, { 'id': 4 + 11 * INSTANCE_OFFSET, 'category': 1, 'is_thing': True, }, { 'id': 6 + 0 * INSTANCE_OFFSET, 'category': 2, 'is_thing': False, }, ], 'seg_map_path': self.seg_map_path } self.gt_mask = BitmapMasks([ (seg_map == 1 + 10 * INSTANCE_OFFSET).astype(np.uint8), (seg_map == 4 + 11 * INSTANCE_OFFSET).astype(np.uint8), ], 10, 10) self.gt_bboxes = np.array([[0, 0, 10, 5], [0, 5, 5, 10]], dtype=np.float32) self.gt_bboxes_labels = np.array([0, 1], dtype=np.int64) self.gt_ignore_flags = np.array([0, 1], dtype=bool) self.gt_seg_map = np.zeros((10, 10), dtype=np.int32) self.gt_seg_map[:5, :10] = 0 self.gt_seg_map[5:10, :5] = 1 self.gt_seg_map[5:10, 5:10] = 2 def tearDown(self): os.remove(self.seg_map_path) @unittest.skipIf(panopticapi is not None, 'panopticapi is installed') def test_init_without_panopticapi(self): # test if panopticapi is not installed from mmdet.datasets.transforms import LoadPanopticAnnotations with self.assertRaisesRegex( ImportError, 'panopticapi is not installed, please install it by'): LoadPanopticAnnotations() def test_transform(self): sys.modules['panopticapi'] = MagicMock() sys.modules['panopticapi.utils'] = MagicMock() from mmdet.datasets.transforms import LoadPanopticAnnotations mock_rgb2id = Mock(return_value=self.seg_map) with patch('panopticapi.utils.rgb2id', mock_rgb2id): # test with all False transform = LoadPanopticAnnotations( with_bbox=False, with_label=False, with_mask=False, with_seg=False) results = transform(copy.deepcopy(self.results)) self.assertDictEqual(results, self.results) # test with with_mask=True transform = LoadPanopticAnnotations( with_bbox=False, with_label=False, with_mask=True, with_seg=False) results = transform(copy.deepcopy(self.results)) self.assertTrue( (results['gt_masks'].masks == self.gt_mask.masks).all()) # test with with_seg=True transform = LoadPanopticAnnotations( with_bbox=False, with_label=False, with_mask=False, with_seg=True) results = transform(copy.deepcopy(self.results)) self.assertNotIn('gt_masks', results) self.assertTrue((results['gt_seg_map'] == self.gt_seg_map).all()) # test with all True transform = LoadPanopticAnnotations( with_bbox=True, with_label=True, with_mask=True, with_seg=True, box_type=None) results = transform(copy.deepcopy(self.results)) self.assertTrue( (results['gt_masks'].masks == self.gt_mask.masks).all()) self.assertTrue((results['gt_bboxes'] == self.gt_bboxes).all()) self.assertTrue( (results['gt_bboxes_labels'] == self.gt_bboxes_labels).all()) self.assertTrue( (results['gt_ignore_flags'] == self.gt_ignore_flags).all()) self.assertTrue((results['gt_seg_map'] == self.gt_seg_map).all()) class TestLoadImageFromNDArray(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.results = {'img': np.zeros((256, 256, 3), dtype=np.uint8)} def test_transform(self): transform = LoadImageFromNDArray() results = transform(copy.deepcopy(self.results)) self.assertEqual(results['img'].shape, (256, 256, 3)) self.assertEqual(results['img'].dtype, np.uint8) self.assertEqual(results['img_shape'], (256, 256)) self.assertEqual(results['ori_shape'], (256, 256)) # to_float32 transform = LoadImageFromNDArray(to_float32=True) results = transform(copy.deepcopy(results)) self.assertEqual(results['img'].dtype, np.float32) def test_repr(self): transform = LoadImageFromNDArray() self.assertEqual( repr(transform), ('LoadImageFromNDArray(' 'ignore_empty=False, ' 'to_float32=False, ' "color_type='color', " "imdecode_backend='cv2', " 'backend_args=None)')) class TestLoadMultiChannelImageFromFiles(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.img_path = [] for i in range(4): img_channel_path = f'./part_{i}.jpg' img_channel = np.zeros((10, 10), dtype=np.uint8) mmcv.imwrite(img_channel, img_channel_path) self.img_path.append(img_channel_path) self.results = {'img_path': self.img_path} def tearDown(self): for filename in self.img_path: os.remove(filename) def test_transform(self): transform = LoadMultiChannelImageFromFiles() results = transform(copy.deepcopy(self.results)) self.assertEqual(results['img'].shape, (10, 10, 4)) self.assertEqual(results['img'].dtype, np.uint8) self.assertEqual(results['img_shape'], (10, 10)) self.assertEqual(results['ori_shape'], (10, 10)) # to_float32 transform = LoadMultiChannelImageFromFiles(to_float32=True) results = transform(copy.deepcopy(results)) self.assertEqual(results['img'].dtype, np.float32) def test_rper(self): transform = LoadMultiChannelImageFromFiles() self.assertEqual( repr(transform), ('LoadMultiChannelImageFromFiles(' 'to_float32=False, ' "color_type='unchanged', " "imdecode_backend='cv2', " "file_client_args={'backend': 'disk'})")) class TestLoadProposals(unittest.TestCase): def test_transform(self): transform = LoadProposals() results = { 'proposals': dict( bboxes=np.zeros((5, 4), dtype=np.int64), scores=np.zeros((5, ), dtype=np.int64)) } results = transform(results) self.assertEqual(results['proposals'].dtype, np.float32) self.assertEqual(results['proposals'].shape[-1], 4) self.assertEqual(results['proposals_scores'].dtype, np.float32) # bboxes.shape[1] should be 4 results = {'proposals': dict(bboxes=np.zeros((5, 5), dtype=np.int64))} with self.assertRaises(AssertionError): transform(results) # bboxes.shape[0] should equal to scores.shape[0] results = { 'proposals': dict( bboxes=np.zeros((5, 4), dtype=np.int64), scores=np.zeros((3, ), dtype=np.int64)) } with self.assertRaises(AssertionError): transform(results) # empty bboxes results = { 'proposals': dict(bboxes=np.zeros((0, 4), dtype=np.float32)) } results = transform(results) excepted_proposals = np.zeros((0, 4), dtype=np.float32) excepted_proposals_scores = np.zeros(0, dtype=np.float32) self.assertTrue((results['proposals'] == excepted_proposals).all()) self.assertTrue( (results['proposals_scores'] == excepted_proposals_scores).all()) transform = LoadProposals(num_max_proposals=2) results = { 'proposals': dict( bboxes=np.zeros((5, 4), dtype=np.int64), scores=np.zeros((5, ), dtype=np.int64)) } results = transform(results) self.assertEqual(results['proposals'].shape[0], 2) def test_repr(self): transform = LoadProposals() self.assertEqual( repr(transform), 'LoadProposals(num_max_proposals=None)') class TestLoadEmptyAnnotations(unittest.TestCase): def test_transform(self): transform = LoadEmptyAnnotations( with_bbox=True, with_label=True, with_mask=True, with_seg=True) results = {'img_shape': (224, 224)} results = transform(results) self.assertEqual(results['gt_bboxes'].dtype, np.float32) self.assertEqual(results['gt_bboxes'].shape[-1], 4) self.assertEqual(results['gt_ignore_flags'].dtype, bool) self.assertEqual(results['gt_bboxes_labels'].dtype, np.int64) self.assertEqual(results['gt_masks'].masks.dtype, np.uint8) self.assertEqual(results['gt_masks'].masks.shape[-2:], results['img_shape']) self.assertEqual(results['gt_seg_map'].dtype, np.uint8) self.assertEqual(results['gt_seg_map'].shape, results['img_shape']) def test_repr(self): transform = LoadEmptyAnnotations() self.assertEqual( repr(transform), 'LoadEmptyAnnotations(with_bbox=True, ' 'with_label=True, ' 'with_mask=False, ' 'with_seg=False, ' 'seg_ignore_label=255)') ================================================ FILE: tests/test_datasets/test_transforms/test_transforms.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import os.path as osp import unittest import mmcv import numpy as np import torch from mmcv.transforms import LoadImageFromFile # yapf:disable from mmdet.datasets.transforms import (CopyPaste, CutOut, Expand, FixShapeResize, MinIoURandomCrop, MixUp, Mosaic, Pad, PhotoMetricDistortion, RandomAffine, RandomCenterCropPad, RandomCrop, RandomErasing, RandomFlip, RandomShift, Resize, SegRescale, YOLOXHSVRandomAug) # yapf:enable from mmdet.evaluation import bbox_overlaps from mmdet.registry import TRANSFORMS from mmdet.structures.bbox import HorizontalBoxes, bbox_project from mmdet.structures.mask import BitmapMasks from .utils import construct_toy_data, create_full_masks, create_random_bboxes try: import albumentations from albumentations import Compose except ImportError: albumentations = None Compose = None # yapf:enable class TestResize(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ rng = np.random.RandomState(0) self.data_info1 = dict( img=np.random.random((1333, 800, 3)), gt_seg_map=np.random.random((1333, 800, 3)), gt_bboxes=np.array([[0, 0, 112, 112]], dtype=np.float32), gt_masks=BitmapMasks( rng.rand(1, 1333, 800), height=1333, width=800)) self.data_info2 = dict( img=np.random.random((300, 400, 3)), gt_bboxes=np.array([[200, 150, 600, 450]], dtype=np.float32), dtype=np.float32) self.data_info3 = dict(img=np.random.random((300, 400, 3))) def test_resize(self): # test keep_ratio is True transform = Resize(scale=(2000, 2000), keep_ratio=True) results = transform(copy.deepcopy(self.data_info1)) self.assertEqual(results['img_shape'], (2000, 1200)) self.assertEqual(results['scale_factor'], (1200 / 800, 2000 / 1333)) # test resize_bboxes/seg/masks transform = Resize(scale_factor=(1.5, 2)) results = transform(copy.deepcopy(self.data_info1)) self.assertTrue((results['gt_bboxes'] == np.array([[0, 0, 168, 224]])).all()) self.assertEqual(results['gt_masks'].height, 2666) self.assertEqual(results['gt_masks'].width, 1200) self.assertEqual(results['gt_seg_map'].shape[:2], (2666, 1200)) # test clip_object_border = False transform = Resize(scale=(200, 150), clip_object_border=False) results = transform(self.data_info2) self.assertTrue((results['gt_bboxes'] == np.array([100, 75, 300, 225])).all()) # test only with image transform = Resize(scale=(200, 150), clip_object_border=False) results = transform(self.data_info3) self.assertTupleEqual(results['img'].shape[:2], (150, 200)) # test geometric transformation with homography matrix transform = Resize(scale_factor=(1.5, 2)) results = transform(copy.deepcopy(self.data_info1)) self.assertTrue((bbox_project( copy.deepcopy(self.data_info1['gt_bboxes']), results['homography_matrix']) == results['gt_bboxes']).all()) def test_resize_use_box_type(self): data_info1 = copy.deepcopy(self.data_info1) data_info1['gt_bboxes'] = HorizontalBoxes(data_info1['gt_bboxes']) data_info2 = copy.deepcopy(self.data_info2) data_info2['gt_bboxes'] = HorizontalBoxes(data_info2['gt_bboxes']) # test keep_ratio is True transform = Resize(scale=(2000, 2000), keep_ratio=True) results = transform(copy.deepcopy(data_info1)) self.assertEqual(results['img_shape'], (2000, 1200)) self.assertEqual(results['scale_factor'], (1200 / 800, 2000 / 1333)) # test resize_bboxes/seg/masks transform = Resize(scale_factor=(1.5, 2)) results = transform(copy.deepcopy(data_info1)) self.assertTrue( (results['gt_bboxes'].numpy() == np.array([[0, 0, 168, 224]])).all()) self.assertEqual(results['gt_masks'].height, 2666) self.assertEqual(results['gt_masks'].width, 1200) self.assertEqual(results['gt_seg_map'].shape[:2], (2666, 1200)) # test clip_object_border = False transform = Resize(scale=(200, 150), clip_object_border=False) results = transform(data_info2) self.assertTrue( (results['gt_bboxes'].numpy() == np.array([100, 75, 300, 225])).all()) # test geometric transformation with homography matrix transform = Resize(scale_factor=(1.5, 2)) results = transform(copy.deepcopy(data_info1)) self.assertTrue((bbox_project( copy.deepcopy(data_info1['gt_bboxes'].numpy()), results['homography_matrix']) == results['gt_bboxes'].numpy() ).all()) def test_repr(self): transform = Resize(scale=(2000, 2000), keep_ratio=True) self.assertEqual( repr(transform), ('Resize(scale=(2000, 2000), ' 'scale_factor=None, keep_ratio=True, ' 'clip_object_border=True), backend=cv2), ' 'interpolation=bilinear)')) class TestFIXShapeResize(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ rng = np.random.RandomState(0) self.data_info1 = dict( img=np.random.random((1333, 800, 3)), gt_seg_map=np.random.random((1333, 800, 3)), gt_bboxes=np.array([[0, 0, 112, 1333]], dtype=np.float32), gt_masks=BitmapMasks( rng.rand(1, 1333, 800), height=1333, width=800)) self.data_info2 = dict( img=np.random.random((300, 400, 3)), gt_bboxes=np.array([[200, 150, 600, 450]], dtype=np.float32), dtype=np.float32) self.data_info3 = dict(img=np.random.random((300, 400, 3))) self.data_info4 = dict( img=np.random.random((600, 800, 3)), gt_bboxes=np.array([[200, 150, 300, 400]], dtype=np.float32), dtype=np.float32) def test_resize(self): # test keep_ratio is True transform = FixShapeResize(width=2000, height=800, keep_ratio=True) results = transform(copy.deepcopy(self.data_info1)) self.assertEqual(results['img_shape'], (800, 2000)) self.assertEqual(results['scale_factor'], (800 / 1333, 800 / 1333)) # test resize_bboxes/seg/masks transform = FixShapeResize(width=2000, height=800, keep_ratio=False) results = transform(copy.deepcopy(self.data_info1)) self.assertTrue((results['gt_bboxes'] == np.array([[0, 0, 280, 800]])).all()) self.assertEqual(results['gt_masks'].height, 800) self.assertEqual(results['gt_masks'].width, 2000) self.assertEqual(results['gt_seg_map'].shape[:2], (800, 2000)) # test clip_object_border = False transform = FixShapeResize( width=200, height=150, clip_object_border=False) results = transform(copy.deepcopy(self.data_info2)) self.assertTrue((results['gt_bboxes'] == np.array([100, 75, 300, 225])).all()) # test only with image transform = FixShapeResize( width=200, height=150, clip_object_border=False) results = transform(self.data_info3) self.assertTupleEqual(results['img'].shape[:2], (150, 200)) # test geometric transformation with homography matrix transform = FixShapeResize(width=400, height=300) results = transform(copy.deepcopy(self.data_info4)) self.assertTrue((bbox_project( copy.deepcopy(self.data_info4['gt_bboxes']), results['homography_matrix']) == results['gt_bboxes']).all()) def test_resize_with_boxlist(self): data_info1 = copy.deepcopy(self.data_info1) data_info1['gt_bboxes'] = HorizontalBoxes(data_info1['gt_bboxes']) data_info2 = copy.deepcopy(self.data_info2) data_info2['gt_bboxes'] = HorizontalBoxes(data_info2['gt_bboxes']) data_info4 = copy.deepcopy(self.data_info4) data_info4['gt_bboxes'] = HorizontalBoxes(data_info4['gt_bboxes']) # test keep_ratio is True transform = FixShapeResize(width=2000, height=800, keep_ratio=True) results = transform(copy.deepcopy(data_info1)) self.assertEqual(results['img_shape'], (800, 2000)) self.assertEqual(results['scale_factor'], (800 / 1333, 800 / 1333)) # test resize_bboxes/seg/masks transform = FixShapeResize(width=2000, height=800, keep_ratio=False) results = transform(copy.deepcopy(data_info1)) self.assertTrue( (results['gt_bboxes'].numpy() == np.array([[0, 0, 280, 800]])).all()) self.assertEqual(results['gt_masks'].height, 800) self.assertEqual(results['gt_masks'].width, 2000) self.assertEqual(results['gt_seg_map'].shape[:2], (800, 2000)) # test clip_object_border = False transform = FixShapeResize( width=200, height=150, clip_object_border=False) results = transform(copy.deepcopy(data_info2)) self.assertTrue( (results['gt_bboxes'].numpy() == np.array([100, 75, 300, 225])).all()) # test only with image transform = FixShapeResize( width=200, height=150, clip_object_border=False) results = transform(self.data_info3) self.assertTupleEqual(results['img'].shape[:2], (150, 200)) # test geometric transformation with homography matrix transform = FixShapeResize(width=400, height=300) results = transform(copy.deepcopy(data_info4)) self.assertTrue((bbox_project( copy.deepcopy(self.data_info4['gt_bboxes']), results['homography_matrix']) == results['gt_bboxes'].numpy() ).all()) def test_repr(self): transform = FixShapeResize(width=2000, height=2000, keep_ratio=True) self.assertEqual( repr(transform), ('FixShapeResize(width=2000, height=2000, ' 'keep_ratio=True, ' 'clip_object_border=True), backend=cv2), ' 'interpolation=bilinear)')) class TestRandomFlip(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ rng = np.random.RandomState(0) self.results1 = { 'img': np.random.random((224, 224, 3)), 'gt_bboxes': np.array([[0, 1, 100, 101]], dtype=np.float32), 'gt_masks': BitmapMasks(rng.rand(1, 224, 224), height=224, width=224), 'gt_seg_map': np.random.random((224, 224)) } self.results2 = {'img': self.results1['img']} def test_transform(self): # test with image, gt_bboxes, gt_masks, gt_seg_map transform = RandomFlip(1.0) results_update = transform.transform(copy.deepcopy(self.results1)) self.assertTrue( (results_update['gt_bboxes'] == np.array([[124, 1, 224, 101]])).all()) # test only with image transform = RandomFlip(1.0) results_update = transform.transform(copy.deepcopy(self.results2)) self.assertTrue( (results_update['img'] == self.results2['img'][:, ::-1]).all()) # test geometric transformation with homography matrix # (1) Horizontal Flip transform = RandomFlip(1.0) results_update = transform.transform(copy.deepcopy(self.results1)) bboxes = copy.deepcopy(self.results1['gt_bboxes']) self.assertTrue((bbox_project( bboxes, results_update['homography_matrix']) == results_update['gt_bboxes'] ).all()) # (2) Vertical Flip transform = RandomFlip(1.0, direction='vertical') results_update = transform.transform(copy.deepcopy(self.results1)) bboxes = copy.deepcopy(self.results1['gt_bboxes']) self.assertTrue((bbox_project( bboxes, results_update['homography_matrix']) == results_update['gt_bboxes'] ).all()) # (3) Diagonal Flip transform = RandomFlip(1.0, direction='diagonal') results_update = transform.transform(copy.deepcopy(self.results1)) bboxes = copy.deepcopy(self.results1['gt_bboxes']) self.assertTrue((bbox_project( bboxes, results_update['homography_matrix']) == results_update['gt_bboxes'] ).all()) def test_transform_use_box_type(self): results1 = copy.deepcopy(self.results1) results1['gt_bboxes'] = HorizontalBoxes(results1['gt_bboxes']) # test with image, gt_bboxes, gt_masks, gt_seg_map transform = RandomFlip(1.0) results_update = transform.transform(copy.deepcopy(results1)) self.assertTrue((results_update['gt_bboxes'].numpy() == np.array( [[124, 1, 224, 101]])).all()) # test geometric transformation with homography matrix # (1) Horizontal Flip transform = RandomFlip(1.0) results_update = transform.transform(copy.deepcopy(results1)) bboxes = copy.deepcopy(results1['gt_bboxes'].numpy()) self.assertTrue((bbox_project(bboxes, results_update['homography_matrix']) == results_update['gt_bboxes'].numpy()).all()) # (2) Vertical Flip transform = RandomFlip(1.0, direction='vertical') results_update = transform.transform(copy.deepcopy(results1)) bboxes = copy.deepcopy(results1['gt_bboxes'].numpy()) self.assertTrue((bbox_project(bboxes, results_update['homography_matrix']) == results_update['gt_bboxes'].numpy()).all()) # (3) Diagonal Flip transform = RandomFlip(1.0, direction='diagonal') results_update = transform.transform(copy.deepcopy(results1)) bboxes = copy.deepcopy(results1['gt_bboxes'].numpy()) self.assertTrue((bbox_project(bboxes, results_update['homography_matrix']) == results_update['gt_bboxes'].numpy()).all()) def test_repr(self): transform = RandomFlip(0.1) transform_str = str(transform) self.assertIsInstance(transform_str, str) class TestPad(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ rng = np.random.RandomState(0) self.results = { 'img': np.random.random((1333, 800, 3)), 'gt_masks': BitmapMasks(rng.rand(4, 1333, 800), height=1333, width=800) } def test_transform(self): # test pad img/gt_masks with size transform = Pad(size=(1200, 2000)) results = transform(copy.deepcopy(self.results)) self.assertEqual(results['img'].shape[:2], (2000, 1200)) self.assertEqual(results['gt_masks'].masks.shape[1:], (2000, 1200)) # test pad img/gt_masks with size_divisor transform = Pad(size_divisor=11) results = transform(copy.deepcopy(self.results)) self.assertEqual(results['img'].shape[:2], (1342, 803)) self.assertEqual(results['gt_masks'].masks.shape[1:], (1342, 803)) # test pad img/gt_masks with pad_to_square transform = Pad(pad_to_square=True) results = transform(copy.deepcopy(self.results)) self.assertEqual(results['img'].shape[:2], (1333, 1333)) self.assertEqual(results['gt_masks'].masks.shape[1:], (1333, 1333)) # test pad img/gt_masks with pad_to_square and size_divisor transform = Pad(pad_to_square=True, size_divisor=11) results = transform(copy.deepcopy(self.results)) self.assertEqual(results['img'].shape[:2], (1342, 1342)) self.assertEqual(results['gt_masks'].masks.shape[1:], (1342, 1342)) # test pad img/gt_masks with pad_to_square and size_divisor transform = Pad(pad_to_square=True, size_divisor=11) results = transform(copy.deepcopy(self.results)) self.assertEqual(results['img'].shape[:2], (1342, 1342)) self.assertEqual(results['gt_masks'].masks.shape[1:], (1342, 1342)) def test_repr(self): transform = Pad( pad_to_square=True, size_divisor=11, padding_mode='edge') self.assertEqual( repr(transform), ('Pad(size=None, size_divisor=11, pad_to_square=True, ' "pad_val={'img': 0, 'seg': 255}), padding_mode=edge)")) class TestMinIoURandomCrop(unittest.TestCase): def test_transform(self): results = dict() img = mmcv.imread( osp.join(osp.dirname(__file__), '../../data/color.jpg'), 'color') results['img'] = img results['img_shape'] = img.shape[:2] gt_bboxes = create_random_bboxes(1, results['img_shape'][1], results['img_shape'][0]) results['gt_labels'] = np.ones(gt_bboxes.shape[0], dtype=np.int64) results['gt_bboxes'] = gt_bboxes transform = MinIoURandomCrop() results = transform.transform(copy.deepcopy(results)) self.assertEqual(results['gt_labels'].shape[0], results['gt_bboxes'].shape[0]) self.assertEqual(results['gt_labels'].dtype, np.int64) self.assertEqual(results['gt_bboxes'].dtype, np.float32) patch = np.array( [0, 0, results['img_shape'][1], results['img_shape'][0]]) ious = bbox_overlaps(patch.reshape(-1, 4), results['gt_bboxes']).reshape(-1) mode = transform.mode if mode == 1: self.assertTrue(np.equal(results['gt_bboxes'], gt_bboxes).all()) else: self.assertTrue((ious >= mode).all()) def test_transform_use_box_type(self): results = dict() img = mmcv.imread( osp.join(osp.dirname(__file__), '../../data/color.jpg'), 'color') results['img'] = img results['img_shape'] = img.shape[:2] gt_bboxes = create_random_bboxes(1, results['img_shape'][1], results['img_shape'][0]) results['gt_labels'] = np.ones(gt_bboxes.shape[0], dtype=np.int64) results['gt_bboxes'] = HorizontalBoxes(gt_bboxes) transform = MinIoURandomCrop() results = transform.transform(copy.deepcopy(results)) self.assertEqual(results['gt_labels'].shape[0], results['gt_bboxes'].shape[0]) self.assertEqual(results['gt_labels'].dtype, np.int64) self.assertEqual(results['gt_bboxes'].dtype, torch.float32) patch = np.array( [0, 0, results['img_shape'][1], results['img_shape'][0]]) ious = bbox_overlaps( patch.reshape(-1, 4), results['gt_bboxes'].numpy()).reshape(-1) mode = transform.mode if mode == 1: self.assertTrue((results['gt_bboxes'].numpy() == gt_bboxes).all()) else: self.assertTrue((ious >= mode).all()) def test_repr(self): transform = MinIoURandomCrop() self.assertEqual( repr(transform), ('MinIoURandomCrop' '(min_ious=(0.1, 0.3, 0.5, 0.7, 0.9), ' 'min_crop_size=0.3, ' 'bbox_clip_border=True)')) class TestPhotoMetricDistortion(unittest.TestCase): def test_transform(self): img = mmcv.imread( osp.join(osp.dirname(__file__), '../../data/color.jpg'), 'color') transform = PhotoMetricDistortion() # test uint8 input results = dict() results['img'] = img results = transform.transform(copy.deepcopy(results)) self.assertEqual(results['img'].dtype, np.float32) # test float32 input results = dict() results['img'] = img.astype(np.float32) results = transform.transform(copy.deepcopy(results)) self.assertEqual(results['img'].dtype, np.float32) def test_repr(self): transform = PhotoMetricDistortion() self.assertEqual( repr(transform), ('PhotoMetricDistortion' '(brightness_delta=32, ' 'contrast_range=(0.5, 1.5), ' 'saturation_range=(0.5, 1.5), ' 'hue_delta=18)')) class TestExpand(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ rng = np.random.RandomState(0) self.results = { 'img': np.random.random((224, 224, 3)), 'img_shape': (224, 224), 'gt_bboxes': np.array([[0, 1, 100, 101]]), 'gt_masks': BitmapMasks(rng.rand(1, 224, 224), height=224, width=224), 'gt_seg_map': np.random.random((224, 224)) } def test_transform(self): transform = Expand() results = transform.transform(copy.deepcopy(self.results)) self.assertEqual( results['img_shape'], (results['gt_masks'].height, results['gt_masks'].width)) self.assertEqual(results['img_shape'], results['gt_seg_map'].shape) def test_transform_use_box_type(self): results = copy.deepcopy(self.results) results['gt_bboxes'] = HorizontalBoxes(results['gt_bboxes']) transform = Expand() results = transform.transform(results) self.assertEqual( results['img_shape'], (results['gt_masks'].height, results['gt_masks'].width)) self.assertEqual(results['img_shape'], results['gt_seg_map'].shape) def test_repr(self): transform = Expand() self.assertEqual( repr(transform), ('Expand' '(mean=(0, 0, 0), to_rgb=True, ' 'ratio_range=(1, 4), ' 'seg_ignore_label=None, ' 'prob=0.5)')) class TestSegRescale(unittest.TestCase): def setUp(self) -> None: seg_map = np.random.randint(0, 255, size=(32, 32), dtype=np.int32) self.results = {'gt_seg_map': seg_map} def test_transform(self): # test scale_factor != 1 transform = SegRescale(scale_factor=2) results = transform(copy.deepcopy(self.results)) self.assertEqual(results['gt_seg_map'].shape[:2], (64, 64)) # test scale_factor = 1 transform = SegRescale(scale_factor=1) results = transform(copy.deepcopy(self.results)) self.assertEqual(results['gt_seg_map'].shape[:2], (32, 32)) def test_repr(self): transform = SegRescale(scale_factor=2) self.assertEqual( repr(transform), ('SegRescale(scale_factor=2, backend=cv2)')) class TestRandomCrop(unittest.TestCase): def test_init(self): # test invalid crop_type with self.assertRaisesRegex(ValueError, 'Invalid crop_type'): RandomCrop(crop_size=(10, 10), crop_type='unknown') crop_type_list = ['absolute', 'absolute_range'] for crop_type in crop_type_list: # test h > 0 and w > 0 for crop_size in [(0, 0), (0, 1), (1, 0)]: with self.assertRaises(AssertionError): RandomCrop(crop_size=crop_size, crop_type=crop_type) # test type(h) = int and type(w) = int for crop_size in [(1.0, 1), (1, 1.0), (1.0, 1.0)]: with self.assertRaises(AssertionError): RandomCrop(crop_size=crop_size, crop_type=crop_type) # test crop_size[0] <= crop_size[1] with self.assertRaises(AssertionError): RandomCrop(crop_size=(10, 5), crop_type='absolute_range') # test h in (0, 1] and w in (0, 1] crop_type_list = ['relative_range', 'relative'] for crop_type in crop_type_list: for crop_size in [(0, 1), (1, 0), (1.1, 0.5), (0.5, 1.1)]: with self.assertRaises(AssertionError): RandomCrop(crop_size=crop_size, crop_type=crop_type) def test_transform(self): # test relative and absolute crop src_results = { 'img': np.random.randint(0, 255, size=(24, 32), dtype=np.int32) } target_shape = (12, 16) for crop_type, crop_size in zip(['relative', 'absolute'], [(0.5, 0.5), (16, 12)]): transform = RandomCrop(crop_size=crop_size, crop_type=crop_type) results = transform(copy.deepcopy(src_results)) print(results['img'].shape[:2]) self.assertEqual(results['img'].shape[:2], target_shape) # test absolute_range crop transform = RandomCrop(crop_size=(10, 20), crop_type='absolute_range') results = transform(copy.deepcopy(src_results)) h, w = results['img'].shape self.assertTrue(10 <= w <= 20) self.assertTrue(10 <= h <= 20) # test relative_range crop transform = RandomCrop( crop_size=(0.5, 0.5), crop_type='relative_range') results = transform(copy.deepcopy(src_results)) h, w = results['img'].shape self.assertTrue(16 <= w <= 32) self.assertTrue(12 <= h <= 24) # test with gt_bboxes, gt_bboxes_labels, gt_ignore_flags, # gt_masks, gt_seg_map img = np.random.randint(0, 255, size=(10, 10), dtype=np.uint8) gt_bboxes = np.array([[0, 0, 7, 7], [2, 3, 9, 9]], dtype=np.float32) gt_bboxes_labels = np.array([0, 1], dtype=np.int64) gt_ignore_flags = np.array([0, 1], dtype=bool) gt_masks_ = np.zeros((2, 10, 10), np.uint8) gt_masks_[0, 0:7, 0:7] = 1 gt_masks_[1, 2:7, 3:8] = 1 gt_masks = BitmapMasks(gt_masks_.copy(), height=10, width=10) gt_seg_map = np.random.randint(0, 255, size=(10, 10), dtype=np.uint8) src_results = { 'img': img, 'gt_bboxes': gt_bboxes, 'gt_bboxes_labels': gt_bboxes_labels, 'gt_ignore_flags': gt_ignore_flags, 'gt_masks': gt_masks, 'gt_seg_map': gt_seg_map } transform = RandomCrop( crop_size=(7, 5), allow_negative_crop=False, recompute_bbox=False, bbox_clip_border=True) results = transform(copy.deepcopy(src_results)) h, w = results['img'].shape self.assertEqual(h, 5) self.assertEqual(w, 7) self.assertEqual(results['gt_bboxes'].shape[0], 2) self.assertEqual(results['gt_bboxes_labels'].shape[0], 2) self.assertEqual(results['gt_ignore_flags'].shape[0], 2) self.assertTupleEqual(results['gt_seg_map'].shape[:2], (5, 7)) # test geometric transformation with homography matrix bboxes = copy.deepcopy(src_results['gt_bboxes']) self.assertTrue((bbox_project(bboxes, results['homography_matrix'], (5, 7)) == results['gt_bboxes']).all()) # test recompute_bbox = True gt_masks_ = np.zeros((2, 10, 10), np.uint8) gt_masks = BitmapMasks(gt_masks_.copy(), height=10, width=10) gt_bboxes = np.array([[0.1, 0.1, 0.2, 0.2]]) src_results = { 'img': img, 'gt_bboxes': gt_bboxes, 'gt_masks': gt_masks } target_gt_bboxes = np.zeros((1, 4), dtype=np.float32) transform = RandomCrop( crop_size=(10, 11), allow_negative_crop=False, recompute_bbox=True, bbox_clip_border=True) results = transform(copy.deepcopy(src_results)) self.assertTrue((results['gt_bboxes'] == target_gt_bboxes).all()) # test bbox_clip_border = False src_results = {'img': img, 'gt_bboxes': gt_bboxes} transform = RandomCrop( crop_size=(10, 11), allow_negative_crop=False, recompute_bbox=True, bbox_clip_border=False) results = transform(copy.deepcopy(src_results)) self.assertTrue( (results['gt_bboxes'] == src_results['gt_bboxes']).all()) # test the crop does not contain any gt-bbox # allow_negative_crop = False img = np.random.randint(0, 255, size=(10, 10), dtype=np.uint8) gt_bboxes = np.zeros((0, 4), dtype=np.float32) src_results = {'img': img, 'gt_bboxes': gt_bboxes} transform = RandomCrop(crop_size=(5, 3), allow_negative_crop=False) results = transform(copy.deepcopy(src_results)) self.assertIsNone(results) # allow_negative_crop = True img = np.random.randint(0, 255, size=(10, 10), dtype=np.uint8) gt_bboxes = np.zeros((0, 4), dtype=np.float32) src_results = {'img': img, 'gt_bboxes': gt_bboxes} transform = RandomCrop(crop_size=(5, 3), allow_negative_crop=True) results = transform(copy.deepcopy(src_results)) self.assertTrue(isinstance(results, dict)) def test_transform_use_box_type(self): # test with gt_bboxes, gt_bboxes_labels, gt_ignore_flags, # gt_masks, gt_seg_map img = np.random.randint(0, 255, size=(10, 10), dtype=np.uint8) gt_bboxes = np.array([[0, 0, 7, 7], [2, 3, 9, 9]], dtype=np.float32) gt_bboxes_labels = np.array([0, 1], dtype=np.int64) gt_ignore_flags = np.array([0, 1], dtype=bool) gt_masks_ = np.zeros((2, 10, 10), np.uint8) gt_masks_[0, 0:7, 0:7] = 1 gt_masks_[1, 2:7, 3:8] = 1 gt_masks = BitmapMasks(gt_masks_.copy(), height=10, width=10) gt_seg_map = np.random.randint(0, 255, size=(10, 10), dtype=np.uint8) src_results = { 'img': img, 'gt_bboxes': HorizontalBoxes(gt_bboxes), 'gt_bboxes_labels': gt_bboxes_labels, 'gt_ignore_flags': gt_ignore_flags, 'gt_masks': gt_masks, 'gt_seg_map': gt_seg_map } transform = RandomCrop( crop_size=(7, 5), allow_negative_crop=False, recompute_bbox=False, bbox_clip_border=True) results = transform(copy.deepcopy(src_results)) h, w = results['img'].shape self.assertEqual(h, 5) self.assertEqual(w, 7) self.assertEqual(results['gt_bboxes'].shape[0], 2) self.assertEqual(results['gt_bboxes_labels'].shape[0], 2) self.assertEqual(results['gt_ignore_flags'].shape[0], 2) self.assertTupleEqual(results['gt_seg_map'].shape[:2], (5, 7)) # test geometric transformation with homography matrix bboxes = copy.deepcopy(src_results['gt_bboxes'].numpy()) print(bboxes, results['gt_bboxes']) self.assertTrue( (bbox_project(bboxes, results['homography_matrix'], (5, 7)) == results['gt_bboxes'].numpy()).all()) # test recompute_bbox = True gt_masks_ = np.zeros((2, 10, 10), np.uint8) gt_masks = BitmapMasks(gt_masks_.copy(), height=10, width=10) gt_bboxes = HorizontalBoxes(np.array([[0.1, 0.1, 0.2, 0.2]])) src_results = { 'img': img, 'gt_bboxes': gt_bboxes, 'gt_masks': gt_masks } target_gt_bboxes = np.zeros((1, 4), dtype=np.float32) transform = RandomCrop( crop_size=(10, 11), allow_negative_crop=False, recompute_bbox=True, bbox_clip_border=True) results = transform(copy.deepcopy(src_results)) self.assertTrue( (results['gt_bboxes'].numpy() == target_gt_bboxes).all()) # test bbox_clip_border = False src_results = {'img': img, 'gt_bboxes': gt_bboxes} transform = RandomCrop( crop_size=(10, 10), allow_negative_crop=False, recompute_bbox=True, bbox_clip_border=False) results = transform(copy.deepcopy(src_results)) self.assertTrue( (results['gt_bboxes'].numpy() == src_results['gt_bboxes'].numpy() ).all()) # test the crop does not contain any gt-bbox # allow_negative_crop = False img = np.random.randint(0, 255, size=(10, 10), dtype=np.uint8) gt_bboxes = HorizontalBoxes(np.zeros((0, 4), dtype=np.float32)) src_results = {'img': img, 'gt_bboxes': gt_bboxes} transform = RandomCrop(crop_size=(5, 2), allow_negative_crop=False) results = transform(copy.deepcopy(src_results)) self.assertIsNone(results) # allow_negative_crop = True img = np.random.randint(0, 255, size=(10, 10), dtype=np.uint8) gt_bboxes = HorizontalBoxes(np.zeros((0, 4), dtype=np.float32)) src_results = {'img': img, 'gt_bboxes': gt_bboxes} transform = RandomCrop(crop_size=(5, 2), allow_negative_crop=True) results = transform(copy.deepcopy(src_results)) self.assertTrue(isinstance(results, dict)) def test_repr(self): crop_type = 'absolute' crop_size = (10, 5) allow_negative_crop = False recompute_bbox = True bbox_clip_border = False transform = RandomCrop( crop_size=crop_size, crop_type=crop_type, allow_negative_crop=allow_negative_crop, recompute_bbox=recompute_bbox, bbox_clip_border=bbox_clip_border) self.assertEqual( repr(transform), f'RandomCrop(crop_size={crop_size}, crop_type={crop_type}, ' f'allow_negative_crop={allow_negative_crop}, ' f'recompute_bbox={recompute_bbox}, ' f'bbox_clip_border={bbox_clip_border})') class TestCutOut(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ img = mmcv.imread( osp.join(osp.dirname(__file__), '../../data/color.jpg'), 'color') self.results = {'img': img} def test_transform(self): # test n_holes with self.assertRaises(AssertionError): transform = CutOut(n_holes=(5, 3), cutout_shape=(8, 8)) with self.assertRaises(AssertionError): transform = CutOut(n_holes=(3, 4, 5), cutout_shape=(8, 8)) # test cutout_shape and cutout_ratio with self.assertRaises(AssertionError): transform = CutOut(n_holes=1, cutout_shape=8) with self.assertRaises(AssertionError): transform = CutOut(n_holes=1, cutout_ratio=0.2) # either of cutout_shape and cutout_ratio should be given with self.assertRaises(AssertionError): transform = CutOut(n_holes=1) with self.assertRaises(AssertionError): transform = CutOut( n_holes=1, cutout_shape=(2, 2), cutout_ratio=(0.4, 0.4)) transform = CutOut(n_holes=1, cutout_shape=(10, 10)) results = transform(copy.deepcopy(self.results)) self.assertTrue(results['img'].sum() < self.results['img'].sum()) transform = CutOut( n_holes=(2, 4), cutout_shape=[(10, 10), (15, 15)], fill_in=(255, 255, 255)) results = transform(copy.deepcopy(self.results)) self.assertTrue(results['img'].sum() > self.results['img'].sum()) transform = CutOut( n_holes=1, cutout_ratio=(0.8, 0.8), fill_in=(255, 255, 255)) results = transform(copy.deepcopy(self.results)) self.assertTrue(results['img'].sum() > self.results['img'].sum()) def test_repr(self): transform = CutOut(n_holes=1, cutout_shape=(10, 10)) self.assertEqual( repr(transform), ('CutOut(n_holes=(1, 1), ' 'cutout_shape=[(10, 10)], ' 'fill_in=(0, 0, 0))')) transform = CutOut( n_holes=1, cutout_ratio=(0.8, 0.8), fill_in=(255, 255, 255)) self.assertEqual( repr(transform), ('CutOut(n_holes=(1, 1), ' 'cutout_ratio=[(0.8, 0.8)], ' 'fill_in=(255, 255, 255))')) class TestMosaic(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ rng = np.random.RandomState(0) self.results = { 'img': np.random.random((224, 224, 3)), 'img_shape': (224, 224), 'gt_bboxes_labels': np.array([1, 2, 3], dtype=np.int64), 'gt_bboxes': np.array([[10, 10, 20, 20], [20, 20, 40, 40], [40, 40, 80, 80]], dtype=np.float32), 'gt_ignore_flags': np.array([0, 0, 1], dtype=bool), 'gt_masks': BitmapMasks(rng.rand(3, 224, 224), height=224, width=224), } def test_transform(self): # test assertion for invalid img_scale with self.assertRaises(AssertionError): transform = Mosaic(img_scale=640) # test assertion for invalid probability with self.assertRaises(AssertionError): transform = Mosaic(prob=1.5) transform = Mosaic(img_scale=(12, 10)) # test assertion for invalid mix_results with self.assertRaises(AssertionError): results = transform(copy.deepcopy(self.results)) self.results['mix_results'] = [copy.deepcopy(self.results)] * 3 results = transform(copy.deepcopy(self.results)) self.assertTrue(results['img'].shape[:2] == (20, 24)) self.assertTrue(results['gt_bboxes_labels'].shape[0] == results['gt_bboxes'].shape[0]) self.assertTrue(results['gt_bboxes_labels'].dtype == np.int64) self.assertTrue(results['gt_bboxes'].dtype == np.float32) self.assertTrue(results['gt_ignore_flags'].dtype == bool) def test_transform_with_no_gt(self): self.results['gt_bboxes'] = np.empty((0, 4), dtype=np.float32) self.results['gt_bboxes_labels'] = np.empty((0, ), dtype=np.int64) self.results['gt_ignore_flags'] = np.empty((0, ), dtype=bool) transform = Mosaic(img_scale=(12, 10)) self.results['mix_results'] = [copy.deepcopy(self.results)] * 3 results = transform(copy.deepcopy(self.results)) self.assertIsInstance(results, dict) self.assertTrue(results['img'].shape[:2] == (20, 24)) self.assertTrue( results['gt_bboxes_labels'].shape[0] == results['gt_bboxes']. shape[0] == results['gt_ignore_flags'].shape[0] == 0) self.assertTrue(results['gt_bboxes_labels'].dtype == np.int64) self.assertTrue(results['gt_bboxes'].dtype == np.float32) self.assertTrue(results['gt_ignore_flags'].dtype == bool) def test_transform_use_box_type(self): transform = Mosaic(img_scale=(12, 10)) results = copy.deepcopy(self.results) results['gt_bboxes'] = HorizontalBoxes(results['gt_bboxes']) results['mix_results'] = [results] * 3 results = transform(results) self.assertTrue(results['img'].shape[:2] == (20, 24)) self.assertTrue(results['gt_bboxes_labels'].shape[0] == results['gt_bboxes'].shape[0]) self.assertTrue(results['gt_bboxes_labels'].dtype == np.int64) self.assertTrue(results['gt_bboxes'].dtype == torch.float32) self.assertTrue(results['gt_ignore_flags'].dtype == bool) def test_repr(self): transform = Mosaic(img_scale=(640, 640), ) self.assertEqual( repr(transform), ('Mosaic(img_scale=(640, 640), ' 'center_ratio_range=(0.5, 1.5), ' 'pad_val=114.0, ' 'prob=1.0)')) class TestMixUp(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ rng = np.random.RandomState(0) self.results = { 'img': np.random.random((224, 224, 3)), 'img_shape': (224, 224), 'gt_bboxes_labels': np.array([1, 2, 3], dtype=np.int64), 'gt_bboxes': np.array([[10, 10, 20, 20], [20, 20, 40, 40], [40, 40, 80, 80]], dtype=np.float32), 'gt_ignore_flags': np.array([0, 0, 1], dtype=bool), 'gt_masks': BitmapMasks(rng.rand(3, 224, 224), height=224, width=224), } def test_transform(self): # test assertion for invalid img_scale with self.assertRaises(AssertionError): transform = MixUp(img_scale=640) transform = MixUp(img_scale=(12, 10)) # test assertion for invalid mix_results with self.assertRaises(AssertionError): results = transform(copy.deepcopy(self.results)) with self.assertRaises(AssertionError): self.results['mix_results'] = [copy.deepcopy(self.results)] * 2 results = transform(copy.deepcopy(self.results)) self.results['mix_results'] = [copy.deepcopy(self.results)] results = transform(copy.deepcopy(self.results)) self.assertTrue(results['img'].shape[:2] == (224, 224)) self.assertTrue(results['gt_bboxes_labels'].shape[0] == results['gt_bboxes'].shape[0]) self.assertTrue(results['gt_bboxes_labels'].dtype == np.int64) self.assertTrue(results['gt_bboxes'].dtype == np.float32) self.assertTrue(results['gt_ignore_flags'].dtype == bool) def test_transform_use_box_type(self): results = copy.deepcopy(self.results) results['gt_bboxes'] = HorizontalBoxes(results['gt_bboxes']) transform = MixUp(img_scale=(12, 10)) results['mix_results'] = [results] results = transform(results) self.assertTrue(results['img'].shape[:2] == (224, 224)) self.assertTrue(results['gt_bboxes_labels'].shape[0] == results['gt_bboxes'].shape[0]) self.assertTrue(results['gt_bboxes_labels'].dtype == np.int64) self.assertTrue(results['gt_bboxes'].dtype == torch.float32) self.assertTrue(results['gt_ignore_flags'].dtype == bool) def test_repr(self): transform = MixUp( img_scale=(640, 640), ratio_range=(0.8, 1.6), pad_val=114.0, ) self.assertEqual( repr(transform), ('MixUp(dynamic_scale=(640, 640), ' 'ratio_range=(0.8, 1.6), ' 'flip_ratio=0.5, ' 'pad_val=114.0, ' 'max_iters=15, ' 'bbox_clip_border=True)')) class TestRandomAffine(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.results = { 'img': np.random.random((224, 224, 3)), 'img_shape': (224, 224), 'gt_bboxes_labels': np.array([1, 2, 3], dtype=np.int64), 'gt_bboxes': np.array([[10, 10, 20, 20], [20, 20, 40, 40], [40, 40, 80, 80]], dtype=np.float32), 'gt_ignore_flags': np.array([0, 0, 1], dtype=bool), } def test_transform(self): # test assertion for invalid translate_ratio with self.assertRaises(AssertionError): transform = RandomAffine(max_translate_ratio=1.5) # test assertion for invalid scaling_ratio_range with self.assertRaises(AssertionError): transform = RandomAffine(scaling_ratio_range=(1.5, 0.5)) with self.assertRaises(AssertionError): transform = RandomAffine(scaling_ratio_range=(0, 0.5)) transform = RandomAffine() results = transform(copy.deepcopy(self.results)) self.assertTrue(results['img'].shape[:2] == (224, 224)) self.assertTrue(results['gt_bboxes_labels'].shape[0] == results['gt_bboxes'].shape[0]) self.assertTrue(results['gt_bboxes_labels'].dtype == np.int64) self.assertTrue(results['gt_bboxes'].dtype == np.float32) self.assertTrue(results['gt_ignore_flags'].dtype == bool) def test_transform_use_box_type(self): results = copy.deepcopy(self.results) results['gt_bboxes'] = HorizontalBoxes(results['gt_bboxes']) transform = RandomAffine() results = transform(copy.deepcopy(results)) self.assertTrue(results['img'].shape[:2] == (224, 224)) self.assertTrue(results['gt_bboxes_labels'].shape[0] == results['gt_bboxes'].shape[0]) self.assertTrue(results['gt_bboxes_labels'].dtype == np.int64) self.assertTrue(results['gt_bboxes'].dtype == torch.float32) self.assertTrue(results['gt_ignore_flags'].dtype == bool) def test_repr(self): transform = RandomAffine( scaling_ratio_range=(0.1, 2), border=(-320, -320), ) self.assertEqual( repr(transform), ('RandomAffine(max_rotate_degree=10.0, ' 'max_translate_ratio=0.1, ' 'scaling_ratio_range=(0.1, 2), ' 'max_shear_degree=2.0, ' 'border=(-320, -320), ' 'border_val=(114, 114, 114), ' 'bbox_clip_border=True)')) class TestYOLOXHSVRandomAug(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ img = mmcv.imread( osp.join(osp.dirname(__file__), '../../data/color.jpg'), 'color') self.results = { 'img': img, 'img_shape': (224, 224), 'gt_bboxes_labels': np.array([1, 2, 3], dtype=np.int64), 'gt_bboxes': np.array([[10, 10, 20, 20], [20, 20, 40, 40], [40, 40, 80, 80]], dtype=np.float32), 'gt_ignore_flags': np.array([0, 0, 1], dtype=bool), } def test_transform(self): transform = YOLOXHSVRandomAug() results = transform(copy.deepcopy(self.results)) self.assertTrue( results['img'].shape[:2] == self.results['img'].shape[:2]) self.assertTrue(results['gt_bboxes_labels'].shape[0] == results['gt_bboxes'].shape[0]) self.assertTrue(results['gt_bboxes_labels'].dtype == np.int64) self.assertTrue(results['gt_bboxes'].dtype == np.float32) self.assertTrue(results['gt_ignore_flags'].dtype == bool) def test_repr(self): transform = YOLOXHSVRandomAug() self.assertEqual( repr(transform), ('YOLOXHSVRandomAug(hue_delta=5, ' 'saturation_delta=30, ' 'value_delta=30)')) class TestRandomCenterCropPad(unittest.TestCase): def test_init(self): # test assertion for invalid crop_size while test_mode=False with self.assertRaises(AssertionError): RandomCenterCropPad( crop_size=(-1, 0), test_mode=False, test_pad_mode=None) # test assertion for invalid ratios while test_mode=False with self.assertRaises(AssertionError): RandomCenterCropPad( crop_size=(511, 511), ratios=(1.0, 1.0), test_mode=False, test_pad_mode=None) # test assertion for invalid mean, std and to_rgb with self.assertRaises(AssertionError): RandomCenterCropPad( crop_size=(511, 511), mean=None, std=None, to_rgb=None, test_mode=False, test_pad_mode=None) # test assertion for invalid crop_size while test_mode=True with self.assertRaises(AssertionError): RandomCenterCropPad( crop_size=(511, 511), ratios=None, border=None, mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True, test_mode=True, test_pad_mode=('logical_or', 127)) # test assertion for invalid ratios while test_mode=True with self.assertRaises(AssertionError): RandomCenterCropPad( crop_size=None, ratios=(0.9, 1.0, 1.1), border=None, mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True, test_mode=True, test_pad_mode=('logical_or', 127)) # test assertion for invalid border while test_mode=True with self.assertRaises(AssertionError): RandomCenterCropPad( crop_size=None, ratios=None, border=128, mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True, test_mode=True, test_pad_mode=('logical_or', 127)) # test assertion for invalid test_pad_mode while test_mode=True with self.assertRaises(AssertionError): RandomCenterCropPad( crop_size=None, ratios=None, border=None, mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True, test_mode=True, test_pad_mode=('do_nothing', 100)) def test_transform(self): results = dict( img_path=osp.join(osp.dirname(__file__), '../../data/color.jpg')) load = LoadImageFromFile(to_float32=True) results = load(results) test_results = copy.deepcopy(results) h, w = results['img_shape'] gt_bboxes = create_random_bboxes(4, w, h) gt_bboxes_labels = np.array([1, 2, 3, 1], dtype=np.int64) gt_ignore_flags = np.array([0, 0, 1, 1], dtype=bool) results['gt_bboxes'] = gt_bboxes results['gt_bboxes_labels'] = gt_bboxes_labels results['gt_ignore_flags'] = gt_ignore_flags crop_module = RandomCenterCropPad( crop_size=(w - 20, h - 20), ratios=(1.0, ), border=128, mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True, test_mode=False, test_pad_mode=None) train_results = crop_module(results) assert train_results['img'].shape[:2] == (h - 20, w - 20) # All bboxes should be reserved after crop assert train_results['img_shape'][:2] == (h - 20, w - 20) assert train_results['gt_bboxes'].shape[0] == 4 assert train_results['gt_bboxes'].dtype == np.float32 crop_module = RandomCenterCropPad( crop_size=None, ratios=None, border=None, mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True, test_mode=True, test_pad_mode=('logical_or', 127)) test_results = crop_module(test_results) assert test_results['img'].shape[:2] == (h | 127, w | 127) assert test_results['img_shape'][:2] == (h | 127, w | 127) assert 'border' in test_results def test_transform_use_box_type(self): results = dict( img_path=osp.join(osp.dirname(__file__), '../../data/color.jpg')) load = LoadImageFromFile(to_float32=True) results = load(results) test_results = copy.deepcopy(results) h, w = results['img_shape'] gt_bboxes = create_random_bboxes(4, w, h) gt_bboxes_labels = np.array([1, 2, 3, 1], dtype=np.int64) gt_ignore_flags = np.array([0, 0, 1, 1], dtype=bool) results['gt_bboxes'] = HorizontalBoxes(gt_bboxes) results['gt_bboxes_labels'] = gt_bboxes_labels results['gt_ignore_flags'] = gt_ignore_flags crop_module = RandomCenterCropPad( crop_size=(w - 20, h - 20), ratios=(1.0, ), border=128, mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True, test_mode=False, test_pad_mode=None) train_results = crop_module(results) assert train_results['img'].shape[:2] == (h - 20, w - 20) # All bboxes should be reserved after crop assert train_results['img_shape'][:2] == (h - 20, w - 20) assert train_results['gt_bboxes'].shape[0] == 4 assert train_results['gt_bboxes'].dtype == torch.float32 crop_module = RandomCenterCropPad( crop_size=None, ratios=None, border=None, mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True, test_mode=True, test_pad_mode=('logical_or', 127)) test_results = crop_module(test_results) assert test_results['img'].shape[:2] == (h | 127, w | 127) assert test_results['img_shape'][:2] == (h | 127, w | 127) assert 'border' in test_results class TestCopyPaste(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ img = mmcv.imread( osp.join(osp.dirname(__file__), '../../data/color.jpg'), 'color') h, w, _ = img.shape dst_bboxes = np.array([[0.2 * w, 0.2 * h, 0.4 * w, 0.4 * h], [0.5 * w, 0.5 * h, 0.6 * w, 0.6 * h]], dtype=np.float32) src_bboxes = np.array([[0.1 * w, 0.1 * h, 0.3 * w, 0.5 * h], [0.4 * w, 0.4 * h, 0.7 * w, 0.7 * h], [0.8 * w, 0.8 * h, 0.9 * w, 0.9 * h]], dtype=np.float32) self.dst_results = { 'img': img.copy(), 'gt_bboxes': dst_bboxes, 'gt_bboxes_labels': np.ones(dst_bboxes.shape[0], dtype=np.int64), 'gt_masks': create_full_masks(dst_bboxes, w, h), 'gt_ignore_flags': np.array([0, 1], dtype=bool), } self.src_results = { 'img': img.copy(), 'gt_bboxes': src_bboxes, 'gt_bboxes_labels': np.ones(src_bboxes.shape[0], dtype=np.int64) * 2, 'gt_masks': create_full_masks(src_bboxes, w, h), 'gt_ignore_flags': np.array([0, 0, 1], dtype=bool), } def test_transform(self): transform = CopyPaste(selected=False) # test assertion for invalid mix_results with self.assertRaises(AssertionError): results = transform(copy.deepcopy(self.dst_results)) results = copy.deepcopy(self.dst_results) results['mix_results'] = [copy.deepcopy(self.src_results)] results = transform(results) self.assertEqual(results['img'].shape[:2], self.dst_results['img'].shape[:2]) # one object of destination image is totally occluded self.assertEqual( results['gt_bboxes'].shape[0], self.dst_results['gt_bboxes'].shape[0] + self.src_results['gt_bboxes'].shape[0] - 1) self.assertEqual( results['gt_bboxes_labels'].shape[0], self.dst_results['gt_bboxes_labels'].shape[0] + self.src_results['gt_bboxes_labels'].shape[0] - 1) self.assertEqual( results['gt_masks'].masks.shape[0], self.dst_results['gt_masks'].masks.shape[0] + self.src_results['gt_masks'].masks.shape[0] - 1) self.assertEqual( results['gt_ignore_flags'].shape[0], self.dst_results['gt_ignore_flags'].shape[0] + self.src_results['gt_ignore_flags'].shape[0] - 1) # the object of destination image is partially occluded ori_bbox = self.dst_results['gt_bboxes'][0] occ_bbox = results['gt_bboxes'][0] ori_mask = self.dst_results['gt_masks'].masks[0] occ_mask = results['gt_masks'].masks[0] self.assertTrue(ori_mask.sum() > occ_mask.sum()) self.assertTrue( np.all(np.abs(occ_bbox - ori_bbox) <= transform.bbox_occluded_thr) or occ_mask.sum() > transform.mask_occluded_thr) # test copypaste with selected objects transform = CopyPaste() results = copy.deepcopy(self.dst_results) results['mix_results'] = [copy.deepcopy(self.src_results)] results = transform(results) # test copypaste with an empty source image results = copy.deepcopy(self.dst_results) valid_inds = [False] * self.src_results['gt_bboxes'].shape[0] results['mix_results'] = [{ 'img': self.src_results['img'].copy(), 'gt_bboxes': self.src_results['gt_bboxes'][valid_inds], 'gt_bboxes_labels': self.src_results['gt_bboxes_labels'][valid_inds], 'gt_masks': self.src_results['gt_masks'][valid_inds], 'gt_ignore_flags': self.src_results['gt_ignore_flags'][valid_inds], }] results = transform(results) def test_transform_use_box_type(self): src_results = copy.deepcopy(self.src_results) src_results['gt_bboxes'] = HorizontalBoxes(src_results['gt_bboxes']) dst_results = copy.deepcopy(self.dst_results) dst_results['gt_bboxes'] = HorizontalBoxes(dst_results['gt_bboxes']) transform = CopyPaste(selected=False) results = copy.deepcopy(dst_results) results['mix_results'] = [copy.deepcopy(src_results)] results = transform(results) self.assertEqual(results['img'].shape[:2], self.dst_results['img'].shape[:2]) # one object of destination image is totally occluded self.assertEqual( results['gt_bboxes'].shape[0], self.dst_results['gt_bboxes'].shape[0] + self.src_results['gt_bboxes'].shape[0] - 1) self.assertEqual( results['gt_bboxes_labels'].shape[0], self.dst_results['gt_bboxes_labels'].shape[0] + self.src_results['gt_bboxes_labels'].shape[0] - 1) self.assertEqual( results['gt_masks'].masks.shape[0], self.dst_results['gt_masks'].masks.shape[0] + self.src_results['gt_masks'].masks.shape[0] - 1) self.assertEqual( results['gt_ignore_flags'].shape[0], self.dst_results['gt_ignore_flags'].shape[0] + self.src_results['gt_ignore_flags'].shape[0] - 1) # the object of destination image is partially occluded ori_bbox = dst_results['gt_bboxes'][0].numpy() occ_bbox = results['gt_bboxes'][0].numpy() ori_mask = dst_results['gt_masks'].masks[0] occ_mask = results['gt_masks'].masks[0] self.assertTrue(ori_mask.sum() > occ_mask.sum()) self.assertTrue( np.all(np.abs(occ_bbox - ori_bbox) <= transform.bbox_occluded_thr) or occ_mask.sum() > transform.mask_occluded_thr) # test copypaste with selected objects transform = CopyPaste() results = copy.deepcopy(dst_results) results['mix_results'] = [copy.deepcopy(src_results)] results = transform(results) # test copypaste with an empty source image results = copy.deepcopy(dst_results) valid_inds = [False] * self.src_results['gt_bboxes'].shape[0] results['mix_results'] = [{ 'img': src_results['img'].copy(), 'gt_bboxes': src_results['gt_bboxes'][valid_inds], 'gt_bboxes_labels': src_results['gt_bboxes_labels'][valid_inds], 'gt_masks': src_results['gt_masks'][valid_inds], 'gt_ignore_flags': src_results['gt_ignore_flags'][valid_inds], }] results = transform(results) def test_repr(self): transform = CopyPaste() self.assertEqual( repr(transform), ('CopyPaste(max_num_pasted=100, ' 'bbox_occluded_thr=10, ' 'mask_occluded_thr=300, ' 'selected=True)')) class TestAlbu(unittest.TestCase): @unittest.skipIf(albumentations is None, 'albumentations is not installed') def test_transform(self): results = dict( img_path=osp.join(osp.dirname(__file__), '../../data/color.jpg')) # Define simple pipeline load = dict(type='LoadImageFromFile') load = TRANSFORMS.build(load) albu_transform = dict( type='Albu', transforms=[dict(type='ChannelShuffle', p=1)]) albu_transform = TRANSFORMS.build(albu_transform) # Execute transforms results = load(results) results = albu_transform(results) self.assertEqual(results['img'].dtype, np.uint8) # test bbox albu_transform = dict( type='Albu', transforms=[dict(type='ChannelShuffle', p=1)], bbox_params=dict( type='BboxParams', format='pascal_voc', label_fields=['gt_bboxes_labels', 'gt_ignore_flags']), keymap={ 'img': 'image', 'gt_bboxes': 'bboxes' }) albu_transform = TRANSFORMS.build(albu_transform) results = { 'img': np.random.random((224, 224, 3)), 'img_shape': (224, 224), 'gt_bboxes_labels': np.array([1, 2, 3], dtype=np.int64), 'gt_bboxes': np.array([[10, 10, 20, 20], [20, 20, 40, 40], [40, 40, 80, 80]], dtype=np.float32), 'gt_ignore_flags': np.array([0, 0, 1], dtype=bool), } results = albu_transform(results) self.assertEqual(results['img'].dtype, np.float64) self.assertEqual(results['gt_bboxes'].dtype, np.float32) self.assertEqual(results['gt_ignore_flags'].dtype, bool) self.assertEqual(results['gt_bboxes_labels'].dtype, np.int64) @unittest.skipIf(albumentations is None, 'albumentations is not installed') def test_repr(self): albu_transform = dict( type='Albu', transforms=[dict(type='ChannelShuffle', p=1)]) albu_transform = TRANSFORMS.build(albu_transform) self.assertEqual( repr(albu_transform), 'Albu(transforms=[' '{\'type\': \'ChannelShuffle\', ' '\'p\': 1}])') class TestCorrupt(unittest.TestCase): def test_transform(self): results = dict( img_path=osp.join(osp.dirname(__file__), '../../data/color.jpg')) # Define simple pipeline load = dict(type='LoadImageFromFile') load = TRANSFORMS.build(load) corrupt_transform = dict(type='Corrupt', corruption='gaussian_blur') corrupt_transform = TRANSFORMS.build(corrupt_transform) # Execute transforms results = load(results) results = corrupt_transform(results) self.assertEqual(results['img'].dtype, np.uint8) def test_repr(self): corrupt_transform = dict(type='Corrupt', corruption='gaussian_blur') corrupt_transform = TRANSFORMS.build(corrupt_transform) self.assertEqual( repr(corrupt_transform), 'Corrupt(corruption=gaussian_blur, ' 'severity=1)') class TestRandomShift(unittest.TestCase): def test_init(self): # test assertion for invalid shift_ratio with self.assertRaises(AssertionError): RandomShift(prob=1.5) # test assertion for invalid max_shift_px with self.assertRaises(AssertionError): RandomShift(max_shift_px=-1) def test_transform(self): results = dict() img = mmcv.imread( osp.join(osp.dirname(__file__), '../../data/color.jpg'), 'color') results['img'] = img h, w, _ = img.shape gt_bboxes = create_random_bboxes(8, w, h) results['gt_bboxes_labels'] = np.ones( gt_bboxes.shape[0], dtype=np.int64) results['gt_bboxes'] = gt_bboxes transform = RandomShift(prob=1.0) results = transform(results) self.assertEqual(results['img'].shape[:2], (h, w)) self.assertEqual(results['gt_bboxes_labels'].shape[0], results['gt_bboxes'].shape[0]) self.assertEqual(results['gt_bboxes_labels'].dtype, np.int64) self.assertEqual(results['gt_bboxes'].dtype, np.float32) def test_transform_use_box_type(self): results = dict() img = mmcv.imread( osp.join(osp.dirname(__file__), '../../data/color.jpg'), 'color') results['img'] = img h, w, _ = img.shape gt_bboxes = create_random_bboxes(8, w, h) results['gt_bboxes_labels'] = np.ones( gt_bboxes.shape[0], dtype=np.int64) results['gt_bboxes'] = HorizontalBoxes(gt_bboxes) transform = RandomShift(prob=1.0) results = transform(results) self.assertEqual(results['img'].shape[:2], (h, w)) self.assertEqual(results['gt_bboxes_labels'].shape[0], results['gt_bboxes'].shape[0]) self.assertEqual(results['gt_bboxes_labels'].dtype, np.int64) self.assertEqual(results['gt_bboxes'].dtype, torch.float32) def test_repr(self): transform = RandomShift() self.assertEqual( repr(transform), ('RandomShift(prob=0.5, ' 'max_shift_px=32, ' 'filter_thr_px=1)')) class TestRandomErasing(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.results = construct_toy_data(poly2mask=True) def test_transform(self): transform = RandomErasing( n_patches=(1, 5), ratio=(0.4, 0.8), img_border_value=0) results = transform(copy.deepcopy(self.results)) self.assertTrue(results['img'].sum() < self.results['img'].sum()) transform = RandomErasing( n_patches=1, ratio=0.999, img_border_value=255) results = transform(copy.deepcopy(self.results)) self.assertTrue(results['img'].sum() > self.results['img'].sum()) # test empty results empty_results = copy.deepcopy(self.results) empty_results['gt_bboxes'] = np.zeros((0, 4), dtype=np.float32) empty_results['gt_bboxes_labels'] = np.zeros((0, ), dtype=np.int64) empty_results['gt_masks'] = empty_results['gt_masks'][False] empty_results['gt_ignore_flags'] = np.zeros((0, ), dtype=bool) empty_results['gt_seg_map'] = np.ones_like( empty_results['gt_seg_map']) * 255 results = transform(copy.deepcopy(empty_results)) self.assertTrue(results['img'].sum() > self.results['img'].sum()) def test_transform_use_box_type(self): src_results = copy.deepcopy(self.results) src_results['gt_bboxes'] = HorizontalBoxes(src_results['gt_bboxes']) transform = RandomErasing( n_patches=(1, 5), ratio=(0.4, 0.8), img_border_value=0) results = transform(copy.deepcopy(src_results)) self.assertTrue(results['img'].sum() < src_results['img'].sum()) transform = RandomErasing( n_patches=1, ratio=0.999, img_border_value=255) results = transform(copy.deepcopy(src_results)) self.assertTrue(results['img'].sum() > src_results['img'].sum()) # test empty results empty_results = copy.deepcopy(src_results) empty_results['gt_bboxes'] = HorizontalBoxes([], dtype=torch.float32) empty_results['gt_bboxes_labels'] = np.zeros((0, ), dtype=np.int64) empty_results['gt_masks'] = empty_results['gt_masks'][False] empty_results['gt_ignore_flags'] = np.zeros((0, ), dtype=bool) empty_results['gt_seg_map'] = np.ones_like( empty_results['gt_seg_map']) * 255 results = transform(copy.deepcopy(empty_results)) self.assertTrue(results['img'].sum() > src_results['img'].sum()) def test_repr(self): transform = RandomErasing(n_patches=(1, 5), ratio=(0, 0.2)) self.assertEqual( repr(transform), ('RandomErasing(n_patches=(1, 5), ' 'ratio=(0, 0.2), ' 'squared=True, ' 'bbox_erased_thr=0.9, ' 'img_border_value=128, ' 'mask_border_value=0, ' 'seg_ignore_label=255)')) ================================================ FILE: tests/test_datasets/test_transforms/test_wrappers.py ================================================ import copy import os.path as osp import unittest from mmcv.transforms import Compose from mmdet.datasets.transforms import MultiBranch, RandomOrder from mmdet.utils import register_all_modules from .utils import construct_toy_data register_all_modules() class TestMultiBranch(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ data_prefix = osp.join(osp.dirname(__file__), '../../data') img_path = osp.join(data_prefix, 'color.jpg') seg_map = osp.join(data_prefix, 'gray.jpg') self.meta_keys = ('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction', 'homography_matrix') self.results = { 'img_path': img_path, 'img_id': 12345, 'img_shape': (300, 400), 'seg_map_path': seg_map, 'instances': [{ 'bbox': [0, 0, 10, 20], 'bbox_label': 1, 'mask': [[0, 0, 0, 20, 10, 20, 10, 0]], 'ignore_flag': 0 }, { 'bbox': [10, 10, 110, 120], 'bbox_label': 2, 'mask': [[10, 10, 110, 10, 110, 120, 110, 10]], 'ignore_flag': 0 }, { 'bbox': [50, 50, 60, 80], 'bbox_label': 2, 'mask': [[50, 50, 60, 50, 60, 80, 50, 80]], 'ignore_flag': 1 }] } self.branch_field = ['sup', 'sup_teacher', 'sup_student'] self.weak_pipeline = [ dict(type='ShearX'), dict(type='PackDetInputs', meta_keys=self.meta_keys) ] self.strong_pipeline = [ dict(type='ShearX'), dict(type='ShearY'), dict(type='PackDetInputs', meta_keys=self.meta_keys) ] self.labeled_pipeline = [ dict(type='LoadImageFromFile'), dict( type='LoadAnnotations', with_bbox=True, with_mask=True, with_seg=True), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='MultiBranch', branch_field=self.branch_field, sup_teacher=self.weak_pipeline, sup_student=self.strong_pipeline), ] self.unlabeled_pipeline = [ dict(type='LoadImageFromFile'), dict(type='Resize', scale=(1333, 800), keep_ratio=True), dict(type='RandomFlip', prob=0.5), dict( type='MultiBranch', branch_field=self.branch_field, unsup_teacher=self.weak_pipeline, unsup_student=self.strong_pipeline), ] def test_transform(self): labeled_pipeline = Compose(self.labeled_pipeline) labeled_results = labeled_pipeline(copy.deepcopy(self.results)) unlabeled_pipeline = Compose(self.unlabeled_pipeline) unlabeled_results = unlabeled_pipeline(copy.deepcopy(self.results)) # test branch sup_teacher and sup_student sup_branches = ['sup_teacher', 'sup_student'] for branch in sup_branches: self.assertIn(branch, labeled_results['data_samples']) self.assertIn('homography_matrix', labeled_results['data_samples'][branch]) self.assertIn('labels', labeled_results['data_samples'][branch].gt_instances) self.assertIn('bboxes', labeled_results['data_samples'][branch].gt_instances) self.assertIn('masks', labeled_results['data_samples'][branch].gt_instances) self.assertIn('gt_sem_seg', labeled_results['data_samples'][branch]) # test branch unsup_teacher and unsup_student unsup_branches = ['unsup_teacher', 'unsup_student'] for branch in unsup_branches: self.assertIn(branch, unlabeled_results['data_samples']) self.assertIn('homography_matrix', unlabeled_results['data_samples'][branch]) self.assertNotIn( 'labels', unlabeled_results['data_samples'][branch].gt_instances) self.assertNotIn( 'bboxes', unlabeled_results['data_samples'][branch].gt_instances) self.assertNotIn( 'masks', unlabeled_results['data_samples'][branch].gt_instances) self.assertNotIn('gt_sem_seg', unlabeled_results['data_samples'][branch]) def test_repr(self): pipeline = [dict(type='PackDetInputs', meta_keys=())] transform = MultiBranch( branch_field=self.branch_field, sup=pipeline, unsup=pipeline) self.assertEqual( repr(transform), ("MultiBranch(branch_pipelines=['sup', 'unsup'])")) class TestRandomOrder(unittest.TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.results = construct_toy_data(poly2mask=True) self.pipeline = [ dict(type='Sharpness'), dict(type='Contrast'), dict(type='Brightness'), dict(type='Rotate'), dict(type='ShearX'), dict(type='TranslateY') ] def test_transform(self): transform = RandomOrder(self.pipeline) results = transform(copy.deepcopy(self.results)) self.assertEqual(results['img_shape'], self.results['img_shape']) self.assertEqual(results['gt_bboxes'].shape, self.results['gt_bboxes'].shape) self.assertEqual(results['gt_bboxes_labels'], self.results['gt_bboxes_labels']) self.assertEqual(results['gt_ignore_flags'], self.results['gt_ignore_flags']) self.assertEqual(results['gt_masks'].masks.shape, self.results['gt_masks'].masks.shape) self.assertEqual(results['gt_seg_map'].shape, self.results['gt_seg_map'].shape) def test_repr(self): transform = RandomOrder(self.pipeline) self.assertEqual( repr(transform), ('RandomOrder(Sharpness, Contrast, ' 'Brightness, Rotate, ShearX, TranslateY, )')) ================================================ FILE: tests/test_datasets/test_transforms/utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np from mmengine.testing import assert_allclose from mmdet.structures.bbox import BaseBoxes, HorizontalBoxes from mmdet.structures.mask import BitmapMasks, PolygonMasks def create_random_bboxes(num_bboxes, img_w, img_h): bboxes_left_top = np.random.uniform(0, 0.5, size=(num_bboxes, 2)) bboxes_right_bottom = np.random.uniform(0.5, 1, size=(num_bboxes, 2)) bboxes = np.concatenate((bboxes_left_top, bboxes_right_bottom), 1) bboxes = (bboxes * np.array([img_w, img_h, img_w, img_h])).astype( np.float32) return bboxes def create_full_masks(gt_bboxes, img_w, img_h): xmin, ymin = gt_bboxes[:, 0:1], gt_bboxes[:, 1:2] xmax, ymax = gt_bboxes[:, 2:3], gt_bboxes[:, 3:4] gt_masks = np.zeros((len(gt_bboxes), img_h, img_w), dtype=np.uint8) for i in range(len(gt_bboxes)): gt_masks[i, int(ymin[i]):int(ymax[i]), int(xmin[i]):int(xmax[i])] = 1 gt_masks = BitmapMasks(gt_masks, img_h, img_w) return gt_masks def construct_toy_data(poly2mask, use_box_type=False): img = np.array([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], dtype=np.uint8) img = np.stack([img, img, img], axis=-1) results = dict() results['img'] = img results['img_shape'] = img.shape[:2] if use_box_type: results['gt_bboxes'] = HorizontalBoxes( np.array([[1, 0, 2, 2]], dtype=np.float32)) else: results['gt_bboxes'] = np.array([[1, 0, 2, 2]], dtype=np.float32) results['gt_bboxes_labels'] = np.array([13], dtype=np.int64) if poly2mask: gt_masks = np.array([[0, 1, 0, 0], [0, 1, 1, 0], [0, 1, 0, 0]], dtype=np.uint8)[None, :, :] results['gt_masks'] = BitmapMasks(gt_masks, 3, 4) else: raw_masks = [[np.array([1, 2, 1, 0, 2, 1], dtype=np.float32)]] results['gt_masks'] = PolygonMasks(raw_masks, 3, 4) results['gt_ignore_flags'] = np.array(np.array([1], dtype=bool)) results['gt_seg_map'] = np.array( [[255, 13, 255, 255], [255, 13, 13, 255], [255, 13, 255, 255]], dtype=np.uint8) return results def check_result_same(results, pipeline_results, check_keys): """Check whether the ``pipeline_results`` is the same with the predefined ``results``. Args: results (dict): Predefined results which should be the standard output of the transform pipeline. pipeline_results (dict): Results processed by the transform pipeline. check_keys (tuple): Keys that need to be checked between results and pipeline_results. """ for key in check_keys: if results.get(key, None) is None: continue if isinstance(results[key], (BitmapMasks, PolygonMasks)): assert_allclose(pipeline_results[key].to_ndarray(), results[key].to_ndarray()) elif isinstance(results[key], BaseBoxes): assert_allclose(pipeline_results[key].tensor, results[key].tensor) else: assert_allclose(pipeline_results[key], results[key]) ================================================ FILE: tests/test_datasets/test_tta.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp from unittest import TestCase import mmcv import pytest from mmdet.datasets.transforms import * # noqa from mmdet.registry import TRANSFORMS class TestMuitiScaleFlipAug(TestCase): def test_exception(self): with pytest.raises(TypeError): tta_transform = dict( type='TestTimeAug', transforms=[dict(type='Resize', keep_ratio=False)], ) TRANSFORMS.build(tta_transform) def test_multi_scale_flip_aug(self): tta_transform = dict( type='TestTimeAug', transforms=[[ dict(type='Resize', scale=scale, keep_ratio=False) for scale in [(256, 256), (512, 512), (1024, 1024)] ], [ dict( type='mmdet.PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ]]) tta_module = TRANSFORMS.build(tta_transform) results = dict() img = mmcv.imread( osp.join(osp.dirname(__file__), '../data/color.jpg'), 'color') results['img_id'] = '1' results['img_path'] = 'data/color.jpg' results['img'] = img results['ori_shape'] = img.shape results['ori_height'] = img.shape[0] results['ori_width'] = img.shape[1] # Set initial values for default meta_keys results['pad_shape'] = img.shape results['scale_factor'] = 1.0 tta_results = tta_module(results.copy()) assert [img.shape for img in tta_results['inputs']] == [(3, 256, 256), (3, 512, 512), (3, 1024, 1024)] tta_transform = dict( type='TestTimeAug', transforms=[ [ dict(type='Resize', scale=scale, keep_ratio=False) for scale in [(256, 256), (512, 512), (1024, 1024)] ], [ dict(type='RandomFlip', prob=0., direction='horizontal'), dict(type='RandomFlip', prob=1., direction='horizontal') ], [ dict( type='mmdet.PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction')) ] ]) tta_module = TRANSFORMS.build(tta_transform) tta_results: dict = tta_module(results.copy()) assert [img.shape for img in tta_results['inputs']] == [(3, 256, 256), (3, 256, 256), (3, 512, 512), (3, 512, 512), (3, 1024, 1024), (3, 1024, 1024)] assert [ data_sample.metainfo['flip'] for data_sample in tta_results['data_samples'] ] == [False, True, False, True, False, True] tta_transform = dict( type='TestTimeAug', transforms=[[ dict(type='Resize', scale=(512, 512), keep_ratio=False) ], [ dict( type='mmdet.PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ]]) tta_module = TRANSFORMS.build(tta_transform) tta_results = tta_module(results.copy()) assert [tta_results['inputs'][0].shape] == [(3, 512, 512)] tta_transform = dict( type='TestTimeAug', transforms=[ [dict(type='Resize', scale=(512, 512), keep_ratio=False)], [ dict(type='RandomFlip', prob=0., direction='horizontal'), dict(type='RandomFlip', prob=1., direction='horizontal') ], [ dict( type='mmdet.PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction')) ] ]) tta_module = TRANSFORMS.build(tta_transform) tta_results = tta_module(results.copy()) assert [img.shape for img in tta_results['inputs']] == [(3, 512, 512), (3, 512, 512)] assert [ data_sample.metainfo['flip'] for data_sample in tta_results['data_samples'] ] == [False, True] tta_transform = dict( type='TestTimeAug', transforms=[[ dict(type='Resize', scale_factor=r, keep_ratio=False) for r in [0.5, 1.0, 2.0] ], [ dict( type='mmdet.PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor')) ]]) tta_module = TRANSFORMS.build(tta_transform) tta_results = tta_module(results.copy()) assert [img.shape for img in tta_results['inputs']] == [(3, 144, 256), (3, 288, 512), (3, 576, 1024)] tta_transform = dict( type='TestTimeAug', transforms=[ [ dict(type='Resize', scale_factor=r, keep_ratio=True) for r in [0.5, 1.0, 2.0] ], [ dict(type='RandomFlip', prob=0., direction='horizontal'), dict(type='RandomFlip', prob=1., direction='horizontal') ], [ dict( type='mmdet.PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction')) ] ]) tta_module = TRANSFORMS.build(tta_transform) tta_results = tta_module(results.copy()) assert [img.shape for img in tta_results['inputs']] == [(3, 144, 256), (3, 144, 256), (3, 288, 512), (3, 288, 512), (3, 576, 1024), (3, 576, 1024)] assert [ data_sample.metainfo['flip'] for data_sample in tta_results['data_samples'] ] == [False, True, False, True, False, True] ================================================ FILE: tests/test_engine/__init__.py ================================================ ================================================ FILE: tests/test_engine/test_hooks/test_checkloss_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase from unittest.mock import Mock import torch from mmdet.engine.hooks import CheckInvalidLossHook class TestCheckInvalidLossHook(TestCase): def test_after_train_iter(self): n = 50 hook = CheckInvalidLossHook(n) runner = Mock() runner.logger = Mock() runner.logger.info = Mock() # Test `after_train_iter` function within the n iteration. runner.iter = 10 outputs = dict(loss=torch.LongTensor([2])) hook.after_train_iter(runner, 10, outputs=outputs) outputs = dict(loss=torch.tensor(float('nan'))) hook.after_train_iter(runner, 10, outputs=outputs) outputs = dict(loss=torch.tensor(float('inf'))) hook.after_train_iter(runner, 10, outputs=outputs) # Test `after_train_iter` at the n iteration. runner.iter = n - 1 outputs = dict(loss=torch.LongTensor([2])) hook.after_train_iter(runner, n - 1, outputs=outputs) outputs = dict(loss=torch.tensor(float('nan'))) with self.assertRaises(AssertionError): hook.after_train_iter(runner, n - 1, outputs=outputs) outputs = dict(loss=torch.tensor(float('inf'))) with self.assertRaises(AssertionError): hook.after_train_iter(runner, n - 1, outputs=outputs) ================================================ FILE: tests/test_engine/test_hooks/test_mean_teacher_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp import tempfile from unittest import TestCase import torch import torch.nn as nn from mmengine.evaluator import BaseMetric from mmengine.model import BaseModel from mmengine.optim import OptimWrapper from mmengine.registry import MODEL_WRAPPERS from mmengine.runner import Runner from torch.utils.data import Dataset from mmdet.registry import DATASETS from mmdet.utils import register_all_modules register_all_modules() class ToyModel(nn.Module): def __init__(self): super().__init__() self.linear = nn.Linear(2, 1) def forward(self, inputs, data_samples, mode='tensor'): labels = torch.stack(data_samples) inputs = torch.stack(inputs) outputs = self.linear(inputs) if mode == 'tensor': return outputs elif mode == 'loss': loss = (labels - outputs).sum() outputs = dict(loss=loss) return outputs else: return outputs class ToyModel1(BaseModel, ToyModel): def __init__(self): super().__init__() def forward(self, *args, **kwargs): return super(BaseModel, self).forward(*args, **kwargs) class ToyModel2(BaseModel): def __init__(self): super().__init__() self.teacher = ToyModel1() self.student = ToyModel1() def forward(self, *args, **kwargs): return self.student(*args, **kwargs) @DATASETS.register_module(force=True) class DummyDataset(Dataset): METAINFO = dict() # type: ignore data = torch.randn(12, 2) label = torch.ones(12) @property def metainfo(self): return self.METAINFO def __len__(self): return self.data.size(0) def __getitem__(self, index): return dict(inputs=self.data[index], data_samples=self.label[index]) class ToyMetric1(BaseMetric): def __init__(self, collect_device='cpu', dummy_metrics=None): super().__init__(collect_device=collect_device) self.dummy_metrics = dummy_metrics def process(self, data_batch, predictions): result = {'acc': 1} self.results.append(result) def compute_metrics(self, results): return dict(acc=1) class TestMeanTeacherHook(TestCase): def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() def tearDown(self): self.temp_dir.cleanup() def test_mean_teacher_hook(self): device = 'cuda:0' if torch.cuda.is_available() else 'cpu' model = ToyModel2().to(device) runner = Runner( model=model, train_dataloader=dict( dataset=DummyDataset(), sampler=dict(type='DefaultSampler', shuffle=True), batch_size=3, num_workers=0), val_dataloader=dict( dataset=DummyDataset(), sampler=dict(type='DefaultSampler', shuffle=False), batch_size=3, num_workers=0), val_evaluator=[ToyMetric1()], work_dir=self.temp_dir.name, default_scope='mmdet', optim_wrapper=OptimWrapper( torch.optim.Adam(ToyModel().parameters())), train_cfg=dict(by_epoch=True, max_epochs=2, val_interval=1), val_cfg=dict(), default_hooks=dict(logger=None), custom_hooks=[dict(type='MeanTeacherHook')], experiment_name='test1') runner.train() self.assertTrue( osp.exists(osp.join(self.temp_dir.name, 'epoch_2.pth'))) # checkpoint = torch.load(osp.join(self.temp_dir.name, 'epoch_2.pth')) # load and testing runner = Runner( model=model, test_dataloader=dict( dataset=DummyDataset(), sampler=dict(type='DefaultSampler', shuffle=True), batch_size=3, num_workers=0), test_evaluator=[ToyMetric1()], test_cfg=dict(), work_dir=self.temp_dir.name, default_scope='mmdet', load_from=osp.join(self.temp_dir.name, 'epoch_2.pth'), default_hooks=dict(logger=None), custom_hooks=[dict(type='MeanTeacherHook')], experiment_name='test2') runner.test() @MODEL_WRAPPERS.register_module() class DummyWrapper(BaseModel): def __init__(self, model): super().__init__() self.module = model def forward(self, *args, **kwargs): return self.module(*args, **kwargs) # with model wrapper runner = Runner( model=DummyWrapper(ToyModel2()), test_dataloader=dict( dataset=DummyDataset(), sampler=dict(type='DefaultSampler', shuffle=True), batch_size=3, num_workers=0), test_evaluator=[ToyMetric1()], test_cfg=dict(), work_dir=self.temp_dir.name, default_scope='mmdet', load_from=osp.join(self.temp_dir.name, 'epoch_2.pth'), default_hooks=dict(logger=None), custom_hooks=[dict(type='MeanTeacherHook')], experiment_name='test3') runner.test() ================================================ FILE: tests/test_engine/test_hooks/test_memory_profiler_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase from unittest.mock import Mock from mmdet.engine.hooks import MemoryProfilerHook class TestMemoryProfilerHook(TestCase): def test_after_train_iter(self): hook = MemoryProfilerHook(2) runner = Mock() runner.logger = Mock() runner.logger.info = Mock() hook.after_train_iter(runner, 0) runner.logger.info.assert_not_called() hook.after_train_iter(runner, 1) runner.logger.info.assert_called_once() def test_after_val_iter(self): hook = MemoryProfilerHook(2) runner = Mock() runner.logger = Mock() runner.logger.info = Mock() hook.after_val_iter(runner, 0) runner.logger.info.assert_not_called() hook.after_val_iter(runner, 1) runner.logger.info.assert_called_once() def test_after_test_iter(self): hook = MemoryProfilerHook(2) runner = Mock() runner.logger = Mock() runner.logger.info = Mock() hook.after_test_iter(runner, 0) runner.logger.info.assert_not_called() hook.after_test_iter(runner, 1) runner.logger.info.assert_called_once() ================================================ FILE: tests/test_engine/test_hooks/test_num_class_check_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from copy import deepcopy from unittest import TestCase from unittest.mock import Mock from mmcv.cnn import VGG from mmengine.dataset import BaseDataset from torch import nn from mmdet.engine.hooks import NumClassCheckHook from mmdet.models.roi_heads.mask_heads import FusedSemanticHead class TestNumClassCheckHook(TestCase): def setUp(self): # Setup NumClassCheckHook hook = NumClassCheckHook() self.hook = hook # Setup runner mock runner = Mock() runner.model = Mock() runner.logger = Mock() runner.logger.warning = Mock() runner.train_dataloader = Mock() runner.val_dataloader = Mock() self.runner = runner # Setup dataset metainfo = dict(classes=None) self.none_classmeta_dataset = BaseDataset( metainfo=metainfo, lazy_init=True) metainfo = dict(classes='class_name') self.str_classmeta_dataset = BaseDataset( metainfo=metainfo, lazy_init=True) metainfo = dict(classes=('bus', 'car')) self.normal_classmeta_dataset = BaseDataset( metainfo=metainfo, lazy_init=True) # Setup valid model valid_model = nn.Module() valid_model.add_module('backbone', VGG(depth=11)) fused_semantic_head = FusedSemanticHead( num_ins=1, fusion_level=0, num_convs=1, in_channels=1, conv_out_channels=1) valid_model.add_module('semantic_head', fused_semantic_head) rpn_head = nn.Module() rpn_head.num_classes = 1 valid_model.add_module('rpn_head', rpn_head) bbox_head = nn.Module() bbox_head.num_classes = 2 valid_model.add_module('bbox_head', bbox_head) self.valid_model = valid_model # Setup invalid model invalid_model = nn.Module() bbox_head = nn.Module() bbox_head.num_classes = 4 invalid_model.add_module('bbox_head', bbox_head) self.invalid_model = invalid_model def test_before_train_epch(self): runner = deepcopy(self.runner) # Test when dataset.metainfo['classes'] is None runner.train_dataloader.dataset = self.none_classmeta_dataset self.hook.before_train_epoch(runner) runner.logger.warning.assert_called_once() # Test when dataset.metainfo['classes'] is a str runner.train_dataloader.dataset = self.str_classmeta_dataset with self.assertRaises(AssertionError): self.hook.before_train_epoch(runner) runner.train_dataloader.dataset = self.normal_classmeta_dataset # Test `num_classes` of model is compatible with dataset runner.model = self.valid_model self.hook.before_train_epoch(runner) # Test `num_classes` of model is not compatible with dataset runner.model = self.invalid_model with self.assertRaises(AssertionError): self.hook.before_train_epoch(runner) def test_before_val_epoch(self): runner = deepcopy(self.runner) # Test when dataset.metainfo['classes'] is None runner.val_dataloader.dataset = self.none_classmeta_dataset self.hook.before_val_epoch(runner) runner.logger.warning.assert_called_once() # Test when dataset.metainfo['classes'] is a str runner.val_dataloader.dataset = self.str_classmeta_dataset with self.assertRaises(AssertionError): self.hook.before_val_epoch(runner) runner.val_dataloader.dataset = self.normal_classmeta_dataset # Test `num_classes` of model is compatible with dataset runner.model = self.valid_model self.hook.before_val_epoch(runner) # Test `num_classes` of model is not compatible with dataset runner.model = self.invalid_model with self.assertRaises(AssertionError): self.hook.before_val_epoch(runner) ================================================ FILE: tests/test_engine/test_hooks/test_sync_norm_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase from unittest.mock import Mock, patch import torch.nn as nn from mmdet.engine.hooks import SyncNormHook class TestSyncNormHook(TestCase): @patch( 'mmdet.engine.hooks.sync_norm_hook.get_dist_info', return_value=(0, 1)) def test_before_val_epoch_non_dist(self, mock): model = nn.Sequential( nn.Conv2d(1, 5, kernel_size=3), nn.BatchNorm2d(5, momentum=0.3), nn.Linear(5, 10)) runner = Mock() runner.model = model hook = SyncNormHook() hook.before_val_epoch(runner) @patch( 'mmdet.engine.hooks.sync_norm_hook.get_dist_info', return_value=(0, 2)) def test_before_val_epoch_dist(self, mock): model = nn.Sequential( nn.Conv2d(1, 5, kernel_size=3), nn.BatchNorm2d(5, momentum=0.3), nn.Linear(5, 10)) runner = Mock() runner.model = model hook = SyncNormHook() hook.before_val_epoch(runner) @patch( 'mmdet.engine.hooks.sync_norm_hook.get_dist_info', return_value=(0, 2)) def test_before_val_epoch_dist_no_norm(self, mock): model = nn.Sequential(nn.Conv2d(1, 5, kernel_size=3), nn.Linear(5, 10)) runner = Mock() runner.model = model hook = SyncNormHook() hook.before_val_epoch(runner) ================================================ FILE: tests/test_engine/test_hooks/test_visualization_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp import shutil import time from unittest import TestCase from unittest.mock import Mock import torch from mmengine.structures import InstanceData from mmdet.engine.hooks import DetVisualizationHook from mmdet.structures import DetDataSample from mmdet.visualization import DetLocalVisualizer def _rand_bboxes(num_boxes, h, w): cx, cy, bw, bh = torch.rand(num_boxes, 4).T tl_x = ((cx * w) - (w * bw / 2)).clamp(0, w) tl_y = ((cy * h) - (h * bh / 2)).clamp(0, h) br_x = ((cx * w) + (w * bw / 2)).clamp(0, w) br_y = ((cy * h) + (h * bh / 2)).clamp(0, h) bboxes = torch.stack([tl_x, tl_y, br_x, br_y], dim=0).T return bboxes class TestVisualizationHook(TestCase): def setUp(self) -> None: DetLocalVisualizer.get_instance('current_visualizer') pred_instances = InstanceData() pred_instances.bboxes = _rand_bboxes(5, 10, 12) pred_instances.labels = torch.randint(0, 2, (5, )) pred_instances.scores = torch.rand((5, )) pred_det_data_sample = DetDataSample() pred_det_data_sample.set_metainfo({ 'img_path': osp.join(osp.dirname(__file__), '../../data/color.jpg') }) pred_det_data_sample.pred_instances = pred_instances self.outputs = [pred_det_data_sample] * 2 def test_after_val_iter(self): runner = Mock() runner.iter = 1 hook = DetVisualizationHook() hook.after_val_iter(runner, 1, {}, self.outputs) def test_after_test_iter(self): runner = Mock() runner.iter = 1 hook = DetVisualizationHook(draw=True) hook.after_test_iter(runner, 1, {}, self.outputs) self.assertEqual(hook._test_index, 2) # test timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime()) test_out_dir = timestamp + '1' runner.work_dir = timestamp runner.timestamp = '1' hook = DetVisualizationHook(draw=False, test_out_dir=test_out_dir) hook.after_test_iter(runner, 1, {}, self.outputs) self.assertTrue(not osp.exists(f'{timestamp}/1/{test_out_dir}')) hook = DetVisualizationHook(draw=True, test_out_dir=test_out_dir) hook.after_test_iter(runner, 1, {}, self.outputs) self.assertTrue(osp.exists(f'{timestamp}/1/{test_out_dir}')) shutil.rmtree(f'{timestamp}') ================================================ FILE: tests/test_engine/test_hooks/test_yolox_mode_switch_hook.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase from unittest.mock import Mock, patch from mmdet.engine.hooks import YOLOXModeSwitchHook class TestYOLOXModeSwitchHook(TestCase): @patch('mmdet.engine.hooks.yolox_mode_switch_hook.is_model_wrapper') def test_is_model_wrapper_and_persistent_workers_on( self, mock_is_model_wrapper): mock_is_model_wrapper.return_value = True runner = Mock() runner.model = Mock() runner.model.module = Mock() runner.model.module.bbox_head.use_l1 = False runner.train_dataloader = Mock() runner.train_dataloader.persistent_workers = True runner.train_dataloader._DataLoader__initialized = True runner.epoch = 284 runner.max_epochs = 300 hook = YOLOXModeSwitchHook(num_last_epochs=15) hook.before_train_epoch(runner) self.assertTrue(hook._restart_dataloader) self.assertTrue(runner.model.module.bbox_head.use_l1) self.assertFalse(runner.train_dataloader._DataLoader__initialized) runner.epoch = 285 hook.before_train_epoch(runner) self.assertTrue(runner.train_dataloader._DataLoader__initialized) def test_not_model_wrapper_and_persistent_workers_off(self): runner = Mock() runner.model = Mock() runner.model.bbox_head.use_l1 = False runner.train_dataloader = Mock() runner.train_dataloader.persistent_workers = False runner.train_dataloader._DataLoader__initialized = True runner.epoch = 284 runner.max_epochs = 300 hook = YOLOXModeSwitchHook(num_last_epochs=15) hook.before_train_epoch(runner) self.assertFalse(hook._restart_dataloader) self.assertTrue(runner.model.bbox_head.use_l1) self.assertTrue(runner.train_dataloader._DataLoader__initialized) runner.epoch = 285 hook.before_train_epoch(runner) self.assertFalse(hook._restart_dataloader) self.assertTrue(runner.train_dataloader._DataLoader__initialized) ================================================ FILE: tests/test_engine/test_optimizers/__init__.py ================================================ ================================================ FILE: tests/test_engine/test_optimizers/test_layer_decay_optimizer_constructor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import torch import torch.nn as nn from mmcv.cnn import ConvModule from mmdet.engine import LearningRateDecayOptimizerConstructor base_lr = 1 decay_rate = 2 base_wd = 0.05 weight_decay = 0.05 expected_stage_wise_lr_wd_convnext = [{ 'weight_decay': 0.0, 'lr_scale': 128 }, { 'weight_decay': 0.0, 'lr_scale': 1 }, { 'weight_decay': 0.05, 'lr_scale': 64 }, { 'weight_decay': 0.0, 'lr_scale': 64 }, { 'weight_decay': 0.05, 'lr_scale': 32 }, { 'weight_decay': 0.0, 'lr_scale': 32 }, { 'weight_decay': 0.05, 'lr_scale': 16 }, { 'weight_decay': 0.0, 'lr_scale': 16 }, { 'weight_decay': 0.05, 'lr_scale': 8 }, { 'weight_decay': 0.0, 'lr_scale': 8 }, { 'weight_decay': 0.05, 'lr_scale': 128 }, { 'weight_decay': 0.05, 'lr_scale': 1 }] expected_layer_wise_lr_wd_convnext = [{ 'weight_decay': 0.0, 'lr_scale': 128 }, { 'weight_decay': 0.0, 'lr_scale': 1 }, { 'weight_decay': 0.05, 'lr_scale': 64 }, { 'weight_decay': 0.0, 'lr_scale': 64 }, { 'weight_decay': 0.05, 'lr_scale': 32 }, { 'weight_decay': 0.0, 'lr_scale': 32 }, { 'weight_decay': 0.05, 'lr_scale': 16 }, { 'weight_decay': 0.0, 'lr_scale': 16 }, { 'weight_decay': 0.05, 'lr_scale': 2 }, { 'weight_decay': 0.0, 'lr_scale': 2 }, { 'weight_decay': 0.05, 'lr_scale': 128 }, { 'weight_decay': 0.05, 'lr_scale': 1 }] class ToyConvNeXt(nn.Module): def __init__(self): super().__init__() self.stages = nn.ModuleList() for i in range(4): stage = nn.Sequential(ConvModule(3, 4, kernel_size=1, bias=True)) self.stages.append(stage) self.norm0 = nn.BatchNorm2d(2) # add some variables to meet unit test coverate rate self.cls_token = nn.Parameter(torch.ones(1)) self.mask_token = nn.Parameter(torch.ones(1)) self.pos_embed = nn.Parameter(torch.ones(1)) self.stem_norm = nn.Parameter(torch.ones(1)) self.downsample_norm0 = nn.BatchNorm2d(2) self.downsample_norm1 = nn.BatchNorm2d(2) self.downsample_norm2 = nn.BatchNorm2d(2) self.lin = nn.Parameter(torch.ones(1)) self.lin.requires_grad = False self.downsample_layers = nn.ModuleList() for _ in range(4): stage = nn.Sequential(nn.Conv2d(3, 4, kernel_size=1, bias=True)) self.downsample_layers.append(stage) class ToyDetector(nn.Module): def __init__(self, backbone): super().__init__() self.backbone = backbone self.head = nn.Conv2d(2, 2, kernel_size=1, groups=2) class PseudoDataParallel(nn.Module): def __init__(self, model): super().__init__() self.module = model def check_optimizer_lr_wd(optimizer, gt_lr_wd): assert isinstance(optimizer, torch.optim.AdamW) assert optimizer.defaults['lr'] == base_lr assert optimizer.defaults['weight_decay'] == base_wd param_groups = optimizer.param_groups print(param_groups) assert len(param_groups) == len(gt_lr_wd) for i, param_dict in enumerate(param_groups): assert param_dict['weight_decay'] == gt_lr_wd[i]['weight_decay'] assert param_dict['lr_scale'] == gt_lr_wd[i]['lr_scale'] assert param_dict['lr_scale'] == param_dict['lr'] def test_learning_rate_decay_optimizer_constructor(): # Test lr wd for ConvNeXT backbone = ToyConvNeXt() model = PseudoDataParallel(ToyDetector(backbone)) optim_wrapper_cfg = dict( type='OptimWrapper', optimizer=dict( type='AdamW', lr=base_lr, betas=(0.9, 0.999), weight_decay=0.05)) # stagewise decay stagewise_paramwise_cfg = dict( decay_rate=decay_rate, decay_type='stage_wise', num_layers=6) optim_constructor = LearningRateDecayOptimizerConstructor( optim_wrapper_cfg, stagewise_paramwise_cfg) optim_wrapper = optim_constructor(model) check_optimizer_lr_wd(optim_wrapper.optimizer, expected_stage_wise_lr_wd_convnext) # layerwise decay layerwise_paramwise_cfg = dict( decay_rate=decay_rate, decay_type='layer_wise', num_layers=6) optim_constructor = LearningRateDecayOptimizerConstructor( optim_wrapper_cfg, layerwise_paramwise_cfg) optim_wrapper = optim_constructor(model) check_optimizer_lr_wd(optim_wrapper.optimizer, expected_layer_wise_lr_wd_convnext) ================================================ FILE: tests/test_engine/test_runner/test_loops.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import tempfile from unittest import TestCase from unittest.mock import Mock import torch import torch.nn as nn from mmengine.evaluator import Evaluator from mmengine.model import BaseModel from mmengine.optim import OptimWrapper from mmengine.runner import Runner from torch.utils.data import Dataset from mmdet.registry import DATASETS from mmdet.utils import register_all_modules register_all_modules() class ToyModel(nn.Module): def __init__(self): super().__init__() self.linear = nn.Linear(2, 1) def forward(self, inputs, data_samples, mode='tensor'): labels = torch.stack(data_samples) inputs = torch.stack(inputs) outputs = self.linear(inputs) if mode == 'tensor': return outputs elif mode == 'loss': loss = (labels - outputs).sum() outputs = dict(loss=loss) return outputs else: return outputs class ToyModel1(BaseModel, ToyModel): def __init__(self): super().__init__() def forward(self, *args, **kwargs): return super(BaseModel, self).forward(*args, **kwargs) class ToyModel2(BaseModel): def __init__(self): super().__init__() self.teacher = ToyModel1() self.student = ToyModel1() self.semi_test_cfg = dict(predict_on='teacher') def forward(self, *args, **kwargs): return self.student(*args, **kwargs) @DATASETS.register_module(force=True) class DummyDataset(Dataset): METAINFO = dict() # type: ignore data = torch.randn(12, 2) label = torch.ones(12) @property def metainfo(self): return self.METAINFO def __len__(self): return self.data.size(0) def __getitem__(self, index): return dict(inputs=self.data[index], data_samples=self.label[index]) class TestTeacherStudentValLoop(TestCase): def setUp(self): self.temp_dir = tempfile.TemporaryDirectory() def tearDown(self): self.temp_dir.cleanup() def test_teacher_student_val_loop(self): device = 'cuda:0' if torch.cuda.is_available() else 'cpu' model = ToyModel2().to(device) evaluator = Mock() evaluator.evaluate = Mock(return_value=dict(acc=0.5)) evaluator.__class__ = Evaluator runner = Runner( model=model, train_dataloader=dict( dataset=dict(type='DummyDataset'), sampler=dict(type='DefaultSampler', shuffle=True), batch_size=3, num_workers=0), val_dataloader=dict( dataset=dict(type='DummyDataset'), sampler=dict(type='DefaultSampler', shuffle=False), batch_size=3, num_workers=0), val_evaluator=evaluator, work_dir=self.temp_dir.name, default_scope='mmdet', optim_wrapper=OptimWrapper( torch.optim.Adam(ToyModel().parameters())), train_cfg=dict(by_epoch=True, max_epochs=2, val_interval=1), val_cfg=dict(type='TeacherStudentValLoop'), default_hooks=dict(logger=dict(type='LoggerHook', interval=1)), experiment_name='test1') runner.train() ================================================ FILE: tests/test_engine/test_schedulers/test_quadratic_warmup.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch import torch.nn.functional as F import torch.optim as optim from mmengine.optim.scheduler import _ParamScheduler from mmengine.testing import assert_allclose from mmdet.engine.schedulers import (QuadraticWarmupLR, QuadraticWarmupMomentum, QuadraticWarmupParamScheduler) class ToyModel(torch.nn.Module): def __init__(self): super().__init__() self.conv1 = torch.nn.Conv2d(1, 1, 1) self.conv2 = torch.nn.Conv2d(1, 1, 1) def forward(self, x): return self.conv2(F.relu(self.conv1(x))) class TestQuadraticWarmupScheduler(TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.model = ToyModel() self.optimizer = optim.SGD( self.model.parameters(), lr=0.05, momentum=0.01, weight_decay=5e-4) def _test_scheduler_value(self, schedulers, targets, epochs=10, param_name='lr'): if isinstance(schedulers, _ParamScheduler): schedulers = [schedulers] for epoch in range(epochs): for param_group, target in zip(self.optimizer.param_groups, targets): print(param_group[param_name]) assert_allclose( target[epoch], param_group[param_name], msg='{} is wrong in epoch {}: expected {}, got {}'.format( param_name, epoch, target[epoch], param_group[param_name]), atol=1e-5, rtol=0) [scheduler.step() for scheduler in schedulers] def test_quadratic_warmup_scheduler(self): with self.assertRaises(ValueError): QuadraticWarmupParamScheduler(self.optimizer, param_name='lr') epochs = 10 iters = 5 warmup_factor = [pow((i + 1) / float(iters), 2) for i in range(iters)] single_targets = [x * 0.05 for x in warmup_factor] + [0.05] * ( epochs - iters) targets = [single_targets, [x * epochs for x in single_targets]] scheduler = QuadraticWarmupParamScheduler( self.optimizer, param_name='lr', end=iters) self._test_scheduler_value(scheduler, targets, epochs) def test_quadratic_warmup_scheduler_convert_iterbased(self): epochs = 10 end = 5 epoch_length = 11 iters = end * epoch_length warmup_factor = [pow((i + 1) / float(iters), 2) for i in range(iters)] single_targets = [x * 0.05 for x in warmup_factor] + [0.05] * ( epochs * epoch_length - iters) targets = [single_targets, [x * epochs for x in single_targets]] scheduler = QuadraticWarmupParamScheduler.build_iter_from_epoch( self.optimizer, param_name='lr', end=end, epoch_length=epoch_length) self._test_scheduler_value(scheduler, targets, epochs * epoch_length) def test_quadratic_warmup_lr(self): epochs = 10 iters = 5 warmup_factor = [pow((i + 1) / float(iters), 2) for i in range(iters)] single_targets = [x * 0.05 for x in warmup_factor] + [0.05] * ( epochs - iters) targets = [single_targets, [x * epochs for x in single_targets]] scheduler = QuadraticWarmupLR(self.optimizer, end=iters) self._test_scheduler_value(scheduler, targets, epochs) def test_quadratic_warmup_momentum(self): epochs = 10 iters = 5 warmup_factor = [pow((i + 1) / float(iters), 2) for i in range(iters)] single_targets = [x * 0.01 for x in warmup_factor] + [0.01] * ( epochs - iters) targets = [single_targets, [x * epochs for x in single_targets]] scheduler = QuadraticWarmupMomentum(self.optimizer, end=iters) self._test_scheduler_value( scheduler, targets, epochs, param_name='momentum') ================================================ FILE: tests/test_evaluation/test_metrics/__init__.py ================================================ ================================================ FILE: tests/test_evaluation/test_metrics/test_cityscapes_metric.py ================================================ import os import os.path as osp import tempfile import unittest import numpy as np import torch from PIL import Image from mmdet.evaluation import CityScapesMetric try: import cityscapesscripts except ImportError: cityscapesscripts = None class TestCityScapesMetric(unittest.TestCase): def setUp(self): self.tmp_dir = tempfile.TemporaryDirectory() def tearDown(self): self.tmp_dir.cleanup() @unittest.skipIf(cityscapesscripts is None, 'cityscapesscripts is not installed.') def test_init(self): # test with outfile_prefix = None with self.assertRaises(AssertionError): CityScapesMetric(outfile_prefix=None) # test with format_only=True, keep_results=False with self.assertRaises(AssertionError): CityScapesMetric( outfile_prefix=self.tmp_dir.name + 'test', format_only=True, keep_results=False) @unittest.skipIf(cityscapesscripts is None, 'cityscapesscripts is not installed.') def test_evaluate(self): dummy_mask1 = np.zeros((1, 20, 20), dtype=np.uint8) dummy_mask1[:, :10, :10] = 1 dummy_mask2 = np.zeros((1, 20, 20), dtype=np.uint8) dummy_mask2[:, :10, :10] = 1 self.outfile_prefix = osp.join(self.tmp_dir.name, 'test') self.seg_prefix = osp.join(self.tmp_dir.name, 'cityscapes/gtFine/val') city = 'lindau' sequenceNb = '000000' frameNb = '000019' img_name1 = f'{city}_{sequenceNb}_{frameNb}_gtFine_instanceIds.png' img_path1 = osp.join(self.seg_prefix, city, img_name1) frameNb = '000020' img_name2 = f'{city}_{sequenceNb}_{frameNb}_gtFine_instanceIds.png' img_path2 = osp.join(self.seg_prefix, city, img_name2) os.makedirs(osp.join(self.seg_prefix, city)) masks1 = np.zeros((20, 20), dtype=np.int32) masks1[:10, :10] = 24 * 1000 Image.fromarray(masks1).save(img_path1) masks2 = np.zeros((20, 20), dtype=np.int32) masks2[:10, :10] = 24 * 1000 + 1 Image.fromarray(masks2).save(img_path2) data_samples = [{ 'img_path': img_path1, 'pred_instances': { 'scores': torch.from_numpy(np.array([1.0])), 'labels': torch.from_numpy(np.array([0])), 'masks': torch.from_numpy(dummy_mask1) } }, { 'img_path': img_path2, 'pred_instances': { 'scores': torch.from_numpy(np.array([0.98])), 'labels': torch.from_numpy(np.array([1])), 'masks': torch.from_numpy(dummy_mask2) } }] target = {'cityscapes/mAP': 0.5, 'cityscapes/AP@50': 0.5} metric = CityScapesMetric( seg_prefix=self.seg_prefix, format_only=False, keep_results=False, outfile_prefix=self.outfile_prefix) metric.dataset_meta = dict( classes=('person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', 'bicycle')) metric.process({}, data_samples) results = metric.evaluate(size=2) self.assertDictEqual(results, target) del metric self.assertTrue(not osp.exists('{self.outfile_prefix}.results')) # test format_only metric = CityScapesMetric( seg_prefix=self.seg_prefix, format_only=True, keep_results=True, outfile_prefix=self.outfile_prefix) metric.dataset_meta = dict( classes=('person', 'rider', 'car', 'truck', 'bus', 'train', 'motorcycle', 'bicycle')) metric.process({}, data_samples) results = metric.evaluate(size=2) self.assertDictEqual(results, dict()) ================================================ FILE: tests/test_evaluation/test_metrics/test_coco_metric.py ================================================ import os.path as osp import tempfile from unittest import TestCase import numpy as np import pycocotools.mask as mask_util import torch from mmengine.fileio import dump from mmdet.evaluation import CocoMetric class TestCocoMetric(TestCase): def _create_dummy_coco_json(self, json_name): dummy_mask = np.zeros((10, 10), order='F', dtype=np.uint8) dummy_mask[:5, :5] = 1 rle_mask = mask_util.encode(dummy_mask) rle_mask['counts'] = rle_mask['counts'].decode('utf-8') image = { 'id': 0, 'width': 640, 'height': 640, 'file_name': 'fake_name.jpg', } annotation_1 = { 'id': 1, 'image_id': 0, 'category_id': 0, 'area': 400, 'bbox': [50, 60, 20, 20], 'iscrowd': 0, 'segmentation': rle_mask, } annotation_2 = { 'id': 2, 'image_id': 0, 'category_id': 0, 'area': 900, 'bbox': [100, 120, 30, 30], 'iscrowd': 0, 'segmentation': rle_mask, } annotation_3 = { 'id': 3, 'image_id': 0, 'category_id': 1, 'area': 1600, 'bbox': [150, 160, 40, 40], 'iscrowd': 0, 'segmentation': rle_mask, } annotation_4 = { 'id': 4, 'image_id': 0, 'category_id': 0, 'area': 10000, 'bbox': [250, 260, 100, 100], 'iscrowd': 0, 'segmentation': rle_mask, } categories = [ { 'id': 0, 'name': 'car', 'supercategory': 'car', }, { 'id': 1, 'name': 'bicycle', 'supercategory': 'bicycle', }, ] fake_json = { 'images': [image], 'annotations': [annotation_1, annotation_2, annotation_3, annotation_4], 'categories': categories } dump(fake_json, json_name) def _create_dummy_results(self): bboxes = np.array([[50, 60, 70, 80], [100, 120, 130, 150], [150, 160, 190, 200], [250, 260, 350, 360]]) scores = np.array([1.0, 0.98, 0.96, 0.95]) labels = np.array([0, 0, 1, 0]) dummy_mask = np.zeros((4, 10, 10), dtype=np.uint8) dummy_mask[:, :5, :5] = 1 return dict( bboxes=torch.from_numpy(bboxes), scores=torch.from_numpy(scores), labels=torch.from_numpy(labels), masks=torch.from_numpy(dummy_mask)) def setUp(self): self.tmp_dir = tempfile.TemporaryDirectory() def tearDown(self): self.tmp_dir.cleanup() def test_init(self): fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_coco_json(fake_json_file) with self.assertRaisesRegex(KeyError, 'metric should be one of'): CocoMetric(ann_file=fake_json_file, metric='unknown') def test_evaluate(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_coco_json(fake_json_file) dummy_pred = self._create_dummy_results() # test single coco dataset evaluation coco_metric = CocoMetric( ann_file=fake_json_file, classwise=False, outfile_prefix=f'{self.tmp_dir.name}/test') coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) coco_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = coco_metric.evaluate(size=1) target = { 'coco/bbox_mAP': 1.0, 'coco/bbox_mAP_50': 1.0, 'coco/bbox_mAP_75': 1.0, 'coco/bbox_mAP_s': 1.0, 'coco/bbox_mAP_m': 1.0, 'coco/bbox_mAP_l': 1.0, } self.assertDictEqual(eval_results, target) self.assertTrue( osp.isfile(osp.join(self.tmp_dir.name, 'test.bbox.json'))) # test box and segm coco dataset evaluation coco_metric = CocoMetric( ann_file=fake_json_file, metric=['bbox', 'segm'], classwise=False, outfile_prefix=f'{self.tmp_dir.name}/test') coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) coco_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = coco_metric.evaluate(size=1) target = { 'coco/bbox_mAP': 1.0, 'coco/bbox_mAP_50': 1.0, 'coco/bbox_mAP_75': 1.0, 'coco/bbox_mAP_s': 1.0, 'coco/bbox_mAP_m': 1.0, 'coco/bbox_mAP_l': 1.0, 'coco/segm_mAP': 1.0, 'coco/segm_mAP_50': 1.0, 'coco/segm_mAP_75': 1.0, 'coco/segm_mAP_s': 1.0, 'coco/segm_mAP_m': 1.0, 'coco/segm_mAP_l': 1.0, } self.assertDictEqual(eval_results, target) self.assertTrue( osp.isfile(osp.join(self.tmp_dir.name, 'test.bbox.json'))) self.assertTrue( osp.isfile(osp.join(self.tmp_dir.name, 'test.segm.json'))) # test invalid custom metric_items with self.assertRaisesRegex(KeyError, 'metric item "invalid" is not supported'): coco_metric = CocoMetric( ann_file=fake_json_file, metric_items=['invalid']) coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) coco_metric.process({}, [ dict( pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640)) ]) coco_metric.evaluate(size=1) # test custom metric_items coco_metric = CocoMetric( ann_file=fake_json_file, metric_items=['mAP_m']) coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) coco_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = coco_metric.evaluate(size=1) target = { 'coco/bbox_mAP_m': 1.0, } self.assertDictEqual(eval_results, target) def test_classwise_evaluate(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_coco_json(fake_json_file) dummy_pred = self._create_dummy_results() # test single coco dataset evaluation coco_metric = CocoMetric( ann_file=fake_json_file, metric='bbox', classwise=True) coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) coco_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = coco_metric.evaluate(size=1) target = { 'coco/bbox_mAP': 1.0, 'coco/bbox_mAP_50': 1.0, 'coco/bbox_mAP_75': 1.0, 'coco/bbox_mAP_s': 1.0, 'coco/bbox_mAP_m': 1.0, 'coco/bbox_mAP_l': 1.0, 'coco/car_precision': 1.0, 'coco/bicycle_precision': 1.0, } self.assertDictEqual(eval_results, target) def test_manually_set_iou_thrs(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_coco_json(fake_json_file) # test single coco dataset evaluation coco_metric = CocoMetric( ann_file=fake_json_file, metric='bbox', iou_thrs=[0.3, 0.6]) coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) self.assertEqual(coco_metric.iou_thrs, [0.3, 0.6]) def test_fast_eval_recall(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_coco_json(fake_json_file) dummy_pred = self._create_dummy_results() # test default proposal nums coco_metric = CocoMetric( ann_file=fake_json_file, metric='proposal_fast') coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) coco_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = coco_metric.evaluate(size=1) target = {'coco/AR@100': 1.0, 'coco/AR@300': 1.0, 'coco/AR@1000': 1.0} self.assertDictEqual(eval_results, target) # test manually set proposal nums coco_metric = CocoMetric( ann_file=fake_json_file, metric='proposal_fast', proposal_nums=(2, 4)) coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) coco_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = coco_metric.evaluate(size=1) target = {'coco/AR@2': 0.5, 'coco/AR@4': 1.0} self.assertDictEqual(eval_results, target) def test_evaluate_proposal(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_coco_json(fake_json_file) dummy_pred = self._create_dummy_results() coco_metric = CocoMetric(ann_file=fake_json_file, metric='proposal') coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) coco_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = coco_metric.evaluate(size=1) print(eval_results) target = { 'coco/AR@100': 1, 'coco/AR@300': 1.0, 'coco/AR@1000': 1.0, 'coco/AR_s@1000': 1.0, 'coco/AR_m@1000': 1.0, 'coco/AR_l@1000': 1.0 } self.assertDictEqual(eval_results, target) def test_empty_results(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_coco_json(fake_json_file) coco_metric = CocoMetric(ann_file=fake_json_file, metric='bbox') coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) bboxes = np.zeros((0, 4)) labels = np.array([]) scores = np.array([]) dummy_mask = np.zeros((0, 10, 10), dtype=np.uint8) empty_pred = dict( bboxes=torch.from_numpy(bboxes), scores=torch.from_numpy(scores), labels=torch.from_numpy(labels), masks=torch.from_numpy(dummy_mask)) coco_metric.process( {}, [dict(pred_instances=empty_pred, img_id=0, ori_shape=(640, 640))]) # coco api Index error will be caught coco_metric.evaluate(size=1) def test_evaluate_without_json(self): dummy_pred = self._create_dummy_results() dummy_mask = np.zeros((10, 10), order='F', dtype=np.uint8) dummy_mask[:5, :5] = 1 rle_mask = mask_util.encode(dummy_mask) rle_mask['counts'] = rle_mask['counts'].decode('utf-8') instances = [{ 'bbox_label': 0, 'bbox': [50, 60, 70, 80], 'ignore_flag': 0, 'mask': rle_mask, }, { 'bbox_label': 0, 'bbox': [100, 120, 130, 150], 'ignore_flag': 0, 'mask': rle_mask, }, { 'bbox_label': 1, 'bbox': [150, 160, 190, 200], 'ignore_flag': 0, 'mask': rle_mask, }, { 'bbox_label': 0, 'bbox': [250, 260, 350, 360], 'ignore_flag': 0, 'mask': rle_mask, }] coco_metric = CocoMetric( ann_file=None, metric=['bbox', 'segm'], classwise=False, outfile_prefix=f'{self.tmp_dir.name}/test') coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) coco_metric.process({}, [ dict( pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640), instances=instances) ]) eval_results = coco_metric.evaluate(size=1) print(eval_results) target = { 'coco/bbox_mAP': 1.0, 'coco/bbox_mAP_50': 1.0, 'coco/bbox_mAP_75': 1.0, 'coco/bbox_mAP_s': 1.0, 'coco/bbox_mAP_m': 1.0, 'coco/bbox_mAP_l': 1.0, 'coco/segm_mAP': 1.0, 'coco/segm_mAP_50': 1.0, 'coco/segm_mAP_75': 1.0, 'coco/segm_mAP_s': 1.0, 'coco/segm_mAP_m': 1.0, 'coco/segm_mAP_l': 1.0, } self.assertDictEqual(eval_results, target) self.assertTrue( osp.isfile(osp.join(self.tmp_dir.name, 'test.bbox.json'))) self.assertTrue( osp.isfile(osp.join(self.tmp_dir.name, 'test.segm.json'))) self.assertTrue( osp.isfile(osp.join(self.tmp_dir.name, 'test.gt.json'))) def test_format_only(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_coco_json(fake_json_file) dummy_pred = self._create_dummy_results() with self.assertRaises(AssertionError): CocoMetric( ann_file=fake_json_file, classwise=False, format_only=True, outfile_prefix=None) coco_metric = CocoMetric( ann_file=fake_json_file, metric='bbox', classwise=False, format_only=True, outfile_prefix=f'{self.tmp_dir.name}/test') coco_metric.dataset_meta = dict(classes=['car', 'bicycle']) coco_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = coco_metric.evaluate(size=1) self.assertDictEqual(eval_results, dict()) self.assertTrue(osp.exists(f'{self.tmp_dir.name}/test.bbox.json')) ================================================ FILE: tests/test_evaluation/test_metrics/test_coco_occluded_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp from tempfile import TemporaryDirectory import mmengine import numpy as np from mmdet.datasets import CocoDataset from mmdet.evaluation import CocoOccludedSeparatedMetric def test_coco_occluded_separated_metric(): ann = [[ 'fake1.jpg', 'person', 8, [219.9, 176.12, 11.14, 34.23], { 'size': [480, 640], 'counts': b'nYW31n>2N2FNbA48Kf=?XBDe=m0OM3M4YOPB8_>L4JXao5' } ]] * 3 dummy_mask = np.zeros((10, 10), dtype=np.uint8) dummy_mask[:5, :5] = 1 rle = { 'size': [480, 640], 'counts': b'nYW31n>2N2FNbA48Kf=?XBDe=m0OM3M4YOPB8_>L4JXao5' } res = [(None, dict( img_id=0, bboxes=np.array([[50, 60, 70, 80]] * 2), masks=[rle] * 2, labels=np.array([0, 1], dtype=np.int64), scores=np.array([0.77, 0.77])))] * 3 tempdir = TemporaryDirectory() ann_path = osp.join(tempdir.name, 'coco_occluded.pkl') mmengine.dump(ann, ann_path) metric = CocoOccludedSeparatedMetric( ann_file='tests/data/coco_sample.json', occluded_ann=ann_path, separated_ann=ann_path, metric=[]) metric.dataset_meta = CocoDataset.METAINFO eval_res = metric.compute_metrics(res) assert isinstance(eval_res, dict) assert eval_res['occluded_recall'] == 100 assert eval_res['separated_recall'] == 100 ================================================ FILE: tests/test_evaluation/test_metrics/test_coco_panoptic_metric.py ================================================ import os import os.path as osp import tempfile import unittest from copy import deepcopy import mmcv import numpy as np import torch from mmengine.fileio import dump from mmdet.evaluation import INSTANCE_OFFSET, CocoPanopticMetric try: import panopticapi except ImportError: panopticapi = None class TestCocoPanopticMetric(unittest.TestCase): def _create_panoptic_gt_annotations(self, ann_file, seg_map_dir): categories = [{ 'id': 0, 'name': 'person', 'supercategory': 'person', 'isthing': 1 }, { 'id': 1, 'name': 'cat', 'supercategory': 'cat', 'isthing': 1 }, { 'id': 2, 'name': 'dog', 'supercategory': 'dog', 'isthing': 1 }, { 'id': 3, 'name': 'wall', 'supercategory': 'wall', 'isthing': 0 }] images = [{ 'id': 0, 'width': 80, 'height': 60, 'file_name': 'fake_name1.jpg', }] annotations = [{ 'segments_info': [{ 'id': 1, 'category_id': 0, 'area': 400, 'bbox': [10, 10, 10, 40], 'iscrowd': 0 }, { 'id': 2, 'category_id': 0, 'area': 400, 'bbox': [30, 10, 10, 40], 'iscrowd': 0 }, { 'id': 3, 'category_id': 2, 'iscrowd': 0, 'bbox': [50, 10, 10, 5], 'area': 50 }, { 'id': 4, 'category_id': 3, 'iscrowd': 0, 'bbox': [0, 0, 80, 60], 'area': 3950 }], 'file_name': 'fake_name1.png', 'image_id': 0 }] gt_json = { 'images': images, 'annotations': annotations, 'categories': categories } # 4 is the id of the background class annotation. gt = np.zeros((60, 80), dtype=np.int64) + 4 gt_bboxes = np.array( [[10, 10, 10, 40], [30, 10, 10, 40], [50, 10, 10, 5]], dtype=np.int64) for i in range(3): x, y, w, h = gt_bboxes[i] gt[y:y + h, x:x + w] = i + 1 # id starts from 1 rgb_gt_seg_map = np.zeros(gt.shape + (3, ), dtype=np.uint8) rgb_gt_seg_map[:, :, 2] = gt // (256 * 256) rgb_gt_seg_map[:, :, 1] = gt % (256 * 256) // 256 rgb_gt_seg_map[:, :, 0] = gt % 256 img_path = osp.join(seg_map_dir, 'fake_name1.png') mmcv.imwrite(rgb_gt_seg_map[:, :, ::-1], img_path) dump(gt_json, ann_file) return gt_json def _create_panoptic_data_samples(self): # predictions # TP for background class, IoU=3576/4324=0.827 # 2 the category id of the background class pred = np.zeros((60, 80), dtype=np.int64) + 2 pred_bboxes = np.array( [ [11, 11, 10, 40], # TP IoU=351/449=0.78 [38, 10, 10, 40], # FP [51, 10, 10, 5] # TP IoU=45/55=0.818 ], dtype=np.int64) pred_labels = np.array([0, 0, 1], dtype=np.int64) for i in range(3): x, y, w, h = pred_bboxes[i] pred[y:y + h, x:x + w] = (i + 1) * INSTANCE_OFFSET + pred_labels[i] data_samples = [{ 'img_id': 0, 'ori_shape': (60, 80), 'img_path': 'xxx/fake_name1.jpg', 'segments_info': [{ 'id': 1, 'category': 0, 'is_thing': 1 }, { 'id': 2, 'category': 0, 'is_thing': 1 }, { 'id': 3, 'category': 1, 'is_thing': 1 }, { 'id': 4, 'category': 2, 'is_thing': 0 }], 'seg_map_path': osp.join(self.gt_seg_dir, 'fake_name1.png'), 'pred_panoptic_seg': { 'sem_seg': torch.from_numpy(pred).unsqueeze(0) }, }] return data_samples def setUp(self): self.tmp_dir = tempfile.TemporaryDirectory() self.gt_json_path = osp.join(self.tmp_dir.name, 'gt.json') self.gt_seg_dir = osp.join(self.tmp_dir.name, 'gt_seg') os.mkdir(self.gt_seg_dir) self._create_panoptic_gt_annotations(self.gt_json_path, self.gt_seg_dir) self.dataset_meta = { 'classes': ('person', 'dog', 'wall'), 'thing_classes': ('person', 'dog'), 'stuff_classes': ('wall', ) } self.target = { 'coco_panoptic/PQ': 67.86874803219071, 'coco_panoptic/SQ': 80.89770126158936, 'coco_panoptic/RQ': 83.33333333333334, 'coco_panoptic/PQ_th': 60.45252075318891, 'coco_panoptic/SQ_th': 79.9959505972869, 'coco_panoptic/RQ_th': 75.0, 'coco_panoptic/PQ_st': 82.70120259019427, 'coco_panoptic/SQ_st': 82.70120259019427, 'coco_panoptic/RQ_st': 100.0 } self.data_samples = self._create_panoptic_data_samples() def tearDown(self): self.tmp_dir.cleanup() @unittest.skipIf(panopticapi is not None, 'panopticapi is installed') def test_init(self): with self.assertRaises(RuntimeError): CocoPanopticMetric() @unittest.skipIf(panopticapi is None, 'panopticapi is not installed') def test_evaluate_without_json(self): # with tmpfile, without json metric = CocoPanopticMetric( ann_file=None, seg_prefix=self.gt_seg_dir, classwise=False, nproc=1, outfile_prefix=None) metric.dataset_meta = self.dataset_meta metric.process({}, deepcopy(self.data_samples)) eval_results = metric.evaluate(size=1) self.assertDictEqual(eval_results, self.target) # without tmpfile and json outfile_prefix = f'{self.tmp_dir.name}/test' metric = CocoPanopticMetric( ann_file=None, seg_prefix=self.gt_seg_dir, classwise=False, nproc=1, outfile_prefix=outfile_prefix) metric.dataset_meta = self.dataset_meta metric.process({}, deepcopy(self.data_samples)) eval_results = metric.evaluate(size=1) self.assertDictEqual(eval_results, self.target) @unittest.skipIf(panopticapi is None, 'panopticapi is not installed') def test_evaluate_with_json(self): # with tmpfile and json metric = CocoPanopticMetric( ann_file=self.gt_json_path, seg_prefix=self.gt_seg_dir, classwise=False, nproc=1, outfile_prefix=None) metric.dataset_meta = self.dataset_meta metric.process({}, deepcopy(self.data_samples)) eval_results = metric.evaluate(size=1) self.assertDictEqual(eval_results, self.target) # classwise metric = CocoPanopticMetric( ann_file=self.gt_json_path, seg_prefix=self.gt_seg_dir, classwise=True, nproc=1, outfile_prefix=None) metric.dataset_meta = self.dataset_meta metric.process({}, deepcopy(self.data_samples)) eval_results = metric.evaluate(size=1) self.assertDictEqual(eval_results, self.target) # without tmpfile, with json outfile_prefix = f'{self.tmp_dir.name}/test1' metric = CocoPanopticMetric( ann_file=self.gt_json_path, seg_prefix=self.gt_seg_dir, classwise=False, nproc=1, outfile_prefix=outfile_prefix) metric.dataset_meta = self.dataset_meta metric.process({}, deepcopy(self.data_samples)) eval_results = metric.evaluate(size=1) self.assertDictEqual(eval_results, self.target) @unittest.skipIf(panopticapi is None, 'panopticapi is not installed') def test_format_only(self): with self.assertRaises(AssertionError): metric = CocoPanopticMetric( ann_file=self.gt_json_path, seg_prefix=self.gt_seg_dir, classwise=False, nproc=1, format_only=True, outfile_prefix=None) outfile_prefix = f'{self.tmp_dir.name}/test' metric = CocoPanopticMetric( ann_file=self.gt_json_path, seg_prefix=self.gt_seg_dir, classwise=False, nproc=1, format_only=True, outfile_prefix=outfile_prefix) metric.dataset_meta = self.dataset_meta metric.process({}, deepcopy(self.data_samples)) eval_results = metric.evaluate(size=1) self.assertDictEqual(eval_results, dict()) self.assertTrue(osp.exists(f'{self.tmp_dir.name}/test.panoptic')) self.assertTrue(osp.exists(f'{self.tmp_dir.name}/test.panoptic.json')) ================================================ FILE: tests/test_evaluation/test_metrics/test_crowdhuman_metric.py ================================================ import os.path as osp import tempfile from unittest import TestCase import numpy as np import torch from mmdet.evaluation import CrowdHumanMetric class TestCrowdHumanMetric(TestCase): def _create_dummy_results(self): bboxes = np.array([[1330, 317, 418, 1338], [792, 24, 723, 2017], [693, 291, 307, 894], [522, 290, 285, 826], [728, 336, 175, 602], [92, 337, 267, 681]]) bboxes[:, 2:4] += bboxes[:, 0:2] scores = np.array([1.0, 1.0, 1.0, 1.0, 1.0, 1.0]) return dict( bboxes=torch.from_numpy(bboxes), scores=torch.from_numpy(scores)) def setUp(self): self.tmp_dir = tempfile.TemporaryDirectory() self.ann_file_path = \ 'tests/data/crowdhuman_dataset/test_annotation_train.odgt', def tearDown(self): self.tmp_dir.cleanup() def test_init(self): with self.assertRaisesRegex(KeyError, 'metric should be one of'): CrowdHumanMetric(ann_file=self.ann_file_path[0], metric='unknown') def test_evaluate(self): # create dummy data dummy_pred = self._create_dummy_results() crowdhuman_metric = CrowdHumanMetric( ann_file=self.ann_file_path[0], outfile_prefix=f'{self.tmp_dir.name}/test') crowdhuman_metric.process({}, [ dict( pred_instances=dummy_pred, img_id='283554,35288000868e92d4', ori_shape=(1640, 1640)) ]) eval_results = crowdhuman_metric.evaluate(size=1) target = { 'crowd_human/mAP': 0.8333, 'crowd_human/mMR': 0.0, 'crowd_human/JI': 1.0 } self.assertDictEqual(eval_results, target) self.assertTrue(osp.isfile(osp.join(self.tmp_dir.name, 'test.json'))) ================================================ FILE: tests/test_evaluation/test_metrics/test_dump_det_results.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp import tempfile from unittest import TestCase import torch from mmengine.fileio import load from torch import Tensor from mmdet.evaluation import DumpDetResults from mmdet.structures.mask import encode_mask_results class TestDumpResults(TestCase): def test_init(self): with self.assertRaisesRegex(ValueError, 'The output file must be a pkl file.'): DumpDetResults(out_file_path='./results.json') def test_process(self): metric = DumpDetResults(out_file_path='./results.pkl') data_samples = [dict(data=(Tensor([1, 2, 3]), Tensor([4, 5, 6])))] metric.process(None, data_samples) self.assertEqual(len(metric.results), 1) self.assertEqual(metric.results[0]['data'][0].device, torch.device('cpu')) metric = DumpDetResults(out_file_path='./results.pkl') masks = torch.zeros(10, 10, 4) data_samples = [ dict(pred_instances=dict(masks=masks), gt_instances=[]) ] metric.process(None, data_samples) self.assertEqual(len(metric.results), 1) self.assertEqual(metric.results[0]['pred_instances']['masks'], encode_mask_results(masks.numpy())) self.assertNotIn('gt_instances', metric.results[0]) def test_compute_metrics(self): temp_dir = tempfile.TemporaryDirectory() path = osp.join(temp_dir.name, 'results.pkl') metric = DumpDetResults(out_file_path=path) data_samples = [dict(data=(Tensor([1, 2, 3]), Tensor([4, 5, 6])))] metric.process(None, data_samples) metric.compute_metrics(metric.results) self.assertTrue(osp.isfile(path)) results = load(path) self.assertEqual(len(results), 1) self.assertEqual(results[0]['data'][0].device, torch.device('cpu')) temp_dir.cleanup() ================================================ FILE: tests/test_evaluation/test_metrics/test_lvis_metric.py ================================================ import os.path as osp import tempfile import unittest import numpy as np import pycocotools.mask as mask_util import torch from mmdet.evaluation.metrics import LVISMetric try: import lvis except ImportError: lvis = None from mmengine.fileio import dump class TestLVISMetric(unittest.TestCase): def _create_dummy_lvis_json(self, json_name): dummy_mask = np.zeros((10, 10), order='F', dtype=np.uint8) dummy_mask[:5, :5] = 1 rle_mask = mask_util.encode(dummy_mask) rle_mask['counts'] = rle_mask['counts'].decode('utf-8') image = { 'id': 0, 'width': 640, 'height': 640, 'neg_category_ids': [], 'not_exhaustive_category_ids': [], 'coco_url': 'http://images.cocodataset.org/val2017/0.jpg', } annotation_1 = { 'id': 1, 'image_id': 0, 'category_id': 1, 'area': 400, 'bbox': [50, 60, 20, 20], 'segmentation': rle_mask, } annotation_2 = { 'id': 2, 'image_id': 0, 'category_id': 1, 'area': 900, 'bbox': [100, 120, 30, 30], 'segmentation': rle_mask, } annotation_3 = { 'id': 3, 'image_id': 0, 'category_id': 2, 'area': 1600, 'bbox': [150, 160, 40, 40], 'segmentation': rle_mask, } annotation_4 = { 'id': 4, 'image_id': 0, 'category_id': 1, 'area': 10000, 'bbox': [250, 260, 100, 100], 'segmentation': rle_mask, } categories = [ { 'id': 1, 'name': 'aerosol_can', 'frequency': 'c', 'image_count': 64 }, { 'id': 2, 'name': 'air_conditioner', 'frequency': 'f', 'image_count': 364 }, ] fake_json = { 'images': [image], 'annotations': [annotation_1, annotation_2, annotation_3, annotation_4], 'categories': categories } dump(fake_json, json_name) def _create_dummy_results(self): bboxes = np.array([[50, 60, 70, 80], [100, 120, 130, 150], [150, 160, 190, 200], [250, 260, 350, 360]]) scores = np.array([1.0, 0.98, 0.96, 0.95]) labels = np.array([0, 0, 1, 0]) dummy_mask = np.zeros((4, 10, 10), dtype=np.uint8) dummy_mask[:, :5, :5] = 1 return dict( bboxes=torch.from_numpy(bboxes), scores=torch.from_numpy(scores), labels=torch.from_numpy(labels), masks=torch.from_numpy(dummy_mask)) def setUp(self): self.tmp_dir = tempfile.TemporaryDirectory() def tearDown(self): self.tmp_dir.cleanup() @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_init(self): fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_lvis_json(fake_json_file) with self.assertRaisesRegex(KeyError, 'metric should be one of'): LVISMetric(ann_file=fake_json_file, metric='unknown') @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_evaluate(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_lvis_json(fake_json_file) dummy_pred = self._create_dummy_results() # test single lvis dataset evaluation lvis_metric = LVISMetric( ann_file=fake_json_file, classwise=False, outfile_prefix=f'{self.tmp_dir.name}/test') lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) lvis_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = { 'lvis/bbox_AP': 1.0, 'lvis/bbox_AP50': 1.0, 'lvis/bbox_AP75': 1.0, 'lvis/bbox_APs': 1.0, 'lvis/bbox_APm': 1.0, 'lvis/bbox_APl': 1.0, 'lvis/bbox_APr': -1.0, 'lvis/bbox_APc': 1.0, 'lvis/bbox_APf': 1.0 } self.assertDictEqual(eval_results, target) self.assertTrue( osp.isfile(osp.join(self.tmp_dir.name, 'test.bbox.json'))) # test box and segm lvis dataset evaluation lvis_metric = LVISMetric( ann_file=fake_json_file, metric=['bbox', 'segm'], classwise=False, outfile_prefix=f'{self.tmp_dir.name}/test') lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) lvis_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = { 'lvis/bbox_AP': 1.0, 'lvis/bbox_AP50': 1.0, 'lvis/bbox_AP75': 1.0, 'lvis/bbox_APs': 1.0, 'lvis/bbox_APm': 1.0, 'lvis/bbox_APl': 1.0, 'lvis/bbox_APr': -1.0, 'lvis/bbox_APc': 1.0, 'lvis/bbox_APf': 1.0, 'lvis/segm_AP': 1.0, 'lvis/segm_AP50': 1.0, 'lvis/segm_AP75': 1.0, 'lvis/segm_APs': 1.0, 'lvis/segm_APm': 1.0, 'lvis/segm_APl': 1.0, 'lvis/segm_APr': -1.0, 'lvis/segm_APc': 1.0, 'lvis/segm_APf': 1.0 } self.assertDictEqual(eval_results, target) self.assertTrue( osp.isfile(osp.join(self.tmp_dir.name, 'test.bbox.json'))) self.assertTrue( osp.isfile(osp.join(self.tmp_dir.name, 'test.segm.json'))) # test invalid custom metric_items with self.assertRaisesRegex( KeyError, "metric should be one of 'bbox', 'segm', 'proposal', " "'proposal_fast', but got invalid."): lvis_metric = LVISMetric( ann_file=fake_json_file, metric=['invalid']) lvis_metric.evaluate(size=1) # test custom metric_items lvis_metric = LVISMetric(ann_file=fake_json_file, metric_items=['APm']) lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) lvis_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = { 'lvis/bbox_APm': 1.0, } self.assertDictEqual(eval_results, target) @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_classwise_evaluate(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_lvis_json(fake_json_file) dummy_pred = self._create_dummy_results() # test single lvis dataset evaluation lvis_metric = LVISMetric( ann_file=fake_json_file, metric='bbox', classwise=True) lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) lvis_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = { 'lvis/bbox_AP': 1.0, 'lvis/bbox_AP50': 1.0, 'lvis/bbox_AP75': 1.0, 'lvis/bbox_APs': 1.0, 'lvis/bbox_APm': 1.0, 'lvis/bbox_APl': 1.0, 'lvis/bbox_APr': -1.0, 'lvis/bbox_APc': 1.0, 'lvis/bbox_APf': 1.0, 'lvis/aerosol_can_precision': 1.0, 'lvis/air_conditioner_precision': 1.0, } self.assertDictEqual(eval_results, target) @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_manually_set_iou_thrs(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_lvis_json(fake_json_file) # test single lvis dataset evaluation lvis_metric = LVISMetric( ann_file=fake_json_file, metric='bbox', iou_thrs=[0.3, 0.6]) lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) self.assertEqual(lvis_metric.iou_thrs, [0.3, 0.6]) @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_fast_eval_recall(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_lvis_json(fake_json_file) dummy_pred = self._create_dummy_results() # test default proposal nums lvis_metric = LVISMetric( ann_file=fake_json_file, metric='proposal_fast') lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) lvis_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = {'lvis/AR@100': 1.0, 'lvis/AR@300': 1.0, 'lvis/AR@1000': 1.0} self.assertDictEqual(eval_results, target) # test manually set proposal nums lvis_metric = LVISMetric( ann_file=fake_json_file, metric='proposal_fast', proposal_nums=(2, 4)) lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) lvis_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = {'lvis/AR@2': 0.5, 'lvis/AR@4': 1.0} self.assertDictEqual(eval_results, target) @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_evaluate_proposal(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_lvis_json(fake_json_file) dummy_pred = self._create_dummy_results() lvis_metric = LVISMetric(ann_file=fake_json_file, metric='proposal') lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) lvis_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) target = { 'lvis/AR@300': 1.0, 'lvis/ARs@300': 1.0, 'lvis/ARm@300': 1.0, 'lvis/ARl@300': 1.0 } self.assertDictEqual(eval_results, target) @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_empty_results(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_lvis_json(fake_json_file) lvis_metric = LVISMetric(ann_file=fake_json_file, metric='bbox') lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) bboxes = np.zeros((0, 4)) labels = np.array([]) scores = np.array([]) dummy_mask = np.zeros((0, 10, 10), dtype=np.uint8) empty_pred = dict( bboxes=torch.from_numpy(bboxes), scores=torch.from_numpy(scores), labels=torch.from_numpy(labels), masks=torch.from_numpy(dummy_mask)) lvis_metric.process( {}, [dict(pred_instances=empty_pred, img_id=0, ori_shape=(640, 640))]) # lvis api Index error will be caught lvis_metric.evaluate(size=1) @unittest.skipIf(lvis is None, 'lvis is not installed.') def test_format_only(self): # create dummy data fake_json_file = osp.join(self.tmp_dir.name, 'fake_data.json') self._create_dummy_lvis_json(fake_json_file) dummy_pred = self._create_dummy_results() with self.assertRaises(AssertionError): LVISMetric( ann_file=fake_json_file, classwise=False, format_only=True, outfile_prefix=None) lvis_metric = LVISMetric( ann_file=fake_json_file, metric='bbox', classwise=False, format_only=True, outfile_prefix=f'{self.tmp_dir.name}/test') lvis_metric.dataset_meta = dict( classes=['aerosol_can', 'air_conditioner']) lvis_metric.process( {}, [dict(pred_instances=dummy_pred, img_id=0, ori_shape=(640, 640))]) eval_results = lvis_metric.evaluate(size=1) self.assertDictEqual(eval_results, dict()) self.assertTrue(osp.exists(f'{self.tmp_dir.name}/test.bbox.json')) ================================================ FILE: tests/test_evaluation/test_metrics/test_openimages_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest import numpy as np import torch from mmdet.datasets import OpenImagesDataset from mmdet.evaluation import OpenImagesMetric from mmdet.utils import register_all_modules class TestOpenImagesMetric(unittest.TestCase): def _create_dummy_results(self): bboxes = np.array([[23.2172, 31.7541, 987.3413, 357.8443], [100, 120, 130, 150], [150, 160, 190, 200], [250, 260, 350, 360]]) scores = np.array([1.0, 0.98, 0.96, 0.95]) labels = np.array([0, 0, 0, 0]) return dict( bboxes=torch.from_numpy(bboxes), scores=torch.from_numpy(scores), labels=torch.from_numpy(labels)) def test_init(self): # test invalid iou_thrs with self.assertRaises(AssertionError): OpenImagesMetric(iou_thrs={'a', 0.5}, ioa_thrs={'b', 0.5}) # test ioa and iou_thrs length not equal with self.assertRaises(AssertionError): OpenImagesMetric(iou_thrs=[0.5, 0.75], ioa_thrs=[0.5]) metric = OpenImagesMetric(iou_thrs=0.6) self.assertEqual(metric.iou_thrs, [0.6]) def test_eval(self): register_all_modules() dataset = OpenImagesDataset( data_root='tests/data/OpenImages/', ann_file='annotations/oidv6-train-annotations-bbox.csv', data_prefix=dict(img='OpenImages/train/'), label_file='annotations/class-descriptions-boxable.csv', hierarchy_file='annotations/bbox_labels_600_hierarchy.json', meta_file='annotations/image-metas.pkl', pipeline=[ dict(type='LoadAnnotations', with_bbox=True), dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'instances')) ]) dataset.full_init() data_sample = dataset[0]['data_samples'].to_dict() data_sample['pred_instances'] = self._create_dummy_results() metric = OpenImagesMetric() metric.dataset_meta = dataset.metainfo metric.process({}, [data_sample]) results = metric.evaluate(size=len(dataset)) targets = {'openimages/AP50': 1.0, 'openimages/mAP': 1.0} self.assertDictEqual(results, targets) # test multi-threshold metric = OpenImagesMetric(iou_thrs=[0.1, 0.5], ioa_thrs=[0.1, 0.5]) metric.dataset_meta = dataset.metainfo metric.process({}, [data_sample]) results = metric.evaluate(size=len(dataset)) targets = { 'openimages/AP10': 1.0, 'openimages/AP50': 1.0, 'openimages/mAP': 1.0 } self.assertDictEqual(results, targets) ================================================ FILE: tests/test_models/test_backbones/__init__.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from .utils import check_norm_state, is_block, is_norm __all__ = ['is_block', 'is_norm', 'check_norm_state'] ================================================ FILE: tests/test_models/test_backbones/test_csp_darknet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from torch.nn.modules.batchnorm import _BatchNorm from mmdet.models.backbones.csp_darknet import CSPDarknet from .utils import check_norm_state, is_norm def test_csp_darknet_backbone(): with pytest.raises(ValueError): # frozen_stages must in range(-1, len(arch_setting) + 1) CSPDarknet(frozen_stages=6) with pytest.raises(AssertionError): # out_indices in range(len(arch_setting) + 1) CSPDarknet(out_indices=[6]) # Test CSPDarknet with first stage frozen frozen_stages = 1 model = CSPDarknet(frozen_stages=frozen_stages) model.train() for mod in model.stem.modules(): for param in mod.parameters(): assert param.requires_grad is False for i in range(1, frozen_stages + 1): layer = getattr(model, f'stage{i}') for mod in layer.modules(): if isinstance(mod, _BatchNorm): assert mod.training is False for param in layer.parameters(): assert param.requires_grad is False # Test CSPDarknet with norm_eval=True model = CSPDarknet(norm_eval=True) model.train() assert check_norm_state(model.modules(), False) # Test CSPDarknet-P5 forward with widen_factor=0.5 model = CSPDarknet(arch='P5', widen_factor=0.25, out_indices=range(0, 5)) model.train() imgs = torch.randn(1, 3, 64, 64) feat = model(imgs) assert len(feat) == 5 assert feat[0].shape == torch.Size((1, 16, 32, 32)) assert feat[1].shape == torch.Size((1, 32, 16, 16)) assert feat[2].shape == torch.Size((1, 64, 8, 8)) assert feat[3].shape == torch.Size((1, 128, 4, 4)) assert feat[4].shape == torch.Size((1, 256, 2, 2)) # Test CSPDarknet-P6 forward with widen_factor=0.5 model = CSPDarknet( arch='P6', widen_factor=0.25, out_indices=range(0, 6), spp_kernal_sizes=(3, 5, 7)) model.train() imgs = torch.randn(1, 3, 128, 128) feat = model(imgs) assert feat[0].shape == torch.Size((1, 16, 64, 64)) assert feat[1].shape == torch.Size((1, 32, 32, 32)) assert feat[2].shape == torch.Size((1, 64, 16, 16)) assert feat[3].shape == torch.Size((1, 128, 8, 8)) assert feat[4].shape == torch.Size((1, 192, 4, 4)) assert feat[5].shape == torch.Size((1, 256, 2, 2)) # Test CSPDarknet forward with dict(type='ReLU') model = CSPDarknet( widen_factor=0.125, act_cfg=dict(type='ReLU'), out_indices=range(0, 5)) model.train() imgs = torch.randn(1, 3, 64, 64) feat = model(imgs) assert len(feat) == 5 assert feat[0].shape == torch.Size((1, 8, 32, 32)) assert feat[1].shape == torch.Size((1, 16, 16, 16)) assert feat[2].shape == torch.Size((1, 32, 8, 8)) assert feat[3].shape == torch.Size((1, 64, 4, 4)) assert feat[4].shape == torch.Size((1, 128, 2, 2)) # Test CSPDarknet with BatchNorm forward model = CSPDarknet(widen_factor=0.125, out_indices=range(0, 5)) for m in model.modules(): if is_norm(m): assert isinstance(m, _BatchNorm) model.train() imgs = torch.randn(1, 3, 64, 64) feat = model(imgs) assert len(feat) == 5 assert feat[0].shape == torch.Size((1, 8, 32, 32)) assert feat[1].shape == torch.Size((1, 16, 16, 16)) assert feat[2].shape == torch.Size((1, 32, 8, 8)) assert feat[3].shape == torch.Size((1, 64, 4, 4)) assert feat[4].shape == torch.Size((1, 128, 2, 2)) # Test CSPDarknet with custom arch forward arch_ovewrite = [[32, 56, 3, True, False], [56, 224, 2, True, False], [224, 512, 1, True, False]] model = CSPDarknet( arch_ovewrite=arch_ovewrite, widen_factor=0.25, out_indices=(0, 1, 2, 3)) model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size((1, 8, 16, 16)) assert feat[1].shape == torch.Size((1, 14, 8, 8)) assert feat[2].shape == torch.Size((1, 56, 4, 4)) assert feat[3].shape == torch.Size((1, 128, 2, 2)) ================================================ FILE: tests/test_models/test_backbones/test_detectors_resnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest from mmdet.models.backbones import DetectoRS_ResNet def test_detectorrs_resnet_backbone(): detectorrs_cfg = dict( depth=50, num_stages=4, out_indices=(0, 1, 2, 3), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch', conv_cfg=dict(type='ConvAWS'), sac=dict(type='SAC', use_deform=True), stage_with_sac=(False, True, True, True), output_img=True) """Test init_weights config""" with pytest.raises(AssertionError): # pretrained and init_cfg cannot be specified at the same time DetectoRS_ResNet( **detectorrs_cfg, pretrained='Pretrained', init_cfg='Pretrained') with pytest.raises(AssertionError): # init_cfg must be a dict DetectoRS_ResNet( **detectorrs_cfg, pretrained=None, init_cfg=['Pretrained']) with pytest.raises(KeyError): # init_cfg must contain the key `type` DetectoRS_ResNet( **detectorrs_cfg, pretrained=None, init_cfg=dict(checkpoint='Pretrained')) with pytest.raises(AssertionError): # init_cfg only support initialize pretrained model way DetectoRS_ResNet( **detectorrs_cfg, pretrained=None, init_cfg=dict(type='Trained')) with pytest.raises(TypeError): # pretrained mast be a str or None model = DetectoRS_ResNet( **detectorrs_cfg, pretrained=['Pretrained'], init_cfg=None) model.init_weights() ================================================ FILE: tests/test_models/test_backbones/test_efficientnet.py ================================================ import pytest import torch from mmdet.models.backbones import EfficientNet def test_efficientnet_backbone(): """Test EfficientNet backbone.""" with pytest.raises(AssertionError): # EfficientNet arch should be a key in EfficientNet.arch_settings EfficientNet(arch='c3') model = EfficientNet(arch='b0', out_indices=(0, 1, 2, 3, 4, 5, 6)) model.train() imgs = torch.randn(2, 3, 32, 32) feat = model(imgs) assert len(feat) == 7 assert feat[0].shape == torch.Size([2, 32, 16, 16]) assert feat[1].shape == torch.Size([2, 16, 16, 16]) assert feat[2].shape == torch.Size([2, 24, 8, 8]) assert feat[3].shape == torch.Size([2, 40, 4, 4]) assert feat[4].shape == torch.Size([2, 112, 2, 2]) assert feat[5].shape == torch.Size([2, 320, 1, 1]) assert feat[6].shape == torch.Size([2, 1280, 1, 1]) ================================================ FILE: tests/test_models/test_backbones/test_hourglass.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmdet.models.backbones.hourglass import HourglassNet def test_hourglass_backbone(): with pytest.raises(AssertionError): # HourglassNet's num_stacks should larger than 0 HourglassNet(num_stacks=0) with pytest.raises(AssertionError): # len(stage_channels) should equal len(stage_blocks) HourglassNet( stage_channels=[256, 256, 384, 384, 384], stage_blocks=[2, 2, 2, 2, 2, 4]) with pytest.raises(AssertionError): # len(stage_channels) should lagrer than downsample_times HourglassNet( downsample_times=5, stage_channels=[256, 256, 384, 384, 384], stage_blocks=[2, 2, 2, 2, 2]) # Test HourglassNet-52 model = HourglassNet( num_stacks=1, stage_channels=(64, 64, 96, 96, 96, 128), feat_channel=64) model.train() imgs = torch.randn(1, 3, 256, 256) feat = model(imgs) assert len(feat) == 1 assert feat[0].shape == torch.Size([1, 64, 64, 64]) # Test HourglassNet-104 model = HourglassNet( num_stacks=2, stage_channels=(64, 64, 96, 96, 96, 128), feat_channel=64) model.train() imgs = torch.randn(1, 3, 256, 256) feat = model(imgs) assert len(feat) == 2 assert feat[0].shape == torch.Size([1, 64, 64, 64]) assert feat[1].shape == torch.Size([1, 64, 64, 64]) ================================================ FILE: tests/test_models/test_backbones/test_hrnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmdet.models.backbones.hrnet import HRModule, HRNet from mmdet.models.backbones.resnet import BasicBlock, Bottleneck @pytest.mark.parametrize('block', [BasicBlock, Bottleneck]) def test_hrmodule(block): # Test multiscale forward num_channles = (32, 64) in_channels = [c * block.expansion for c in num_channles] hrmodule = HRModule( num_branches=2, blocks=block, in_channels=in_channels, num_blocks=(4, 4), num_channels=num_channles, ) feats = [ torch.randn(1, in_channels[0], 64, 64), torch.randn(1, in_channels[1], 32, 32) ] feats = hrmodule(feats) assert len(feats) == 2 assert feats[0].shape == torch.Size([1, in_channels[0], 64, 64]) assert feats[1].shape == torch.Size([1, in_channels[1], 32, 32]) # Test single scale forward num_channles = (32, 64) in_channels = [c * block.expansion for c in num_channles] hrmodule = HRModule( num_branches=2, blocks=block, in_channels=in_channels, num_blocks=(4, 4), num_channels=num_channles, multiscale_output=False, ) feats = [ torch.randn(1, in_channels[0], 64, 64), torch.randn(1, in_channels[1], 32, 32) ] feats = hrmodule(feats) assert len(feats) == 1 assert feats[0].shape == torch.Size([1, in_channels[0], 64, 64]) def test_hrnet_backbone(): # only have 3 stages extra = dict( stage1=dict( num_modules=1, num_branches=1, block='BOTTLENECK', num_blocks=(4, ), num_channels=(64, )), stage2=dict( num_modules=1, num_branches=2, block='BASIC', num_blocks=(4, 4), num_channels=(32, 64)), stage3=dict( num_modules=4, num_branches=3, block='BASIC', num_blocks=(4, 4, 4), num_channels=(32, 64, 128))) with pytest.raises(AssertionError): # HRNet now only support 4 stages HRNet(extra=extra) extra['stage4'] = dict( num_modules=3, num_branches=3, # should be 4 block='BASIC', num_blocks=(4, 4, 4, 4), num_channels=(32, 64, 128, 256)) with pytest.raises(AssertionError): # len(num_blocks) should equal num_branches HRNet(extra=extra) extra['stage4']['num_branches'] = 4 # Test hrnetv2p_w32 model = HRNet(extra=extra) model.init_weights() model.train() imgs = torch.randn(1, 3, 256, 256) feats = model(imgs) assert len(feats) == 4 assert feats[0].shape == torch.Size([1, 32, 64, 64]) assert feats[3].shape == torch.Size([1, 256, 8, 8]) # Test single scale output model = HRNet(extra=extra, multiscale_output=False) model.init_weights() model.train() imgs = torch.randn(1, 3, 256, 256) feats = model(imgs) assert len(feats) == 1 assert feats[0].shape == torch.Size([1, 32, 64, 64]) ================================================ FILE: tests/test_models/test_backbones/test_mobilenet_v2.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from torch.nn.modules import GroupNorm from torch.nn.modules.batchnorm import _BatchNorm from mmdet.models.backbones.mobilenet_v2 import MobileNetV2 from .utils import check_norm_state, is_block, is_norm def test_mobilenetv2_backbone(): with pytest.raises(ValueError): # frozen_stages must in range(-1, 8) MobileNetV2(frozen_stages=8) with pytest.raises(ValueError): # out_indices in range(-1, 8) MobileNetV2(out_indices=[8]) # Test MobileNetV2 with first stage frozen frozen_stages = 1 model = MobileNetV2(frozen_stages=frozen_stages) model.train() for mod in model.conv1.modules(): for param in mod.parameters(): assert param.requires_grad is False for i in range(1, frozen_stages + 1): layer = getattr(model, f'layer{i}') for mod in layer.modules(): if isinstance(mod, _BatchNorm): assert mod.training is False for param in layer.parameters(): assert param.requires_grad is False # Test MobileNetV2 with norm_eval=True model = MobileNetV2(norm_eval=True) model.train() assert check_norm_state(model.modules(), False) # Test MobileNetV2 forward with widen_factor=1.0 model = MobileNetV2(widen_factor=1.0, out_indices=range(0, 8)) model.train() assert check_norm_state(model.modules(), True) imgs = torch.randn(1, 3, 224, 224) feat = model(imgs) assert len(feat) == 8 assert feat[0].shape == torch.Size((1, 16, 112, 112)) assert feat[1].shape == torch.Size((1, 24, 56, 56)) assert feat[2].shape == torch.Size((1, 32, 28, 28)) assert feat[3].shape == torch.Size((1, 64, 14, 14)) assert feat[4].shape == torch.Size((1, 96, 14, 14)) assert feat[5].shape == torch.Size((1, 160, 7, 7)) assert feat[6].shape == torch.Size((1, 320, 7, 7)) assert feat[7].shape == torch.Size((1, 1280, 7, 7)) # Test MobileNetV2 forward with widen_factor=0.5 model = MobileNetV2(widen_factor=0.5, out_indices=range(0, 7)) model.train() imgs = torch.randn(1, 3, 224, 224) feat = model(imgs) assert len(feat) == 7 assert feat[0].shape == torch.Size((1, 8, 112, 112)) assert feat[1].shape == torch.Size((1, 16, 56, 56)) assert feat[2].shape == torch.Size((1, 16, 28, 28)) assert feat[3].shape == torch.Size((1, 32, 14, 14)) assert feat[4].shape == torch.Size((1, 48, 14, 14)) assert feat[5].shape == torch.Size((1, 80, 7, 7)) assert feat[6].shape == torch.Size((1, 160, 7, 7)) # Test MobileNetV2 forward with widen_factor=2.0 model = MobileNetV2(widen_factor=2.0, out_indices=range(0, 8)) model.train() imgs = torch.randn(1, 3, 224, 224) feat = model(imgs) assert feat[0].shape == torch.Size((1, 32, 112, 112)) assert feat[1].shape == torch.Size((1, 48, 56, 56)) assert feat[2].shape == torch.Size((1, 64, 28, 28)) assert feat[3].shape == torch.Size((1, 128, 14, 14)) assert feat[4].shape == torch.Size((1, 192, 14, 14)) assert feat[5].shape == torch.Size((1, 320, 7, 7)) assert feat[6].shape == torch.Size((1, 640, 7, 7)) assert feat[7].shape == torch.Size((1, 2560, 7, 7)) # Test MobileNetV2 forward with dict(type='ReLU') model = MobileNetV2( widen_factor=1.0, act_cfg=dict(type='ReLU'), out_indices=range(0, 7)) model.train() imgs = torch.randn(1, 3, 224, 224) feat = model(imgs) assert len(feat) == 7 assert feat[0].shape == torch.Size((1, 16, 112, 112)) assert feat[1].shape == torch.Size((1, 24, 56, 56)) assert feat[2].shape == torch.Size((1, 32, 28, 28)) assert feat[3].shape == torch.Size((1, 64, 14, 14)) assert feat[4].shape == torch.Size((1, 96, 14, 14)) assert feat[5].shape == torch.Size((1, 160, 7, 7)) assert feat[6].shape == torch.Size((1, 320, 7, 7)) # Test MobileNetV2 with BatchNorm forward model = MobileNetV2(widen_factor=1.0, out_indices=range(0, 7)) for m in model.modules(): if is_norm(m): assert isinstance(m, _BatchNorm) model.train() imgs = torch.randn(1, 3, 224, 224) feat = model(imgs) assert len(feat) == 7 assert feat[0].shape == torch.Size((1, 16, 112, 112)) assert feat[1].shape == torch.Size((1, 24, 56, 56)) assert feat[2].shape == torch.Size((1, 32, 28, 28)) assert feat[3].shape == torch.Size((1, 64, 14, 14)) assert feat[4].shape == torch.Size((1, 96, 14, 14)) assert feat[5].shape == torch.Size((1, 160, 7, 7)) assert feat[6].shape == torch.Size((1, 320, 7, 7)) # Test MobileNetV2 with GroupNorm forward model = MobileNetV2( widen_factor=1.0, norm_cfg=dict(type='GN', num_groups=2, requires_grad=True), out_indices=range(0, 7)) for m in model.modules(): if is_norm(m): assert isinstance(m, GroupNorm) model.train() imgs = torch.randn(1, 3, 224, 224) feat = model(imgs) assert len(feat) == 7 assert feat[0].shape == torch.Size((1, 16, 112, 112)) assert feat[1].shape == torch.Size((1, 24, 56, 56)) assert feat[2].shape == torch.Size((1, 32, 28, 28)) assert feat[3].shape == torch.Size((1, 64, 14, 14)) assert feat[4].shape == torch.Size((1, 96, 14, 14)) assert feat[5].shape == torch.Size((1, 160, 7, 7)) assert feat[6].shape == torch.Size((1, 320, 7, 7)) # Test MobileNetV2 with layers 1, 3, 5 out forward model = MobileNetV2(widen_factor=1.0, out_indices=(0, 2, 4)) model.train() imgs = torch.randn(1, 3, 224, 224) feat = model(imgs) assert len(feat) == 3 assert feat[0].shape == torch.Size((1, 16, 112, 112)) assert feat[1].shape == torch.Size((1, 32, 28, 28)) assert feat[2].shape == torch.Size((1, 96, 14, 14)) # Test MobileNetV2 with checkpoint forward model = MobileNetV2( widen_factor=1.0, with_cp=True, out_indices=range(0, 7)) for m in model.modules(): if is_block(m): assert m.with_cp model.train() imgs = torch.randn(1, 3, 224, 224) feat = model(imgs) assert len(feat) == 7 assert feat[0].shape == torch.Size((1, 16, 112, 112)) assert feat[1].shape == torch.Size((1, 24, 56, 56)) assert feat[2].shape == torch.Size((1, 32, 28, 28)) assert feat[3].shape == torch.Size((1, 64, 14, 14)) assert feat[4].shape == torch.Size((1, 96, 14, 14)) assert feat[5].shape == torch.Size((1, 160, 7, 7)) assert feat[6].shape == torch.Size((1, 320, 7, 7)) ================================================ FILE: tests/test_models/test_backbones/test_pvt.py ================================================ import pytest import torch from mmdet.models.backbones.pvt import (PVTEncoderLayer, PyramidVisionTransformer, PyramidVisionTransformerV2) def test_pvt_block(): # test PVT structure and forward block = PVTEncoderLayer( embed_dims=64, num_heads=4, feedforward_channels=256) assert block.ffn.embed_dims == 64 assert block.attn.num_heads == 4 assert block.ffn.feedforward_channels == 256 x = torch.randn(1, 56 * 56, 64) x_out = block(x, (56, 56)) assert x_out.shape == torch.Size([1, 56 * 56, 64]) def test_pvt(): """Test PVT backbone.""" with pytest.raises(TypeError): # Pretrained arg must be str or None. PyramidVisionTransformer(pretrained=123) # test pretrained image size with pytest.raises(AssertionError): PyramidVisionTransformer(pretrain_img_size=(224, 224, 224)) # Test absolute position embedding temp = torch.randn((1, 3, 224, 224)) model = PyramidVisionTransformer( pretrain_img_size=224, use_abs_pos_embed=True) model.init_weights() model(temp) # Test normal inference temp = torch.randn((1, 3, 32, 32)) model = PyramidVisionTransformer() outs = model(temp) assert outs[0].shape == (1, 64, 8, 8) assert outs[1].shape == (1, 128, 4, 4) assert outs[2].shape == (1, 320, 2, 2) assert outs[3].shape == (1, 512, 1, 1) # Test abnormal inference size temp = torch.randn((1, 3, 33, 33)) model = PyramidVisionTransformer() outs = model(temp) assert outs[0].shape == (1, 64, 8, 8) assert outs[1].shape == (1, 128, 4, 4) assert outs[2].shape == (1, 320, 2, 2) assert outs[3].shape == (1, 512, 1, 1) # Test abnormal inference size temp = torch.randn((1, 3, 112, 137)) model = PyramidVisionTransformer() outs = model(temp) assert outs[0].shape == (1, 64, 28, 34) assert outs[1].shape == (1, 128, 14, 17) assert outs[2].shape == (1, 320, 7, 8) assert outs[3].shape == (1, 512, 3, 4) def test_pvtv2(): """Test PVTv2 backbone.""" with pytest.raises(TypeError): # Pretrained arg must be str or None. PyramidVisionTransformerV2(pretrained=123) # test pretrained image size with pytest.raises(AssertionError): PyramidVisionTransformerV2(pretrain_img_size=(224, 224, 224)) # Test normal inference temp = torch.randn((1, 3, 32, 32)) model = PyramidVisionTransformerV2() outs = model(temp) assert outs[0].shape == (1, 64, 8, 8) assert outs[1].shape == (1, 128, 4, 4) assert outs[2].shape == (1, 320, 2, 2) assert outs[3].shape == (1, 512, 1, 1) # Test abnormal inference size temp = torch.randn((1, 3, 31, 31)) model = PyramidVisionTransformerV2() outs = model(temp) assert outs[0].shape == (1, 64, 8, 8) assert outs[1].shape == (1, 128, 4, 4) assert outs[2].shape == (1, 320, 2, 2) assert outs[3].shape == (1, 512, 1, 1) # Test abnormal inference size temp = torch.randn((1, 3, 112, 137)) model = PyramidVisionTransformerV2() outs = model(temp) assert outs[0].shape == (1, 64, 28, 35) assert outs[1].shape == (1, 128, 14, 18) assert outs[2].shape == (1, 320, 7, 9) assert outs[3].shape == (1, 512, 4, 5) ================================================ FILE: tests/test_models/test_backbones/test_regnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmdet.models.backbones import RegNet regnet_test_data = [ ('regnetx_400mf', dict(w0=24, wa=24.48, wm=2.54, group_w=16, depth=22, bot_mul=1.0), [32, 64, 160, 384]), ('regnetx_800mf', dict(w0=56, wa=35.73, wm=2.28, group_w=16, depth=16, bot_mul=1.0), [64, 128, 288, 672]), ('regnetx_1.6gf', dict(w0=80, wa=34.01, wm=2.25, group_w=24, depth=18, bot_mul=1.0), [72, 168, 408, 912]), ('regnetx_3.2gf', dict(w0=88, wa=26.31, wm=2.25, group_w=48, depth=25, bot_mul=1.0), [96, 192, 432, 1008]), ('regnetx_4.0gf', dict(w0=96, wa=38.65, wm=2.43, group_w=40, depth=23, bot_mul=1.0), [80, 240, 560, 1360]), ('regnetx_6.4gf', dict(w0=184, wa=60.83, wm=2.07, group_w=56, depth=17, bot_mul=1.0), [168, 392, 784, 1624]), ('regnetx_8.0gf', dict(w0=80, wa=49.56, wm=2.88, group_w=120, depth=23, bot_mul=1.0), [80, 240, 720, 1920]), ('regnetx_12gf', dict(w0=168, wa=73.36, wm=2.37, group_w=112, depth=19, bot_mul=1.0), [224, 448, 896, 2240]), ] @pytest.mark.parametrize('arch_name,arch,out_channels', regnet_test_data) def test_regnet_backbone(arch_name, arch, out_channels): with pytest.raises(AssertionError): # ResNeXt depth should be in [50, 101, 152] RegNet(arch_name + '233') # Test RegNet with arch_name model = RegNet(arch_name) model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, out_channels[0], 8, 8]) assert feat[1].shape == torch.Size([1, out_channels[1], 4, 4]) assert feat[2].shape == torch.Size([1, out_channels[2], 2, 2]) assert feat[3].shape == torch.Size([1, out_channels[3], 1, 1]) # Test RegNet with arch model = RegNet(arch) assert feat[0].shape == torch.Size([1, out_channels[0], 8, 8]) assert feat[1].shape == torch.Size([1, out_channels[1], 4, 4]) assert feat[2].shape == torch.Size([1, out_channels[2], 2, 2]) assert feat[3].shape == torch.Size([1, out_channels[3], 1, 1]) ================================================ FILE: tests/test_models/test_backbones/test_renext.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmdet.models.backbones import ResNeXt from mmdet.models.backbones.resnext import Bottleneck as BottleneckX from .utils import is_block def test_renext_bottleneck(): with pytest.raises(AssertionError): # Style must be in ['pytorch', 'caffe'] BottleneckX(64, 64, groups=32, base_width=4, style='tensorflow') # Test ResNeXt Bottleneck structure block = BottleneckX( 64, 64, groups=32, base_width=4, stride=2, style='pytorch') assert block.conv2.stride == (2, 2) assert block.conv2.groups == 32 assert block.conv2.out_channels == 128 # Test ResNeXt Bottleneck with DCN dcn = dict(type='DCN', deform_groups=1, fallback_on_stride=False) with pytest.raises(AssertionError): # conv_cfg must be None if dcn is not None BottleneckX( 64, 64, groups=32, base_width=4, dcn=dcn, conv_cfg=dict(type='Conv')) BottleneckX(64, 64, dcn=dcn) # Test ResNeXt Bottleneck forward block = BottleneckX(64, 16, groups=32, base_width=4) x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) # Test ResNeXt Bottleneck forward with plugins plugins = [ dict( cfg=dict( type='GeneralizedAttention', spatial_range=-1, num_heads=8, attention_type='0010', kv_stride=2), stages=(False, False, True, True), position='after_conv2') ] block = BottleneckX(64, 16, groups=32, base_width=4, plugins=plugins) x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) def test_resnext_backbone(): with pytest.raises(KeyError): # ResNeXt depth should be in [50, 101, 152] ResNeXt(depth=18) # Test ResNeXt with group 32, base_width 4 model = ResNeXt(depth=50, groups=32, base_width=4) for m in model.modules(): if is_block(m): assert m.conv2.groups == 32 model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, 256, 8, 8]) assert feat[1].shape == torch.Size([1, 512, 4, 4]) assert feat[2].shape == torch.Size([1, 1024, 2, 2]) assert feat[3].shape == torch.Size([1, 2048, 1, 1]) regnet_test_data = [ ('regnetx_400mf', dict(w0=24, wa=24.48, wm=2.54, group_w=16, depth=22, bot_mul=1.0), [32, 64, 160, 384]), ('regnetx_800mf', dict(w0=56, wa=35.73, wm=2.28, group_w=16, depth=16, bot_mul=1.0), [64, 128, 288, 672]), ('regnetx_1.6gf', dict(w0=80, wa=34.01, wm=2.25, group_w=24, depth=18, bot_mul=1.0), [72, 168, 408, 912]), ('regnetx_3.2gf', dict(w0=88, wa=26.31, wm=2.25, group_w=48, depth=25, bot_mul=1.0), [96, 192, 432, 1008]), ('regnetx_4.0gf', dict(w0=96, wa=38.65, wm=2.43, group_w=40, depth=23, bot_mul=1.0), [80, 240, 560, 1360]), ('regnetx_6.4gf', dict(w0=184, wa=60.83, wm=2.07, group_w=56, depth=17, bot_mul=1.0), [168, 392, 784, 1624]), ('regnetx_8.0gf', dict(w0=80, wa=49.56, wm=2.88, group_w=120, depth=23, bot_mul=1.0), [80, 240, 720, 1920]), ('regnetx_12gf', dict(w0=168, wa=73.36, wm=2.37, group_w=112, depth=19, bot_mul=1.0), [224, 448, 896, 2240]), ] ================================================ FILE: tests/test_models/test_backbones/test_res2net.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmdet.models.backbones import Res2Net from mmdet.models.backbones.res2net import Bottle2neck from .utils import is_block def test_res2net_bottle2neck(): with pytest.raises(AssertionError): # Style must be in ['pytorch', 'caffe'] Bottle2neck(64, 64, base_width=26, scales=4, style='tensorflow') with pytest.raises(AssertionError): # Scale must be larger than 1 Bottle2neck(64, 64, base_width=26, scales=1, style='pytorch') # Test Res2Net Bottle2neck structure block = Bottle2neck( 64, 64, base_width=26, stride=2, scales=4, style='pytorch') assert block.scales == 4 # Test Res2Net Bottle2neck with DCN dcn = dict(type='DCN', deform_groups=1, fallback_on_stride=False) with pytest.raises(AssertionError): # conv_cfg must be None if dcn is not None Bottle2neck( 64, 64, base_width=26, scales=4, dcn=dcn, conv_cfg=dict(type='Conv')) Bottle2neck(64, 64, dcn=dcn) # Test Res2Net Bottle2neck forward block = Bottle2neck(64, 16, base_width=26, scales=4) x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) def test_res2net_backbone(): with pytest.raises(KeyError): # Res2Net depth should be in [50, 101, 152] Res2Net(depth=18) # Test Res2Net with scales 4, base_width 26 model = Res2Net(depth=50, scales=4, base_width=26) for m in model.modules(): if is_block(m): assert m.scales == 4 model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, 256, 8, 8]) assert feat[1].shape == torch.Size([1, 512, 4, 4]) assert feat[2].shape == torch.Size([1, 1024, 2, 2]) assert feat[3].shape == torch.Size([1, 2048, 1, 1]) ================================================ FILE: tests/test_models/test_backbones/test_resnest.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmdet.models.backbones import ResNeSt from mmdet.models.backbones.resnest import Bottleneck as BottleneckS def test_resnest_bottleneck(): with pytest.raises(AssertionError): # Style must be in ['pytorch', 'caffe'] BottleneckS(64, 64, radix=2, reduction_factor=4, style='tensorflow') # Test ResNeSt Bottleneck structure block = BottleneckS( 2, 4, radix=2, reduction_factor=4, stride=2, style='pytorch') assert block.avd_layer.stride == 2 assert block.conv2.channels == 4 # Test ResNeSt Bottleneck forward block = BottleneckS(16, 4, radix=2, reduction_factor=4) x = torch.randn(2, 16, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([2, 16, 56, 56]) def test_resnest_backbone(): with pytest.raises(KeyError): # ResNeSt depth should be in [50, 101, 152, 200] ResNeSt(depth=18) # Test ResNeSt with radix 2, reduction_factor 4 model = ResNeSt( depth=50, base_channels=4, radix=2, reduction_factor=4, out_indices=(0, 1, 2, 3)) model.train() imgs = torch.randn(2, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([2, 16, 8, 8]) assert feat[1].shape == torch.Size([2, 32, 4, 4]) assert feat[2].shape == torch.Size([2, 64, 2, 2]) assert feat[3].shape == torch.Size([2, 128, 1, 1]) ================================================ FILE: tests/test_models/test_backbones/test_resnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmcv.ops import DeformConv2dPack from torch.nn.modules import AvgPool2d, GroupNorm from torch.nn.modules.batchnorm import _BatchNorm from mmdet.models.backbones import ResNet, ResNetV1d from mmdet.models.backbones.resnet import BasicBlock, Bottleneck from mmdet.models.layers import ResLayer, SimplifiedBasicBlock from .utils import check_norm_state, is_block, is_norm def assert_params_all_zeros(module) -> bool: """Check if the parameters of the module is all zeros. Args: module (nn.Module): The module to be checked. Returns: bool: Whether the parameters of the module is all zeros. """ weight_data = module.weight.data is_weight_zero = weight_data.allclose( weight_data.new_zeros(weight_data.size())) if hasattr(module, 'bias') and module.bias is not None: bias_data = module.bias.data is_bias_zero = bias_data.allclose( bias_data.new_zeros(bias_data.size())) else: is_bias_zero = True return is_weight_zero and is_bias_zero def test_resnet_basic_block(): with pytest.raises(AssertionError): # Not implemented yet. dcn = dict(type='DCN', deform_groups=1, fallback_on_stride=False) BasicBlock(64, 64, dcn=dcn) with pytest.raises(AssertionError): # Not implemented yet. plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv3') ] BasicBlock(64, 64, plugins=plugins) with pytest.raises(AssertionError): # Not implemented yet plugins = [ dict( cfg=dict( type='GeneralizedAttention', spatial_range=-1, num_heads=8, attention_type='0010', kv_stride=2), position='after_conv2') ] BasicBlock(64, 64, plugins=plugins) # test BasicBlock structure and forward block = BasicBlock(64, 64) assert block.conv1.in_channels == 64 assert block.conv1.out_channels == 64 assert block.conv1.kernel_size == (3, 3) assert block.conv2.in_channels == 64 assert block.conv2.out_channels == 64 assert block.conv2.kernel_size == (3, 3) x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) # Test BasicBlock with checkpoint forward block = BasicBlock(64, 64, with_cp=True) assert block.with_cp x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) def test_resnet_bottleneck(): with pytest.raises(AssertionError): # Style must be in ['pytorch', 'caffe'] Bottleneck(64, 64, style='tensorflow') with pytest.raises(AssertionError): # Allowed positions are 'after_conv1', 'after_conv2', 'after_conv3' plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv4') ] Bottleneck(64, 16, plugins=plugins) with pytest.raises(AssertionError): # Need to specify different postfix to avoid duplicate plugin name plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv3'), dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv3') ] Bottleneck(64, 16, plugins=plugins) with pytest.raises(KeyError): # Plugin type is not supported plugins = [dict(cfg=dict(type='WrongPlugin'), position='after_conv3')] Bottleneck(64, 16, plugins=plugins) # Test Bottleneck with checkpoint forward block = Bottleneck(64, 16, with_cp=True) assert block.with_cp x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) # Test Bottleneck style block = Bottleneck(64, 64, stride=2, style='pytorch') assert block.conv1.stride == (1, 1) assert block.conv2.stride == (2, 2) block = Bottleneck(64, 64, stride=2, style='caffe') assert block.conv1.stride == (2, 2) assert block.conv2.stride == (1, 1) # Test Bottleneck DCN dcn = dict(type='DCN', deform_groups=1, fallback_on_stride=False) with pytest.raises(AssertionError): Bottleneck(64, 64, dcn=dcn, conv_cfg=dict(type='Conv')) block = Bottleneck(64, 64, dcn=dcn) assert isinstance(block.conv2, DeformConv2dPack) # Test Bottleneck forward block = Bottleneck(64, 16) x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) # Test Bottleneck with 1 ContextBlock after conv3 plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv3') ] block = Bottleneck(64, 16, plugins=plugins) assert block.context_block.in_channels == 64 x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) # Test Bottleneck with 1 GeneralizedAttention after conv2 plugins = [ dict( cfg=dict( type='GeneralizedAttention', spatial_range=-1, num_heads=8, attention_type='0010', kv_stride=2), position='after_conv2') ] block = Bottleneck(64, 16, plugins=plugins) assert block.gen_attention_block.in_channels == 16 x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) # Test Bottleneck with 1 GeneralizedAttention after conv2, 1 NonLocal2D # after conv2, 1 ContextBlock after conv3 plugins = [ dict( cfg=dict( type='GeneralizedAttention', spatial_range=-1, num_heads=8, attention_type='0010', kv_stride=2), position='after_conv2'), dict(cfg=dict(type='NonLocal2d'), position='after_conv2'), dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv3') ] block = Bottleneck(64, 16, plugins=plugins) assert block.gen_attention_block.in_channels == 16 assert block.nonlocal_block.in_channels == 16 assert block.context_block.in_channels == 64 x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) # Test Bottleneck with 1 ContextBlock after conv2, 2 ContextBlock after # conv3 plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16, postfix=1), position='after_conv2'), dict( cfg=dict(type='ContextBlock', ratio=1. / 16, postfix=2), position='after_conv3'), dict( cfg=dict(type='ContextBlock', ratio=1. / 16, postfix=3), position='after_conv3') ] block = Bottleneck(64, 16, plugins=plugins) assert block.context_block1.in_channels == 16 assert block.context_block2.in_channels == 64 assert block.context_block3.in_channels == 64 x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) def test_simplied_basic_block(): with pytest.raises(AssertionError): # Not implemented yet. dcn = dict(type='DCN', deform_groups=1, fallback_on_stride=False) SimplifiedBasicBlock(64, 64, dcn=dcn) with pytest.raises(AssertionError): # Not implemented yet. plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv3') ] SimplifiedBasicBlock(64, 64, plugins=plugins) with pytest.raises(AssertionError): # Not implemented yet plugins = [ dict( cfg=dict( type='GeneralizedAttention', spatial_range=-1, num_heads=8, attention_type='0010', kv_stride=2), position='after_conv2') ] SimplifiedBasicBlock(64, 64, plugins=plugins) with pytest.raises(AssertionError): # Not implemented yet SimplifiedBasicBlock(64, 64, with_cp=True) # test SimplifiedBasicBlock structure and forward block = SimplifiedBasicBlock(64, 64) assert block.conv1.in_channels == 64 assert block.conv1.out_channels == 64 assert block.conv1.kernel_size == (3, 3) assert block.conv2.in_channels == 64 assert block.conv2.out_channels == 64 assert block.conv2.kernel_size == (3, 3) x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) # test SimplifiedBasicBlock without norm block = SimplifiedBasicBlock(64, 64, norm_cfg=None) assert block.norm1 is None assert block.norm2 is None x_out = block(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) def test_resnet_res_layer(): # Test ResLayer of 3 Bottleneck w\o downsample layer = ResLayer(Bottleneck, 64, 16, 3) assert len(layer) == 3 assert layer[0].conv1.in_channels == 64 assert layer[0].conv1.out_channels == 16 for i in range(1, len(layer)): assert layer[i].conv1.in_channels == 64 assert layer[i].conv1.out_channels == 16 for i in range(len(layer)): assert layer[i].downsample is None x = torch.randn(1, 64, 56, 56) x_out = layer(x) assert x_out.shape == torch.Size([1, 64, 56, 56]) # Test ResLayer of 3 Bottleneck with downsample layer = ResLayer(Bottleneck, 64, 64, 3) assert layer[0].downsample[0].out_channels == 256 for i in range(1, len(layer)): assert layer[i].downsample is None x = torch.randn(1, 64, 56, 56) x_out = layer(x) assert x_out.shape == torch.Size([1, 256, 56, 56]) # Test ResLayer of 3 Bottleneck with stride=2 layer = ResLayer(Bottleneck, 64, 64, 3, stride=2) assert layer[0].downsample[0].out_channels == 256 assert layer[0].downsample[0].stride == (2, 2) for i in range(1, len(layer)): assert layer[i].downsample is None x = torch.randn(1, 64, 56, 56) x_out = layer(x) assert x_out.shape == torch.Size([1, 256, 28, 28]) # Test ResLayer of 3 Bottleneck with stride=2 and average downsample layer = ResLayer(Bottleneck, 64, 64, 3, stride=2, avg_down=True) assert isinstance(layer[0].downsample[0], AvgPool2d) assert layer[0].downsample[1].out_channels == 256 assert layer[0].downsample[1].stride == (1, 1) for i in range(1, len(layer)): assert layer[i].downsample is None x = torch.randn(1, 64, 56, 56) x_out = layer(x) assert x_out.shape == torch.Size([1, 256, 28, 28]) # Test ResLayer of 3 BasicBlock with stride=2 and downsample_first=False layer = ResLayer(BasicBlock, 64, 64, 3, stride=2, downsample_first=False) assert layer[2].downsample[0].out_channels == 64 assert layer[2].downsample[0].stride == (2, 2) for i in range(len(layer) - 1): assert layer[i].downsample is None x = torch.randn(1, 64, 56, 56) x_out = layer(x) assert x_out.shape == torch.Size([1, 64, 28, 28]) def test_resnest_stem(): # Test default stem_channels model = ResNet(50) assert model.stem_channels == 64 assert model.conv1.out_channels == 64 assert model.norm1.num_features == 64 # Test default stem_channels, with base_channels=3 model = ResNet(50, base_channels=3) assert model.stem_channels == 3 assert model.conv1.out_channels == 3 assert model.norm1.num_features == 3 assert model.layer1[0].conv1.in_channels == 3 # Test stem_channels=3 model = ResNet(50, stem_channels=3) assert model.stem_channels == 3 assert model.conv1.out_channels == 3 assert model.norm1.num_features == 3 assert model.layer1[0].conv1.in_channels == 3 # Test stem_channels=3, with base_channels=2 model = ResNet(50, stem_channels=3, base_channels=2) assert model.stem_channels == 3 assert model.conv1.out_channels == 3 assert model.norm1.num_features == 3 assert model.layer1[0].conv1.in_channels == 3 # Test V1d stem_channels model = ResNetV1d(depth=50, stem_channels=6) model.train() assert model.stem[0].out_channels == 3 assert model.stem[1].num_features == 3 assert model.stem[3].out_channels == 3 assert model.stem[4].num_features == 3 assert model.stem[6].out_channels == 6 assert model.stem[7].num_features == 6 assert model.layer1[0].conv1.in_channels == 6 def test_resnet_backbone(): """Test resnet backbone.""" with pytest.raises(KeyError): # ResNet depth should be in [18, 34, 50, 101, 152] ResNet(20) with pytest.raises(AssertionError): # In ResNet: 1 <= num_stages <= 4 ResNet(50, num_stages=0) with pytest.raises(AssertionError): # len(stage_with_dcn) == num_stages dcn = dict(type='DCN', deform_groups=1, fallback_on_stride=False) ResNet(50, dcn=dcn, stage_with_dcn=(True, )) with pytest.raises(AssertionError): # len(stage_with_plugin) == num_stages plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16), stages=(False, True, True), position='after_conv3') ] ResNet(50, plugins=plugins) with pytest.raises(AssertionError): # In ResNet: 1 <= num_stages <= 4 ResNet(50, num_stages=5) with pytest.raises(AssertionError): # len(strides) == len(dilations) == num_stages ResNet(50, strides=(1, ), dilations=(1, 1), num_stages=3) with pytest.raises(TypeError): # pretrained must be a string path model = ResNet(50, pretrained=0) with pytest.raises(AssertionError): # Style must be in ['pytorch', 'caffe'] ResNet(50, style='tensorflow') # Test ResNet50 norm_eval=True model = ResNet(50, norm_eval=True, base_channels=1) model.train() assert check_norm_state(model.modules(), False) # Test ResNet50 with torchvision pretrained weight model = ResNet( depth=50, norm_eval=True, pretrained='torchvision://resnet50') model.train() assert check_norm_state(model.modules(), False) # Test ResNet50 with first stage frozen frozen_stages = 1 model = ResNet(50, frozen_stages=frozen_stages, base_channels=1) model.train() assert model.norm1.training is False for layer in [model.conv1, model.norm1]: for param in layer.parameters(): assert param.requires_grad is False for i in range(1, frozen_stages + 1): layer = getattr(model, f'layer{i}') for mod in layer.modules(): if isinstance(mod, _BatchNorm): assert mod.training is False for param in layer.parameters(): assert param.requires_grad is False # Test ResNet50V1d with first stage frozen model = ResNetV1d(depth=50, frozen_stages=frozen_stages, base_channels=2) assert len(model.stem) == 9 model.train() assert check_norm_state(model.stem, False) for param in model.stem.parameters(): assert param.requires_grad is False for i in range(1, frozen_stages + 1): layer = getattr(model, f'layer{i}') for mod in layer.modules(): if isinstance(mod, _BatchNorm): assert mod.training is False for param in layer.parameters(): assert param.requires_grad is False # Test ResNet18 forward model = ResNet(18) model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, 64, 8, 8]) assert feat[1].shape == torch.Size([1, 128, 4, 4]) assert feat[2].shape == torch.Size([1, 256, 2, 2]) assert feat[3].shape == torch.Size([1, 512, 1, 1]) # Test ResNet18 with checkpoint forward model = ResNet(18, with_cp=True) for m in model.modules(): if is_block(m): assert m.with_cp # Test ResNet50 with BatchNorm forward model = ResNet(50, base_channels=1) for m in model.modules(): if is_norm(m): assert isinstance(m, _BatchNorm) model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, 4, 8, 8]) assert feat[1].shape == torch.Size([1, 8, 4, 4]) assert feat[2].shape == torch.Size([1, 16, 2, 2]) assert feat[3].shape == torch.Size([1, 32, 1, 1]) # Test ResNet50 with layers 1, 2, 3 out forward model = ResNet(50, out_indices=(0, 1, 2), base_channels=1) model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 3 assert feat[0].shape == torch.Size([1, 4, 8, 8]) assert feat[1].shape == torch.Size([1, 8, 4, 4]) assert feat[2].shape == torch.Size([1, 16, 2, 2]) # Test ResNet50 with checkpoint forward model = ResNet(50, with_cp=True, base_channels=1) for m in model.modules(): if is_block(m): assert m.with_cp model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, 4, 8, 8]) assert feat[1].shape == torch.Size([1, 8, 4, 4]) assert feat[2].shape == torch.Size([1, 16, 2, 2]) assert feat[3].shape == torch.Size([1, 32, 1, 1]) # Test ResNet50 with GroupNorm forward model = ResNet( 50, base_channels=4, norm_cfg=dict(type='GN', num_groups=2, requires_grad=True)) for m in model.modules(): if is_norm(m): assert isinstance(m, GroupNorm) model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, 16, 8, 8]) assert feat[1].shape == torch.Size([1, 32, 4, 4]) assert feat[2].shape == torch.Size([1, 64, 2, 2]) assert feat[3].shape == torch.Size([1, 128, 1, 1]) # Test ResNet50 with 1 GeneralizedAttention after conv2, 1 NonLocal2D # after conv2, 1 ContextBlock after conv3 in layers 2, 3, 4 plugins = [ dict( cfg=dict( type='GeneralizedAttention', spatial_range=-1, num_heads=8, attention_type='0010', kv_stride=2), stages=(False, True, True, True), position='after_conv2'), dict(cfg=dict(type='NonLocal2d'), position='after_conv2'), dict( cfg=dict(type='ContextBlock', ratio=1. / 16), stages=(False, True, True, False), position='after_conv3') ] model = ResNet(50, plugins=plugins, base_channels=8) for m in model.layer1.modules(): if is_block(m): assert not hasattr(m, 'context_block') assert not hasattr(m, 'gen_attention_block') assert m.nonlocal_block.in_channels == 8 for m in model.layer2.modules(): if is_block(m): assert m.nonlocal_block.in_channels == 16 assert m.gen_attention_block.in_channels == 16 assert m.context_block.in_channels == 64 for m in model.layer3.modules(): if is_block(m): assert m.nonlocal_block.in_channels == 32 assert m.gen_attention_block.in_channels == 32 assert m.context_block.in_channels == 128 for m in model.layer4.modules(): if is_block(m): assert m.nonlocal_block.in_channels == 64 assert m.gen_attention_block.in_channels == 64 assert not hasattr(m, 'context_block') model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, 32, 8, 8]) assert feat[1].shape == torch.Size([1, 64, 4, 4]) assert feat[2].shape == torch.Size([1, 128, 2, 2]) assert feat[3].shape == torch.Size([1, 256, 1, 1]) # Test ResNet50 with 1 ContextBlock after conv2, 1 ContextBlock after # conv3 in layers 2, 3, 4 plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16, postfix=1), stages=(False, True, True, False), position='after_conv3'), dict( cfg=dict(type='ContextBlock', ratio=1. / 16, postfix=2), stages=(False, True, True, False), position='after_conv3') ] model = ResNet(50, plugins=plugins, base_channels=8) for m in model.layer1.modules(): if is_block(m): assert not hasattr(m, 'context_block') assert not hasattr(m, 'context_block1') assert not hasattr(m, 'context_block2') for m in model.layer2.modules(): if is_block(m): assert not hasattr(m, 'context_block') assert m.context_block1.in_channels == 64 assert m.context_block2.in_channels == 64 for m in model.layer3.modules(): if is_block(m): assert not hasattr(m, 'context_block') assert m.context_block1.in_channels == 128 assert m.context_block2.in_channels == 128 for m in model.layer4.modules(): if is_block(m): assert not hasattr(m, 'context_block') assert not hasattr(m, 'context_block1') assert not hasattr(m, 'context_block2') model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, 32, 8, 8]) assert feat[1].shape == torch.Size([1, 64, 4, 4]) assert feat[2].shape == torch.Size([1, 128, 2, 2]) assert feat[3].shape == torch.Size([1, 256, 1, 1]) # Test ResNet50 zero initialization of residual model = ResNet(50, zero_init_residual=True, base_channels=1) model.init_weights() for m in model.modules(): if isinstance(m, Bottleneck): assert assert_params_all_zeros(m.norm3) elif isinstance(m, BasicBlock): assert assert_params_all_zeros(m.norm2) model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, 4, 8, 8]) assert feat[1].shape == torch.Size([1, 8, 4, 4]) assert feat[2].shape == torch.Size([1, 16, 2, 2]) assert feat[3].shape == torch.Size([1, 32, 1, 1]) # Test ResNetV1d forward model = ResNetV1d(depth=50, base_channels=2) model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 4 assert feat[0].shape == torch.Size([1, 8, 8, 8]) assert feat[1].shape == torch.Size([1, 16, 4, 4]) assert feat[2].shape == torch.Size([1, 32, 2, 2]) assert feat[3].shape == torch.Size([1, 64, 1, 1]) ================================================ FILE: tests/test_models/test_backbones/test_swin.py ================================================ import pytest import torch from mmdet.models.backbones.swin import SwinBlock, SwinTransformer def test_swin_block(): # test SwinBlock structure and forward block = SwinBlock(embed_dims=64, num_heads=4, feedforward_channels=256) assert block.ffn.embed_dims == 64 assert block.attn.w_msa.num_heads == 4 assert block.ffn.feedforward_channels == 256 x = torch.randn(1, 56 * 56, 64) x_out = block(x, (56, 56)) assert x_out.shape == torch.Size([1, 56 * 56, 64]) # Test BasicBlock with checkpoint forward block = SwinBlock( embed_dims=64, num_heads=4, feedforward_channels=256, with_cp=True) assert block.with_cp x = torch.randn(1, 56 * 56, 64) x_out = block(x, (56, 56)) assert x_out.shape == torch.Size([1, 56 * 56, 64]) def test_swin_transformer(): """Test Swin Transformer backbone.""" with pytest.raises(TypeError): # Pretrained arg must be str or None. SwinTransformer(pretrained=123) with pytest.raises(AssertionError): # Because swin uses non-overlapping patch embed, so the stride of patch # embed must be equal to patch size. SwinTransformer(strides=(2, 2, 2, 2), patch_size=4) # test pretrained image size with pytest.raises(AssertionError): SwinTransformer(pretrain_img_size=(224, 224, 224)) # Test absolute position embedding temp = torch.randn((1, 3, 224, 224)) model = SwinTransformer(pretrain_img_size=224, use_abs_pos_embed=True) model.init_weights() model(temp) # Test patch norm model = SwinTransformer(patch_norm=False) model(temp) # Test normal inference temp = torch.randn((1, 3, 32, 32)) model = SwinTransformer() outs = model(temp) assert outs[0].shape == (1, 96, 8, 8) assert outs[1].shape == (1, 192, 4, 4) assert outs[2].shape == (1, 384, 2, 2) assert outs[3].shape == (1, 768, 1, 1) # Test abnormal inference size temp = torch.randn((1, 3, 31, 31)) model = SwinTransformer() outs = model(temp) assert outs[0].shape == (1, 96, 8, 8) assert outs[1].shape == (1, 192, 4, 4) assert outs[2].shape == (1, 384, 2, 2) assert outs[3].shape == (1, 768, 1, 1) # Test abnormal inference size temp = torch.randn((1, 3, 112, 137)) model = SwinTransformer() outs = model(temp) assert outs[0].shape == (1, 96, 28, 35) assert outs[1].shape == (1, 192, 14, 18) assert outs[2].shape == (1, 384, 7, 9) assert outs[3].shape == (1, 768, 4, 5) model = SwinTransformer(frozen_stages=4) model.train() for p in model.parameters(): assert not p.requires_grad ================================================ FILE: tests/test_models/test_backbones/test_trident_resnet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmdet.models.backbones import TridentResNet from mmdet.models.backbones.trident_resnet import TridentBottleneck def test_trident_resnet_bottleneck(): trident_dilations = (1, 2, 3) test_branch_idx = 1 concat_output = True trident_build_config = (trident_dilations, test_branch_idx, concat_output) with pytest.raises(AssertionError): # Style must be in ['pytorch', 'caffe'] TridentBottleneck( *trident_build_config, inplanes=64, planes=64, style='tensorflow') with pytest.raises(AssertionError): # Allowed positions are 'after_conv1', 'after_conv2', 'after_conv3' plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv4') ] TridentBottleneck( *trident_build_config, inplanes=64, planes=16, plugins=plugins) with pytest.raises(AssertionError): # Need to specify different postfix to avoid duplicate plugin name plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv3'), dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv3') ] TridentBottleneck( *trident_build_config, inplanes=64, planes=16, plugins=plugins) with pytest.raises(KeyError): # Plugin type is not supported plugins = [dict(cfg=dict(type='WrongPlugin'), position='after_conv3')] TridentBottleneck( *trident_build_config, inplanes=64, planes=16, plugins=plugins) # Test Bottleneck with checkpoint forward block = TridentBottleneck( *trident_build_config, inplanes=64, planes=16, with_cp=True) assert block.with_cp x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([block.num_branch, 64, 56, 56]) # Test Bottleneck style block = TridentBottleneck( *trident_build_config, inplanes=64, planes=64, stride=2, style='pytorch') assert block.conv1.stride == (1, 1) assert block.conv2.stride == (2, 2) block = TridentBottleneck( *trident_build_config, inplanes=64, planes=64, stride=2, style='caffe') assert block.conv1.stride == (2, 2) assert block.conv2.stride == (1, 1) # Test Bottleneck forward block = TridentBottleneck(*trident_build_config, inplanes=64, planes=16) x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([block.num_branch, 64, 56, 56]) # Test Bottleneck with 1 ContextBlock after conv3 plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv3') ] block = TridentBottleneck( *trident_build_config, inplanes=64, planes=16, plugins=plugins) assert block.context_block.in_channels == 64 x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([block.num_branch, 64, 56, 56]) # Test Bottleneck with 1 GeneralizedAttention after conv2 plugins = [ dict( cfg=dict( type='GeneralizedAttention', spatial_range=-1, num_heads=8, attention_type='0010', kv_stride=2), position='after_conv2') ] block = TridentBottleneck( *trident_build_config, inplanes=64, planes=16, plugins=plugins) assert block.gen_attention_block.in_channels == 16 x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([block.num_branch, 64, 56, 56]) # Test Bottleneck with 1 GeneralizedAttention after conv2, 1 NonLocal2D # after conv2, 1 ContextBlock after conv3 plugins = [ dict( cfg=dict( type='GeneralizedAttention', spatial_range=-1, num_heads=8, attention_type='0010', kv_stride=2), position='after_conv2'), dict(cfg=dict(type='NonLocal2d'), position='after_conv2'), dict( cfg=dict(type='ContextBlock', ratio=1. / 16), position='after_conv3') ] block = TridentBottleneck( *trident_build_config, inplanes=64, planes=16, plugins=plugins) assert block.gen_attention_block.in_channels == 16 assert block.nonlocal_block.in_channels == 16 assert block.context_block.in_channels == 64 x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([block.num_branch, 64, 56, 56]) # Test Bottleneck with 1 ContextBlock after conv2, 2 ContextBlock after # conv3 plugins = [ dict( cfg=dict(type='ContextBlock', ratio=1. / 16, postfix=1), position='after_conv2'), dict( cfg=dict(type='ContextBlock', ratio=1. / 16, postfix=2), position='after_conv3'), dict( cfg=dict(type='ContextBlock', ratio=1. / 16, postfix=3), position='after_conv3') ] block = TridentBottleneck( *trident_build_config, inplanes=64, planes=16, plugins=plugins) assert block.context_block1.in_channels == 16 assert block.context_block2.in_channels == 64 assert block.context_block3.in_channels == 64 x = torch.randn(1, 64, 56, 56) x_out = block(x) assert x_out.shape == torch.Size([block.num_branch, 64, 56, 56]) def test_trident_resnet_backbone(): tridentresnet_config = dict( num_branch=3, test_branch_idx=1, strides=(1, 2, 2), dilations=(1, 1, 1), trident_dilations=(1, 2, 3), out_indices=(2, ), ) """Test tridentresnet backbone.""" with pytest.raises(AssertionError): # TridentResNet depth should be in [50, 101, 152] TridentResNet(18, **tridentresnet_config) with pytest.raises(AssertionError): # In TridentResNet: num_stages == 3 TridentResNet(50, num_stages=4, **tridentresnet_config) model = TridentResNet(50, num_stages=3, **tridentresnet_config) model.train() imgs = torch.randn(1, 3, 32, 32) feat = model(imgs) assert len(feat) == 1 assert feat[0].shape == torch.Size([3, 1024, 2, 2]) ================================================ FILE: tests/test_models/test_backbones/utils.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from torch.nn.modules import GroupNorm from torch.nn.modules.batchnorm import _BatchNorm from mmdet.models.backbones.res2net import Bottle2neck from mmdet.models.backbones.resnet import BasicBlock, Bottleneck from mmdet.models.backbones.resnext import Bottleneck as BottleneckX from mmdet.models.layers import SimplifiedBasicBlock def is_block(modules): """Check if is ResNet building block.""" if isinstance(modules, (BasicBlock, Bottleneck, BottleneckX, Bottle2neck, SimplifiedBasicBlock)): return True return False def is_norm(modules): """Check if is one of the norms.""" if isinstance(modules, (GroupNorm, _BatchNorm)): return True return False def check_norm_state(modules, train_state): """Check if norm layer is in correct train state.""" for mod in modules: if isinstance(mod, _BatchNorm): if mod.training != train_state: return False return True ================================================ FILE: tests/test_models/test_data_preprocessors/test_batch_resize.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase from mmdet.models.data_preprocessors import BatchResize, DetDataPreprocessor from mmdet.testing import demo_mm_inputs class TestDetDataPreprocessor(TestCase): def test_batch_resize(self): processor = DetDataPreprocessor( mean=[103.53, 116.28, 123.675], std=[57.375, 57.12, 58.395], bgr_to_rgb=False, batch_augments=[ dict(type='BatchResize', scale=(32, 32), pad_size_divisor=32) ]) self.assertTrue(isinstance(processor.batch_augments[0], BatchResize)) packed_inputs = demo_mm_inputs( 2, [[3, 10, 11], [3, 9, 24]], use_box_type=True) data = processor(packed_inputs, training=True) batch_inputs, batch_data_samples = data['inputs'], data['data_samples'] self.assertEqual(batch_inputs.shape[-2:], (32, 32)) self.assertEqual(batch_data_samples[0].scale_factor, batch_data_samples[1].scale_factor) ================================================ FILE: tests/test_models/test_data_preprocessors/test_boxinst_preprocessor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmdet.models.data_preprocessors import BoxInstDataPreprocessor from mmdet.structures import DetDataSample from mmdet.testing import demo_mm_inputs class TestBoxInstDataPreprocessor(TestCase): def test_forward(self): processor = BoxInstDataPreprocessor(mean=[0, 0, 0], std=[1, 1, 1]) data = { 'inputs': [torch.randint(0, 256, (3, 256, 256))], 'data_samples': [DetDataSample()] } # Test evaluation mode out_data = processor(data) batch_inputs, batch_data_samples = out_data['inputs'], out_data[ 'data_samples'] self.assertEqual(batch_inputs.shape, (1, 3, 256, 256)) self.assertEqual(len(batch_data_samples), 1) # Test traning mode without gt bboxes packed_inputs = demo_mm_inputs( 2, [[3, 256, 256], [3, 128, 128]], num_items=[0, 0]) out_data = processor(packed_inputs, training=True) batch_inputs, batch_data_samples = out_data['inputs'], out_data[ 'data_samples'] self.assertEqual(batch_inputs.shape, (2, 3, 256, 256)) self.assertEqual(len(batch_data_samples), 2) self.assertEqual(len(batch_data_samples[0].gt_instances.masks), 0) self.assertEqual( len(batch_data_samples[0].gt_instances.pairwise_masks), 0) self.assertEqual(len(batch_data_samples[1].gt_instances.masks), 0) self.assertEqual( len(batch_data_samples[1].gt_instances.pairwise_masks), 0) # Test traning mode with gt bboxes packed_inputs = demo_mm_inputs( 2, [[3, 256, 256], [3, 128, 128]], num_items=[2, 1]) out_data = processor(packed_inputs, training=True) batch_inputs, batch_data_samples = out_data['inputs'], out_data[ 'data_samples'] self.assertEqual(batch_inputs.shape, (2, 3, 256, 256)) self.assertEqual(len(batch_data_samples), 2) self.assertEqual(len(batch_data_samples[0].gt_instances.masks), 2) self.assertEqual( len(batch_data_samples[0].gt_instances.pairwise_masks), 2) self.assertEqual(len(batch_data_samples[1].gt_instances.masks), 1) self.assertEqual( len(batch_data_samples[1].gt_instances.pairwise_masks), 1) ================================================ FILE: tests/test_models/test_data_preprocessors/test_data_preprocessor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.logging import MessageHub from mmdet.models.data_preprocessors import (BatchFixedSizePad, BatchSyncRandomResize, DetDataPreprocessor, MultiBranchDataPreprocessor) from mmdet.structures import DetDataSample from mmdet.testing import demo_mm_inputs class TestDetDataPreprocessor(TestCase): def test_init(self): # test mean is None processor = DetDataPreprocessor() self.assertTrue(not hasattr(processor, 'mean')) self.assertTrue(processor._enable_normalize is False) # test mean is not None processor = DetDataPreprocessor(mean=[0, 0, 0], std=[1, 1, 1]) self.assertTrue(hasattr(processor, 'mean')) self.assertTrue(hasattr(processor, 'std')) self.assertTrue(processor._enable_normalize) # please specify both mean and std with self.assertRaises(AssertionError): DetDataPreprocessor(mean=[0, 0, 0]) # bgr2rgb and rgb2bgr cannot be set to True at the same time with self.assertRaises(AssertionError): DetDataPreprocessor(bgr_to_rgb=True, rgb_to_bgr=True) def test_forward(self): processor = DetDataPreprocessor(mean=[0, 0, 0], std=[1, 1, 1]) data = { 'inputs': [torch.randint(0, 256, (3, 11, 10))], 'data_samples': [DetDataSample()] } out_data = processor(data) batch_inputs, batch_data_samples = out_data['inputs'], out_data[ 'data_samples'] self.assertEqual(batch_inputs.shape, (1, 3, 11, 10)) self.assertEqual(len(batch_data_samples), 1) # test channel_conversion processor = DetDataPreprocessor( mean=[0., 0., 0.], std=[1., 1., 1.], bgr_to_rgb=True) out_data = processor(data) batch_inputs, batch_data_samples = out_data['inputs'], out_data[ 'data_samples'] self.assertEqual(batch_inputs.shape, (1, 3, 11, 10)) self.assertEqual(len(batch_data_samples), 1) # test padding data = { 'inputs': [ torch.randint(0, 256, (3, 10, 11)), torch.randint(0, 256, (3, 9, 14)) ] } processor = DetDataPreprocessor( mean=[0., 0., 0.], std=[1., 1., 1.], bgr_to_rgb=True) out_data = processor(data) batch_inputs, batch_data_samples = out_data['inputs'], out_data[ 'data_samples'] self.assertEqual(batch_inputs.shape, (2, 3, 10, 14)) self.assertIsNone(batch_data_samples) # test pad_size_divisor data = { 'inputs': [ torch.randint(0, 256, (3, 10, 11)), torch.randint(0, 256, (3, 9, 24)) ], 'data_samples': [DetDataSample()] * 2 } processor = DetDataPreprocessor( mean=[0., 0., 0.], std=[1., 1., 1.], pad_size_divisor=5) out_data = processor(data) batch_inputs, batch_data_samples = out_data['inputs'], out_data[ 'data_samples'] self.assertEqual(batch_inputs.shape, (2, 3, 10, 25)) self.assertEqual(len(batch_data_samples), 2) for data_samples, expected_shape in zip(batch_data_samples, [(10, 15), (10, 25)]): self.assertEqual(data_samples.pad_shape, expected_shape) # test pad_mask=True and pad_seg=True processor = DetDataPreprocessor( pad_mask=True, mask_pad_value=0, pad_seg=True, seg_pad_value=0) packed_inputs = demo_mm_inputs( 2, [[3, 10, 11], [3, 9, 24]], with_mask=True, with_semantic=True, use_box_type=True) packed_inputs['data_samples'][0].gt_sem_seg.sem_seg = torch.randint( 0, 256, (1, 10, 11)) packed_inputs['data_samples'][1].gt_sem_seg.sem_seg = torch.randint( 0, 256, (1, 9, 24)) mask_pad_sums = [ x.gt_instances.masks.masks.sum() for x in packed_inputs['data_samples'] ] seg_pad_sums = [ x.gt_sem_seg.sem_seg.sum() for x in packed_inputs['data_samples'] ] batch_data_samples = processor( packed_inputs, training=True)['data_samples'] for data_samples, expected_shape, mask_pad_sum, seg_pad_sum in zip( batch_data_samples, [(10, 24), (10, 24)], mask_pad_sums, seg_pad_sums): self.assertEqual(data_samples.gt_instances.masks.masks.shape[-2:], expected_shape) self.assertEqual(data_samples.gt_sem_seg.sem_seg.shape[-2:], expected_shape) self.assertEqual(data_samples.gt_instances.masks.masks.sum(), mask_pad_sum) self.assertEqual(data_samples.gt_sem_seg.sem_seg.sum(), seg_pad_sum) def test_batch_sync_random_resize(self): processor = DetDataPreprocessor(batch_augments=[ dict( type='BatchSyncRandomResize', random_size_range=(320, 320), size_divisor=32, interval=1) ]) self.assertTrue( isinstance(processor.batch_augments[0], BatchSyncRandomResize)) message_hub = MessageHub.get_instance('test_batch_sync_random_resize') message_hub.update_info('iter', 0) packed_inputs = demo_mm_inputs( 2, [[3, 128, 128], [3, 128, 128]], use_box_type=True) batch_inputs = processor(packed_inputs, training=True)['inputs'] self.assertEqual(batch_inputs.shape, (2, 3, 128, 128)) # resize after one iter message_hub.update_info('iter', 1) packed_inputs = demo_mm_inputs( 2, [[3, 128, 128], [3, 128, 128]], use_box_type=True) batch_inputs = processor(packed_inputs, training=True)['inputs'] self.assertEqual(batch_inputs.shape, (2, 3, 320, 320)) packed_inputs = demo_mm_inputs( 2, [[3, 128, 128], [3, 128, 128]], use_box_type=True) batch_inputs = processor(packed_inputs, training=False)['inputs'] self.assertEqual(batch_inputs.shape, (2, 3, 128, 128)) def test_batch_fixed_size_pad(self): # test pad_mask=False and pad_seg=False processor = DetDataPreprocessor( pad_mask=False, pad_seg=False, batch_augments=[ dict( type='BatchFixedSizePad', size=(32, 32), img_pad_value=0, pad_mask=True, mask_pad_value=0, pad_seg=True, seg_pad_value=0) ]) self.assertTrue( isinstance(processor.batch_augments[0], BatchFixedSizePad)) packed_inputs = demo_mm_inputs( 2, [[3, 10, 11], [3, 9, 24]], with_mask=True, with_semantic=True, use_box_type=True) packed_inputs['data_samples'][0].gt_sem_seg.sem_seg = torch.randint( 0, 256, (1, 10, 11)) packed_inputs['data_samples'][1].gt_sem_seg.sem_seg = torch.randint( 0, 256, (1, 9, 24)) mask_pad_sums = [ x.gt_instances.masks.masks.sum() for x in packed_inputs['data_samples'] ] seg_pad_sums = [ x.gt_sem_seg.sem_seg.sum() for x in packed_inputs['data_samples'] ] data = processor(packed_inputs, training=True) batch_inputs, batch_data_samples = data['inputs'], data['data_samples'] self.assertEqual(batch_inputs.shape[-2:], (32, 32)) for data_samples, expected_shape, mask_pad_sum, seg_pad_sum in zip( batch_data_samples, [(32, 32), (32, 32)], mask_pad_sums, seg_pad_sums): self.assertEqual(data_samples.gt_instances.masks.masks.shape[-2:], expected_shape) self.assertEqual(data_samples.gt_sem_seg.sem_seg.shape[-2:], expected_shape) self.assertEqual(data_samples.gt_instances.masks.masks.sum(), mask_pad_sum) self.assertEqual(data_samples.gt_sem_seg.sem_seg.sum(), seg_pad_sum) # test pad_mask=True and pad_seg=True processor = DetDataPreprocessor( pad_mask=True, pad_seg=True, seg_pad_value=0, mask_pad_value=0, batch_augments=[ dict( type='BatchFixedSizePad', size=(32, 32), img_pad_value=0, pad_mask=True, mask_pad_value=0, pad_seg=True, seg_pad_value=0) ]) self.assertTrue( isinstance(processor.batch_augments[0], BatchFixedSizePad)) packed_inputs = demo_mm_inputs( 2, [[3, 10, 11], [3, 9, 24]], with_mask=True, with_semantic=True, use_box_type=True) packed_inputs['data_samples'][0].gt_sem_seg.sem_seg = torch.randint( 0, 256, (1, 10, 11)) packed_inputs['data_samples'][1].gt_sem_seg.sem_seg = torch.randint( 0, 256, (1, 9, 24)) mask_pad_sums = [ x.gt_instances.masks.masks.sum() for x in packed_inputs['data_samples'] ] seg_pad_sums = [ x.gt_sem_seg.sem_seg.sum() for x in packed_inputs['data_samples'] ] data = processor(packed_inputs, training=True) batch_inputs, batch_data_samples = data['inputs'], data['data_samples'] self.assertEqual(batch_inputs.shape[-2:], (32, 32)) for data_samples, expected_shape, mask_pad_sum, seg_pad_sum in zip( batch_data_samples, [(32, 32), (32, 32)], mask_pad_sums, seg_pad_sums): self.assertEqual(data_samples.gt_instances.masks.masks.shape[-2:], expected_shape) self.assertEqual(data_samples.gt_sem_seg.sem_seg.shape[-2:], expected_shape) self.assertEqual(data_samples.gt_instances.masks.masks.sum(), mask_pad_sum) self.assertEqual(data_samples.gt_sem_seg.sem_seg.sum(), seg_pad_sum) # test negative pad/no pad processor = DetDataPreprocessor( pad_mask=True, pad_seg=True, seg_pad_value=0, mask_pad_value=0, batch_augments=[ dict( type='BatchFixedSizePad', size=(5, 5), img_pad_value=0, pad_mask=True, mask_pad_value=1, pad_seg=True, seg_pad_value=1) ]) self.assertTrue( isinstance(processor.batch_augments[0], BatchFixedSizePad)) packed_inputs = demo_mm_inputs( 2, [[3, 10, 11], [3, 9, 24]], with_mask=True, with_semantic=True, use_box_type=True) packed_inputs['data_samples'][0].gt_sem_seg.sem_seg = torch.randint( 0, 256, (1, 10, 11)) packed_inputs['data_samples'][1].gt_sem_seg.sem_seg = torch.randint( 0, 256, (1, 9, 24)) mask_pad_sums = [ x.gt_instances.masks.masks.sum() for x in packed_inputs['data_samples'] ] seg_pad_sums = [ x.gt_sem_seg.sem_seg.sum() for x in packed_inputs['data_samples'] ] data = processor(packed_inputs, training=True) batch_inputs, batch_data_samples = data['inputs'], data['data_samples'] self.assertEqual(batch_inputs.shape[-2:], (10, 24)) for data_samples, expected_shape, mask_pad_sum, seg_pad_sum in zip( batch_data_samples, [(10, 24), (10, 24)], mask_pad_sums, seg_pad_sums): self.assertEqual(data_samples.gt_instances.masks.masks.shape[-2:], expected_shape) self.assertEqual(data_samples.gt_sem_seg.sem_seg.shape[-2:], expected_shape) self.assertEqual(data_samples.gt_instances.masks.masks.sum(), mask_pad_sum) self.assertEqual(data_samples.gt_sem_seg.sem_seg.sum(), seg_pad_sum) class TestMultiBranchDataPreprocessor(TestCase): def setUp(self): """Setup the model and optimizer which are used in every test method. TestCase calls functions in this order: setUp() -> testMethod() -> tearDown() -> cleanUp() """ self.data_preprocessor = dict( type='DetDataPreprocessor', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], bgr_to_rgb=True, pad_size_divisor=32) self.multi_data = { 'inputs': { 'sup': [torch.randint(0, 256, (3, 224, 224))], 'unsup_teacher': [ torch.randint(0, 256, (3, 400, 600)), torch.randint(0, 256, (3, 600, 400)) ], 'unsup_student': [ torch.randint(0, 256, (3, 700, 500)), torch.randint(0, 256, (3, 500, 700)) ] }, 'data_samples': { 'sup': [DetDataSample()], 'unsup_teacher': [DetDataSample(), DetDataSample()], 'unsup_student': [DetDataSample(), DetDataSample()], } } self.data = { 'inputs': [torch.randint(0, 256, (3, 224, 224))], 'data_samples': [DetDataSample()] } def test_multi_data_preprocessor(self): processor = MultiBranchDataPreprocessor(self.data_preprocessor) # test processing multi_data when training multi_data = processor(self.multi_data, training=True) self.assertEqual(multi_data['inputs']['sup'].shape, (1, 3, 224, 224)) self.assertEqual(multi_data['inputs']['unsup_teacher'].shape, (2, 3, 608, 608)) self.assertEqual(multi_data['inputs']['unsup_student'].shape, (2, 3, 704, 704)) self.assertEqual(len(multi_data['data_samples']['sup']), 1) self.assertEqual(len(multi_data['data_samples']['unsup_teacher']), 2) self.assertEqual(len(multi_data['data_samples']['unsup_student']), 2) # test processing data when testing data = processor(self.data) self.assertEqual(data['inputs'].shape, (1, 3, 224, 224)) self.assertEqual(len(data['data_samples']), 1) ================================================ FILE: tests/test_models/test_dense_heads/test_anchor_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import AnchorHead class TestAnchorHead(TestCase): def test_anchor_head_loss(self): """Tests anchor head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] cfg = Config( dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=0, pos_weight=-1, debug=False)) anchor_head = AnchorHead(num_classes=4, in_channels=1, train_cfg=cfg) # Anchor head expects a multiple levels of features per image feats = ( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2))) for i in range(len(anchor_head.prior_generator.strides))) cls_scores, bbox_preds = anchor_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = anchor_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # there should be no box loss. empty_cls_loss = sum(empty_gt_losses['loss_cls']) empty_box_loss = sum(empty_gt_losses['loss_bbox']) assert empty_cls_loss.item() > 0, 'cls loss should be non-zero' assert empty_box_loss.item() == 0, ( 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = anchor_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']) onegt_box_loss = sum(one_gt_losses['loss_bbox']) assert onegt_cls_loss.item() > 0, 'cls loss should be non-zero' assert onegt_box_loss.item() > 0, 'box loss should be non-zero' ================================================ FILE: tests/test_models/test_dense_heads/test_atss_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import ATSSHead class TestATSSHead(TestCase): def test_atss_head_loss(self): """Tests atss head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1 }] cfg = Config( dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False)) atss_head = ATSSHead( num_classes=4, in_channels=1, stacked_convs=1, feat_channels=1, norm_cfg=None, train_cfg=cfg, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0)) feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [8, 16, 32, 64, 128] ] cls_scores, bbox_preds, centernesses = atss_head.forward(feat) # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = atss_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = sum(empty_gt_losses['loss_cls']) empty_box_loss = sum(empty_gt_losses['loss_bbox']) empty_centerness_loss = sum(empty_gt_losses['loss_centerness']) self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_centerness_loss.item(), 0, 'there should be no centerness loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = atss_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']) onegt_box_loss = sum(one_gt_losses['loss_bbox']) onegt_centerness_loss = sum(one_gt_losses['loss_centerness']) self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') self.assertGreater(onegt_centerness_loss.item(), 0, 'centerness loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_autoassign_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.models.dense_heads import AutoAssignHead class TestAutoAssignHead(TestCase): def test_autoassign_head_loss(self): """Tests autoassign head loss when truth is empty and non-empty.""" s = 300 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] autoassign_head = AutoAssignHead( num_classes=4, in_channels=1, stacked_convs=1, feat_channels=1, strides=[8, 16, 32, 64, 128], loss_bbox=dict(type='GIoULoss', loss_weight=5.0), norm_cfg=None) # Fcos head expects a multiple levels of features per image feats = ( torch.rand(1, 1, s // stride[1], s // stride[0]) for stride in autoassign_head.prior_generator.strides) cls_scores, bbox_preds, centernesses = autoassign_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = autoassign_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) # When there is no truth, the neg loss should be nonzero but # pos loss and center loss should be zero empty_pos_loss = empty_gt_losses['loss_pos'].item() empty_neg_loss = empty_gt_losses['loss_neg'].item() empty_ctr_loss = empty_gt_losses['loss_center'].item() self.assertGreater(empty_neg_loss, 0, 'neg loss should be non-zero') self.assertEqual( empty_pos_loss, 0, 'there should be no pos loss when there are no true boxes') self.assertEqual( empty_ctr_loss, 0, 'there should be no centerness loss when there are no true boxes') # When truth is non-empty then all pos, neg loss and center loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = autoassign_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) onegt_pos_loss = one_gt_losses['loss_pos'].item() onegt_neg_loss = one_gt_losses['loss_neg'].item() onegt_ctr_loss = one_gt_losses['loss_center'].item() self.assertGreater(onegt_pos_loss, 0, 'pos loss should be non-zero') self.assertGreater(onegt_neg_loss, 0, 'neg loss should be non-zero') self.assertGreater(onegt_ctr_loss, 0, 'center loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_boxinst_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import numpy as np import torch from mmengine import MessageHub from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.dense_heads import BoxInstBboxHead, BoxInstMaskHead from mmdet.structures.mask import BitmapMasks def _rand_masks(num_items, bboxes, img_w, img_h): rng = np.random.RandomState(0) masks = np.zeros((num_items, img_h, img_w), dtype=np.float32) for i, bbox in enumerate(bboxes): bbox = bbox.astype(np.int32) mask = (rng.rand(1, bbox[3] - bbox[1], bbox[2] - bbox[0]) > 0.3).astype(np.int64) masks[i:i + 1, bbox[1]:bbox[3], bbox[0]:bbox[2]] = mask return BitmapMasks(masks, height=img_h, width=img_w) def _fake_mask_feature_head(): mask_feature_head = ConfigDict( in_channels=1, feat_channels=1, start_level=0, end_level=2, out_channels=8, mask_stride=8, num_stacked_convs=4, norm_cfg=dict(type='BN', requires_grad=True)) return mask_feature_head class TestBoxInstHead(TestCase): def test_boxinst_maskhead_loss(self): """Tests boxinst maskhead loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] boxinst_bboxhead = BoxInstBboxHead( num_classes=4, in_channels=1, feat_channels=1, stacked_convs=1, norm_cfg=None) mask_feature_head = _fake_mask_feature_head() boxinst_maskhead = BoxInstMaskHead( mask_feature_head=mask_feature_head, loss_mask=dict( type='DiceLoss', use_sigmoid=True, activate=True, eps=5e-6, loss_weight=1.0)) # Fcos head expects a multiple levels of features per image feats = [] for i in range(len(boxinst_bboxhead.strides)): feats.append( torch.rand(1, 1, s // (2**(i + 3)), s // (2**(i + 3)))) feats = tuple(feats) cls_scores, bbox_preds, centernesses, param_preds =\ boxinst_bboxhead.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) gt_instances.masks = _rand_masks(0, gt_instances.bboxes.numpy(), s, s) gt_instances.pairwise_masks = _rand_masks( 0, gt_instances.bboxes.numpy(), s // 4, s // 4).to_tensor( dtype=torch.float32, device='cpu').unsqueeze(1).repeat(1, 8, 1, 1) message_hub = MessageHub.get_instance('runtime_info') message_hub.update_info('iter', 1) _ = boxinst_bboxhead.loss_by_feat(cls_scores, bbox_preds, centernesses, param_preds, [gt_instances], img_metas) # When truth is empty then all mask loss # should be zero for random inputs positive_infos = boxinst_bboxhead.get_positive_infos() mask_outs = boxinst_maskhead.forward(feats, positive_infos) empty_gt_mask_losses = boxinst_maskhead.loss_by_feat( *mask_outs, [gt_instances], img_metas, positive_infos) loss_mask_project = empty_gt_mask_losses['loss_mask_project'] loss_mask_pairwise = empty_gt_mask_losses['loss_mask_pairwise'] self.assertEqual(loss_mask_project, 0, 'mask project loss should be zero') self.assertEqual(loss_mask_pairwise, 0, 'mask pairwise loss should be zero') # When truth is non-empty then all cls, box loss and centerness loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor([[0.111, 0.222, 25.6667, 29.8757]]) gt_instances.labels = torch.LongTensor([2]) gt_instances.masks = _rand_masks(1, gt_instances.bboxes.numpy(), s, s) gt_instances.pairwise_masks = _rand_masks( 1, gt_instances.bboxes.numpy(), s // 4, s // 4).to_tensor( dtype=torch.float32, device='cpu').unsqueeze(1).repeat(1, 8, 1, 1) _ = boxinst_bboxhead.loss_by_feat(cls_scores, bbox_preds, centernesses, param_preds, [gt_instances], img_metas) positive_infos = boxinst_bboxhead.get_positive_infos() mask_outs = boxinst_maskhead.forward(feats, positive_infos) one_gt_mask_losses = boxinst_maskhead.loss_by_feat( *mask_outs, [gt_instances], img_metas, positive_infos) loss_mask_project = one_gt_mask_losses['loss_mask_project'] loss_mask_pairwise = one_gt_mask_losses['loss_mask_pairwise'] self.assertGreater(loss_mask_project, 0, 'mask project loss should be nonzero') self.assertGreater(loss_mask_pairwise, 0, 'mask pairwise loss should be nonzero') ================================================ FILE: tests/test_models/test_dense_heads/test_cascade_rpn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.dense_heads import CascadeRPNHead from mmdet.structures import DetDataSample rpn_weight = 0.7 cascade_rpn_config = ConfigDict( dict( num_stages=2, num_classes=1, stages=[ dict( type='StageCascadeRPNHead', in_channels=1, feat_channels=1, anchor_generator=dict( type='AnchorGenerator', scales=[8], ratios=[1.0], strides=[4, 8, 16, 32, 64]), adapt_cfg=dict(type='dilation', dilation=3), bridged_feature=True, with_cls=False, reg_decoded_bbox=True, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=(.0, .0, .0, .0), target_stds=(0.1, 0.1, 0.5, 0.5)), loss_bbox=dict( type='IoULoss', linear=True, loss_weight=10.0 * rpn_weight)), dict( type='StageCascadeRPNHead', in_channels=1, feat_channels=1, adapt_cfg=dict(type='offset'), bridged_feature=False, with_cls=True, reg_decoded_bbox=True, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=(.0, .0, .0, .0), target_stds=(0.05, 0.05, 0.1, 0.1)), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0 * rpn_weight), loss_bbox=dict( type='IoULoss', linear=True, loss_weight=10.0 * rpn_weight)) ], train_cfg=[ dict( assigner=dict( type='RegionAssigner', center_ratio=0.2, ignore_ratio=0.5), allowed_border=-1, pos_weight=-1, debug=False), dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.7, min_pos_iou=0.3, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=-1, pos_weight=-1, debug=False) ], test_cfg=dict(max_per_img=300, nms=dict(iou_threshold=0.8)))) class TestStageCascadeRPNHead(TestCase): def test_cascade_rpn_head_loss(self): """Tests cascade rpn head loss when truth is empty and non-empty.""" cascade_rpn_head = CascadeRPNHead(**cascade_rpn_config) s = 256 feats = [ torch.rand(1, 1, s // stride[1], s // stride[0]) for stride in cascade_rpn_head.stages[0].prior_generator.strides ] img_metas = { 'img_shape': (s, s), 'pad_shape': (s, s), 'scale_factor': 1, } sample = DetDataSample() sample.set_metainfo(img_metas) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) sample.gt_instances = gt_instances empty_gt_losses = cascade_rpn_head.loss(feats, [sample]) for key, loss in empty_gt_losses.items(): loss = sum(loss) if 'cls' in key: self.assertGreater(loss.item(), 0, 'cls loss should be non-zero') elif 'reg' in key: self.assertEqual( loss.item(), 0, 'there should be no reg loss when no ground true boxes') # When truth is non-empty then all cls, box loss and centerness loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([0]) sample.gt_instances = gt_instances one_gt_losses = cascade_rpn_head.loss(feats, [sample]) for loss in one_gt_losses.values(): loss = sum(loss) self.assertGreater( loss.item(), 0, 'cls loss, or box loss, or iou loss should be non-zero') def test_cascade_rpn_head_loss_and_predict(self): """Tests cascade rpn head loss and predict function.""" cascade_rpn_head = CascadeRPNHead(**cascade_rpn_config) s = 256 feats = [ torch.rand(1, 1, s // stride[1], s // stride[0]) for stride in cascade_rpn_head.stages[0].prior_generator.strides ] img_metas = { 'img_shape': (s, s), 'pad_shape': (s, s), 'scale_factor': 1, } sample = DetDataSample() sample.set_metainfo(img_metas) gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) sample.gt_instances = gt_instances proposal_cfg = ConfigDict( dict(max_per_img=300, nms=dict(iou_threshold=0.8))) cascade_rpn_head.loss_and_predict(feats, [sample], proposal_cfg) def test_cascade_rpn_head_predict(self): """Tests cascade rpn head predict function.""" cascade_rpn_head = CascadeRPNHead(**cascade_rpn_config) s = 256 feats = [ torch.rand(1, 1, s // stride[1], s // stride[0]) for stride in cascade_rpn_head.stages[0].prior_generator.strides ] img_metas = { 'img_shape': (s, s), 'pad_shape': (s, s), 'scale_factor': 1, } sample = DetDataSample() sample.set_metainfo(img_metas) cascade_rpn_head.predict(feats, [sample]) ================================================ FILE: tests/test_models/test_dense_heads/test_centernet_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.dense_heads import CenterNetHead class TestCenterNetHead(TestCase): def test_center_head_loss(self): """Tests center head loss when truth is empty and non-empty.""" s = 256 img_metas = [{'batch_input_shape': (s, s, 3)}] test_cfg = dict(topK=100, max_per_img=100) centernet_head = CenterNetHead( num_classes=4, in_channels=1, feat_channels=4, test_cfg=test_cfg) feat = [torch.rand(1, 1, s, s)] center_out, wh_out, offset_out = centernet_head.forward(feat) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = centernet_head.loss_by_feat(center_out, wh_out, offset_out, [gt_instances], img_metas) loss_center = empty_gt_losses['loss_center_heatmap'] loss_wh = empty_gt_losses['loss_wh'] loss_offset = empty_gt_losses['loss_offset'] assert loss_center.item() > 0, 'loss_center should be non-zero' assert loss_wh.item() == 0, ( 'there should be no loss_wh when there are no true boxes') assert loss_offset.item() == 0, ( 'there should be no loss_offset when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = centernet_head.loss_by_feat(center_out, wh_out, offset_out, [gt_instances], img_metas) loss_center = one_gt_losses['loss_center_heatmap'] loss_wh = one_gt_losses['loss_wh'] loss_offset = one_gt_losses['loss_offset'] assert loss_center.item() > 0, 'loss_center should be non-zero' assert loss_wh.item() > 0, 'loss_wh should be non-zero' assert loss_offset.item() > 0, 'loss_offset should be non-zero' def test_centernet_head_get_targets(self): """Tests center head generating and decoding the heatmap.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'batch_input_shape': (s, s), }] test_cfg = ConfigDict( dict(topk=100, local_maximum_kernel=3, max_per_img=100)) gt_bboxes = [ torch.Tensor([[10, 20, 200, 240], [40, 50, 100, 200], [10, 20, 100, 240]]) ] gt_labels = [torch.LongTensor([1, 1, 2])] centernet_head = CenterNetHead( num_classes=4, in_channels=1, feat_channels=4, test_cfg=test_cfg) self.feat_shape = (1, 1, s // 4, s // 4) targets, _ = centernet_head.get_targets(gt_bboxes, gt_labels, self.feat_shape, img_metas[0]['img_shape']) center_target = targets['center_heatmap_target'] wh_target = targets['wh_target'] offset_target = targets['offset_target'] # make sure assign target right for i in range(len(gt_bboxes[0])): bbox, label = gt_bboxes[0][i] / 4, gt_labels[0][i] ctx, cty = sum(bbox[0::2]) / 2, sum(bbox[1::2]) / 2 int_ctx, int_cty = int(sum(bbox[0::2]) / 2), int( sum(bbox[1::2]) / 2) w, h = bbox[2] - bbox[0], bbox[3] - bbox[1] x_off = ctx - int(ctx) y_off = cty - int(cty) assert center_target[0, label, int_cty, int_ctx] == 1 assert wh_target[0, 0, int_cty, int_ctx] == w assert wh_target[0, 1, int_cty, int_ctx] == h assert offset_target[0, 0, int_cty, int_ctx] == x_off assert offset_target[0, 1, int_cty, int_ctx] == y_off def test_centernet_head_get_results(self): """Tests center head generating and decoding the heatmap.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'batch_input_shape': (s, s), 'border': (0, 0, 0, 0), }] test_cfg = ConfigDict( dict( topk=100, local_maximum_kernel=3, max_per_img=100, nms=dict(type='nms', iou_threshold=0.5))) gt_bboxes = [ torch.Tensor([[10, 20, 200, 240], [40, 50, 100, 200], [10, 20, 100, 240]]) ] gt_labels = [torch.LongTensor([1, 1, 2])] centernet_head = CenterNetHead( num_classes=4, in_channels=1, feat_channels=4, test_cfg=test_cfg) self.feat_shape = (1, 1, s // 4, s // 4) targets, _ = centernet_head.get_targets(gt_bboxes, gt_labels, self.feat_shape, img_metas[0]['img_shape']) center_target = targets['center_heatmap_target'] wh_target = targets['wh_target'] offset_target = targets['offset_target'] # make sure get_bboxes is right detections = centernet_head.predict_by_feat([center_target], [wh_target], [offset_target], img_metas, rescale=True, with_nms=False) pred_instances = detections[0] out_bboxes = pred_instances.bboxes[:3] out_clses = pred_instances.labels[:3] for bbox, cls in zip(out_bboxes, out_clses): flag = False for gt_bbox, gt_cls in zip(gt_bboxes[0], gt_labels[0]): if (bbox[:4] == gt_bbox[:4]).all(): flag = True assert flag, 'get_bboxes is wrong' detections = centernet_head.predict_by_feat([center_target], [wh_target], [offset_target], img_metas, rescale=True, with_nms=True) pred_instances = detections[0] out_bboxes = pred_instances.bboxes[:3] out_clses = pred_instances.labels[:3] for bbox, cls in zip(out_bboxes, out_clses): flag = False for gt_bbox, gt_cls in zip(gt_bboxes[0], gt_labels[0]): if (bbox[:4] == gt_bbox[:4]).all(): flag = True assert flag, 'get_bboxes is wrong' ================================================ FILE: tests/test_models/test_dense_heads/test_centernet_update_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.models.dense_heads import CenterNetUpdateHead class TestCenterNetUpdateHead(TestCase): def test_centernet_update_head_loss(self): """Tests fcos head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] centernet_head = CenterNetUpdateHead( num_classes=4, in_channels=1, feat_channels=1, stacked_convs=1, norm_cfg=None) # Fcos head expects a multiple levels of features per image feats = ( torch.rand(1, 1, s // stride[1], s // stride[0]) for stride in centernet_head.prior_generator.strides) cls_scores, bbox_preds = centernet_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = centernet_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # box loss and centerness loss should be zero empty_cls_loss = empty_gt_losses['loss_cls'].item() empty_box_loss = empty_gt_losses['loss_bbox'].item() self.assertGreater(empty_cls_loss, 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss, 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then all cls, box loss and centerness loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = centernet_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'].item() onegt_box_loss = one_gt_losses['loss_bbox'].item() self.assertGreater(onegt_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss, 0, 'box loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_centripetal_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.models.dense_heads import CentripetalHead class TestCentripetalHead(TestCase): def test_centripetal_head_loss(self): """Tests corner head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, 'batch_input_shape': (s, s, 3) }] centripetal_head = CentripetalHead( num_classes=4, in_channels=1, corner_emb_channels=0) # Corner head expects a multiple levels of features per image feat = [ torch.rand(1, 1, s // 4, s // 4) for _ in range(centripetal_head.num_feat_levels) ] forward_outputs = centripetal_head.forward(feat) # Test that empty ground truth encourages the network # to predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) gt_bboxes_ignore = None empty_gt_losses = centripetal_head.loss_by_feat( *forward_outputs, [gt_instances], img_metas, gt_bboxes_ignore) empty_det_loss = sum(empty_gt_losses['det_loss']) empty_guiding_loss = sum(empty_gt_losses['guiding_loss']) empty_centripetal_loss = sum(empty_gt_losses['centripetal_loss']) empty_off_loss = sum(empty_gt_losses['off_loss']) self.assertTrue(empty_det_loss.item() > 0, 'det loss should be non-zero') self.assertTrue( empty_guiding_loss.item() == 0, 'there should be no guiding loss when there are no true boxes') self.assertTrue( empty_centripetal_loss.item() == 0, 'there should be no centripetal loss when there are no true boxes') self.assertTrue( empty_off_loss.item() == 0, 'there should be no box loss when there are no true boxes') gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874], [123.6667, 123.8757, 138.6326, 251.8874]]) gt_instances.labels = torch.LongTensor([2, 3]) two_gt_losses = centripetal_head.loss_by_feat(*forward_outputs, [gt_instances], img_metas, gt_bboxes_ignore) twogt_det_loss = sum(two_gt_losses['det_loss']) twogt_guiding_loss = sum(two_gt_losses['guiding_loss']) twogt_centripetal_loss = sum(two_gt_losses['centripetal_loss']) twogt_off_loss = sum(two_gt_losses['off_loss']) assert twogt_det_loss.item() > 0, 'det loss should be non-zero' assert twogt_guiding_loss.item() > 0, 'push loss should be non-zero' assert twogt_centripetal_loss.item( ) > 0, 'pull loss should be non-zero' assert twogt_off_loss.item() > 0, 'off loss should be non-zero' ================================================ FILE: tests/test_models/test_dense_heads/test_condinst_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import numpy as np import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.dense_heads import CondInstBboxHead, CondInstMaskHead from mmdet.structures.mask import BitmapMasks def _rand_masks(num_items, bboxes, img_w, img_h): rng = np.random.RandomState(0) masks = np.zeros((num_items, img_h, img_w), dtype=np.float32) for i, bbox in enumerate(bboxes): bbox = bbox.astype(np.int32) mask = (rng.rand(1, bbox[3] - bbox[1], bbox[2] - bbox[0]) > 0.3).astype(np.int64) masks[i:i + 1, bbox[1]:bbox[3], bbox[0]:bbox[2]] = mask return BitmapMasks(masks, height=img_h, width=img_w) def _fake_mask_feature_head(): mask_feature_head = ConfigDict( in_channels=1, feat_channels=1, start_level=0, end_level=2, out_channels=8, mask_stride=8, num_stacked_convs=4, norm_cfg=dict(type='BN', requires_grad=True)) return mask_feature_head class TestCondInstHead(TestCase): def test_condinst_bboxhead_loss(self): """Tests condinst bboxhead loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] condinst_bboxhead = CondInstBboxHead( num_classes=4, in_channels=1, feat_channels=1, stacked_convs=1, norm_cfg=None) # Fcos head expects a multiple levels of features per image feats = ( torch.rand(1, 1, s // stride[1], s // stride[0]) for stride in condinst_bboxhead.prior_generator.strides) cls_scores, bbox_preds, centernesses, param_preds =\ condinst_bboxhead.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) gt_instances.masks = _rand_masks(0, gt_instances.bboxes.numpy(), s, s) empty_gt_losses = condinst_bboxhead.loss_by_feat( cls_scores, bbox_preds, centernesses, param_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # box loss and centerness loss should be zero empty_cls_loss = empty_gt_losses['loss_cls'].item() empty_box_loss = empty_gt_losses['loss_bbox'].item() empty_ctr_loss = empty_gt_losses['loss_centerness'].item() self.assertGreater(empty_cls_loss, 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss, 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_ctr_loss, 0, 'there should be no centerness loss when there are no true boxes') # When truth is non-empty then all cls, box loss and centerness loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) gt_instances.masks = _rand_masks(1, gt_instances.bboxes.numpy(), s, s) one_gt_losses = condinst_bboxhead.loss_by_feat(cls_scores, bbox_preds, centernesses, param_preds, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'].item() onegt_box_loss = one_gt_losses['loss_bbox'].item() onegt_ctr_loss = one_gt_losses['loss_centerness'].item() self.assertGreater(onegt_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss, 0, 'box loss should be non-zero') self.assertGreater(onegt_ctr_loss, 0, 'centerness loss should be non-zero') # Test the `center_sampling` works fine. condinst_bboxhead.center_sampling = True ctrsamp_losses = condinst_bboxhead.loss_by_feat( cls_scores, bbox_preds, centernesses, param_preds, [gt_instances], img_metas) ctrsamp_cls_loss = ctrsamp_losses['loss_cls'].item() ctrsamp_box_loss = ctrsamp_losses['loss_bbox'].item() ctrsamp_ctr_loss = ctrsamp_losses['loss_centerness'].item() self.assertGreater(ctrsamp_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(ctrsamp_box_loss, 0, 'box loss should be non-zero') self.assertGreater(ctrsamp_ctr_loss, 0, 'centerness loss should be non-zero') # Test the `norm_on_bbox` works fine. condinst_bboxhead.norm_on_bbox = True normbox_losses = condinst_bboxhead.loss_by_feat( cls_scores, bbox_preds, centernesses, param_preds, [gt_instances], img_metas) normbox_cls_loss = normbox_losses['loss_cls'].item() normbox_box_loss = normbox_losses['loss_bbox'].item() normbox_ctr_loss = normbox_losses['loss_centerness'].item() self.assertGreater(normbox_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(normbox_box_loss, 0, 'box loss should be non-zero') self.assertGreater(normbox_ctr_loss, 0, 'centerness loss should be non-zero') def test_condinst_maskhead_loss(self): """Tests condinst maskhead loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] condinst_bboxhead = CondInstBboxHead( num_classes=4, in_channels=1, feat_channels=1, stacked_convs=1, norm_cfg=None) mask_feature_head = _fake_mask_feature_head() condinst_maskhead = CondInstMaskHead( mask_feature_head=mask_feature_head, loss_mask=dict( type='DiceLoss', use_sigmoid=True, activate=True, eps=5e-6, loss_weight=1.0)) # Fcos head expects a multiple levels of features per image feats = [] for i in range(len(condinst_bboxhead.strides)): feats.append( torch.rand(1, 1, s // (2**(i + 3)), s // (2**(i + 3)))) feats = tuple(feats) cls_scores, bbox_preds, centernesses, param_preds =\ condinst_bboxhead.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) gt_instances.masks = _rand_masks(0, gt_instances.bboxes.numpy(), s, s) _ = condinst_bboxhead.loss_by_feat(cls_scores, bbox_preds, centernesses, param_preds, [gt_instances], img_metas) # When truth is empty then all mask loss # should be zero for random inputs positive_infos = condinst_bboxhead.get_positive_infos() mask_outs = condinst_maskhead.forward(feats, positive_infos) empty_gt_mask_losses = condinst_maskhead.loss_by_feat( *mask_outs, [gt_instances], img_metas, positive_infos) loss_mask = empty_gt_mask_losses['loss_mask'] self.assertEqual(loss_mask, 0, 'mask loss should be zero') # When truth is non-empty then all cls, box loss and centerness loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) gt_instances.masks = _rand_masks(1, gt_instances.bboxes.numpy(), s, s) _ = condinst_bboxhead.loss_by_feat(cls_scores, bbox_preds, centernesses, param_preds, [gt_instances], img_metas) positive_infos = condinst_bboxhead.get_positive_infos() mask_outs = condinst_maskhead.forward(feats, positive_infos) one_gt_mask_losses = condinst_maskhead.loss_by_feat( *mask_outs, [gt_instances], img_metas, positive_infos) loss_mask = one_gt_mask_losses['loss_mask'] self.assertGreater(loss_mask, 0, 'mask loss should be nonzero') ================================================ FILE: tests/test_models/test_dense_heads/test_corner_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.evaluation import bbox_overlaps from mmdet.models.dense_heads import CornerHead class TestCornerHead(TestCase): def test_corner_head_loss(self): """Tests corner head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, 'batch_input_shape': (s, s, 3) }] corner_head = CornerHead(num_classes=4, in_channels=1) # Corner head expects a multiple levels of features per image feat = [ torch.rand(1, 1, s // 4, s // 4) for _ in range(corner_head.num_feat_levels) ] forward_outputs = corner_head.forward(feat) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) gt_bboxes_ignore = None empty_gt_losses = corner_head.loss_by_feat(*forward_outputs, [gt_instances], img_metas, gt_bboxes_ignore) empty_det_loss = sum(empty_gt_losses['det_loss']) empty_push_loss = sum(empty_gt_losses['push_loss']) empty_pull_loss = sum(empty_gt_losses['pull_loss']) empty_off_loss = sum(empty_gt_losses['off_loss']) self.assertTrue(empty_det_loss.item() > 0, 'det loss should be non-zero') self.assertTrue( empty_push_loss.item() == 0, 'there should be no push loss when there are no true boxes') self.assertTrue( empty_pull_loss.item() == 0, 'there should be no pull loss when there are no true boxes') self.assertTrue( empty_off_loss.item() == 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = corner_head.loss_by_feat(*forward_outputs, [gt_instances], img_metas, gt_bboxes_ignore) onegt_det_loss = sum(one_gt_losses['det_loss']) onegt_push_loss = sum(one_gt_losses['push_loss']) onegt_pull_loss = sum(one_gt_losses['pull_loss']) onegt_off_loss = sum(one_gt_losses['off_loss']) self.assertTrue(onegt_det_loss.item() > 0, 'det loss should be non-zero') self.assertTrue( onegt_push_loss.item() == 0, 'there should be no push loss when there are only one true box') self.assertTrue(onegt_pull_loss.item() > 0, 'pull loss should be non-zero') self.assertTrue(onegt_off_loss.item() > 0, 'off loss should be non-zero') gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874], [123.6667, 123.8757, 138.6326, 251.8874]]) gt_instances.labels = torch.LongTensor([2, 3]) two_gt_losses = corner_head.loss_by_feat(*forward_outputs, [gt_instances], img_metas, gt_bboxes_ignore) twogt_det_loss = sum(two_gt_losses['det_loss']) twogt_push_loss = sum(two_gt_losses['push_loss']) twogt_pull_loss = sum(two_gt_losses['pull_loss']) twogt_off_loss = sum(two_gt_losses['off_loss']) self.assertTrue(twogt_det_loss.item() > 0, 'det loss should be non-zero') # F.relu limits push loss larger than or equal to 0. self.assertTrue(twogt_push_loss.item() >= 0, 'push loss should be non-zero') self.assertTrue(twogt_pull_loss.item() > 0, 'pull loss should be non-zero') self.assertTrue(twogt_off_loss.item() > 0, 'off loss should be non-zero') def test_corner_head_encode_and_decode_heatmap(self): """Tests corner head generating and decoding the heatmap.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, 'batch_input_shape': (s, s, 3), 'border': (0, 0, 0, 0) }] gt_bboxes = [ torch.Tensor([[10, 20, 200, 240], [40, 50, 100, 200], [10, 20, 200, 240]]) ] gt_labels = [torch.LongTensor([1, 1, 2])] corner_head = CornerHead( num_classes=4, in_channels=1, corner_emb_channels=1) feat = [ torch.rand(1, 1, s // 4, s // 4) for _ in range(corner_head.num_feat_levels) ] targets = corner_head.get_targets( gt_bboxes, gt_labels, feat[0].shape, img_metas[0]['batch_input_shape'], with_corner_emb=corner_head.with_corner_emb) gt_tl_heatmap = targets['topleft_heatmap'] gt_br_heatmap = targets['bottomright_heatmap'] gt_tl_offset = targets['topleft_offset'] gt_br_offset = targets['bottomright_offset'] embedding = targets['corner_embedding'] [top, left], [bottom, right] = embedding[0][0] gt_tl_embedding_heatmap = torch.zeros([1, 1, s // 4, s // 4]) gt_br_embedding_heatmap = torch.zeros([1, 1, s // 4, s // 4]) gt_tl_embedding_heatmap[0, 0, top, left] = 1 gt_br_embedding_heatmap[0, 0, bottom, right] = 1 batch_bboxes, batch_scores, batch_clses = corner_head._decode_heatmap( tl_heat=gt_tl_heatmap, br_heat=gt_br_heatmap, tl_off=gt_tl_offset, br_off=gt_br_offset, tl_emb=gt_tl_embedding_heatmap, br_emb=gt_br_embedding_heatmap, img_meta=img_metas[0], k=100, kernel=3, distance_threshold=0.5) bboxes = batch_bboxes.view(-1, 4) scores = batch_scores.view(-1, 1) clses = batch_clses.view(-1, 1) idx = scores.argsort(dim=0, descending=True) bboxes = bboxes[idx].view(-1, 4) scores = scores[idx].view(-1) clses = clses[idx].view(-1) valid_bboxes = bboxes[torch.where(scores > 0.05)] valid_labels = clses[torch.where(scores > 0.05)] max_coordinate = valid_bboxes.max() offsets = valid_labels.to(valid_bboxes) * (max_coordinate + 1) gt_offsets = gt_labels[0].to(gt_bboxes[0]) * (max_coordinate + 1) offset_bboxes = valid_bboxes + offsets[:, None] offset_gtbboxes = gt_bboxes[0] + gt_offsets[:, None] iou_matrix = bbox_overlaps(offset_bboxes.numpy(), offset_gtbboxes.numpy()) self.assertEqual((iou_matrix == 1).sum(), 3) ================================================ FILE: tests/test_models/test_dense_heads/test_ddod_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import DDODHead class TestDDODHead(TestCase): def test_ddod_head_loss(self): """Tests ddod head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1 }] cfg = Config( dict( assigner=dict(type='ATSSAssigner', topk=9, alpha=0.8), reg_assigner=dict(type='ATSSAssigner', topk=9, alpha=0.5), allowed_border=-1, pos_weight=-1, debug=False)) atss_head = DDODHead( num_classes=4, in_channels=1, stacked_convs=1, feat_channels=1, use_dcn=False, norm_cfg=None, train_cfg=cfg, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), loss_iou=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0)) feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [8, 16, 32, 64, 128] ] cls_scores, bbox_preds, centernesses = atss_head.forward(feat) # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = atss_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = sum(empty_gt_losses['loss_cls']) empty_box_loss = sum(empty_gt_losses['loss_bbox']) empty_centerness_loss = sum(empty_gt_losses['loss_iou']) self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_centerness_loss.item(), 0, 'there should be no centerness loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = atss_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']) onegt_box_loss = sum(one_gt_losses['loss_bbox']) onegt_centerness_loss = sum(one_gt_losses['loss_iou']) self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') self.assertGreater(onegt_centerness_loss.item(), 0, 'centerness loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_embedding_rpn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import pytest import torch from mmengine.structures import InstanceData from mmdet.models.dense_heads import EmbeddingRPNHead from mmdet.structures import DetDataSample class TestEmbeddingRPNHead(TestCase): def test_init(self): """Test init rpn head.""" rpn_head = EmbeddingRPNHead( num_proposals=100, proposal_feature_channel=256) rpn_head.init_weights() self.assertTrue(rpn_head.init_proposal_bboxes) self.assertTrue(rpn_head.init_proposal_features) def test_loss_and_predict(self): s = 256 img_meta = { 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, } rpn_head = EmbeddingRPNHead( num_proposals=100, proposal_feature_channel=256) feats = [ torch.rand(2, 1, s // (2**(i + 2)), s // (2**(i + 2))) for i in range(5) ] data_sample = DetDataSample() data_sample.set_metainfo(img_meta) # test predict result_list = rpn_head.predict(feats, [data_sample]) self.assertTrue(isinstance(result_list, list)) self.assertTrue(isinstance(result_list[0], InstanceData)) # test loss_and_predict result_list = rpn_head.loss_and_predict(feats, [data_sample]) self.assertTrue(isinstance(result_list, tuple)) self.assertTrue(isinstance(result_list[0], dict)) self.assertEqual(len(result_list[0]), 0) self.assertTrue(isinstance(result_list[1], list)) self.assertTrue(isinstance(result_list[1][0], InstanceData)) # test loss with pytest.raises(NotImplementedError): rpn_head.loss(feats, [data_sample]) ================================================ FILE: tests/test_models/test_dense_heads/test_fcos_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.models.dense_heads import FCOSHead class TestFCOSHead(TestCase): def test_fcos_head_loss(self): """Tests fcos head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] fcos_head = FCOSHead( num_classes=4, in_channels=1, feat_channels=1, stacked_convs=1, norm_cfg=None) # Fcos head expects a multiple levels of features per image feats = ( torch.rand(1, 1, s // stride[1], s // stride[0]) for stride in fcos_head.prior_generator.strides) cls_scores, bbox_preds, centernesses = fcos_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = fcos_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # box loss and centerness loss should be zero empty_cls_loss = empty_gt_losses['loss_cls'].item() empty_box_loss = empty_gt_losses['loss_bbox'].item() empty_ctr_loss = empty_gt_losses['loss_centerness'].item() self.assertGreater(empty_cls_loss, 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss, 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_ctr_loss, 0, 'there should be no centerness loss when there are no true boxes') # When truth is non-empty then all cls, box loss and centerness loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = fcos_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'].item() onegt_box_loss = one_gt_losses['loss_bbox'].item() onegt_ctr_loss = one_gt_losses['loss_centerness'].item() self.assertGreater(onegt_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss, 0, 'box loss should be non-zero') self.assertGreater(onegt_ctr_loss, 0, 'centerness loss should be non-zero') # Test the `center_sampling` works fine. fcos_head.center_sampling = True ctrsamp_losses = fcos_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) ctrsamp_cls_loss = ctrsamp_losses['loss_cls'].item() ctrsamp_box_loss = ctrsamp_losses['loss_bbox'].item() ctrsamp_ctr_loss = ctrsamp_losses['loss_centerness'].item() self.assertGreater(ctrsamp_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(ctrsamp_box_loss, 0, 'box loss should be non-zero') self.assertGreater(ctrsamp_ctr_loss, 0, 'centerness loss should be non-zero') # Test the `norm_on_bbox` works fine. fcos_head.norm_on_bbox = True normbox_losses = fcos_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) normbox_cls_loss = normbox_losses['loss_cls'].item() normbox_box_loss = normbox_losses['loss_bbox'].item() normbox_ctr_loss = normbox_losses['loss_centerness'].item() self.assertGreater(normbox_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(normbox_box_loss, 0, 'box loss should be non-zero') self.assertGreater(normbox_ctr_loss, 0, 'centerness loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_fovea_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.models.dense_heads import FoveaHead class TestFOVEAHead(TestCase): def test_fovea_head_loss(self): """Tests anchor head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] fovea_head = FoveaHead(num_classes=4, in_channels=1) # Anchor head expects a multiple levels of features per image feats = ( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2))) for i in range(len(fovea_head.prior_generator.strides))) cls_scores, bbox_preds = fovea_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = fovea_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # there should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_box_loss = empty_gt_losses['loss_bbox'] self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = fovea_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_box_loss = one_gt_losses['loss_bbox'] self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_free_anchor_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import FreeAnchorRetinaHead class TestFreeAnchorRetinaHead(TestCase): def test_free_anchor_head_loss(self): """Tests rpn head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] anchor_head = FreeAnchorRetinaHead(num_classes=1, in_channels=1) # Anchor head expects a multiple levels of features per image feats = ( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2))) for i in range(len(anchor_head.prior_generator.strides))) cls_scores, bbox_preds = anchor_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = anchor_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # there should be no box loss. positive_bag_loss = empty_gt_losses['positive_bag_loss'] negative_bag_loss = empty_gt_losses['negative_bag_loss'] self.assertGreater(negative_bag_loss.item(), 0, 'negative_bag loss should be non-zero') self.assertEqual( positive_bag_loss.item(), 0, 'there should be no positive_bag loss when there are no true boxes' ) # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([0]) one_gt_losses = anchor_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['positive_bag_loss'] onegt_box_loss = one_gt_losses['negative_bag_loss'] self.assertGreater(onegt_cls_loss.item(), 0, 'positive bag loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'negative bag loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_fsaf_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from math import ceil from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet.models.dense_heads import FSAFHead class TestFSAFHead(TestCase): def test_fsaf_head_loss(self): """Tests fsaf head loss when truth is empty and non-empty.""" s = 300 img_metas = [{ 'img_shape': (s, s), 'pad_shape': (s, s), 'scale_factor': 1, }] cfg = Config( dict( assigner=dict( type='CenterRegionAssigner', pos_scale=0.2, neg_scale=0.2, min_pos_iof=0.01), allowed_border=-1, pos_weight=-1, debug=False)) fsaf_head = FSAFHead( num_classes=4, in_channels=1, stacked_convs=1, feat_channels=1, reg_decoded_bbox=True, anchor_generator=dict( type='AnchorGenerator', octave_base_scale=1, scales_per_octave=1, ratios=[1.0], strides=[8, 16, 32, 64, 128]), bbox_coder=dict(type='TBLRBBoxCoder', normalizer=4.0), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0, reduction='none'), loss_bbox=dict( type='IoULoss', eps=1e-6, loss_weight=1.0, reduction='none'), train_cfg=cfg) # FSAF head expects a multiple levels of features per image feats = ( torch.rand(1, 1, ceil(s / stride[0]), ceil(s / stride[0])) for stride in fsaf_head.prior_generator.strides) cls_scores, bbox_preds = fsaf_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = fsaf_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # box loss should be zero empty_cls_loss = sum(empty_gt_losses['loss_cls']) empty_box_loss = sum(empty_gt_losses['loss_bbox']) self.assertGreater(empty_cls_loss, 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = fsaf_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']) onegt_box_loss = sum(one_gt_losses['loss_bbox']) self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_ga_retina_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import ConfigDict from mmdet.models.dense_heads import GARetinaHead ga_retina_head_config = ConfigDict( dict( num_classes=4, in_channels=4, feat_channels=4, stacked_convs=1, approx_anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), square_anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], scales=[4], strides=[8, 16, 32, 64, 128]), anchor_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loc_filter_thr=0.01, loss_loc=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_shape=dict(type='BoundedIoULoss', beta=0.2, loss_weight=1.0), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=0.04, loss_weight=1.0), train_cfg=dict( ga_assigner=dict( type='ApproxMaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0.4, ignore_iof_thr=-1), ga_sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.0, ignore_iof_thr=-1), allowed_border=-1, pos_weight=-1, center_ratio=0.2, ignore_ratio=0.5, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100))) class TestGARetinaHead(TestCase): def test_ga_retina_head_init_and_forward(self): """The GARetinaHead inherit loss and prediction function from GuidedAchorHead. Here, we only test GARetinaHet initialization and forward. """ # Test initializaion ga_retina_head = GARetinaHead(**ga_retina_head_config) # Test forward s = 256 feats = ( torch.rand(1, 4, s // stride[1], s // stride[0]) for stride in ga_retina_head.square_anchor_generator.strides) ga_retina_head(feats) ================================================ FILE: tests/test_models/test_dense_heads/test_ga_rpn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.dense_heads import GARPNHead ga_rpn_config = ConfigDict( dict( num_classes=1, in_channels=4, feat_channels=4, approx_anchor_generator=dict( type='AnchorGenerator', octave_base_scale=8, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[4, 8, 16, 32, 64]), square_anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], scales=[8], strides=[4, 8, 16, 32, 64]), anchor_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[0.07, 0.07, 0.14, 0.14]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[0.07, 0.07, 0.11, 0.11]), loc_filter_thr=0.01, loss_loc=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_shape=dict(type='BoundedIoULoss', beta=0.2, loss_weight=1.0), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=1.0, loss_weight=1.0), train_cfg=dict( ga_assigner=dict( type='ApproxMaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, ignore_iof_thr=-1), ga_sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=-1, center_ratio=0.2, ignore_ratio=0.5, pos_weight=-1, debug=False), test_cfg=dict( nms_pre=1000, ms_post=1000, max_per_img=300, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0))) class TestGARPNHead(TestCase): def test_ga_rpn_head_loss(self): """Tests ga rpn head loss.""" s = 256 img_metas = [{ 'img_shape': (s, s), 'pad_shape': (s, s), 'scale_factor': (1, 1) }] ga_rpn_head = GARPNHead(**ga_rpn_config) feats = ( torch.rand(1, 4, s // stride[1], s // stride[0]) for stride in ga_rpn_head.square_anchor_generator.strides) outs = ga_rpn_head(feats) # When truth is non-empty then all cls, box loss and centerness loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([0]) one_gt_losses = ga_rpn_head.loss_by_feat(*outs, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_rpn_cls']).item() onegt_box_loss = sum(one_gt_losses['loss_rpn_bbox']).item() onegt_shape_loss = sum(one_gt_losses['loss_anchor_shape']).item() onegt_loc_loss = sum(one_gt_losses['loss_anchor_loc']).item() self.assertGreater(onegt_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss, 0, 'box loss should be non-zero') self.assertGreater(onegt_shape_loss, 0, 'shape loss should be non-zero') self.assertGreater(onegt_loc_loss, 0, 'location loss should be non-zero') def test_ga_rpn_head_predict_by_feat(self): s = 256 img_metas = [{ 'img_shape': (s, s), 'pad_shape': (s, s), 'scale_factor': (1, 1) }] ga_rpn_head = GARPNHead(**ga_rpn_config) feats = ( torch.rand(1, 4, s // stride[1], s // stride[0]) for stride in ga_rpn_head.square_anchor_generator.strides) outs = ga_rpn_head(feats) cfg = ConfigDict( dict( nms_pre=2000, nms_post=1000, max_per_img=300, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0)) ga_rpn_head.predict_by_feat( *outs, batch_img_metas=img_metas, cfg=cfg, rescale=True) ================================================ FILE: tests/test_models/test_dense_heads/test_gfl_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import GFLHead class TestGFLHead(TestCase): def test_gfl_head_loss(self): """Tests gfl head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1 }] train_cfg = Config( dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False)) gfl_head = GFLHead( num_classes=4, in_channels=1, stacked_convs=1, train_cfg=train_cfg, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), loss_cls=dict( type='QualityFocalLoss', use_sigmoid=True, beta=2.0, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0)) feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [4, 8, 16, 32, 64] ] cls_scores, bbox_preds = gfl_head.forward(feat) # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = gfl_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = sum(empty_gt_losses['loss_cls']) empty_box_loss = sum(empty_gt_losses['loss_bbox']) empty_dfl_loss = sum(empty_gt_losses['loss_dfl']) self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_dfl_loss.item(), 0, 'there should be no dfl loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = gfl_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']) onegt_box_loss = sum(one_gt_losses['loss_bbox']) onegt_dfl_loss = sum(one_gt_losses['loss_dfl']) self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') self.assertGreater(onegt_dfl_loss.item(), 0, 'dfl loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_guided_anchor_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.dense_heads import GuidedAnchorHead guided_anchor_head_config = ConfigDict( dict( num_classes=4, in_channels=4, feat_channels=4, approx_anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), square_anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], scales=[4], strides=[8, 16, 32, 64, 128]), anchor_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loc_filter_thr=0.01, loss_loc=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_shape=dict(type='BoundedIoULoss', beta=0.2, loss_weight=1.0), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=0.04, loss_weight=1.0), train_cfg=dict( ga_assigner=dict( type='ApproxMaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0.4, ignore_iof_thr=-1), ga_sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.0, ignore_iof_thr=-1), allowed_border=-1, pos_weight=-1, center_ratio=0.2, ignore_ratio=0.5, debug=False), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100))) class TestGuidedAnchorHead(TestCase): def test_guided_anchor_head_loss(self): """Tests guided anchor loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s), 'pad_shape': (s, s), 'scale_factor': (1, 1) }] guided_anchor_head = GuidedAnchorHead(**guided_anchor_head_config) feats = ( torch.rand(1, 4, s // stride[1], s // stride[0]) for stride in guided_anchor_head.square_anchor_generator.strides) outs = guided_anchor_head(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = guided_anchor_head.loss_by_feat( *outs, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # box shape and location loss should be zero empty_cls_loss = sum(empty_gt_losses['loss_cls']).item() empty_box_loss = sum(empty_gt_losses['loss_bbox']).item() empty_shape_loss = sum(empty_gt_losses['loss_shape']).item() empty_loc_loss = sum(empty_gt_losses['loss_loc']).item() self.assertGreater(empty_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(empty_loc_loss, 0, 'location loss should be non-zero') self.assertEqual( empty_box_loss, 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_shape_loss, 0, 'there should be no shape loss when there are no true boxes') # When truth is non-empty then all cls, box loss and centerness loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = guided_anchor_head.loss_by_feat( *outs, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']).item() onegt_box_loss = sum(one_gt_losses['loss_bbox']).item() onegt_shape_loss = sum(one_gt_losses['loss_shape']).item() onegt_loc_loss = sum(one_gt_losses['loss_loc']).item() self.assertGreater(onegt_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss, 0, 'box loss should be non-zero') self.assertGreater(onegt_shape_loss, 0, 'shape loss should be non-zero') self.assertGreater(onegt_loc_loss, 0, 'location loss should be non-zero') def test_guided_anchor_head_predict_by_feat(self): s = 256 img_metas = [{ 'img_shape': (s, s), 'pad_shape': (s, s), 'scale_factor': (1, 1) }] guided_anchor_head = GuidedAnchorHead(**guided_anchor_head_config) feats = ( torch.rand(1, 4, s // stride[1], s // stride[0]) for stride in guided_anchor_head.square_anchor_generator.strides) outs = guided_anchor_head(feats) guided_anchor_head.predict_by_feat( *outs, batch_img_metas=img_metas, rescale=True) ================================================ FILE: tests/test_models/test_dense_heads/test_lad_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import numpy as np import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import LADHead, lad_head from mmdet.models.dense_heads.lad_head import levels_to_images class TestLADHead(TestCase): def test_lad_head_loss(self): """Tests lad head loss when truth is empty and non-empty.""" class mock_skm: def GaussianMixture(self, *args, **kwargs): return self def fit(self, loss): pass def predict(self, loss): components = np.zeros_like(loss, dtype=np.long) return components.reshape(-1) def score_samples(self, loss): scores = np.random.random(len(loss)) return scores lad_head.skm = mock_skm() s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1 }] train_cfg = Config( dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.1, neg_iou_thr=0.1, min_pos_iou=0, ignore_iof_thr=-1), allowed_border=-1, pos_weight=-1, debug=False)) # since Focal Loss is not supported on CPU # since Focal Loss is not supported on CPU lad = LADHead( num_classes=4, in_channels=1, train_cfg=train_cfg, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=1.3), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=0.5)) teacher_model = LADHead( num_classes=4, in_channels=1, train_cfg=train_cfg, loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=1.3), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=0.5)) feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [4, 8, 16, 32, 64] ] lad.init_weights() teacher_model.init_weights() # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) batch_gt_instances_ignore = None outs_teacher = teacher_model(feat) label_assignment_results = teacher_model.get_label_assignment( *outs_teacher, [gt_instances], img_metas, batch_gt_instances_ignore) outs = teacher_model(feat) empty_gt_losses = lad.loss_by_feat(*outs, [gt_instances], img_metas, batch_gt_instances_ignore, label_assignment_results) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_box_loss = empty_gt_losses['loss_bbox'] empty_iou_loss = empty_gt_losses['loss_iou'] self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_iou_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) batch_gt_instances_ignore = None label_assignment_results = teacher_model.get_label_assignment( *outs_teacher, [gt_instances], img_metas, batch_gt_instances_ignore) one_gt_losses = lad.loss_by_feat(*outs, [gt_instances], img_metas, batch_gt_instances_ignore, label_assignment_results) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_box_loss = one_gt_losses['loss_bbox'] onegt_iou_loss = one_gt_losses['loss_iou'] self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') self.assertGreater(onegt_iou_loss.item(), 0, 'box loss should be non-zero') n, c, h, w = 10, 4, 20, 20 mlvl_tensor = [torch.ones(n, c, h, w) for i in range(5)] results = levels_to_images(mlvl_tensor) self.assertEqual(len(results), n) self.assertEqual(results[0].size(), (h * w * 5, c)) self.assertTrue(lad.with_score_voting) lad = LADHead( num_classes=4, in_channels=1, train_cfg=train_cfg, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=1.3), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=0.5)) cls_scores = [torch.ones(2, 4, 5, 5)] bbox_preds = [torch.ones(2, 4, 5, 5)] iou_preds = [torch.ones(2, 1, 5, 5)] cfg = Config( dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) rescale = False lad.predict_by_feat( cls_scores, bbox_preds, iou_preds, img_metas, cfg, rescale=rescale) ================================================ FILE: tests/test_models/test_dense_heads/test_ld_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import GFLHead, LDHead class TestLDHead(TestCase): def test_ld_head_loss(self): """Tests ld head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1 }] train_cfg = Config( dict( assigner=dict(type='ATSSAssigner', topk=9, ignore_iof_thr=0.1), allowed_border=-1, pos_weight=-1, debug=False)) ld_head = LDHead( num_classes=4, in_channels=1, train_cfg=train_cfg, loss_ld=dict( type='KnowledgeDistillationKLDivLoss', loss_weight=1.0), loss_cls=dict( type='QualityFocalLoss', use_sigmoid=True, beta=2.0, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128])) teacher_model = GFLHead( num_classes=4, in_channels=1, train_cfg=train_cfg, loss_cls=dict( type='QualityFocalLoss', use_sigmoid=True, beta=2.0, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128])) feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [4, 8, 16, 32, 64] ] cls_scores, bbox_preds = ld_head.forward(feat) rand_soft_target = teacher_model.forward(feat)[1] # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) batch_gt_instances_ignore = None empty_gt_losses = ld_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas, rand_soft_target, batch_gt_instances_ignore) # When there is no truth, the cls loss should be nonzero, ld loss # should be non-negative but there should be no box loss. empty_cls_loss = sum(empty_gt_losses['loss_cls']) empty_box_loss = sum(empty_gt_losses['loss_bbox']) empty_ld_loss = sum(empty_gt_losses['loss_ld']) self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') self.assertGreaterEqual(empty_ld_loss.item(), 0, 'ld loss should be non-negative') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) batch_gt_instances_ignore = None one_gt_losses = ld_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas, rand_soft_target, batch_gt_instances_ignore) onegt_cls_loss = sum(one_gt_losses['loss_cls']) onegt_box_loss = sum(one_gt_losses['loss_bbox']) self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') batch_gt_instances_ignore = gt_instances # When truth is non-empty but ignored then the cls loss should be # nonzero, but there should be no box loss. ignore_gt_losses = ld_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas, rand_soft_target, batch_gt_instances_ignore) ignore_cls_loss = sum(ignore_gt_losses['loss_cls']) ignore_box_loss = sum(ignore_gt_losses['loss_bbox']) self.assertGreater(ignore_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual(ignore_box_loss.item(), 0, 'gt bbox ignored loss should be zero') # When truth is non-empty and not ignored then both cls and box loss # should be nonzero for random inputs batch_gt_instances_ignore = InstanceData() batch_gt_instances_ignore.bboxes = torch.randn(1, 4) not_ignore_gt_losses = ld_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas, rand_soft_target, batch_gt_instances_ignore) not_ignore_cls_loss = sum(not_ignore_gt_losses['loss_cls']) not_ignore_box_loss = sum(not_ignore_gt_losses['loss_bbox']) self.assertGreater(not_ignore_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreaterEqual(not_ignore_box_loss.item(), 0, 'gt bbox not ignored loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_nasfcos_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.models.dense_heads import NASFCOSHead class TestNASFCOSHead(TestCase): def test_nasfcos_head_loss(self): """Tests nasfcos head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] nasfcos_head = NASFCOSHead( num_classes=4, in_channels=2, # the same as `deform_groups` in dconv3x3_config feat_channels=2, norm_cfg=None) # Nasfcos head expects a multiple levels of features per image feats = ( torch.rand(1, 2, s // stride[1], s // stride[0]).float() for stride in nasfcos_head.prior_generator.strides) cls_scores, bbox_preds, centernesses = nasfcos_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = nasfcos_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # box loss and centerness loss should be zero empty_cls_loss = empty_gt_losses['loss_cls'].item() empty_box_loss = empty_gt_losses['loss_bbox'].item() empty_ctr_loss = empty_gt_losses['loss_centerness'].item() self.assertGreater(empty_cls_loss, 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss, 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_ctr_loss, 0, 'there should be no centerness loss when there are no true boxes') # When truth is non-empty then all cls, box loss and centerness loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = nasfcos_head.loss_by_feat(cls_scores, bbox_preds, centernesses, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'].item() onegt_box_loss = one_gt_losses['loss_bbox'].item() onegt_ctr_loss = one_gt_losses['loss_centerness'].item() self.assertGreater(onegt_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss, 0, 'box loss should be non-zero') self.assertGreater(onegt_ctr_loss, 0, 'centerness loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_paa_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import numpy as np import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import PAAHead, paa_head from mmdet.models.utils import levels_to_images class TestPAAHead(TestCase): def test_paa_head_loss(self): """Tests paa head loss when truth is empty and non-empty.""" class mock_skm: def GaussianMixture(self, *args, **kwargs): return self def fit(self, loss): pass def predict(self, loss): components = np.zeros_like(loss, dtype=np.long) return components.reshape(-1) def score_samples(self, loss): scores = np.random.random(len(loss)) return scores paa_head.skm = mock_skm() s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] train_cfg = Config( dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.1, neg_iou_thr=0.1, min_pos_iou=0, ignore_iof_thr=-1), allowed_border=-1, pos_weight=-1, debug=False)) # since Focal Loss is not supported on CPU paa = PAAHead( num_classes=4, in_channels=1, train_cfg=train_cfg, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=1.3), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=0.5)) feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [4, 8, 16, 32, 64] ] paa.init_weights() cls_scores, bbox_preds, iou_preds = paa(feat) # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = paa.loss_by_feat(cls_scores, bbox_preds, iou_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_box_loss = empty_gt_losses['loss_bbox'] empty_iou_loss = empty_gt_losses['loss_iou'] self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_iou_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = paa.loss_by_feat(cls_scores, bbox_preds, iou_preds, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_box_loss = one_gt_losses['loss_bbox'] onegt_iou_loss = one_gt_losses['loss_iou'] self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') self.assertGreater(onegt_iou_loss.item(), 0, 'box loss should be non-zero') n, c, h, w = 10, 4, 20, 20 mlvl_tensor = [torch.ones(n, c, h, w) for i in range(5)] results = levels_to_images(mlvl_tensor) self.assertEqual(len(results), n) self.assertEqual(results[0].size(), (h * w * 5, c)) self.assertTrue(paa.with_score_voting) paa = PAAHead( num_classes=4, in_channels=1, train_cfg=train_cfg, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8]), loss_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=1.3), loss_centerness=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=0.5)) cls_scores = [torch.ones(2, 4, 5, 5)] bbox_preds = [torch.ones(2, 4, 5, 5)] iou_preds = [torch.ones(2, 1, 5, 5)] cfg = Config( dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) rescale = False paa.predict_by_feat( cls_scores, bbox_preds, iou_preds, img_metas, cfg, rescale=rescale) ================================================ FILE: tests/test_models/test_dense_heads/test_pisa_retinanet_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from math import ceil from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import PISARetinaHead class TestPISARetinaHead(TestCase): def test_pisa_reitnanet_head_loss(self): """Tests pisa retinanet head loss when truth is empty and non-empty.""" s = 300 img_metas = [{ 'img_shape': (s, s), 'pad_shape': (s, s), 'scale_factor': 1, }] cfg = Config( dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0, ignore_iof_thr=-1), isr=dict(k=2., bias=0.), carl=dict(k=1., bias=0.2), sampler=dict(type='PseudoSampler'), allowed_border=-1, pos_weight=-1, debug=False)) pisa_retinanet_head = PISARetinaHead( num_classes=4, in_channels=1, stacked_convs=1, feat_channels=256, anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0]), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='SmoothL1Loss', beta=0.11, loss_weight=1.0), train_cfg=cfg) # pisa retina head expects a multiple levels of features per image feats = ( torch.rand(1, 1, ceil(s / stride[0]), ceil(s / stride[0])) for stride in pisa_retinanet_head.prior_generator.strides) cls_scores, bbox_preds = pisa_retinanet_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = pisa_retinanet_head.loss_by_feat( cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, cls_loss and box_loss should all be zero. empty_cls_loss = empty_gt_losses['loss_cls'] empty_box_loss = empty_gt_losses['loss_bbox'] empty_carl_loss = empty_gt_losses['loss_carl'] self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_carl_loss.item(), 0, 'there should be no carl loss when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = pisa_retinanet_head.loss_by_feat( cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_box_loss = one_gt_losses['loss_bbox'] onegt_carl_loss = one_gt_losses['loss_carl'] self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') self.assertGreater(onegt_carl_loss.item(), 0, 'carl loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_pisa_ssd_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from math import ceil from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import PISASSDHead class TestPISASSDHead(TestCase): def test_pisa_ssd_head_loss(self): """Tests pisa ssd head loss when truth is empty and non-empty.""" s = 300 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] cfg = Config( dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0., ignore_iof_thr=-1, gt_max_assign_all=False), sampler=dict(type='PseudoSampler'), smoothl1_beta=1., allowed_border=-1, pos_weight=-1, neg_pos_ratio=3, debug=False)) pisa_ssd_head = PISASSDHead( num_classes=4, in_channels=(1, 1, 1, 1, 1, 1), anchor_generator=dict( type='SSDAnchorGenerator', scale_major=False, input_size=s, basesize_ratio_range=(0.15, 0.9), strides=[8, 16, 32, 64, 100, 300], ratios=[[2], [2, 3], [2, 3], [2, 3], [2], [2]]), train_cfg=cfg) # PISA SSD head expects a multiple levels of features per image feats = ( torch.rand(1, 1, ceil(s / stride[0]), ceil(s / stride[0])) for stride in pisa_ssd_head.prior_generator.strides) cls_scores, bbox_preds = pisa_ssd_head.forward(feats) # test without isr and carl # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = pisa_ssd_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, cls_loss and box_loss should all be zero. empty_cls_loss = sum(empty_gt_losses['loss_cls']) empty_box_loss = sum(empty_gt_losses['loss_bbox']) self.assertEqual( empty_cls_loss.item(), 0, 'there should be no cls loss when there are no true boxes') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = pisa_ssd_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']) onegt_box_loss = sum(one_gt_losses['loss_bbox']) self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') pisa_ssd_head.train_cfg.update( dict(isr=dict(k=2., bias=0.), carl=dict(k=1., bias=0.2))) # test with isr and carl # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = pisa_ssd_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, cls_loss and box_loss should all be zero. empty_cls_loss = sum(empty_gt_losses['loss_cls']) empty_box_loss = sum(empty_gt_losses['loss_bbox']) self.assertEqual( empty_cls_loss.item(), 0, 'there should be no cls loss when there are no true boxes') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = pisa_ssd_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']) onegt_box_loss = sum(one_gt_losses['loss_bbox']) self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_reppoints_head.py ================================================ import unittest import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from parameterized import parameterized from mmdet.models.dense_heads import RepPointsHead from mmdet.structures import DetDataSample class TestRepPointsHead(unittest.TestCase): @parameterized.expand(['moment', 'minmax', 'partial_minmax']) def test_head_loss(self, transform_method='moment'): cfg = ConfigDict( dict( num_classes=2, in_channels=32, point_feat_channels=10, num_points=9, gradient_mul=0.1, point_strides=[8, 16, 32, 64, 128], point_base_scale=4, loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox_init=dict( type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=0.5), loss_bbox_refine=dict( type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.0), use_grid_points=False, center_init=True, transform_method=transform_method, moment_mul=0.01, init_cfg=dict( type='Normal', layer='Conv2d', std=0.01, override=dict( type='Normal', name='reppoints_cls_out', std=0.01, bias_prob=0.01)), train_cfg=dict( init=dict( assigner=dict( type='PointAssigner', scale=4, pos_num=1), allowed_border=-1, pos_weight=-1, debug=False), refine=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0, ignore_iof_thr=-1), allowed_border=-1, pos_weight=-1, debug=False)), test_cfg=dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100))) reppoints_head = RepPointsHead(**cfg) s = 256 img_metas = [{ 'img_shape': (s, s), 'scale_factor': (1, 1), 'pad_shape': (s, s), 'batch_input_shape': (s, s) }] x = [ torch.rand(1, 32, s // 2**(i + 2), s // 2**(i + 2)) for i in range(5) ] # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) gt_bboxes_ignore = None reppoints_head.train() forward_outputs = reppoints_head.forward(x) empty_gt_losses = reppoints_head.loss_by_feat(*forward_outputs, [gt_instances], img_metas, gt_bboxes_ignore) # When there is no truth, the cls loss should be nonzero but there # should be no pts loss. for key, losses in empty_gt_losses.items(): for loss in losses: if 'cls' in key: self.assertGreater(loss.item(), 0, 'cls loss should be non-zero') elif 'pts' in key: self.assertEqual( loss.item(), 0, 'there should be no reg loss when no ground true boxes' ) # When truth is non-empty then both cls and pts loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = reppoints_head.loss_by_feat(*forward_outputs, [gt_instances], img_metas, gt_bboxes_ignore) # loss_cls should all be non-zero self.assertTrue( all([loss.item() > 0 for loss in one_gt_losses['loss_cls']])) # only one level loss_pts_init is non-zero cnt_non_zero = 0 for loss in one_gt_losses['loss_pts_init']: if loss.item() != 0: cnt_non_zero += 1 self.assertEqual(cnt_non_zero, 1) # only one level loss_pts_refine is non-zero cnt_non_zero = 0 for loss in one_gt_losses['loss_pts_init']: if loss.item() != 0: cnt_non_zero += 1 self.assertEqual(cnt_non_zero, 1) # test loss samples = DetDataSample() samples.set_metainfo(img_metas[0]) samples.gt_instances = gt_instances reppoints_head.loss(x, [samples]) # test only predict reppoints_head.eval() reppoints_head.predict(x, [samples], rescale=True) ================================================ FILE: tests/test_models/test_dense_heads/test_retina_sepBN_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import RetinaSepBNHead class TestRetinaSepBNHead(TestCase): def test_init(self): """Test init RetinaSepBN head.""" anchor_head = RetinaSepBNHead(num_classes=1, num_ins=1, in_channels=1) anchor_head.init_weights() self.assertTrue(anchor_head.cls_convs) self.assertTrue(anchor_head.reg_convs) self.assertTrue(anchor_head.retina_cls) self.assertTrue(anchor_head.retina_reg) def test_retina_sepbn_head_loss(self): """Tests RetinaSepBN head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] cfg = Config( dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0, ignore_iof_thr=-1), sampler=dict(type='PseudoSampler' ), # Focal loss should use PseudoSampler allowed_border=-1, pos_weight=-1, debug=False)) anchor_head = RetinaSepBNHead( num_classes=4, num_ins=5, in_channels=1, train_cfg=cfg) # Anchor head expects a multiple levels of features per image feats = [] for i in range(len(anchor_head.prior_generator.strides)): feats.append( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2)))) cls_scores, bbox_preds = anchor_head.forward(tuple(feats)) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = anchor_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # there should be no box loss. empty_cls_loss = sum(empty_gt_losses['loss_cls']) empty_box_loss = sum(empty_gt_losses['loss_bbox']) self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = anchor_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']) onegt_box_loss = sum(one_gt_losses['loss_bbox']) self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_rpn_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import pytest import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import RPNHead class TestRPNHead(TestCase): def test_init(self): """Test init rpn head.""" rpn_head = RPNHead(num_classes=1, in_channels=1) self.assertTrue(rpn_head.rpn_conv) self.assertTrue(rpn_head.rpn_cls) self.assertTrue(rpn_head.rpn_reg) # rpn_head.num_convs > 1 rpn_head = RPNHead(num_classes=1, in_channels=1, num_convs=2) self.assertTrue(rpn_head.rpn_conv) self.assertTrue(rpn_head.rpn_cls) self.assertTrue(rpn_head.rpn_reg) def test_rpn_head_loss(self): """Tests rpn head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] cfg = Config( dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.7, neg_iou_thr=0.3, min_pos_iou=0.3, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=256, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), allowed_border=0, pos_weight=-1, debug=False)) rpn_head = RPNHead(num_classes=1, in_channels=1, train_cfg=cfg) # Anchor head expects a multiple levels of features per image feats = ( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2))) for i in range(len(rpn_head.prior_generator.strides))) cls_scores, bbox_preds = rpn_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = rpn_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # there should be no box loss. empty_cls_loss = sum(empty_gt_losses['loss_rpn_cls']) empty_box_loss = sum(empty_gt_losses['loss_rpn_bbox']) self.assertGreater(empty_cls_loss.item(), 0, 'rpn cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([0]) one_gt_losses = rpn_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_rpn_cls']) onegt_box_loss = sum(one_gt_losses['loss_rpn_bbox']) self.assertGreater(onegt_cls_loss.item(), 0, 'rpn cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'rpn box loss should be non-zero') # When there is no valid anchor, the loss will be None, # and this will raise a ValueError. img_metas = [{ 'img_shape': (8, 8, 3), 'pad_shape': (8, 8, 3), 'scale_factor': 1, }] with pytest.raises(ValueError): rpn_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) def test_bbox_post_process(self): """Test the length of detection instance results is 0.""" from mmengine.config import ConfigDict cfg = ConfigDict( nms_pre=1000, max_per_img=1000, nms=dict(type='nms', iou_threshold=0.7), min_bbox_size=0) rpn_head = RPNHead(num_classes=1, in_channels=1) results = InstanceData(metainfo=dict()) results.bboxes = torch.zeros((0, 4)) results.scores = torch.zeros(0) results = rpn_head._bbox_post_process(results, cfg, img_meta=dict()) self.assertEqual(len(results), 0) self.assertEqual(results.bboxes.size(), (0, 4)) self.assertEqual(results.scores.size(), (0, )) self.assertEqual(results.labels.size(), (0, )) ================================================ FILE: tests/test_models/test_dense_heads/test_sabl_retina_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.dense_heads import SABLRetinaHead class TestSABLRetinaHead(TestCase): def test_sabl_retina_head(self): """Tests sabl retina head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s), 'pad_shape': (s, s), 'scale_factor': [1, 1], }] train_cfg = ConfigDict( dict( assigner=dict( type='ApproxMaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.4, min_pos_iou=0.0, ignore_iof_thr=-1), allowed_border=-1, pos_weight=-1, debug=False)) sabl_retina_head = SABLRetinaHead( num_classes=4, in_channels=1, feat_channels=1, stacked_convs=1, approx_anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), square_anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], scales=[4], strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='BucketingBBoxCoder', num_buckets=14, scale_factor=3.0), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox_cls=dict( type='CrossEntropyLoss', use_sigmoid=True, loss_weight=1.5), loss_bbox_reg=dict( type='SmoothL1Loss', beta=1.0 / 9.0, loss_weight=1.5), train_cfg=train_cfg) # Fcos head expects a multiple levels of features per image feats = ( torch.rand(1, 1, s // stride[1], s // stride[0]) for stride in sabl_retina_head.square_anchor_generator.strides) outs = sabl_retina_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = sabl_retina_head.loss_by_feat( *outs, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but # box loss and centerness loss should be zero empty_cls_loss = sum(empty_gt_losses['loss_cls']).item() empty_box_cls_loss = sum(empty_gt_losses['loss_bbox_cls']).item() empty_box_reg_loss = sum(empty_gt_losses['loss_bbox_reg']).item() self.assertGreater(empty_cls_loss, 0, 'cls loss should be non-zero') self.assertEqual( empty_box_cls_loss, 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_box_reg_loss, 0, 'there should be no centerness loss when there are no true boxes') # When truth is non-empty then all cls, box loss and centerness loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = sabl_retina_head.loss_by_feat(*outs, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']).item() onegt_box_cls_loss = sum(one_gt_losses['loss_bbox_cls']).item() onegt_box_reg_loss = sum(one_gt_losses['loss_bbox_reg']).item() self.assertGreater(onegt_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_cls_loss, 0, 'box loss should be non-zero') self.assertGreater(onegt_box_reg_loss, 0, 'centerness loss should be non-zero') test_cfg = ConfigDict( dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100)) # test predict_by_feat sabl_retina_head.predict_by_feat( *outs, batch_img_metas=img_metas, cfg=test_cfg, rescale=True) ================================================ FILE: tests/test_models/test_dense_heads/test_solo_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import numpy as np import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from parameterized import parameterized from mmdet import * # noqa from mmdet.models.dense_heads import (DecoupledSOLOHead, DecoupledSOLOLightHead, SOLOHead) from mmdet.structures.mask import BitmapMasks def _rand_masks(num_items, bboxes, img_w, img_h): rng = np.random.RandomState(0) masks = np.zeros((num_items, img_h, img_w)) for i, bbox in enumerate(bboxes): bbox = bbox.astype(np.int32) mask = (rng.rand(1, bbox[3] - bbox[1], bbox[2] - bbox[0]) > 0.3).astype(np.int64) masks[i:i + 1, bbox[1]:bbox[3], bbox[0]:bbox[2]] = mask return BitmapMasks(masks, height=img_h, width=img_w) class TestSOLOHead(TestCase): @parameterized.expand([(SOLOHead, ), (DecoupledSOLOHead, ), (DecoupledSOLOLightHead, )]) def test_mask_head_loss(self, MaskHead): """Tests mask head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'ori_shape': (s, s, 3), 'scale_factor': 1, 'batch_input_shape': (s, s, 3) }] mask_head = MaskHead(num_classes=4, in_channels=1) # SOLO head expects a multiple levels of features per image feats = [] for i in range(len(mask_head.strides)): feats.append( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2)))) feats = tuple(feats) mask_outs = mask_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty(0, 4) gt_instances.labels = torch.LongTensor([]) gt_instances.masks = _rand_masks(0, gt_instances.bboxes.numpy(), s, s) empty_gt_losses = mask_head.loss_by_feat( *mask_outs, batch_gt_instances=[gt_instances], batch_img_metas=img_metas) # When there is no truth, the cls loss should be nonzero but # there should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_mask_loss = empty_gt_losses['loss_mask'] self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_mask_loss.item(), 0, 'there should be no mask loss when there are no true mask') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) gt_instances.masks = _rand_masks(1, gt_instances.bboxes.numpy(), s, s) one_gt_losses = mask_head.loss_by_feat( *mask_outs, batch_gt_instances=[gt_instances], batch_img_metas=img_metas) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_mask_loss = one_gt_losses['loss_mask'] self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_mask_loss.item(), 0, 'mask loss should be non-zero') def test_solo_head_empty_result(self): s = 256 img_metas = { 'img_shape': (s, s, 3), 'ori_shape': (s, s, 3), 'scale_factor': 1, 'batch_input_shape': (s, s, 3) } mask_head = SOLOHead(num_classes=4, in_channels=1) cls_scores = torch.empty(0, 80) mask_preds = torch.empty(0, 16, 16) test_cfg = ConfigDict( score_thr=0.1, mask_thr=0.5, ) results = mask_head._predict_by_feat_single( cls_scores=cls_scores, mask_preds=mask_preds, img_meta=img_metas, cfg=test_cfg) self.assertIsInstance(results, InstanceData) self.assertEqual(len(results), 0) def test_decoupled_solo_head_empty_result(self): s = 256 img_metas = { 'img_shape': (s, s, 3), 'ori_shape': (s, s, 3), 'scale_factor': 1, 'batch_input_shape': (s, s, 3) } mask_head = DecoupledSOLOHead(num_classes=4, in_channels=1) cls_scores = torch.empty(0, 80) mask_preds_x = torch.empty(0, 16, 16) mask_preds_y = torch.empty(0, 16, 16) test_cfg = ConfigDict( score_thr=0.1, mask_thr=0.5, ) results = mask_head._predict_by_feat_single( cls_scores=cls_scores, mask_preds_x=mask_preds_x, mask_preds_y=mask_preds_y, img_meta=img_metas, cfg=test_cfg) self.assertIsInstance(results, InstanceData) self.assertEqual(len(results), 0) ================================================ FILE: tests/test_models/test_dense_heads/test_solov2_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import numpy as np import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import SOLOV2Head from mmdet.structures.mask import BitmapMasks def _rand_masks(num_items, bboxes, img_w, img_h): rng = np.random.RandomState(0) masks = np.zeros((num_items, img_h, img_w)) for i, bbox in enumerate(bboxes): bbox = bbox.astype(np.int32) mask = (rng.rand(1, bbox[3] - bbox[1], bbox[2] - bbox[0]) > 0.3).astype(np.int64) masks[i:i + 1, bbox[1]:bbox[3], bbox[0]:bbox[2]] = mask return BitmapMasks(masks, height=img_h, width=img_w) def _fake_mask_feature_head(): mask_feature_head = ConfigDict( feat_channels=128, start_level=0, end_level=3, out_channels=256, mask_stride=4, norm_cfg=dict(type='GN', num_groups=32, requires_grad=True)) return mask_feature_head class TestSOLOv2Head(TestCase): def test_solov2_head_loss(self): """Tests mask head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'ori_shape': (s, s, 3), 'scale_factor': 1, 'batch_input_shape': (s, s, 3) }] mask_feature_head = _fake_mask_feature_head() mask_head = SOLOV2Head( num_classes=4, in_channels=1, mask_feature_head=mask_feature_head) # SOLO head expects a multiple levels of features per image feats = [] for i in range(len(mask_head.strides)): feats.append( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2)))) feats = tuple(feats) mask_outs = mask_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty(0, 4) gt_instances.labels = torch.LongTensor([]) gt_instances.masks = _rand_masks(0, gt_instances.bboxes.numpy(), s, s) empty_gt_losses = mask_head.loss_by_feat( *mask_outs, batch_gt_instances=[gt_instances], batch_img_metas=img_metas) # When there is no truth, the cls loss should be nonzero but # there should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_mask_loss = empty_gt_losses['loss_mask'] self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_mask_loss.item(), 0, 'there should be no mask loss when there are no true mask') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) gt_instances.masks = _rand_masks(1, gt_instances.bboxes.numpy(), s, s) one_gt_losses = mask_head.loss_by_feat( *mask_outs, batch_gt_instances=[gt_instances], batch_img_metas=img_metas) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_mask_loss = one_gt_losses['loss_mask'] self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_mask_loss.item(), 0, 'mask loss should be non-zero') def test_solov2_head_empty_result(self): s = 256 img_metas = { 'img_shape': (s, s, 3), 'ori_shape': (s, s, 3), 'scale_factor': 1, 'batch_input_shape': (s, s, 3) } mask_feature_head = _fake_mask_feature_head() mask_head = SOLOV2Head( num_classes=4, in_channels=1, mask_feature_head=mask_feature_head) kernel_preds = torch.empty(0, 128) cls_scores = torch.empty(0, 80) mask_feats = torch.empty(0, 16, 16) test_cfg = ConfigDict( score_thr=0.1, mask_thr=0.5, ) results = mask_head._predict_by_feat_single( kernel_preds=kernel_preds, cls_scores=cls_scores, mask_feats=mask_feats, img_meta=img_metas, cfg=test_cfg) self.assertIsInstance(results, InstanceData) self.assertEqual(len(results), 0) ================================================ FILE: tests/test_models/test_dense_heads/test_ssd_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from math import ceil from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import SSDHead class TestSSDHead(TestCase): def test_ssd_head_loss(self): """Tests ssd head loss when truth is empty and non-empty.""" s = 300 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1, }] cfg = Config( dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0., ignore_iof_thr=-1, gt_max_assign_all=False), sampler=dict(type='PseudoSampler'), smoothl1_beta=1., allowed_border=-1, pos_weight=-1, neg_pos_ratio=3, debug=False)) ssd_head = SSDHead( num_classes=4, in_channels=(1, 1, 1, 1, 1, 1), stacked_convs=1, feat_channels=1, use_depthwise=True, anchor_generator=dict( type='SSDAnchorGenerator', scale_major=False, input_size=s, basesize_ratio_range=(0.15, 0.9), strides=[8, 16, 32, 64, 100, 300], ratios=[[2], [2, 3], [2, 3], [2, 3], [2], [2]]), train_cfg=cfg) # SSD head expects a multiple levels of features per image feats = ( torch.rand(1, 1, ceil(s / stride[0]), ceil(s / stride[0])) for stride in ssd_head.prior_generator.strides) cls_scores, bbox_preds = ssd_head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = ssd_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, cls_loss and box_loss should all be zero. empty_cls_loss = sum(empty_gt_losses['loss_cls']) empty_box_loss = sum(empty_gt_losses['loss_bbox']) self.assertEqual( empty_cls_loss.item(), 0, 'there should be no cls loss when there are no true boxes') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = ssd_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = sum(one_gt_losses['loss_cls']) onegt_box_loss = sum(one_gt_losses['loss_bbox']) self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_tood_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import Config, MessageHub from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import TOODHead def _tood_head(anchor_type): """Set type of tood head.""" train_cfg = Config( dict( initial_epoch=4, initial_assigner=dict(type='ATSSAssigner', topk=9), assigner=dict(type='TaskAlignedAssigner', topk=13), alpha=1, beta=6, allowed_border=-1, pos_weight=-1, debug=False)) test_cfg = Config( dict( nms_pre=1000, min_bbox_size=0, score_thr=0.05, nms=dict(type='nms', iou_threshold=0.6), max_per_img=100)) tood_head = TOODHead( num_classes=80, in_channels=1, stacked_convs=1, feat_channels=8, # the same as `la_down_rate` in TaskDecomposition norm_cfg=None, anchor_type=anchor_type, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], octave_base_scale=8, scales_per_octave=1, strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[0.1, 0.1, 0.2, 0.2]), initial_loss_cls=dict( type='FocalLoss', use_sigmoid=True, activated=True, # use probability instead of logit as input gamma=2.0, alpha=0.25, loss_weight=1.0), loss_cls=dict( type='QualityFocalLoss', use_sigmoid=True, activated=True, # use probability instead of logit as input beta=2.0, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=2.0), train_cfg=train_cfg, test_cfg=test_cfg) return tood_head class TestTOODHead(TestCase): def test_tood_head_anchor_free_loss(self): """Tests tood head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1 }] tood_head = _tood_head('anchor_free') tood_head.init_weights() feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [8, 16, 32, 64, 128] ] cls_scores, bbox_preds = tood_head(feat) message_hub = MessageHub.get_instance('runtime_info') message_hub.update_info('epoch', 0) # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) gt_bboxes_ignore = None empty_gt_losses = tood_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas, gt_bboxes_ignore) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_box_loss = empty_gt_losses['loss_bbox'] self.assertGreater( sum(empty_cls_loss).item(), 0, 'cls loss should be non-zero') self.assertEqual( sum(empty_box_loss).item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) gt_bboxes_ignore = None one_gt_losses = tood_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas, gt_bboxes_ignore) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_box_loss = one_gt_losses['loss_bbox'] self.assertGreater( sum(onegt_cls_loss).item(), 0, 'cls loss should be non-zero') self.assertGreater( sum(onegt_box_loss).item(), 0, 'box loss should be non-zero') # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) gt_bboxes_ignore = None empty_gt_losses = tood_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas, gt_bboxes_ignore) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_box_loss = empty_gt_losses['loss_bbox'] self.assertGreater( sum(empty_cls_loss).item(), 0, 'cls loss should be non-zero') self.assertEqual( sum(empty_box_loss).item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) gt_bboxes_ignore = None one_gt_losses = tood_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas, gt_bboxes_ignore) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_box_loss = one_gt_losses['loss_bbox'] self.assertGreater( sum(onegt_cls_loss).item(), 0, 'cls loss should be non-zero') self.assertGreater( sum(onegt_box_loss).item(), 0, 'box loss should be non-zero') def test_tood_head_anchor_based_loss(self): """Tests tood head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'pad_shape': (s, s, 3), 'scale_factor': 1 }] tood_head = _tood_head('anchor_based') tood_head.init_weights() feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [8, 16, 32, 64, 128] ] cls_scores, bbox_preds = tood_head(feat) message_hub = MessageHub.get_instance('runtime_info') message_hub.update_info('epoch', 0) # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) gt_bboxes_ignore = None empty_gt_losses = tood_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas, gt_bboxes_ignore) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_box_loss = empty_gt_losses['loss_bbox'] self.assertGreater( sum(empty_cls_loss).item(), 0, 'cls loss should be non-zero') self.assertEqual( sum(empty_box_loss).item(), 0, 'there should be no box loss when there are no true boxes') ================================================ FILE: tests/test_models/test_dense_heads/test_vfnet_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import VFNetHead class TestVFNetHead(TestCase): def test_vfnet_head_loss(self): """Tests vfnet head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, 'pad_shape': (s, s, 3) }] train_cfg = Config( dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False)) # since VarFocal Loss is not supported on CPU vfnet_head = VFNetHead( num_classes=4, in_channels=1, train_cfg=train_cfg, loss_cls=dict( type='VarifocalLoss', use_sigmoid=True, loss_weight=1.0)) feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [4, 8, 16, 32, 64] ] cls_scores, bbox_preds, bbox_preds_refine = vfnet_head.forward(feat) # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = vfnet_head.loss_by_feat(cls_scores, bbox_preds, bbox_preds_refine, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_box_loss = empty_gt_losses['loss_bbox'] self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = vfnet_head.loss_by_feat(cls_scores, bbox_preds, bbox_preds_refine, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_box_loss = one_gt_losses['loss_bbox'] self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') def test_vfnet_head_loss_without_atss(self): """Tests vfnet head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, 'pad_shape': (s, s, 3) }] train_cfg = Config( dict( assigner=dict(type='ATSSAssigner', topk=9), allowed_border=-1, pos_weight=-1, debug=False)) # since VarFocal Loss is not supported on CPU vfnet_head = VFNetHead( num_classes=4, in_channels=1, train_cfg=train_cfg, use_atss=False, loss_cls=dict( type='VarifocalLoss', use_sigmoid=True, loss_weight=1.0)) feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [4, 8, 16, 32, 64] ] cls_scores, bbox_preds, bbox_preds_refine = vfnet_head.forward(feat) # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = vfnet_head.loss_by_feat(cls_scores, bbox_preds, bbox_preds_refine, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_box_loss = empty_gt_losses['loss_bbox'] self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = vfnet_head.loss_by_feat(cls_scores, bbox_preds, bbox_preds_refine, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_box_loss = one_gt_losses['loss_bbox'] self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_yolo_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import Config from mmengine.structures import InstanceData from mmdet.models.dense_heads import YOLOV3Head class TestYOLOV3Head(TestCase): def test_yolo_head_loss(self): """Tests YOLO head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] head = YOLOV3Head( num_classes=4, in_channels=[1, 1, 1], out_channels=[1, 1, 1], train_cfg=Config( dict( assigner=dict( type='GridAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0)))) head.init_weights() # YOLO head expects a multiple levels of features per image feats = [ torch.rand(1, 1, s // stride[1], s // stride[0]) for stride in head.prior_generator.strides ] predmaps, = head.forward(feats) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = head.loss_by_feat(predmaps, [gt_instances], img_metas) # When there is no truth, the conf loss should be nonzero but # cls loss and xy&wh loss should be zero empty_cls_loss = sum(empty_gt_losses['loss_cls']).item() empty_conf_loss = sum(empty_gt_losses['loss_conf']).item() empty_xy_loss = sum(empty_gt_losses['loss_xy']).item() empty_wh_loss = sum(empty_gt_losses['loss_wh']).item() self.assertGreater(empty_conf_loss, 0, 'conf loss should be non-zero') self.assertEqual( empty_cls_loss, 0, 'there should be no cls loss when there are no true boxes') self.assertEqual( empty_xy_loss, 0, 'there should be no xy loss when there are no true boxes') self.assertEqual( empty_wh_loss, 0, 'there should be no wh loss when there are no true boxes') # When truth is non-empty then all conf, cls loss and xywh loss # should be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = head.loss_by_feat(predmaps, [gt_instances], img_metas) one_gt_cls_loss = sum(one_gt_losses['loss_cls']).item() one_gt_conf_loss = sum(one_gt_losses['loss_conf']).item() one_gt_xy_loss = sum(one_gt_losses['loss_xy']).item() one_gt_wh_loss = sum(one_gt_losses['loss_wh']).item() self.assertGreater(one_gt_conf_loss, 0, 'conf loss should be non-zero') self.assertGreater(one_gt_cls_loss, 0, 'cls loss should be non-zero') self.assertGreater(one_gt_xy_loss, 0, 'xy loss should be non-zero') self.assertGreater(one_gt_wh_loss, 0, 'wh loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_yolof_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import Config from mmengine.structures import InstanceData from mmdet import * # noqa from mmdet.models.dense_heads import YOLOFHead class TestYOLOFHead(TestCase): def test_yolof_head_loss(self): """Tests yolof head loss when truth is empty and non-empty.""" s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, 'pad_shape': (s, s, 3) }] train_cfg = Config( dict( assigner=dict( type='UniformAssigner', pos_ignore_thr=0.15, neg_ignore_thr=0.7), allowed_border=-1, pos_weight=-1, debug=False)) yolof_head = YOLOFHead( num_classes=4, in_channels=1, feat_channels=1, reg_decoded_bbox=True, train_cfg=train_cfg, anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], scales=[1, 2, 4, 8, 16], strides=[32]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1., 1., 1., 1.], add_ctr_clamp=True, ctr_clamp=32), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=1.0), loss_bbox=dict(type='GIoULoss', loss_weight=1.0)) feat = [torch.rand(1, 1, s // 32, s // 32)] cls_scores, bbox_preds = yolof_head.forward(feat) # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) empty_gt_losses = yolof_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'] empty_box_loss = empty_gt_losses['loss_bbox'] self.assertGreater(empty_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) one_gt_losses = yolof_head.loss_by_feat(cls_scores, bbox_preds, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'] onegt_box_loss = one_gt_losses['loss_bbox'] self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') ================================================ FILE: tests/test_models/test_dense_heads/test_yolox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmcv.cnn import ConvModule, DepthwiseSeparableConvModule from mmengine.config import Config from mmengine.model import bias_init_with_prob from mmengine.structures import InstanceData from mmengine.testing import assert_allclose from mmdet.models.dense_heads import YOLOXHead class TestYOLOXHead(TestCase): def test_init_weights(self): head = YOLOXHead( num_classes=4, in_channels=1, stacked_convs=1, use_depthwise=False) head.init_weights() bias_init = bias_init_with_prob(0.01) for conv_cls, conv_obj in zip(head.multi_level_conv_cls, head.multi_level_conv_obj): assert_allclose(conv_cls.bias.data, torch.ones_like(conv_cls.bias.data) * bias_init) assert_allclose(conv_obj.bias.data, torch.ones_like(conv_obj.bias.data) * bias_init) def test_predict_by_feat(self): s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': (1.0, 1.0), }] test_cfg = Config( dict(score_thr=0.01, nms=dict(type='nms', iou_threshold=0.65))) head = YOLOXHead( num_classes=4, in_channels=1, stacked_convs=1, use_depthwise=False, test_cfg=test_cfg) feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [4, 8, 16] ] cls_scores, bbox_preds, objectnesses = head.forward(feat) head.predict_by_feat( cls_scores, bbox_preds, objectnesses, img_metas, cfg=test_cfg, rescale=True, with_nms=True) head.predict_by_feat( cls_scores, bbox_preds, objectnesses, img_metas, cfg=test_cfg, rescale=False, with_nms=False) def test_loss_by_feat(self): s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] train_cfg = Config( dict( assigner=dict( type='SimOTAAssigner', center_radius=2.5, candidate_topk=10, iou_weight=3.0, cls_weight=1.0))) head = YOLOXHead( num_classes=4, in_channels=1, stacked_convs=1, use_depthwise=False, train_cfg=train_cfg) assert not head.use_l1 assert isinstance(head.multi_level_cls_convs[0][0], ConvModule) feat = [ torch.rand(1, 1, s // feat_size, s // feat_size) for feat_size in [4, 8, 16] ] cls_scores, bbox_preds, objectnesses = head.forward(feat) # Test that empty ground truth encourages the network to predict # background gt_instances = InstanceData( bboxes=torch.empty((0, 4)), labels=torch.LongTensor([])) empty_gt_losses = head.loss_by_feat(cls_scores, bbox_preds, objectnesses, [gt_instances], img_metas) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. empty_cls_loss = empty_gt_losses['loss_cls'].sum() empty_box_loss = empty_gt_losses['loss_bbox'].sum() empty_obj_loss = empty_gt_losses['loss_obj'].sum() self.assertEqual( empty_cls_loss.item(), 0, 'there should be no cls loss when there are no true boxes') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when there are no true boxes') self.assertGreater(empty_obj_loss.item(), 0, 'objectness loss should be non-zero') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs head = YOLOXHead( num_classes=4, in_channels=1, stacked_convs=1, use_depthwise=True, train_cfg=train_cfg) assert isinstance(head.multi_level_cls_convs[0][0], DepthwiseSeparableConvModule) head.use_l1 = True gt_instances = InstanceData( bboxes=torch.Tensor([[23.6667, 23.8757, 238.6326, 151.8874]]), labels=torch.LongTensor([2])) one_gt_losses = head.loss_by_feat(cls_scores, bbox_preds, objectnesses, [gt_instances], img_metas) onegt_cls_loss = one_gt_losses['loss_cls'].sum() onegt_box_loss = one_gt_losses['loss_bbox'].sum() onegt_obj_loss = one_gt_losses['loss_obj'].sum() onegt_l1_loss = one_gt_losses['loss_l1'].sum() self.assertGreater(onegt_cls_loss.item(), 0, 'cls loss should be non-zero') self.assertGreater(onegt_box_loss.item(), 0, 'box loss should be non-zero') self.assertGreater(onegt_obj_loss.item(), 0, 'obj loss should be non-zero') self.assertGreater(onegt_l1_loss.item(), 0, 'l1 loss should be non-zero') # Test groud truth out of bound gt_instances = InstanceData( bboxes=torch.Tensor([[s * 4, s * 4, s * 4 + 10, s * 4 + 10]]), labels=torch.LongTensor([2])) empty_gt_losses = head.loss_by_feat(cls_scores, bbox_preds, objectnesses, [gt_instances], img_metas) # When gt_bboxes out of bound, the assign results should be empty, # so the cls and bbox loss should be zero. empty_cls_loss = empty_gt_losses['loss_cls'].sum() empty_box_loss = empty_gt_losses['loss_bbox'].sum() empty_obj_loss = empty_gt_losses['loss_obj'].sum() self.assertEqual( empty_cls_loss.item(), 0, 'there should be no cls loss when gt_bboxes out of bound') self.assertEqual( empty_box_loss.item(), 0, 'there should be no box loss when gt_bboxes out of bound') self.assertGreater(empty_obj_loss.item(), 0, 'objectness loss should be non-zero') ================================================ FILE: tests/test_models/test_detectors/test_conditional_detr.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.testing import get_detector_cfg from mmdet.utils import register_all_modules class TestConditionalDETR(TestCase): def setUp(self) -> None: register_all_modules() def test_conditional_detr_head_loss(self): """Tests transformer head loss when truth is empty and non-empty.""" s = 256 metainfo = { 'img_shape': (s, s), 'scale_factor': (1, 1), 'pad_shape': (s, s), 'batch_input_shape': (s, s) } img_metas = DetDataSample() img_metas.set_metainfo(metainfo) batch_data_samples = [] batch_data_samples.append(img_metas) config = get_detector_cfg( 'conditional_detr/conditional-detr_r50_8xb2-50e_coco.py') model = MODELS.build(config) model.init_weights() random_image = torch.rand(1, 3, s, s) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) img_metas.gt_instances = gt_instances batch_data_samples1 = [] batch_data_samples1.append(img_metas) empty_gt_losses = model.loss( random_image, batch_data_samples=batch_data_samples1) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. for key, loss in empty_gt_losses.items(): if 'cls' in key: self.assertGreater(loss.item(), 0, 'cls loss should be non-zero') elif 'bbox' in key: self.assertEqual( loss.item(), 0, 'there should be no box loss when no ground true boxes') elif 'iou' in key: self.assertEqual( loss.item(), 0, 'there should be no iou loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) img_metas.gt_instances = gt_instances batch_data_samples2 = [] batch_data_samples2.append(img_metas) one_gt_losses = model.loss( random_image, batch_data_samples=batch_data_samples2) for loss in one_gt_losses.values(): self.assertGreater( loss.item(), 0, 'cls loss, or box loss, or iou loss should be non-zero') model.eval() # test _forward model._forward(random_image, batch_data_samples=batch_data_samples2) # test only predict model.predict( random_image, batch_data_samples=batch_data_samples2, rescale=True) ================================================ FILE: tests/test_models/test_detectors/test_cornernet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from mmengine.config import ConfigDict from mmdet.structures import DetDataSample from mmdet.testing import demo_mm_inputs, get_detector_cfg from mmdet.utils import register_all_modules class TestCornerNet(TestCase): def setUp(self) -> None: register_all_modules() model_cfg = get_detector_cfg( 'cornernet/cornernet_hourglass104_8xb6-210e-mstest_coco.py') backbone = dict( type='ResNet', depth=18, num_stages=4, out_indices=(3, ), norm_cfg=dict(type='BN', requires_grad=True), norm_eval=True, style='pytorch') neck = dict( type='FPN', in_channels=[512], out_channels=256, start_level=0, add_extra_convs='on_input', num_outs=1) model_cfg.backbone = ConfigDict(**backbone) model_cfg.neck = ConfigDict(**neck) model_cfg.bbox_head.num_feat_levels = 1 self.model_cfg = model_cfg def test_init(self): model = get_detector_cfg( 'cornernet/cornernet_hourglass104_8xb6-210e-mstest_coco.py') model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) self.assertTrue(detector.bbox_head is not None) self.assertTrue(detector.backbone is not None) self.assertTrue(not hasattr(detector, 'neck')) @unittest.skipIf(not torch.cuda.is_available(), 'test requires GPU and torch+cuda') def test_cornernet_forward_loss_mode(self): from mmdet.registry import MODELS detector = MODELS.build(self.model_cfg) detector.init_weights() packed_inputs = demo_mm_inputs(2, [[3, 511, 511], [3, 511, 511]]) data = detector.data_preprocessor(packed_inputs, True) losses = detector.forward(**data, mode='loss') assert isinstance(losses, dict) @unittest.skipIf(not torch.cuda.is_available(), 'test requires GPU and torch+cuda') def test_cornernet_forward_predict_mode(self): from mmdet.registry import MODELS detector = MODELS.build(self.model_cfg) detector.init_weights() packed_inputs = demo_mm_inputs(2, [[3, 512, 512], [3, 512, 512]]) data = detector.data_preprocessor(packed_inputs, False) # Test forward test detector.eval() with torch.no_grad(): batch_results = detector.forward(**data, mode='predict') assert len(batch_results) == 2 assert isinstance(batch_results[0], DetDataSample) @unittest.skipIf(not torch.cuda.is_available(), 'test requires GPU and torch+cuda') def test_cornernet_forward_tensor_mode(self): from mmdet.registry import MODELS detector = MODELS.build(self.model_cfg) detector.init_weights() packed_inputs = demo_mm_inputs(2, [[3, 512, 512], [3, 512, 512]]) data = detector.data_preprocessor(packed_inputs, False) batch_results = detector.forward(**data, mode='tensor') assert isinstance(batch_results, tuple) ================================================ FILE: tests/test_models/test_detectors/test_dab_detr.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.testing import get_detector_cfg from mmdet.utils import register_all_modules class TestDABDETR(TestCase): def setUp(self) -> None: register_all_modules() def test_dab_detr_head_loss(self): """Tests transformer head loss when truth is empty and non-empty.""" s = 256 metainfo = { 'img_shape': (s, s), 'scale_factor': (1, 1), 'pad_shape': (s, s), 'batch_input_shape': (s, s) } img_metas = DetDataSample() img_metas.set_metainfo(metainfo) batch_data_samples = [] batch_data_samples.append(img_metas) config = get_detector_cfg('dab_detr/dab-detr_r50_8xb2-50e_coco.py') model = MODELS.build(config) model.init_weights() random_image = torch.rand(1, 3, s, s) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) img_metas.gt_instances = gt_instances batch_data_samples1 = [] batch_data_samples1.append(img_metas) empty_gt_losses = model.loss( random_image, batch_data_samples=batch_data_samples1) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. for key, loss in empty_gt_losses.items(): if 'cls' in key: self.assertGreater(loss.item(), 0, 'cls loss should be non-zero') elif 'bbox' in key: self.assertEqual( loss.item(), 0, 'there should be no box loss when no ground true boxes') elif 'iou' in key: self.assertEqual( loss.item(), 0, 'there should be no iou loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) img_metas.gt_instances = gt_instances batch_data_samples2 = [] batch_data_samples2.append(img_metas) one_gt_losses = model.loss( random_image, batch_data_samples=batch_data_samples2) for loss in one_gt_losses.values(): self.assertGreater( loss.item(), 0, 'cls loss, or box loss, or iou loss should be non-zero') model.eval() # test _forward model._forward(random_image, batch_data_samples=batch_data_samples2) # test only predict model.predict( random_image, batch_data_samples=batch_data_samples2, rescale=True) ================================================ FILE: tests/test_models/test_detectors/test_deformable_detr.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.testing import get_detector_cfg from mmdet.utils import register_all_modules class TestDeformableDETR(TestCase): def setUp(self): register_all_modules() def test_deformable_detr_head_loss(self): """Tests transformer head loss when truth is empty and non-empty.""" s = 256 metainfo = { 'img_shape': (s, s), 'scale_factor': (1, 1), 'pad_shape': (s, s), 'batch_input_shape': (s, s) } img_metas = DetDataSample() img_metas.set_metainfo(metainfo) batch_data_samples = [] batch_data_samples.append(img_metas) configs = [ get_detector_cfg( 'deformable_detr/deformable-detr_r50_16xb2-50e_coco.py'), get_detector_cfg( 'deformable_detr/deformable-detr-refine_r50_16xb2-50e_coco.py' # noqa ), get_detector_cfg( 'deformable_detr/deformable-detr-refine-twostage_r50_16xb2-50e_coco.py' # noqa ) ] for config in configs: model = MODELS.build(config) model.init_weights() random_image = torch.rand(1, 3, s, s) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) img_metas.gt_instances = gt_instances batch_data_samples1 = [] batch_data_samples1.append(img_metas) empty_gt_losses = model.loss( random_image, batch_data_samples=batch_data_samples1) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. for key, loss in empty_gt_losses.items(): if 'cls' in key: self.assertGreater(loss.item(), 0, 'cls loss should be non-zero') elif 'bbox' in key: self.assertEqual( loss.item(), 0, 'there should be no box loss when no ground true boxes' ) elif 'iou' in key: self.assertEqual( loss.item(), 0, 'there should be no iou loss when no ground true boxes' ) # When truth is non-empty then both cls and box loss should # be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) img_metas.gt_instances = gt_instances batch_data_samples2 = [] batch_data_samples2.append(img_metas) one_gt_losses = model.loss( random_image, batch_data_samples=batch_data_samples2) for loss in one_gt_losses.values(): self.assertGreater( loss.item(), 0, 'cls loss, or box loss, or iou loss should be non-zero') model.eval() # test _forward model._forward( random_image, batch_data_samples=batch_data_samples2) # test only predict model.predict( random_image, batch_data_samples=batch_data_samples2, rescale=True) ================================================ FILE: tests/test_models/test_detectors/test_detr.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.testing import get_detector_cfg from mmdet.utils import register_all_modules class TestDETR(TestCase): def setUp(self) -> None: register_all_modules() def test_detr_head_loss(self): """Tests transformer head loss when truth is empty and non-empty.""" s = 256 metainfo = { 'img_shape': (s, s), 'scale_factor': (1, 1), 'pad_shape': (s, s), 'batch_input_shape': (s, s) } img_metas = DetDataSample() img_metas.set_metainfo(metainfo) batch_data_samples = [] batch_data_samples.append(img_metas) config = get_detector_cfg('detr/detr_r50_8xb2-150e_coco.py') model = MODELS.build(config) model.init_weights() random_image = torch.rand(1, 3, s, s) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) img_metas.gt_instances = gt_instances batch_data_samples1 = [] batch_data_samples1.append(img_metas) empty_gt_losses = model.loss( random_image, batch_data_samples=batch_data_samples1) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. for key, loss in empty_gt_losses.items(): if 'cls' in key: self.assertGreater(loss.item(), 0, 'cls loss should be non-zero') elif 'bbox' in key: self.assertEqual( loss.item(), 0, 'there should be no box loss when no ground true boxes') elif 'iou' in key: self.assertEqual( loss.item(), 0, 'there should be no iou loss when there are no true boxes') # When truth is non-empty then both cls and box loss should be nonzero # for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) img_metas.gt_instances = gt_instances batch_data_samples2 = [] batch_data_samples2.append(img_metas) one_gt_losses = model.loss( random_image, batch_data_samples=batch_data_samples2) for loss in one_gt_losses.values(): self.assertGreater( loss.item(), 0, 'cls loss, or box loss, or iou loss should be non-zero') model.eval() # test _forward model._forward(random_image, batch_data_samples=batch_data_samples2) # test only predict model.predict( random_image, batch_data_samples=batch_data_samples2, rescale=True) ================================================ FILE: tests/test_models/test_detectors/test_dino.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.testing import get_detector_cfg from mmdet.utils import register_all_modules class TestDINO(TestCase): def setUp(self): register_all_modules() def test_dino_head_loss(self): """Tests transformer head loss when truth is empty and non-empty.""" s = 256 metainfo = { 'img_shape': (s, s), 'scale_factor': (1, 1), 'pad_shape': (s, s), 'batch_input_shape': (s, s) } data_sample = DetDataSample() data_sample.set_metainfo(metainfo) configs = [get_detector_cfg('dino/dino-4scale_r50_8xb2-12e_coco.py')] for config in configs: model = MODELS.build(config) model.init_weights() random_image = torch.rand(1, 3, s, s) # Test that empty ground truth encourages the network to # predict background gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)) gt_instances.labels = torch.LongTensor([]) data_sample.gt_instances = gt_instances batch_data_samples_1 = [data_sample] empty_gt_losses = model.loss( random_image, batch_data_samples=batch_data_samples_1) # When there is no truth, the cls loss should be nonzero but there # should be no box loss. for key, loss in empty_gt_losses.items(): _loss = loss.item() if 'bbox' in key or 'iou' in key or 'dn' in key: self.assertEqual( _loss, 0, f'there should be no {key}({_loss}) ' f'when no ground true boxes') elif 'cls' in key: self.assertGreater(_loss, 0, f'{key}({_loss}) should be non-zero') # When truth is non-empty then both cls and box loss should # be nonzero for random inputs gt_instances = InstanceData() gt_instances.bboxes = torch.Tensor( [[23.6667, 23.8757, 238.6326, 151.8874]]) gt_instances.labels = torch.LongTensor([2]) data_sample.gt_instances = gt_instances batch_data_samples_2 = [data_sample] one_gt_losses = model.loss( random_image, batch_data_samples=batch_data_samples_2) for loss in one_gt_losses.values(): self.assertGreater( loss.item(), 0, 'cls loss, or box loss, or iou loss should be non-zero') model.eval() # test _forward model._forward( random_image, batch_data_samples=batch_data_samples_2) # test only predict model.predict( random_image, batch_data_samples=batch_data_samples_2, rescale=True) ================================================ FILE: tests/test_models/test_detectors/test_kd_single_stage.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet import * # noqa from mmdet.structures import DetDataSample from mmdet.testing import demo_mm_inputs, get_detector_cfg from mmdet.utils import register_all_modules class TestKDSingleStageDetector(TestCase): def setUp(self): register_all_modules() @parameterized.expand(['ld/ld_r18-gflv1-r101_fpn_1x_coco.py']) def test_init(self, cfg_file): model = get_detector_cfg(cfg_file) model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) self.assertTrue(detector.backbone) self.assertTrue(detector.neck) self.assertTrue(detector.bbox_head) @parameterized.expand([('ld/ld_r18-gflv1-r101_fpn_1x_coco.py', ('cpu', 'cuda'))]) def test_single_stage_forward_train(self, cfg_file, devices): model = get_detector_cfg(cfg_file) model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, True) # Test forward train losses = detector.forward(**data, mode='loss') self.assertIsInstance(losses, dict) @parameterized.expand([('ld/ld_r18-gflv1-r101_fpn_1x_coco.py', ('cpu', 'cuda'))]) def test_single_stage_forward_test(self, cfg_file, devices): model = get_detector_cfg(cfg_file) model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, False) # Test forward test detector.eval() with torch.no_grad(): batch_results = detector.forward(**data, mode='predict') self.assertEqual(len(batch_results), 2) self.assertIsInstance(batch_results[0], DetDataSample) ================================================ FILE: tests/test_models/test_detectors/test_maskformer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest import torch from parameterized import parameterized from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.testing._utils import demo_mm_inputs, get_detector_cfg from mmdet.utils import register_all_modules class TestMaskFormer(unittest.TestCase): def setUp(self): register_all_modules() def _create_model_cfg(self): cfg_path = 'maskformer/maskformer_r50_ms-16xb1-75e_coco.py' model_cfg = get_detector_cfg(cfg_path) base_channels = 32 model_cfg.backbone.depth = 18 model_cfg.backbone.init_cfg = None model_cfg.backbone.base_channels = base_channels model_cfg.panoptic_head.in_channels = [ base_channels * 2**i for i in range(4) ] model_cfg.panoptic_head.feat_channels = base_channels model_cfg.panoptic_head.out_channels = base_channels model_cfg.panoptic_head.pixel_decoder.encoder.\ layer_cfg.self_attn_cfg.embed_dims = base_channels model_cfg.panoptic_head.pixel_decoder.encoder.\ layer_cfg.ffn_cfg.embed_dims = base_channels model_cfg.panoptic_head.pixel_decoder.encoder.\ layer_cfg.ffn_cfg.feedforward_channels = base_channels * 8 model_cfg.panoptic_head.pixel_decoder.\ positional_encoding.num_feats = base_channels // 2 model_cfg.panoptic_head.positional_encoding.\ num_feats = base_channels // 2 model_cfg.panoptic_head.transformer_decoder.\ layer_cfg.self_attn_cfg.embed_dims = base_channels model_cfg.panoptic_head.transformer_decoder. \ layer_cfg.cross_attn_cfg.embed_dims = base_channels model_cfg.panoptic_head.transformer_decoder.\ layer_cfg.ffn_cfg.embed_dims = base_channels model_cfg.panoptic_head.transformer_decoder.\ layer_cfg.ffn_cfg.feedforward_channels = base_channels * 8 return model_cfg def test_init(self): model_cfg = self._create_model_cfg() detector = MODELS.build(model_cfg) detector.init_weights() assert detector.backbone assert detector.panoptic_head @parameterized.expand([('cpu', ), ('cuda', )]) def test_forward_loss_mode(self, device): model_cfg = self._create_model_cfg() detector = MODELS.build(model_cfg) if device == 'cuda' and not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.to(device) packed_inputs = demo_mm_inputs( 2, image_shapes=[(3, 128, 127), (3, 91, 92)], sem_seg_output_strides=1, with_mask=True, with_semantic=True) data = detector.data_preprocessor(packed_inputs, True) # Test loss mode losses = detector.forward(**data, mode='loss') self.assertIsInstance(losses, dict) @parameterized.expand([('cpu', ), ('cuda', )]) def test_forward_predict_mode(self, device): model_cfg = self._create_model_cfg() detector = MODELS.build(model_cfg) if device == 'cuda' and not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.to(device) packed_inputs = demo_mm_inputs( 2, image_shapes=[(3, 128, 127), (3, 91, 92)], sem_seg_output_strides=1, with_mask=True, with_semantic=True) data = detector.data_preprocessor(packed_inputs, False) # Test forward test detector.eval() with torch.no_grad(): batch_results = detector.forward(**data, mode='predict') self.assertEqual(len(batch_results), 2) self.assertIsInstance(batch_results[0], DetDataSample) @parameterized.expand([('cpu', ), ('cuda', )]) def test_forward_tensor_mode(self, device): model_cfg = self._create_model_cfg() detector = MODELS.build(model_cfg) if device == 'cuda' and not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.to(device) packed_inputs = demo_mm_inputs( 2, [[3, 128, 128], [3, 125, 130]], sem_seg_output_strides=1, with_mask=True, with_semantic=True) data = detector.data_preprocessor(packed_inputs, False) out = detector.forward(**data, mode='tensor') self.assertIsInstance(out, tuple) class TestMask2Former(unittest.TestCase): def setUp(self): register_all_modules() def _create_model_cfg(self, cfg_path): model_cfg = get_detector_cfg(cfg_path) base_channels = 32 model_cfg.backbone.depth = 18 model_cfg.backbone.init_cfg = None model_cfg.backbone.base_channels = base_channels model_cfg.panoptic_head.in_channels = [ base_channels * 2**i for i in range(4) ] model_cfg.panoptic_head.feat_channels = base_channels model_cfg.panoptic_head.out_channels = base_channels model_cfg.panoptic_head.pixel_decoder.encoder.\ layer_cfg.self_attn_cfg.embed_dims = base_channels model_cfg.panoptic_head.pixel_decoder.encoder.\ layer_cfg.ffn_cfg.embed_dims = base_channels model_cfg.panoptic_head.pixel_decoder.encoder.\ layer_cfg.ffn_cfg.feedforward_channels = base_channels * 4 model_cfg.panoptic_head.pixel_decoder.\ positional_encoding.num_feats = base_channels // 2 model_cfg.panoptic_head.positional_encoding.\ num_feats = base_channels // 2 model_cfg.panoptic_head.transformer_decoder.\ layer_cfg.self_attn_cfg.embed_dims = base_channels model_cfg.panoptic_head.transformer_decoder. \ layer_cfg.cross_attn_cfg.embed_dims = base_channels model_cfg.panoptic_head.transformer_decoder.\ layer_cfg.ffn_cfg.embed_dims = base_channels model_cfg.panoptic_head.transformer_decoder.\ layer_cfg.ffn_cfg.feedforward_channels = base_channels * 8 return model_cfg def test_init(self): model_cfg = self._create_model_cfg( 'mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py') detector = MODELS.build(model_cfg) detector.init_weights() assert detector.backbone assert detector.panoptic_head @parameterized.expand([ ('cpu', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py'), ('cpu', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco.py'), ('cuda', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py'), ('cuda', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco.py') ]) def test_forward_loss_mode(self, device, cfg_path): print(device, cfg_path) with_semantic = 'panoptic' in cfg_path model_cfg = self._create_model_cfg(cfg_path) detector = MODELS.build(model_cfg) if device == 'cuda' and not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.to(device) packed_inputs = demo_mm_inputs( 2, image_shapes=[(3, 128, 127), (3, 91, 92)], sem_seg_output_strides=1, with_mask=True, with_semantic=with_semantic) data = detector.data_preprocessor(packed_inputs, True) # Test loss mode losses = detector.forward(**data, mode='loss') self.assertIsInstance(losses, dict) @parameterized.expand([ ('cpu', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py'), ('cpu', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco.py'), ('cuda', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py'), ('cuda', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco.py') ]) def test_forward_predict_mode(self, device, cfg_path): with_semantic = 'panoptic' in cfg_path model_cfg = self._create_model_cfg(cfg_path) detector = MODELS.build(model_cfg) if device == 'cuda' and not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.to(device) packed_inputs = demo_mm_inputs( 2, image_shapes=[(3, 128, 127), (3, 91, 92)], sem_seg_output_strides=1, with_mask=True, with_semantic=with_semantic) data = detector.data_preprocessor(packed_inputs, False) # Test forward test detector.eval() with torch.no_grad(): batch_results = detector.forward(**data, mode='predict') self.assertEqual(len(batch_results), 2) self.assertIsInstance(batch_results[0], DetDataSample) @parameterized.expand([ ('cpu', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py'), ('cpu', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco.py'), ('cuda', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco-panoptic.py'), ('cuda', 'mask2former/mask2former_r50_8xb2-lsj-50e_coco.py') ]) def test_forward_tensor_mode(self, device, cfg_path): with_semantic = 'panoptic' in cfg_path model_cfg = self._create_model_cfg(cfg_path) detector = MODELS.build(model_cfg) if device == 'cuda' and not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.to(device) packed_inputs = demo_mm_inputs( 2, [[3, 128, 128], [3, 125, 130]], sem_seg_output_strides=1, with_mask=True, with_semantic=with_semantic) data = detector.data_preprocessor(packed_inputs, False) out = detector.forward(**data, mode='tensor') self.assertIsInstance(out, tuple) ================================================ FILE: tests/test_models/test_detectors/test_panoptic_two_stage_segmentor.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest import torch from parameterized import parameterized from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.testing._utils import demo_mm_inputs, get_detector_cfg from mmdet.utils import register_all_modules class TestTwoStagePanopticSegmentor(unittest.TestCase): def setUp(self): register_all_modules() def _create_model_cfg(self): cfg_file = 'panoptic_fpn/panoptic-fpn_r50_fpn_1x_coco.py' model_cfg = get_detector_cfg(cfg_file) model_cfg.backbone.depth = 18 model_cfg.neck.in_channels = [64, 128, 256, 512] model_cfg.backbone.init_cfg = None return model_cfg def test_init(self): model_cfg = self._create_model_cfg() detector = MODELS.build(model_cfg) assert detector.backbone assert detector.neck assert detector.rpn_head assert detector.roi_head assert detector.roi_head.mask_head assert detector.with_semantic_head assert detector.with_panoptic_fusion_head @parameterized.expand([('cpu', ), ('cuda', )]) def test_forward_loss_mode(self, device): model_cfg = self._create_model_cfg() detector = MODELS.build(model_cfg) if device == 'cuda' and not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.to(device) packed_inputs = demo_mm_inputs( 2, image_shapes=[(3, 128, 127), (3, 91, 92)], sem_seg_output_strides=1, with_mask=True, with_semantic=True) data = detector.data_preprocessor(packed_inputs, True) # Test loss mode losses = detector.forward(**data, mode='loss') self.assertIsInstance(losses, dict) @parameterized.expand([('cpu', ), ('cuda', )]) def test_forward_predict_mode(self, device): model_cfg = self._create_model_cfg() detector = MODELS.build(model_cfg) if device == 'cuda' and not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.to(device) packed_inputs = demo_mm_inputs( 2, image_shapes=[(3, 128, 127), (3, 91, 92)], sem_seg_output_strides=1, with_mask=True, with_semantic=True) data = detector.data_preprocessor(packed_inputs, False) # Test forward test detector.eval() with torch.no_grad(): batch_results = detector.forward(**data, mode='predict') self.assertEqual(len(batch_results), 2) self.assertIsInstance(batch_results[0], DetDataSample) @parameterized.expand([('cpu', ), ('cuda', )]) def test_forward_tensor_mode(self, device): model_cfg = self._create_model_cfg() detector = MODELS.build(model_cfg) if device == 'cuda' and not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.to(device) packed_inputs = demo_mm_inputs( 2, [[3, 128, 128], [3, 125, 130]], sem_seg_output_strides=1, with_mask=True, with_semantic=True) data = detector.data_preprocessor(packed_inputs, False) out = detector.forward(**data, mode='tensor') self.assertIsInstance(out, tuple) ================================================ FILE: tests/test_models/test_detectors/test_rpn.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.structures import DetDataSample from mmdet.testing import demo_mm_inputs, get_detector_cfg from mmdet.utils import register_all_modules class TestRPN(TestCase): def setUp(self): register_all_modules() @parameterized.expand(['rpn/rpn_r50_fpn_1x_coco.py']) def test_init(self, cfg_file): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) self.assertTrue(detector.backbone) self.assertTrue(detector.neck) self.assertTrue(detector.bbox_head) # if rpn.num_classes > 1, force set rpn.num_classes = 1 model.rpn_head.num_classes = 2 detector = MODELS.build(model) self.assertEqual(detector.bbox_head.num_classes, 1) @parameterized.expand([('rpn/rpn_r50_fpn_1x_coco.py', ('cpu', 'cuda'))]) def test_rpn_forward_loss_mode(self, cfg_file, devices): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, True) # Test forward train losses = detector.forward(**data, mode='loss') self.assertIsInstance(losses, dict) @parameterized.expand([('rpn/rpn_r50_fpn_1x_coco.py', ('cpu', 'cuda'))]) def test_rpn_forward_predict_mode(self, cfg_file, devices): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, False) # Test forward test detector.eval() with torch.no_grad(): batch_results = detector.forward(**data, mode='predict') self.assertEqual(len(batch_results), 2) self.assertIsInstance(batch_results[0], DetDataSample) @parameterized.expand([('rpn/rpn_r50_fpn_1x_coco.py', ('cpu', 'cuda'))]) def test_rpn_forward_tensor_mode(self, cfg_file, devices): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, False) batch_results = detector.forward(**data, mode='tensor') self.assertIsInstance(batch_results, tuple) ================================================ FILE: tests/test_models/test_detectors/test_semi_base.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase from mmengine.registry import MODELS from parameterized import parameterized from mmdet.testing import get_detector_cfg from mmdet.utils import register_all_modules register_all_modules() class TestSemiBase(TestCase): @parameterized.expand([ 'soft_teacher/' 'soft-teacher_faster-rcnn_r50-caffe_fpn_180k_semi-0.1-coco.py', ]) def test_init(self, cfg_file): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.detector.backbone.depth = 18 model.detector.neck.in_channels = [64, 128, 256, 512] model.detector.backbone.init_cfg = None model = MODELS.build(model) self.assertTrue(model.teacher.backbone) self.assertTrue(model.teacher.neck) self.assertTrue(model.teacher.rpn_head) self.assertTrue(model.teacher.roi_head) self.assertTrue(model.student.backbone) self.assertTrue(model.student.neck) self.assertTrue(model.student.rpn_head) self.assertTrue(model.student.roi_head) ================================================ FILE: tests/test_models/test_detectors/test_single_stage.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import time import unittest from unittest import TestCase import torch from mmengine.logging import MessageHub from parameterized import parameterized from mmdet.structures import DetDataSample from mmdet.testing import demo_mm_inputs, get_detector_cfg from mmdet.utils import register_all_modules class TestSingleStageDetector(TestCase): def setUp(self): register_all_modules() @parameterized.expand([ 'retinanet/retinanet_r18_fpn_1x_coco.py', 'centernet/centernet_r18_8xb16-crop512-140e_coco.py', 'fsaf/fsaf_r50_fpn_1x_coco.py', 'yolox/yolox_tiny_8xb8-300e_coco.py', 'yolo/yolov3_mobilenetv2_8xb24-320-300e_coco.py', 'reppoints/reppoints-minmax_r50_fpn-gn_head-gn_1x_coco.py', ]) def test_init(self, cfg_file): model = get_detector_cfg(cfg_file) model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) self.assertTrue(detector.backbone) self.assertTrue(detector.neck) self.assertTrue(detector.bbox_head) @parameterized.expand([ ('retinanet/retinanet_r18_fpn_1x_coco.py', ('cpu', 'cuda')), ('centernet/centernet_r18_8xb16-crop512-140e_coco.py', ('cpu', 'cuda')), ('fsaf/fsaf_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('yolox/yolox_tiny_8xb8-300e_coco.py', ('cpu', 'cuda')), ('yolo/yolov3_mobilenetv2_8xb24-320-300e_coco.py', ('cpu', 'cuda')), ('reppoints/reppoints-minmax_r50_fpn-gn_head-gn_1x_coco.py', ('cpu', 'cuda')), ]) def test_single_stage_forward_loss_mode(self, cfg_file, devices): message_hub = MessageHub.get_instance( f'test_single_stage_forward_loss_mode-{time.time()}') message_hub.update_info('iter', 0) message_hub.update_info('epoch', 0) model = get_detector_cfg(cfg_file) model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) detector.init_weights() if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, True) losses = detector.forward(**data, mode='loss') self.assertIsInstance(losses, dict) @parameterized.expand([ ('retinanet/retinanet_r18_fpn_1x_coco.py', ('cpu', 'cuda')), ('centernet/centernet_r18_8xb16-crop512-140e_coco.py', ('cpu', 'cuda')), ('fsaf/fsaf_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('yolox/yolox_tiny_8xb8-300e_coco.py', ('cpu', 'cuda')), ('yolo/yolov3_mobilenetv2_8xb24-320-300e_coco.py', ('cpu', 'cuda')), ('reppoints/reppoints-minmax_r50_fpn-gn_head-gn_1x_coco.py', ('cpu', 'cuda')), ]) def test_single_stage_forward_predict_mode(self, cfg_file, devices): model = get_detector_cfg(cfg_file) model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, False) # Test forward test detector.eval() with torch.no_grad(): batch_results = detector.forward(**data, mode='predict') self.assertEqual(len(batch_results), 2) self.assertIsInstance(batch_results[0], DetDataSample) @parameterized.expand([ ('retinanet/retinanet_r18_fpn_1x_coco.py', ('cpu', 'cuda')), ('centernet/centernet_r18_8xb16-crop512-140e_coco.py', ('cpu', 'cuda')), ('fsaf/fsaf_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('yolox/yolox_tiny_8xb8-300e_coco.py', ('cpu', 'cuda')), ('yolo/yolov3_mobilenetv2_8xb24-320-300e_coco.py', ('cpu', 'cuda')), ('reppoints/reppoints-minmax_r50_fpn-gn_head-gn_1x_coco.py', ('cpu', 'cuda')), ]) def test_single_stage_forward_tensor_mode(self, cfg_file, devices): model = get_detector_cfg(cfg_file) model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, False) batch_results = detector.forward(**data, mode='tensor') self.assertIsInstance(batch_results, tuple) ================================================ FILE: tests/test_models/test_detectors/test_single_stage_instance_seg.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.structures import DetDataSample from mmdet.testing import demo_mm_inputs, get_detector_cfg from mmdet.utils import register_all_modules class TestSingleStageInstanceSegmentor(TestCase): def setUp(self): register_all_modules() @parameterized.expand([ 'solo/solo_r50_fpn_1x_coco.py', 'solo/decoupled-solo_r50_fpn_1x_coco.py', 'solo/decoupled-solo-light_r50_fpn_3x_coco.py', 'solov2/solov2_r50_fpn_1x_coco.py', 'solov2/solov2-light_r18_fpn_ms-3x_coco.py', 'yolact/yolact_r50_1xb8-55e_coco.py', ]) def test_init(self, cfg_file): model = get_detector_cfg(cfg_file) model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) self.assertTrue(detector.backbone) self.assertTrue(detector.neck) self.assertTrue(detector.mask_head) if detector.with_bbox: self.assertTrue(detector.bbox_head) @parameterized.expand([ ('solo/solo_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('solo/decoupled-solo_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('solo/decoupled-solo-light_r50_fpn_3x_coco.py', ('cpu', 'cuda')), ('solov2/solov2_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('solov2/solov2-light_r18_fpn_ms-3x_coco.py', ('cpu', 'cuda')), ('yolact/yolact_r50_1xb8-55e_coco.py', ('cpu', 'cuda')), ]) def test_single_stage_forward_loss_mode(self, cfg_file, devices): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) detector.init_weights() if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs( 2, [[3, 128, 128], [3, 125, 130]], with_mask=True) data = detector.data_preprocessor(packed_inputs, True) losses = detector.forward(**data, mode='loss') self.assertIsInstance(losses, dict) @parameterized.expand([ ('solo/solo_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('solo/decoupled-solo_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('solo/decoupled-solo-light_r50_fpn_3x_coco.py', ('cpu', 'cuda')), ('solov2/solov2_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('solov2/solov2-light_r18_fpn_ms-3x_coco.py', ('cpu', 'cuda')), ('yolact/yolact_r50_1xb8-55e_coco.py', ('cpu', 'cuda')), ]) def test_single_stage_forward_predict_mode(self, cfg_file, devices): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs( 2, [[3, 128, 128], [3, 125, 130]], with_mask=True) data = detector.data_preprocessor(packed_inputs, False) # Test forward test detector.eval() with torch.no_grad(): batch_results = detector.forward(**data, mode='predict') self.assertEqual(len(batch_results), 2) self.assertIsInstance(batch_results[0], DetDataSample) @parameterized.expand([ ('solo/solo_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('solo/decoupled-solo_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('solo/decoupled-solo-light_r50_fpn_3x_coco.py', ('cpu', 'cuda')), ('solov2/solov2_r50_fpn_1x_coco.py', ('cpu', 'cuda')), ('solov2/solov2-light_r18_fpn_ms-3x_coco.py', ('cpu', 'cuda')), ('yolact/yolact_r50_1xb8-55e_coco.py', ('cpu', 'cuda')), ]) def test_single_stage_forward_tensor_mode(self, cfg_file, devices): model = get_detector_cfg(cfg_file) model.backbone.init_cfg = None from mmdet.registry import MODELS assert all([device in ['cpu', 'cuda'] for device in devices]) for device in devices: detector = MODELS.build(model) if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, False) batch_results = detector.forward(**data, mode='tensor') self.assertIsInstance(batch_results, tuple) ================================================ FILE: tests/test_models/test_detectors/test_two_stage.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.structures import DetDataSample from mmdet.testing import demo_mm_inputs, get_detector_cfg from mmdet.utils import register_all_modules class TestTwoStageBBox(TestCase): def setUp(self): register_all_modules() @parameterized.expand([ 'faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py', 'cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py', 'sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py', ]) def test_init(self, cfg_file): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) self.assertTrue(detector.backbone) self.assertTrue(detector.neck) self.assertTrue(detector.rpn_head) self.assertTrue(detector.roi_head) # if rpn.num_classes > 1, force set rpn.num_classes = 1 if hasattr(model.rpn_head, 'num_classes'): model.rpn_head.num_classes = 2 detector = MODELS.build(model) self.assertEqual(detector.rpn_head.num_classes, 1) @parameterized.expand([ 'faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py', 'cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py', 'sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py', ]) def test_two_stage_forward_loss_mode(self, cfg_file): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, True) # Test loss mode losses = detector.forward(**data, mode='loss') self.assertIsInstance(losses, dict) @parameterized.expand([ 'faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py', 'cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py', 'sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py', ]) def test_two_stage_forward_predict_mode(self, cfg_file): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) data = detector.data_preprocessor(packed_inputs, False) # Test forward test detector.eval() with torch.no_grad(): with torch.no_grad(): batch_results = detector.forward(**data, mode='predict') self.assertEqual(len(batch_results), 2) self.assertIsInstance(batch_results[0], DetDataSample) # TODO: Awaiting refactoring # @parameterized.expand([ # 'faster_rcnn/faster-rcnn_r50_fpn_1x_coco.py', # 'cascade_rcnn/cascade-rcnn_r50_fpn_1x_coco.py', # 'sparse_rcnn/sparse-rcnn_r50_fpn_1x_coco.py', # ]) # def test_two_stage_forward_tensor_mode(self, cfg_file): # model = get_detector_cfg(cfg_file) # # backbone convert to ResNet18 # model.backbone.depth = 18 # model.neck.in_channels = [64, 128, 256, 512] # model.backbone.init_cfg = None # # from mmdet.models import build_detector # detector = build_detector(model) # # if not torch.cuda.is_available(): # return unittest.skip('test requires GPU and torch+cuda') # detector = detector.cuda() # # packed_inputs = demo_mm_inputs(2, [[3, 128, 128], [3, 125, 130]]) # data = detector.data_preprocessor(packed_inputs, False) # out = detector.forward(**data, mode='tensor') # self.assertIsInstance(out, tuple) class TestTwoStageMask(TestCase): def setUp(self): register_all_modules() @parameterized.expand([ 'mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py', 'cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py', 'queryinst/queryinst_r50_fpn_1x_coco.py' ]) def test_init(self, cfg_file): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) self.assertTrue(detector.backbone) self.assertTrue(detector.neck) self.assertTrue(detector.rpn_head) self.assertTrue(detector.roi_head) self.assertTrue(detector.roi_head.mask_head) # if rpn.num_classes > 1, force set rpn.num_classes = 1 if hasattr(model.rpn_head, 'num_classes'): model.rpn_head.num_classes = 2 detector = MODELS.build(model) self.assertEqual(detector.rpn_head.num_classes, 1) @parameterized.expand([ 'mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py', 'cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py', 'queryinst/queryinst_r50_fpn_1x_coco.py' ]) def test_two_stage_forward_loss_mode(self, cfg_file): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs( 2, [[3, 128, 128], [3, 125, 130]], with_mask=True) data = detector.data_preprocessor(packed_inputs, True) # Test loss mode losses = detector.forward(**data, mode='loss') self.assertIsInstance(losses, dict) @parameterized.expand([ 'mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py', 'cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py', 'queryinst/queryinst_r50_fpn_1x_coco.py' ]) def test_two_stage_forward_predict_mode(self, cfg_file): model = get_detector_cfg(cfg_file) # backbone convert to ResNet18 model.backbone.depth = 18 model.neck.in_channels = [64, 128, 256, 512] model.backbone.init_cfg = None from mmdet.registry import MODELS detector = MODELS.build(model) if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') detector = detector.cuda() packed_inputs = demo_mm_inputs(2, [[3, 256, 256], [3, 255, 260]]) data = detector.data_preprocessor(packed_inputs, False) # Test forward test detector.eval() with torch.no_grad(): batch_results = detector.forward(**data, mode='predict') self.assertEqual(len(batch_results), 2) self.assertIsInstance(batch_results[0], DetDataSample) # TODO: Awaiting refactoring # @parameterized.expand([ # 'mask_rcnn/mask-rcnn_r50_fpn_1x_coco.py', # 'cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py', # 'queryinst/queryinst_r50_fpn_1x_coco.py' # ]) # def test_two_stage_forward_tensor_mode(self, cfg_file): # model = get_detector_cfg(cfg_file) # # backbone convert to ResNet18 # model.backbone.depth = 18 # model.neck.in_channels = [64, 128, 256, 512] # model.backbone.init_cfg = None # # from mmdet.models import build_detector # detector = build_detector(model) # # if not torch.cuda.is_available(): # return unittest.skip('test requires GPU and torch+cuda') # detector = detector.cuda() # # packed_inputs = demo_mm_inputs( # 2, [[3, 128, 128], [3, 125, 130]], with_mask=True) # data = detector.data_preprocessor(packed_inputs, False) # # # out = detector.forward(**data, mode='tensor') # # self.assertIsInstance(out, tuple) ================================================ FILE: tests/test_models/test_layers/__init__.py ================================================ ================================================ FILE: tests/test_models/test_layers/test_brick_wrappers.py ================================================ from unittest.mock import patch import torch import torch.nn as nn import torch.nn.functional as F from mmdet.models.layers import AdaptiveAvgPool2d, adaptive_avg_pool2d if torch.__version__ != 'parrots': torch_version = '1.7' else: torch_version = 'parrots' @patch('torch.__version__', torch_version) def test_adaptive_avg_pool2d(): # Test the empty batch dimension # Test the two input conditions x_empty = torch.randn(0, 3, 4, 5) # 1. tuple[int, int] wrapper_out = adaptive_avg_pool2d(x_empty, (2, 2)) assert wrapper_out.shape == (0, 3, 2, 2) # 2. int wrapper_out = adaptive_avg_pool2d(x_empty, 2) assert wrapper_out.shape == (0, 3, 2, 2) # wrapper op with 3-dim input x_normal = torch.randn(3, 3, 4, 5) wrapper_out = adaptive_avg_pool2d(x_normal, (2, 2)) ref_out = F.adaptive_avg_pool2d(x_normal, (2, 2)) assert wrapper_out.shape == (3, 3, 2, 2) assert torch.equal(wrapper_out, ref_out) wrapper_out = adaptive_avg_pool2d(x_normal, 2) ref_out = F.adaptive_avg_pool2d(x_normal, 2) assert wrapper_out.shape == (3, 3, 2, 2) assert torch.equal(wrapper_out, ref_out) @patch('torch.__version__', torch_version) def test_AdaptiveAvgPool2d(): # Test the empty batch dimension x_empty = torch.randn(0, 3, 4, 5) # Test the four input conditions # 1. tuple[int, int] wrapper = AdaptiveAvgPool2d((2, 2)) wrapper_out = wrapper(x_empty) assert wrapper_out.shape == (0, 3, 2, 2) # 2. int wrapper = AdaptiveAvgPool2d(2) wrapper_out = wrapper(x_empty) assert wrapper_out.shape == (0, 3, 2, 2) # 3. tuple[None, int] wrapper = AdaptiveAvgPool2d((None, 2)) wrapper_out = wrapper(x_empty) assert wrapper_out.shape == (0, 3, 4, 2) # 3. tuple[int, None] wrapper = AdaptiveAvgPool2d((2, None)) wrapper_out = wrapper(x_empty) assert wrapper_out.shape == (0, 3, 2, 5) # Test the normal batch dimension x_normal = torch.randn(3, 3, 4, 5) wrapper = AdaptiveAvgPool2d((2, 2)) ref = nn.AdaptiveAvgPool2d((2, 2)) wrapper_out = wrapper(x_normal) ref_out = ref(x_normal) assert wrapper_out.shape == (3, 3, 2, 2) assert torch.equal(wrapper_out, ref_out) wrapper = AdaptiveAvgPool2d(2) ref = nn.AdaptiveAvgPool2d(2) wrapper_out = wrapper(x_normal) ref_out = ref(x_normal) assert wrapper_out.shape == (3, 3, 2, 2) assert torch.equal(wrapper_out, ref_out) wrapper = AdaptiveAvgPool2d((None, 2)) ref = nn.AdaptiveAvgPool2d((None, 2)) wrapper_out = wrapper(x_normal) ref_out = ref(x_normal) assert wrapper_out.shape == (3, 3, 4, 2) assert torch.equal(wrapper_out, ref_out) wrapper = AdaptiveAvgPool2d((2, None)) ref = nn.AdaptiveAvgPool2d((2, None)) wrapper_out = wrapper(x_normal) ref_out = ref(x_normal) assert wrapper_out.shape == (3, 3, 2, 5) assert torch.equal(wrapper_out, ref_out) ================================================ FILE: tests/test_models/test_layers/test_conv_upsample.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmdet.models.layers import ConvUpsample @pytest.mark.parametrize('num_layers', [0, 1, 2]) def test_conv_upsample(num_layers): num_upsample = num_layers if num_layers > 0 else 0 num_layers = num_layers if num_layers > 0 else 1 layer = ConvUpsample( 10, 5, num_layers=num_layers, num_upsample=num_upsample, conv_cfg=None, norm_cfg=None) size = 5 x = torch.randn((1, 10, size, size)) size = size * pow(2, num_upsample) x = layer(x) assert x.shape[-2:] == (size, size) ================================================ FILE: tests/test_models/test_layers/test_ema.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import itertools import math from unittest import TestCase import torch import torch.nn as nn from mmengine.testing import assert_allclose from mmdet.models.layers import ExpMomentumEMA class TestEMA(TestCase): def test_exp_momentum_ema(self): model = nn.Sequential(nn.Conv2d(1, 5, kernel_size=3), nn.Linear(5, 10)) # Test invalid gamma with self.assertRaisesRegex(AssertionError, 'gamma must be greater than 0'): ExpMomentumEMA(model, gamma=-1) # Test EMA model = torch.nn.Sequential( torch.nn.Conv2d(1, 5, kernel_size=3), torch.nn.Linear(5, 10)) momentum = 0.1 gamma = 4 ema_model = ExpMomentumEMA(model, momentum=momentum, gamma=gamma) averaged_params = [ torch.zeros_like(param) for param in model.parameters() ] n_updates = 10 for i in range(n_updates): updated_averaged_params = [] for p, p_avg in zip(model.parameters(), averaged_params): p.detach().add_(torch.randn_like(p)) if i == 0: updated_averaged_params.append(p.clone()) else: m = (1 - momentum) * math.exp(-(1 + i) / gamma) + momentum updated_averaged_params.append( (p_avg * (1 - m) + p * m).clone()) ema_model.update_parameters(model) averaged_params = updated_averaged_params for p_target, p_ema in zip(averaged_params, ema_model.parameters()): assert_allclose(p_target, p_ema) def test_exp_momentum_ema_update_buffer(self): model = nn.Sequential( nn.Conv2d(1, 5, kernel_size=3), nn.BatchNorm2d(5, momentum=0.3), nn.Linear(5, 10)) # Test invalid gamma with self.assertRaisesRegex(AssertionError, 'gamma must be greater than 0'): ExpMomentumEMA(model, gamma=-1) # Test EMA with momentum annealing. momentum = 0.1 gamma = 4 ema_model = ExpMomentumEMA( model, gamma=gamma, momentum=momentum, update_buffers=True) averaged_params = [ torch.zeros_like(param) for param in itertools.chain(model.parameters(), model.buffers()) if param.size() != torch.Size([]) ] n_updates = 10 for i in range(n_updates): updated_averaged_params = [] params = [ param for param in itertools.chain(model.parameters(), model.buffers()) if param.size() != torch.Size([]) ] for p, p_avg in zip(params, averaged_params): p.detach().add_(torch.randn_like(p)) if i == 0: updated_averaged_params.append(p.clone()) else: m = (1 - momentum) * math.exp(-(1 + i) / gamma) + momentum updated_averaged_params.append( (p_avg * (1 - m) + p * m).clone()) ema_model.update_parameters(model) averaged_params = updated_averaged_params ema_params = [ param for param in itertools.chain(ema_model.module.parameters(), ema_model.module.buffers()) if param.size() != torch.Size([]) ] for p_target, p_ema in zip(averaged_params, ema_params): assert_allclose(p_target, p_ema) ================================================ FILE: tests/test_models/test_layers/test_inverted_residual.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmcv.cnn import is_norm from torch.nn.modules import GroupNorm from mmdet.models.layers import InvertedResidual, SELayer def test_inverted_residual(): with pytest.raises(AssertionError): # stride must be in [1, 2] InvertedResidual(16, 16, 32, stride=3) with pytest.raises(AssertionError): # se_cfg must be None or dict InvertedResidual(16, 16, 32, se_cfg=list()) with pytest.raises(AssertionError): # in_channeld and mid_channels must be the same if # with_expand_conv is False InvertedResidual(16, 16, 32, with_expand_conv=False) # Test InvertedResidual forward, stride=1 block = InvertedResidual(16, 16, 32, stride=1) x = torch.randn(1, 16, 56, 56) x_out = block(x) assert getattr(block, 'se', None) is None assert block.with_res_shortcut assert x_out.shape == torch.Size((1, 16, 56, 56)) # Test InvertedResidual forward, stride=2 block = InvertedResidual(16, 16, 32, stride=2) x = torch.randn(1, 16, 56, 56) x_out = block(x) assert not block.with_res_shortcut assert x_out.shape == torch.Size((1, 16, 28, 28)) # Test InvertedResidual forward with se layer se_cfg = dict(channels=32) block = InvertedResidual(16, 16, 32, stride=1, se_cfg=se_cfg) x = torch.randn(1, 16, 56, 56) x_out = block(x) assert isinstance(block.se, SELayer) assert x_out.shape == torch.Size((1, 16, 56, 56)) # Test InvertedResidual forward, with_expand_conv=False block = InvertedResidual(32, 16, 32, with_expand_conv=False) x = torch.randn(1, 32, 56, 56) x_out = block(x) assert getattr(block, 'expand_conv', None) is None assert x_out.shape == torch.Size((1, 16, 56, 56)) # Test InvertedResidual forward with GroupNorm block = InvertedResidual( 16, 16, 32, norm_cfg=dict(type='GN', num_groups=2)) x = torch.randn(1, 16, 56, 56) x_out = block(x) for m in block.modules(): if is_norm(m): assert isinstance(m, GroupNorm) assert x_out.shape == torch.Size((1, 16, 56, 56)) # Test InvertedResidual forward with HSigmoid block = InvertedResidual(16, 16, 32, act_cfg=dict(type='HSigmoid')) x = torch.randn(1, 16, 56, 56) x_out = block(x) assert x_out.shape == torch.Size((1, 16, 56, 56)) # Test InvertedResidual forward with checkpoint block = InvertedResidual(16, 16, 32, with_cp=True) x = torch.randn(1, 16, 56, 56) x_out = block(x) assert block.with_cp assert x_out.shape == torch.Size((1, 16, 56, 56)) ================================================ FILE: tests/test_models/test_layers/test_plugins.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest import pytest import torch from mmengine.config import ConfigDict from mmdet.models.layers import DropBlock from mmdet.registry import MODELS from mmdet.utils import register_all_modules register_all_modules() def test_dropblock(): feat = torch.rand(1, 1, 11, 11) drop_prob = 1.0 dropblock = DropBlock(drop_prob, block_size=11, warmup_iters=0) out_feat = dropblock(feat) assert (out_feat == 0).all() and out_feat.shape == feat.shape drop_prob = 0.5 dropblock = DropBlock(drop_prob, block_size=5, warmup_iters=0) out_feat = dropblock(feat) assert out_feat.shape == feat.shape # drop_prob must be (0,1] with pytest.raises(AssertionError): DropBlock(1.5, 3) # block_size cannot be an even number with pytest.raises(AssertionError): DropBlock(0.5, 2) # warmup_iters cannot be less than 0 with pytest.raises(AssertionError): DropBlock(0.5, 3, -1) class TestPixelDecoder(unittest.TestCase): def test_forward(self): base_channels = 64 pixel_decoder_cfg = ConfigDict( dict( type='PixelDecoder', in_channels=[base_channels * 2**i for i in range(4)], feat_channels=base_channels, out_channels=base_channels, norm_cfg=dict(type='GN', num_groups=32), act_cfg=dict(type='ReLU'))) self = MODELS.build(pixel_decoder_cfg) self.init_weights() img_metas = [{}, {}] feats = [ torch.rand( (2, base_channels * 2**i, 4 * 2**(3 - i), 5 * 2**(3 - i))) for i in range(4) ] mask_feature, memory = self(feats, img_metas) assert (memory == feats[-1]).all() assert mask_feature.shape == feats[0].shape class TestTransformerEncoderPixelDecoder(unittest.TestCase): def test_forward(self): base_channels = 64 pixel_decoder_cfg = ConfigDict( dict( type='TransformerEncoderPixelDecoder', in_channels=[base_channels * 2**i for i in range(4)], feat_channels=base_channels, out_channels=base_channels, norm_cfg=dict(type='GN', num_groups=32), act_cfg=dict(type='ReLU'), encoder=dict( # DetrTransformerEncoder num_layers=6, layer_cfg=dict( # DetrTransformerEncoderLayer self_attn_cfg=dict( # MultiheadAttention embed_dims=base_channels, num_heads=8, attn_drop=0.1, proj_drop=0.1, dropout_layer=None, batch_first=True), ffn_cfg=dict( embed_dims=base_channels, feedforward_channels=base_channels * 8, num_fcs=2, act_cfg=dict(type='ReLU', inplace=True), ffn_drop=0.1, dropout_layer=None, add_identity=True), norm_cfg=dict(type='LN'), init_cfg=None), init_cfg=None), positional_encoding=dict( num_feats=base_channels // 2, normalize=True))) self = MODELS.build(pixel_decoder_cfg) self.init_weights() img_metas = [{ 'batch_input_shape': (128, 160), 'img_shape': (120, 160), }, { 'batch_input_shape': (128, 160), 'img_shape': (125, 160), }] feats = [ torch.rand( (2, base_channels * 2**i, 4 * 2**(3 - i), 5 * 2**(3 - i))) for i in range(4) ] mask_feature, memory = self(feats, img_metas) assert memory.shape[-2:] == feats[-1].shape[-2:] assert mask_feature.shape == feats[0].shape class TestMSDeformAttnPixelDecoder(unittest.TestCase): def test_forward(self): base_channels = 64 pixel_decoder_cfg = ConfigDict( dict( type='MSDeformAttnPixelDecoder', in_channels=[base_channels * 2**i for i in range(4)], strides=[4, 8, 16, 32], feat_channels=base_channels, out_channels=base_channels, num_outs=3, norm_cfg=dict(type='GN', num_groups=32), act_cfg=dict(type='ReLU'), encoder=dict( # DeformableDetrTransformerEncoder num_layers=6, layer_cfg=dict( # DeformableDetrTransformerEncoderLayer self_attn_cfg=dict( # MultiScaleDeformableAttention embed_dims=base_channels, num_heads=8, num_levels=3, num_points=4, im2col_step=64, dropout=0.0, batch_first=True, norm_cfg=None, init_cfg=None), ffn_cfg=dict( embed_dims=base_channels, feedforward_channels=base_channels * 4, num_fcs=2, ffn_drop=0.0, act_cfg=dict(type='ReLU', inplace=True))), init_cfg=None), positional_encoding=dict( num_feats=base_channels // 2, normalize=True), init_cfg=None)) self = MODELS.build(pixel_decoder_cfg) self.init_weights() feats = [ torch.rand( (2, base_channels * 2**i, 4 * 2**(3 - i), 5 * 2**(3 - i))) for i in range(4) ] mask_feature, multi_scale_features = self(feats) assert mask_feature.shape == feats[0].shape assert len(multi_scale_features) == 3 multi_scale_features = multi_scale_features[::-1] for i in range(3): assert multi_scale_features[i].shape[-2:] == feats[i + 1].shape[-2:] ================================================ FILE: tests/test_models/test_layers/test_position_encoding.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmdet.models.layers import (LearnedPositionalEncoding, SinePositionalEncoding) def test_sine_positional_encoding(num_feats=16, batch_size=2): # test invalid type of scale with pytest.raises(AssertionError): module = SinePositionalEncoding( num_feats, scale=(3., ), normalize=True) module = SinePositionalEncoding(num_feats) h, w = 10, 6 mask = (torch.rand(batch_size, h, w) > 0.5).to(torch.int) assert not module.normalize out = module(mask) assert out.shape == (batch_size, num_feats * 2, h, w) # set normalize module = SinePositionalEncoding(num_feats, normalize=True) assert module.normalize out = module(mask) assert out.shape == (batch_size, num_feats * 2, h, w) def test_learned_positional_encoding(num_feats=16, row_num_embed=10, col_num_embed=10, batch_size=2): module = LearnedPositionalEncoding(num_feats, row_num_embed, col_num_embed) assert module.row_embed.weight.shape == (row_num_embed, num_feats) assert module.col_embed.weight.shape == (col_num_embed, num_feats) h, w = 10, 6 mask = torch.rand(batch_size, h, w) > 0.5 out = module(mask) assert out.shape == (batch_size, num_feats * 2, h, w) ================================================ FILE: tests/test_models/test_layers/test_se_layer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch import torch.nn.functional as F from mmengine.model import constant_init from mmdet.models.layers import DyReLU, SELayer def test_se_layer(): with pytest.raises(AssertionError): # act_cfg sequence length must equal to 2 SELayer(channels=32, act_cfg=(dict(type='ReLU'), )) with pytest.raises(AssertionError): # act_cfg sequence must be a tuple of dict SELayer(channels=32, act_cfg=[dict(type='ReLU'), dict(type='ReLU')]) # Test SELayer forward layer = SELayer(channels=32) layer.init_weights() layer.train() x = torch.randn((1, 32, 10, 10)) x_out = layer(x) assert x_out.shape == torch.Size((1, 32, 10, 10)) def test_dyrelu(): with pytest.raises(AssertionError): # act_cfg sequence length must equal to 2 DyReLU(channels=32, act_cfg=(dict(type='ReLU'), )) with pytest.raises(AssertionError): # act_cfg sequence must be a tuple of dict DyReLU(channels=32, act_cfg=[dict(type='ReLU'), dict(type='ReLU')]) # Test DyReLU forward layer = DyReLU(channels=32) layer.init_weights() layer.train() x = torch.randn((1, 32, 10, 10)) x_out = layer(x) assert x_out.shape == torch.Size((1, 32, 10, 10)) # DyReLU should act as standard (static) ReLU # when eliminating the effect of SE-like module layer = DyReLU(channels=32) constant_init(layer.conv2.conv, 0) layer.train() x = torch.randn((1, 32, 10, 10)) x_out = layer(x) relu_out = F.relu(x) assert torch.equal(x_out, relu_out) ================================================ FILE: tests/test_models/test_layers/test_transformer.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmengine.config import ConfigDict from mmdet.models.layers.transformer import (AdaptivePadding, DetrTransformerDecoder, DetrTransformerEncoder, PatchEmbed, PatchMerging) def test_adaptive_padding(): for padding in ('same', 'corner'): kernel_size = 16 stride = 16 dilation = 1 input = torch.rand(1, 1, 15, 17) pool = AdaptivePadding( kernel_size=kernel_size, stride=stride, dilation=dilation, padding=padding) out = pool(input) # padding to divisible by 16 assert (out.shape[2], out.shape[3]) == (16, 32) input = torch.rand(1, 1, 16, 17) out = pool(input) # padding to divisible by 16 assert (out.shape[2], out.shape[3]) == (16, 32) kernel_size = (2, 2) stride = (2, 2) dilation = (1, 1) adap_pad = AdaptivePadding( kernel_size=kernel_size, stride=stride, dilation=dilation, padding=padding) input = torch.rand(1, 1, 11, 13) out = adap_pad(input) # padding to divisible by 2 assert (out.shape[2], out.shape[3]) == (12, 14) kernel_size = (2, 2) stride = (10, 10) dilation = (1, 1) adap_pad = AdaptivePadding( kernel_size=kernel_size, stride=stride, dilation=dilation, padding=padding) input = torch.rand(1, 1, 10, 13) out = adap_pad(input) # no padding assert (out.shape[2], out.shape[3]) == (10, 13) kernel_size = (11, 11) adap_pad = AdaptivePadding( kernel_size=kernel_size, stride=stride, dilation=dilation, padding=padding) input = torch.rand(1, 1, 11, 13) out = adap_pad(input) # all padding assert (out.shape[2], out.shape[3]) == (21, 21) # test padding as kernel is (7,9) input = torch.rand(1, 1, 11, 13) stride = (3, 4) kernel_size = (4, 5) dilation = (2, 2) # actually (7, 9) adap_pad = AdaptivePadding( kernel_size=kernel_size, stride=stride, dilation=dilation, padding=padding) dilation_out = adap_pad(input) assert (dilation_out.shape[2], dilation_out.shape[3]) == (16, 21) kernel_size = (7, 9) dilation = (1, 1) adap_pad = AdaptivePadding( kernel_size=kernel_size, stride=stride, dilation=dilation, padding=padding) kernel79_out = adap_pad(input) assert (kernel79_out.shape[2], kernel79_out.shape[3]) == (16, 21) assert kernel79_out.shape == dilation_out.shape # assert only support "same" "corner" with pytest.raises(AssertionError): AdaptivePadding( kernel_size=kernel_size, stride=stride, dilation=dilation, padding=1) def test_patch_embed(): B = 2 H = 3 W = 4 C = 3 embed_dims = 10 kernel_size = 3 stride = 1 dummy_input = torch.rand(B, C, H, W) patch_merge_1 = PatchEmbed( in_channels=C, embed_dims=embed_dims, kernel_size=kernel_size, stride=stride, padding=0, dilation=1, norm_cfg=None) x1, shape = patch_merge_1(dummy_input) # test out shape assert x1.shape == (2, 2, 10) # test outsize is correct assert shape == (1, 2) # test L = out_h * out_w assert shape[0] * shape[1] == x1.shape[1] B = 2 H = 10 W = 10 C = 3 embed_dims = 10 kernel_size = 5 stride = 2 dummy_input = torch.rand(B, C, H, W) # test dilation patch_merge_2 = PatchEmbed( in_channels=C, embed_dims=embed_dims, kernel_size=kernel_size, stride=stride, padding=0, dilation=2, norm_cfg=None, ) x2, shape = patch_merge_2(dummy_input) # test out shape assert x2.shape == (2, 1, 10) # test outsize is correct assert shape == (1, 1) # test L = out_h * out_w assert shape[0] * shape[1] == x2.shape[1] stride = 2 input_size = (10, 10) dummy_input = torch.rand(B, C, H, W) # test stride and norm patch_merge_3 = PatchEmbed( in_channels=C, embed_dims=embed_dims, kernel_size=kernel_size, stride=stride, padding=0, dilation=2, norm_cfg=dict(type='LN'), input_size=input_size) x3, shape = patch_merge_3(dummy_input) # test out shape assert x3.shape == (2, 1, 10) # test outsize is correct assert shape == (1, 1) # test L = out_h * out_w assert shape[0] * shape[1] == x3.shape[1] # test the init_out_size with nn.Unfold assert patch_merge_3.init_out_size[1] == (input_size[0] - 2 * 4 - 1) // 2 + 1 assert patch_merge_3.init_out_size[0] == (input_size[0] - 2 * 4 - 1) // 2 + 1 H = 11 W = 12 input_size = (H, W) dummy_input = torch.rand(B, C, H, W) # test stride and norm patch_merge_3 = PatchEmbed( in_channels=C, embed_dims=embed_dims, kernel_size=kernel_size, stride=stride, padding=0, dilation=2, norm_cfg=dict(type='LN'), input_size=input_size) _, shape = patch_merge_3(dummy_input) # when input_size equal to real input # the out_size should be equal to `init_out_size` assert shape == patch_merge_3.init_out_size input_size = (H, W) dummy_input = torch.rand(B, C, H, W) # test stride and norm patch_merge_3 = PatchEmbed( in_channels=C, embed_dims=embed_dims, kernel_size=kernel_size, stride=stride, padding=0, dilation=2, norm_cfg=dict(type='LN'), input_size=input_size) _, shape = patch_merge_3(dummy_input) # when input_size equal to real input # the out_size should be equal to `init_out_size` assert shape == patch_merge_3.init_out_size # test adap padding for padding in ('same', 'corner'): in_c = 2 embed_dims = 3 B = 2 # test stride is 1 input_size = (5, 5) kernel_size = (5, 5) stride = (1, 1) dilation = 1 bias = False x = torch.rand(B, in_c, *input_size) patch_embed = PatchEmbed( in_channels=in_c, embed_dims=embed_dims, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) x_out, out_size = patch_embed(x) assert x_out.size() == (B, 25, 3) assert out_size == (5, 5) assert x_out.size(1) == out_size[0] * out_size[1] # test kernel_size == stride input_size = (5, 5) kernel_size = (5, 5) stride = (5, 5) dilation = 1 bias = False x = torch.rand(B, in_c, *input_size) patch_embed = PatchEmbed( in_channels=in_c, embed_dims=embed_dims, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) x_out, out_size = patch_embed(x) assert x_out.size() == (B, 1, 3) assert out_size == (1, 1) assert x_out.size(1) == out_size[0] * out_size[1] # test kernel_size == stride input_size = (6, 5) kernel_size = (5, 5) stride = (5, 5) dilation = 1 bias = False x = torch.rand(B, in_c, *input_size) patch_embed = PatchEmbed( in_channels=in_c, embed_dims=embed_dims, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) x_out, out_size = patch_embed(x) assert x_out.size() == (B, 2, 3) assert out_size == (2, 1) assert x_out.size(1) == out_size[0] * out_size[1] # test different kernel_size with different stride input_size = (6, 5) kernel_size = (6, 2) stride = (6, 2) dilation = 1 bias = False x = torch.rand(B, in_c, *input_size) patch_embed = PatchEmbed( in_channels=in_c, embed_dims=embed_dims, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) x_out, out_size = patch_embed(x) assert x_out.size() == (B, 3, 3) assert out_size == (1, 3) assert x_out.size(1) == out_size[0] * out_size[1] def test_patch_merging(): # Test the model with int padding in_c = 3 out_c = 4 kernel_size = 3 stride = 3 padding = 1 dilation = 1 bias = False # test the case `pad_to_stride` is False patch_merge = PatchMerging( in_channels=in_c, out_channels=out_c, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) B, L, C = 1, 100, 3 input_size = (10, 10) x = torch.rand(B, L, C) x_out, out_size = patch_merge(x, input_size) assert x_out.size() == (1, 16, 4) assert out_size == (4, 4) # assert out size is consistent with real output assert x_out.size(1) == out_size[0] * out_size[1] in_c = 4 out_c = 5 kernel_size = 6 stride = 3 padding = 2 dilation = 2 bias = False patch_merge = PatchMerging( in_channels=in_c, out_channels=out_c, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) B, L, C = 1, 100, 4 input_size = (10, 10) x = torch.rand(B, L, C) x_out, out_size = patch_merge(x, input_size) assert x_out.size() == (1, 4, 5) assert out_size == (2, 2) # assert out size is consistent with real output assert x_out.size(1) == out_size[0] * out_size[1] # Test with adaptive padding for padding in ('same', 'corner'): in_c = 2 out_c = 3 B = 2 # test stride is 1 input_size = (5, 5) kernel_size = (5, 5) stride = (1, 1) dilation = 1 bias = False L = input_size[0] * input_size[1] x = torch.rand(B, L, in_c) patch_merge = PatchMerging( in_channels=in_c, out_channels=out_c, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) x_out, out_size = patch_merge(x, input_size) assert x_out.size() == (B, 25, 3) assert out_size == (5, 5) assert x_out.size(1) == out_size[0] * out_size[1] # test kernel_size == stride input_size = (5, 5) kernel_size = (5, 5) stride = (5, 5) dilation = 1 bias = False L = input_size[0] * input_size[1] x = torch.rand(B, L, in_c) patch_merge = PatchMerging( in_channels=in_c, out_channels=out_c, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) x_out, out_size = patch_merge(x, input_size) assert x_out.size() == (B, 1, 3) assert out_size == (1, 1) assert x_out.size(1) == out_size[0] * out_size[1] # test kernel_size == stride input_size = (6, 5) kernel_size = (5, 5) stride = (5, 5) dilation = 1 bias = False L = input_size[0] * input_size[1] x = torch.rand(B, L, in_c) patch_merge = PatchMerging( in_channels=in_c, out_channels=out_c, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) x_out, out_size = patch_merge(x, input_size) assert x_out.size() == (B, 2, 3) assert out_size == (2, 1) assert x_out.size(1) == out_size[0] * out_size[1] # test different kernel_size with different stride input_size = (6, 5) kernel_size = (6, 2) stride = (6, 2) dilation = 1 bias = False L = input_size[0] * input_size[1] x = torch.rand(B, L, in_c) patch_merge = PatchMerging( in_channels=in_c, out_channels=out_c, kernel_size=kernel_size, stride=stride, padding=padding, dilation=dilation, bias=bias) x_out, out_size = patch_merge(x, input_size) assert x_out.size() == (B, 3, 3) assert out_size == (1, 3) assert x_out.size(1) == out_size[0] * out_size[1] def test_detr_transformer_encoder_decoder(): config = ConfigDict( num_layers=6, layer_cfg=dict( # DetrTransformerDecoderLayer self_attn_cfg=dict( # MultiheadAttention embed_dims=256, num_heads=8, dropout=0.1), cross_attn_cfg=dict( # MultiheadAttention embed_dims=256, num_heads=8, dropout=0.1), ffn_cfg=dict( embed_dims=256, feedforward_channels=2048, num_fcs=2, ffn_drop=0.1, act_cfg=dict(type='ReLU', inplace=True)))) assert len(DetrTransformerDecoder(**config).layers) == 6 assert DetrTransformerDecoder(**config) config = ConfigDict( dict( num_layers=6, layer_cfg=dict( # DetrTransformerEncoderLayer self_attn_cfg=dict( # MultiheadAttention embed_dims=256, num_heads=8, dropout=0.1), ffn_cfg=dict( embed_dims=256, feedforward_channels=2048, num_fcs=2, ffn_drop=0.1, act_cfg=dict(type='ReLU', inplace=True))))) assert len(DetrTransformerEncoder(**config).layers) == 6 assert DetrTransformerEncoder(**config) ================================================ FILE: tests/test_models/test_losses/test_gaussian_focal_loss.py ================================================ import unittest import torch from mmdet.models.losses import GaussianFocalLoss class TestGaussianFocalLoss(unittest.TestCase): def test_forward(self): pred = torch.rand((10, 4)) target = torch.rand((10, 4)) gaussian_focal_loss = GaussianFocalLoss() loss1 = gaussian_focal_loss(pred, target) self.assertIsInstance(loss1, torch.Tensor) loss2 = gaussian_focal_loss(pred, target, avg_factor=0.5) self.assertIsInstance(loss2, torch.Tensor) # test reduction gaussian_focal_loss = GaussianFocalLoss(reduction='none') loss = gaussian_focal_loss(pred, target) self.assertTrue(loss.shape == (10, 4)) # test reduction_override loss = gaussian_focal_loss(pred, target, reduction_override='mean') self.assertTrue(loss.ndim == 0) # Only supports None, 'none', 'mean', 'sum' with self.assertRaises(AssertionError): gaussian_focal_loss(pred, target, reduction_override='max') # test pos_inds pos_inds = (torch.rand(5) * 8).long() pos_labels = (torch.rand(5) * 2).long() gaussian_focal_loss = GaussianFocalLoss() loss = gaussian_focal_loss(pred, target, pos_inds, pos_labels) self.assertIsInstance(loss, torch.Tensor) ================================================ FILE: tests/test_models/test_losses/test_loss.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch import torch.nn.functional as F from mmengine.utils import digit_version from mmdet.models.losses import (BalancedL1Loss, CrossEntropyLoss, DiceLoss, DistributionFocalLoss, FocalLoss, GaussianFocalLoss, KnowledgeDistillationKLDivLoss, L1Loss, MSELoss, QualityFocalLoss, SeesawLoss, SmoothL1Loss, VarifocalLoss) from mmdet.models.losses.ghm_loss import GHMC, GHMR from mmdet.models.losses.iou_loss import (BoundedIoULoss, CIoULoss, DIoULoss, EIoULoss, GIoULoss, IoULoss) @pytest.mark.parametrize( 'loss_class', [IoULoss, BoundedIoULoss, GIoULoss, DIoULoss, CIoULoss, EIoULoss]) def test_iou_type_loss_zeros_weight(loss_class): pred = torch.rand((10, 4)) target = torch.rand((10, 4)) weight = torch.zeros(10) loss = loss_class()(pred, target, weight) assert loss == 0. @pytest.mark.parametrize('loss_class', [ BalancedL1Loss, BoundedIoULoss, CIoULoss, CrossEntropyLoss, DIoULoss, EIoULoss, FocalLoss, DistributionFocalLoss, MSELoss, SeesawLoss, GaussianFocalLoss, GIoULoss, QualityFocalLoss, IoULoss, L1Loss, VarifocalLoss, GHMR, GHMC, SmoothL1Loss, KnowledgeDistillationKLDivLoss, DiceLoss ]) def test_loss_with_reduction_override(loss_class): pred = torch.rand((10, 4)) target = torch.rand((10, 4)), weight = None with pytest.raises(AssertionError): # only reduction_override from [None, 'none', 'mean', 'sum'] # is not allowed reduction_override = True loss_class()( pred, target, weight, reduction_override=reduction_override) @pytest.mark.parametrize('loss_class', [QualityFocalLoss]) @pytest.mark.parametrize('activated', [False, True]) def test_QualityFocalLoss_Loss(loss_class, activated): input_shape = (4, 5) pred = torch.rand(input_shape) label = torch.Tensor([0, 1, 2, 0]).long() quality_label = torch.rand(input_shape[0]) original_loss = loss_class(activated=activated)(pred, (label, quality_label)) assert isinstance(original_loss, torch.Tensor) target = torch.nn.functional.one_hot(label, 5) target = target * quality_label.reshape(input_shape[0], 1) new_loss = loss_class(activated=activated)(pred, target) assert isinstance(new_loss, torch.Tensor) assert new_loss == original_loss @pytest.mark.parametrize('loss_class', [ IoULoss, BoundedIoULoss, GIoULoss, DIoULoss, CIoULoss, EIoULoss, MSELoss, L1Loss, SmoothL1Loss, BalancedL1Loss ]) @pytest.mark.parametrize('input_shape', [(10, 4), (0, 4)]) def test_regression_losses(loss_class, input_shape): pred = torch.rand(input_shape) target = torch.rand(input_shape) weight = torch.rand(input_shape) # Test loss forward loss = loss_class()(pred, target) assert isinstance(loss, torch.Tensor) # Test loss forward with weight loss = loss_class()(pred, target, weight) assert isinstance(loss, torch.Tensor) # Test loss forward with reduction_override loss = loss_class()(pred, target, reduction_override='mean') assert isinstance(loss, torch.Tensor) # Test loss forward with avg_factor loss = loss_class()(pred, target, avg_factor=10) assert isinstance(loss, torch.Tensor) with pytest.raises(ValueError): # loss can evaluate with avg_factor only if # reduction is None, 'none' or 'mean'. reduction_override = 'sum' loss_class()( pred, target, avg_factor=10, reduction_override=reduction_override) # Test loss forward with avg_factor and reduction for reduction_override in [None, 'none', 'mean']: loss_class()( pred, target, avg_factor=10, reduction_override=reduction_override) assert isinstance(loss, torch.Tensor) @pytest.mark.parametrize('loss_class', [CrossEntropyLoss]) @pytest.mark.parametrize('input_shape', [(10, 5), (0, 5)]) def test_classification_losses(loss_class, input_shape): if input_shape[0] == 0 and digit_version( torch.__version__) < digit_version('1.5.0'): pytest.skip( f'CELoss in PyTorch {torch.__version__} does not support empty' f'tensor.') pred = torch.rand(input_shape) target = torch.randint(0, 5, (input_shape[0], )) # Test loss forward loss = loss_class()(pred, target) assert isinstance(loss, torch.Tensor) # Test loss forward with reduction_override loss = loss_class()(pred, target, reduction_override='mean') assert isinstance(loss, torch.Tensor) # Test loss forward with avg_factor loss = loss_class()(pred, target, avg_factor=10) assert isinstance(loss, torch.Tensor) with pytest.raises(ValueError): # loss can evaluate with avg_factor only if # reduction is None, 'none' or 'mean'. reduction_override = 'sum' loss_class()( pred, target, avg_factor=10, reduction_override=reduction_override) # Test loss forward with avg_factor and reduction for reduction_override in [None, 'none', 'mean']: loss_class()( pred, target, avg_factor=10, reduction_override=reduction_override) assert isinstance(loss, torch.Tensor) @pytest.mark.parametrize('loss_class', [FocalLoss]) @pytest.mark.parametrize('input_shape', [(10, 5), (3, 5, 40, 40)]) def test_FocalLoss_loss(loss_class, input_shape): pred = torch.rand(input_shape) target = torch.randint(0, 5, (input_shape[0], )) if len(input_shape) == 4: B, N, W, H = input_shape target = F.one_hot(torch.randint(0, 5, (B * W * H, )), 5).reshape(B, W, H, N).permute(0, 3, 1, 2) # Test loss forward loss = loss_class()(pred, target) assert isinstance(loss, torch.Tensor) # Test loss forward with reduction_override loss = loss_class()(pred, target, reduction_override='mean') assert isinstance(loss, torch.Tensor) # Test loss forward with avg_factor loss = loss_class()(pred, target, avg_factor=10) assert isinstance(loss, torch.Tensor) with pytest.raises(ValueError): # loss can evaluate with avg_factor only if # reduction is None, 'none' or 'mean'. reduction_override = 'sum' loss_class()( pred, target, avg_factor=10, reduction_override=reduction_override) # Test loss forward with avg_factor and reduction for reduction_override in [None, 'none', 'mean']: loss_class()( pred, target, avg_factor=10, reduction_override=reduction_override) assert isinstance(loss, torch.Tensor) @pytest.mark.parametrize('loss_class', [GHMR]) @pytest.mark.parametrize('input_shape', [(10, 4), (0, 4)]) def test_GHMR_loss(loss_class, input_shape): pred = torch.rand(input_shape) target = torch.rand(input_shape) weight = torch.rand(input_shape) # Test loss forward loss = loss_class()(pred, target, weight) assert isinstance(loss, torch.Tensor) @pytest.mark.parametrize('use_sigmoid', [True, False]) @pytest.mark.parametrize('reduction', ['sum', 'mean', None]) @pytest.mark.parametrize('avg_non_ignore', [True, False]) def test_loss_with_ignore_index(use_sigmoid, reduction, avg_non_ignore): # Test cross_entropy loss loss_class = CrossEntropyLoss( use_sigmoid=use_sigmoid, use_mask=False, ignore_index=255, avg_non_ignore=avg_non_ignore) pred = torch.rand((10, 5)) target = torch.randint(0, 5, (10, )) ignored_indices = torch.randint(0, 10, (2, ), dtype=torch.long) target[ignored_indices] = 255 # Test loss forward with default ignore loss_with_ignore = loss_class(pred, target, reduction_override=reduction) assert isinstance(loss_with_ignore, torch.Tensor) # Test loss forward with forward ignore target[ignored_indices] = 255 loss_with_forward_ignore = loss_class( pred, target, ignore_index=255, reduction_override=reduction) assert isinstance(loss_with_forward_ignore, torch.Tensor) # Verify correctness if avg_non_ignore: # manually remove the ignored elements not_ignored_indices = (target != 255) pred = pred[not_ignored_indices] target = target[not_ignored_indices] loss = loss_class(pred, target, reduction_override=reduction) assert torch.allclose(loss, loss_with_ignore) assert torch.allclose(loss, loss_with_forward_ignore) # test ignore all target pred = torch.rand((10, 5)) target = torch.ones((10, ), dtype=torch.long) * 255 loss = loss_class(pred, target, reduction_override=reduction) assert loss == 0 @pytest.mark.parametrize('naive_dice', [True, False]) def test_dice_loss(naive_dice): loss_class = DiceLoss pred = torch.rand((10, 4, 4)) target = torch.rand((10, 4, 4)) weight = torch.rand((10)) # Test loss forward loss = loss_class(naive_dice=naive_dice)(pred, target) assert isinstance(loss, torch.Tensor) # Test loss forward with weight loss = loss_class(naive_dice=naive_dice)(pred, target, weight) assert isinstance(loss, torch.Tensor) # Test loss forward with reduction_override loss = loss_class(naive_dice=naive_dice)( pred, target, reduction_override='mean') assert isinstance(loss, torch.Tensor) # Test loss forward with avg_factor loss = loss_class(naive_dice=naive_dice)(pred, target, avg_factor=10) assert isinstance(loss, torch.Tensor) with pytest.raises(ValueError): # loss can evaluate with avg_factor only if # reduction is None, 'none' or 'mean'. reduction_override = 'sum' loss_class(naive_dice=naive_dice)( pred, target, avg_factor=10, reduction_override=reduction_override) # Test loss forward with avg_factor and reduction for reduction_override in [None, 'none', 'mean']: loss_class(naive_dice=naive_dice)( pred, target, avg_factor=10, reduction_override=reduction_override) assert isinstance(loss, torch.Tensor) # Test loss forward with has_acted=False and use_sigmoid=False with pytest.raises(NotImplementedError): loss_class( use_sigmoid=False, activate=True, naive_dice=naive_dice)(pred, target) # Test loss forward with weight.ndim != loss.ndim with pytest.raises(AssertionError): weight = torch.rand((2, 8)) loss_class(naive_dice=naive_dice)(pred, target, weight) # Test loss forward with len(weight) != len(pred) with pytest.raises(AssertionError): weight = torch.rand((8)) loss_class(naive_dice=naive_dice)(pred, target, weight) ================================================ FILE: tests/test_models/test_necks/test_ct_resnet_neck.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest import torch from mmdet.models.necks import CTResNetNeck class TestCTResNetNeck(unittest.TestCase): def test_init(self): # num_filters/num_kernels must be same length with self.assertRaises(AssertionError): CTResNetNeck( in_channels=10, num_deconv_filters=(10, 10), num_deconv_kernels=(4, )) ct_resnet_neck = CTResNetNeck( in_channels=16, num_deconv_filters=(8, 8), num_deconv_kernels=(4, 4), use_dcn=False) ct_resnet_neck.init_weights() def test_forward(self): in_channels = 16 num_filters = (8, 8) num_kernels = (4, 4) feat = torch.rand(1, 16, 4, 4) ct_resnet_neck = CTResNetNeck( in_channels=in_channels, num_deconv_filters=num_filters, num_deconv_kernels=num_kernels, use_dcn=False) # feat must be list or tuple with self.assertRaises(AssertionError): ct_resnet_neck(feat) out_feat = ct_resnet_neck([feat])[0] self.assertEqual(out_feat.shape, (1, num_filters[-1], 16, 16)) if torch.cuda.is_available(): # test dcn ct_resnet_neck = CTResNetNeck( in_channels=in_channels, num_deconv_filters=num_filters, num_deconv_kernels=num_kernels) ct_resnet_neck = ct_resnet_neck.cuda() feat = feat.cuda() out_feat = ct_resnet_neck([feat])[0] self.assertEqual(out_feat.shape, (1, num_filters[-1], 16, 16)) ================================================ FILE: tests/test_models/test_necks/test_necks.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from torch.nn.modules.batchnorm import _BatchNorm from mmdet.models.necks import (FPG, FPN, FPN_CARAFE, NASFCOS_FPN, NASFPN, SSH, YOLOXPAFPN, ChannelMapper, DilatedEncoder, DyHead, SSDNeck, YOLOV3Neck) def test_fpn(): """Tests fpn.""" s = 64 in_channels = [8, 16, 32, 64] feat_sizes = [s // 2**i for i in range(4)] # [64, 32, 16, 8] out_channels = 8 # end_level=-1 is equal to end_level=3 FPN(in_channels=in_channels, out_channels=out_channels, start_level=0, end_level=-1, num_outs=5) FPN(in_channels=in_channels, out_channels=out_channels, start_level=0, end_level=3, num_outs=5) # `num_outs` is not equal to end_level - start_level + 1 with pytest.raises(AssertionError): FPN(in_channels=in_channels, out_channels=out_channels, start_level=1, end_level=2, num_outs=3) # `num_outs` is not equal to len(in_channels) - start_level with pytest.raises(AssertionError): FPN(in_channels=in_channels, out_channels=out_channels, start_level=1, num_outs=2) # `end_level` is larger than len(in_channels) - 1 with pytest.raises(AssertionError): FPN(in_channels=in_channels, out_channels=out_channels, start_level=1, end_level=4, num_outs=2) # `num_outs` is not equal to end_level - start_level with pytest.raises(AssertionError): FPN(in_channels=in_channels, out_channels=out_channels, start_level=1, end_level=3, num_outs=1) # Invalid `add_extra_convs` option with pytest.raises(AssertionError): FPN(in_channels=in_channels, out_channels=out_channels, start_level=1, add_extra_convs='on_xxx', num_outs=5) fpn_model = FPN( in_channels=in_channels, out_channels=out_channels, start_level=1, add_extra_convs=True, num_outs=5) # FPN expects a multiple levels of features per image feats = [ torch.rand(1, in_channels[i], feat_sizes[i], feat_sizes[i]) for i in range(len(in_channels)) ] outs = fpn_model(feats) assert fpn_model.add_extra_convs == 'on_input' assert len(outs) == fpn_model.num_outs for i in range(fpn_model.num_outs): outs[i].shape[1] == out_channels outs[i].shape[2] == outs[i].shape[3] == s // (2**i) # Tests for fpn with no extra convs (pooling is used instead) fpn_model = FPN( in_channels=in_channels, out_channels=out_channels, start_level=1, add_extra_convs=False, num_outs=5) outs = fpn_model(feats) assert len(outs) == fpn_model.num_outs assert not fpn_model.add_extra_convs for i in range(fpn_model.num_outs): outs[i].shape[1] == out_channels outs[i].shape[2] == outs[i].shape[3] == s // (2**i) # Tests for fpn with lateral bns fpn_model = FPN( in_channels=in_channels, out_channels=out_channels, start_level=1, add_extra_convs=True, no_norm_on_lateral=False, norm_cfg=dict(type='BN', requires_grad=True), num_outs=5) outs = fpn_model(feats) assert len(outs) == fpn_model.num_outs assert fpn_model.add_extra_convs == 'on_input' for i in range(fpn_model.num_outs): outs[i].shape[1] == out_channels outs[i].shape[2] == outs[i].shape[3] == s // (2**i) bn_exist = False for m in fpn_model.modules(): if isinstance(m, _BatchNorm): bn_exist = True assert bn_exist # Bilinear upsample fpn_model = FPN( in_channels=in_channels, out_channels=out_channels, start_level=1, add_extra_convs=True, upsample_cfg=dict(mode='bilinear', align_corners=True), num_outs=5) fpn_model(feats) outs = fpn_model(feats) assert len(outs) == fpn_model.num_outs assert fpn_model.add_extra_convs == 'on_input' for i in range(fpn_model.num_outs): outs[i].shape[1] == out_channels outs[i].shape[2] == outs[i].shape[3] == s // (2**i) # Scale factor instead of fixed upsample size upsample fpn_model = FPN( in_channels=in_channels, out_channels=out_channels, start_level=1, add_extra_convs=True, upsample_cfg=dict(scale_factor=2), num_outs=5) outs = fpn_model(feats) assert len(outs) == fpn_model.num_outs for i in range(fpn_model.num_outs): outs[i].shape[1] == out_channels outs[i].shape[2] == outs[i].shape[3] == s // (2**i) # Extra convs source is 'inputs' fpn_model = FPN( in_channels=in_channels, out_channels=out_channels, add_extra_convs='on_input', start_level=1, num_outs=5) assert fpn_model.add_extra_convs == 'on_input' outs = fpn_model(feats) assert len(outs) == fpn_model.num_outs for i in range(fpn_model.num_outs): outs[i].shape[1] == out_channels outs[i].shape[2] == outs[i].shape[3] == s // (2**i) # Extra convs source is 'laterals' fpn_model = FPN( in_channels=in_channels, out_channels=out_channels, add_extra_convs='on_lateral', start_level=1, num_outs=5) assert fpn_model.add_extra_convs == 'on_lateral' outs = fpn_model(feats) assert len(outs) == fpn_model.num_outs for i in range(fpn_model.num_outs): outs[i].shape[1] == out_channels outs[i].shape[2] == outs[i].shape[3] == s // (2**i) # Extra convs source is 'outputs' fpn_model = FPN( in_channels=in_channels, out_channels=out_channels, add_extra_convs='on_output', start_level=1, num_outs=5) assert fpn_model.add_extra_convs == 'on_output' outs = fpn_model(feats) assert len(outs) == fpn_model.num_outs for i in range(fpn_model.num_outs): outs[i].shape[1] == out_channels outs[i].shape[2] == outs[i].shape[3] == s // (2**i) def test_channel_mapper(): """Tests ChannelMapper.""" s = 64 in_channels = [8, 16, 32, 64] feat_sizes = [s // 2**i for i in range(4)] # [64, 32, 16, 8] out_channels = 8 kernel_size = 3 feats = [ torch.rand(1, in_channels[i], feat_sizes[i], feat_sizes[i]) for i in range(len(in_channels)) ] # in_channels must be a list with pytest.raises(AssertionError): channel_mapper = ChannelMapper( in_channels=10, out_channels=out_channels, kernel_size=kernel_size) # the length of channel_mapper's inputs must be equal to the length of # in_channels with pytest.raises(AssertionError): channel_mapper = ChannelMapper( in_channels=in_channels[:-1], out_channels=out_channels, kernel_size=kernel_size) channel_mapper(feats) channel_mapper = ChannelMapper( in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size) outs = channel_mapper(feats) assert len(outs) == len(feats) for i in range(len(feats)): outs[i].shape[1] == out_channels outs[i].shape[2] == outs[i].shape[3] == s // (2**i) def test_dilated_encoder(): in_channels = 16 out_channels = 32 out_shape = 34 dilated_encoder = DilatedEncoder(in_channels, out_channels, 16, 2, [2, 4, 6, 8]) feat = [torch.rand(1, in_channels, 34, 34)] out_feat = dilated_encoder(feat)[0] assert out_feat.shape == (1, out_channels, out_shape, out_shape) def test_yolov3_neck(): # num_scales, in_channels, out_channels must be same length with pytest.raises(AssertionError): YOLOV3Neck(num_scales=3, in_channels=[16, 8, 4], out_channels=[8, 4]) # len(feats) must equal to num_scales with pytest.raises(AssertionError): neck = YOLOV3Neck( num_scales=3, in_channels=[16, 8, 4], out_channels=[8, 4, 2]) feats = (torch.rand(1, 4, 16, 16), torch.rand(1, 8, 16, 16)) neck(feats) # test normal channels s = 32 in_channels = [16, 8, 4] out_channels = [8, 4, 2] feat_sizes = [s // 2**i for i in range(len(in_channels) - 1, -1, -1)] feats = [ torch.rand(1, in_channels[i], feat_sizes[i], feat_sizes[i]) for i in range(len(in_channels) - 1, -1, -1) ] neck = YOLOV3Neck( num_scales=3, in_channels=in_channels, out_channels=out_channels) outs = neck(feats) assert len(outs) == len(feats) for i in range(len(outs)): assert outs[i].shape == \ (1, out_channels[i], feat_sizes[i], feat_sizes[i]) # test more flexible setting s = 32 in_channels = [32, 8, 16] out_channels = [19, 21, 5] feat_sizes = [s // 2**i for i in range(len(in_channels) - 1, -1, -1)] feats = [ torch.rand(1, in_channels[i], feat_sizes[i], feat_sizes[i]) for i in range(len(in_channels) - 1, -1, -1) ] neck = YOLOV3Neck( num_scales=3, in_channels=in_channels, out_channels=out_channels) outs = neck(feats) assert len(outs) == len(feats) for i in range(len(outs)): assert outs[i].shape == \ (1, out_channels[i], feat_sizes[i], feat_sizes[i]) def test_ssd_neck(): # level_strides/level_paddings must be same length with pytest.raises(AssertionError): SSDNeck( in_channels=[8, 16], out_channels=[8, 16, 32], level_strides=[2], level_paddings=[2, 1]) # length of out_channels must larger than in_channels with pytest.raises(AssertionError): SSDNeck( in_channels=[8, 16], out_channels=[8], level_strides=[2], level_paddings=[2]) # len(out_channels) - len(in_channels) must equal to len(level_strides) with pytest.raises(AssertionError): SSDNeck( in_channels=[8, 16], out_channels=[4, 16, 64], level_strides=[2, 2], level_paddings=[2, 2]) # in_channels must be same with out_channels[:len(in_channels)] with pytest.raises(AssertionError): SSDNeck( in_channels=[8, 16], out_channels=[4, 16, 64], level_strides=[2], level_paddings=[2]) ssd_neck = SSDNeck( in_channels=[4], out_channels=[4, 8, 16], level_strides=[2, 1], level_paddings=[1, 0]) feats = (torch.rand(1, 4, 16, 16), ) outs = ssd_neck(feats) assert outs[0].shape == (1, 4, 16, 16) assert outs[1].shape == (1, 8, 8, 8) assert outs[2].shape == (1, 16, 6, 6) # test SSD-Lite Neck ssd_neck = SSDNeck( in_channels=[4, 8], out_channels=[4, 8, 16], level_strides=[1], level_paddings=[1], l2_norm_scale=None, use_depthwise=True, norm_cfg=dict(type='BN'), act_cfg=dict(type='ReLU6')) assert not hasattr(ssd_neck, 'l2_norm') from mmcv.cnn.bricks import DepthwiseSeparableConvModule assert isinstance(ssd_neck.extra_layers[0][-1], DepthwiseSeparableConvModule) feats = (torch.rand(1, 4, 8, 8), torch.rand(1, 8, 8, 8)) outs = ssd_neck(feats) assert outs[0].shape == (1, 4, 8, 8) assert outs[1].shape == (1, 8, 8, 8) assert outs[2].shape == (1, 16, 8, 8) def test_yolox_pafpn(): s = 64 in_channels = [8, 16, 32, 64] feat_sizes = [s // 2**i for i in range(4)] # [64, 32, 16, 8] out_channels = 24 feats = [ torch.rand(1, in_channels[i], feat_sizes[i], feat_sizes[i]) for i in range(len(in_channels)) ] neck = YOLOXPAFPN(in_channels=in_channels, out_channels=out_channels) outs = neck(feats) assert len(outs) == len(feats) for i in range(len(feats)): assert outs[i].shape[1] == out_channels assert outs[i].shape[2] == outs[i].shape[3] == s // (2**i) # test depth-wise neck = YOLOXPAFPN( in_channels=in_channels, out_channels=out_channels, use_depthwise=True) from mmcv.cnn.bricks import DepthwiseSeparableConvModule assert isinstance(neck.downsamples[0], DepthwiseSeparableConvModule) outs = neck(feats) assert len(outs) == len(feats) for i in range(len(feats)): assert outs[i].shape[1] == out_channels assert outs[i].shape[2] == outs[i].shape[3] == s // (2**i) def test_dyhead(): s = 64 in_channels = 8 out_channels = 16 feat_sizes = [s // 2**i for i in range(4)] # [64, 32, 16, 8] feats = [ torch.rand(1, in_channels, feat_sizes[i], feat_sizes[i]) for i in range(len(feat_sizes)) ] neck = DyHead( in_channels=in_channels, out_channels=out_channels, num_blocks=3) outs = neck(feats) assert len(outs) == len(feats) for i in range(len(outs)): assert outs[i].shape[1] == out_channels assert outs[i].shape[2] == outs[i].shape[3] == s // (2**i) feat = torch.rand(1, 8, 4, 4) # input feat must be tuple or list with pytest.raises(AssertionError): neck(feat) def test_fpg(): # end_level=-1 is equal to end_level=3 norm_cfg = dict(type='BN', requires_grad=True) FPG(in_channels=[8, 16, 32, 64], out_channels=8, inter_channels=8, num_outs=5, add_extra_convs=True, start_level=1, end_level=-1, stack_times=9, paths=['bu'] * 9, same_down_trans=None, same_up_trans=dict( type='conv', kernel_size=3, stride=2, padding=1, norm_cfg=norm_cfg, inplace=False, order=('act', 'conv', 'norm')), across_lateral_trans=dict( type='conv', kernel_size=1, norm_cfg=norm_cfg, inplace=False, order=('act', 'conv', 'norm')), across_down_trans=dict( type='interpolation_conv', mode='nearest', kernel_size=3, norm_cfg=norm_cfg, order=('act', 'conv', 'norm'), inplace=False), across_up_trans=None, across_skip_trans=dict( type='conv', kernel_size=1, norm_cfg=norm_cfg, inplace=False, order=('act', 'conv', 'norm')), output_trans=dict( type='last_conv', kernel_size=3, order=('act', 'conv', 'norm'), inplace=False), norm_cfg=norm_cfg, skip_inds=[(0, 1, 2, 3), (0, 1, 2), (0, 1), (0, ), ()]) FPG(in_channels=[8, 16, 32, 64], out_channels=8, inter_channels=8, num_outs=5, add_extra_convs=True, start_level=1, end_level=3, stack_times=9, paths=['bu'] * 9, same_down_trans=None, same_up_trans=dict( type='conv', kernel_size=3, stride=2, padding=1, norm_cfg=norm_cfg, inplace=False, order=('act', 'conv', 'norm')), across_lateral_trans=dict( type='conv', kernel_size=1, norm_cfg=norm_cfg, inplace=False, order=('act', 'conv', 'norm')), across_down_trans=dict( type='interpolation_conv', mode='nearest', kernel_size=3, norm_cfg=norm_cfg, order=('act', 'conv', 'norm'), inplace=False), across_up_trans=None, across_skip_trans=dict( type='conv', kernel_size=1, norm_cfg=norm_cfg, inplace=False, order=('act', 'conv', 'norm')), output_trans=dict( type='last_conv', kernel_size=3, order=('act', 'conv', 'norm'), inplace=False), norm_cfg=norm_cfg, skip_inds=[(0, 1, 2, 3), (0, 1, 2), (0, 1), (0, ), ()]) # `end_level` is larger than len(in_channels) - 1 with pytest.raises(AssertionError): FPG(in_channels=[8, 16, 32, 64], out_channels=8, stack_times=9, paths=['bu'] * 9, start_level=1, end_level=4, num_outs=2, skip_inds=[(0, 1, 2, 3), (0, 1, 2), (0, 1), (0, ), ()]) # `num_outs` is not equal to end_level - start_level + 1 with pytest.raises(AssertionError): FPG(in_channels=[8, 16, 32, 64], out_channels=8, stack_times=9, paths=['bu'] * 9, start_level=1, end_level=2, num_outs=3, skip_inds=[(0, 1, 2, 3), (0, 1, 2), (0, 1), (0, ), ()]) def test_fpn_carafe(): # end_level=-1 is equal to end_level=3 FPN_CARAFE( in_channels=[8, 16, 32, 64], out_channels=8, start_level=0, end_level=3, num_outs=4) FPN_CARAFE( in_channels=[8, 16, 32, 64], out_channels=8, start_level=0, end_level=-1, num_outs=4) # `end_level` is larger than len(in_channels) - 1 with pytest.raises(AssertionError): FPN_CARAFE( in_channels=[8, 16, 32, 64], out_channels=8, start_level=1, end_level=4, num_outs=2) # `num_outs` is not equal to end_level - start_level + 1 with pytest.raises(AssertionError): FPN_CARAFE( in_channels=[8, 16, 32, 64], out_channels=8, start_level=1, end_level=2, num_outs=3) def test_nas_fpn(): # end_level=-1 is equal to end_level=3 NASFPN( in_channels=[8, 16, 32, 64], out_channels=8, stack_times=9, start_level=0, end_level=3, num_outs=4) NASFPN( in_channels=[8, 16, 32, 64], out_channels=8, stack_times=9, start_level=0, end_level=-1, num_outs=4) # `end_level` is larger than len(in_channels) - 1 with pytest.raises(AssertionError): NASFPN( in_channels=[8, 16, 32, 64], out_channels=8, stack_times=9, start_level=1, end_level=4, num_outs=2) # `num_outs` is not equal to end_level - start_level + 1 with pytest.raises(AssertionError): NASFPN( in_channels=[8, 16, 32, 64], out_channels=8, stack_times=9, start_level=1, end_level=2, num_outs=3) def test_nasfcos_fpn(): # end_level=-1 is equal to end_level=3 NASFCOS_FPN( in_channels=[8, 16, 32, 64], out_channels=8, start_level=0, end_level=3, num_outs=4) NASFCOS_FPN( in_channels=[8, 16, 32, 64], out_channels=8, start_level=0, end_level=-1, num_outs=4) # `end_level` is larger than len(in_channels) - 1 with pytest.raises(AssertionError): NASFCOS_FPN( in_channels=[8, 16, 32, 64], out_channels=8, start_level=1, end_level=4, num_outs=2) # `num_outs` is not equal to end_level - start_level + 1 with pytest.raises(AssertionError): NASFCOS_FPN( in_channels=[8, 16, 32, 64], out_channels=8, start_level=1, end_level=2, num_outs=3) def test_ssh_neck(): """Tests ssh.""" s = 64 in_channels = [8, 16, 32, 64] feat_sizes = [s // 2**i for i in range(4)] # [64, 32, 16, 8] out_channels = [16, 32, 64, 128] ssh_model = SSH( num_scales=4, in_channels=in_channels, out_channels=out_channels) feats = [ torch.rand(1, in_channels[i], feat_sizes[i], feat_sizes[i]) for i in range(len(in_channels)) ] outs = ssh_model(feats) assert len(outs) == len(feats) for i in range(len(outs)): assert outs[i].shape == \ (1, out_channels[i], feat_sizes[i], feat_sizes[i]) ================================================ FILE: tests/test_models/test_roi_heads/test_bbox_heads/test_bbox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.roi_heads.bbox_heads import (BBoxHead, Shared2FCBBoxHead, Shared4Conv1FCBBoxHead) from mmdet.models.task_modules.samplers import SamplingResult class TestBboxHead(TestCase): def test_init(self): # Shared2FCBBoxHead bbox_head = Shared2FCBBoxHead( in_channels=1, fc_out_channels=1, num_classes=4) self.assertTrue(bbox_head.fc_cls) self.assertTrue(bbox_head.fc_reg) self.assertEqual(len(bbox_head.shared_fcs), 2) # Shared4Conv1FCBBoxHead bbox_head = Shared4Conv1FCBBoxHead( in_channels=1, fc_out_channels=1, num_classes=4) self.assertTrue(bbox_head.fc_cls) self.assertTrue(bbox_head.fc_reg) self.assertEqual(len(bbox_head.shared_convs), 4) self.assertEqual(len(bbox_head.shared_fcs), 1) def test_bbox_head_get_results(self): num_classes = 6 bbox_head = BBoxHead(reg_class_agnostic=True, num_classes=num_classes) s = 128 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] num_samples = 2 rois = [torch.rand((num_samples, 5))] cls_scores = [torch.rand((num_samples, num_classes + 1))] bbox_preds = [torch.rand((num_samples, 4))] # with nms rcnn_test_cfg = ConfigDict( score_thr=0., nms=dict(type='nms', iou_threshold=0.5), max_per_img=100) result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas, rcnn_test_cfg=rcnn_test_cfg) self.assertLessEqual(len(result_list[0]), num_samples * num_classes) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(result_list[0].bboxes.shape[1], 4) self.assertEqual(len(result_list[0].scores.shape), 1) self.assertEqual(len(result_list[0].labels.shape), 1) # without nms result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), num_samples) self.assertEqual(result_list[0].bboxes.shape, bbox_preds[0].shape) self.assertEqual(result_list[0].scores.shape, cls_scores[0].shape) self.assertIsNone(result_list[0].get('label', None)) # num_samples is 0 num_samples = 0 rois = [torch.rand((num_samples, 5))] cls_scores = [torch.rand((num_samples, num_classes + 1))] bbox_preds = [torch.rand((num_samples, 4))] # with nms rcnn_test_cfg = ConfigDict( score_thr=0., nms=dict(type='nms', iou_threshold=0.5), max_per_img=100) result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas, rcnn_test_cfg=rcnn_test_cfg) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), 0) self.assertEqual(result_list[0].bboxes.shape[1], 4) # without nms result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), 0) self.assertEqual(result_list[0].bboxes.shape, bbox_preds[0].shape) self.assertIsNone(result_list[0].get('label', None)) def test_bbox_head_refine_bboxes(self): num_classes = 6 bbox_head = BBoxHead(reg_class_agnostic=True, num_classes=num_classes) s = 128 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] sampling_results = [SamplingResult.random()] num_samples = 20 rois = torch.rand((num_samples, 4)) roi_img_ids = torch.zeros(num_samples, 1) rois = torch.cat((roi_img_ids, rois), dim=1) cls_scores = torch.rand((num_samples, num_classes + 1)) bbox_preds = torch.rand((num_samples, 4)) labels = torch.randint(0, num_classes + 1, (num_samples, )).long() bbox_targets = (labels, None, None, None) bbox_results = dict( rois=rois, bbox_pred=bbox_preds, cls_score=cls_scores, bbox_targets=bbox_targets) bbox_list = bbox_head.refine_bboxes( sampling_results=sampling_results, bbox_results=bbox_results, batch_img_metas=img_metas) self.assertGreaterEqual(num_samples, len(bbox_list[0])) self.assertIsInstance(bbox_list[0], InstanceData) self.assertEqual(bbox_list[0].bboxes.shape[1], 4) ================================================ FILE: tests/test_models/test_roi_heads/test_bbox_heads/test_double_bbox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.models.roi_heads.bbox_heads import DoubleConvFCBBoxHead class TestDoubleBboxHead(TestCase): @parameterized.expand(['cpu', 'cuda']) def test_forward_loss(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') double_bbox_head = DoubleConvFCBBoxHead( num_convs=4, num_fcs=2, in_channels=1, conv_out_channels=4, fc_out_channels=4) double_bbox_head = double_bbox_head.to(device=device) num_samples = 4 feats = torch.rand((num_samples, 1, 7, 7)).to(device) double_bbox_head(x_cls=feats, x_reg=feats) ================================================ FILE: tests/test_models/test_roi_heads/test_bbox_heads/test_multi_instance_bbox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.roi_heads.bbox_heads import MultiInstanceBBoxHead class TestMultiInstanceBBoxHead(TestCase): def test_init(self): bbox_head = MultiInstanceBBoxHead( num_instance=2, with_refine=True, num_shared_fcs=2, in_channels=1, fc_out_channels=1, num_classes=4) self.assertTrue(bbox_head.shared_fcs_ref) self.assertTrue(bbox_head.fc_reg) self.assertTrue(bbox_head.fc_cls) self.assertEqual(len(bbox_head.shared_fcs), 2) self.assertEqual(len(bbox_head.fc_reg), 2) self.assertEqual(len(bbox_head.fc_cls), 2) def test_bbox_head_get_results(self): num_classes = 1 num_instance = 2 bbox_head = MultiInstanceBBoxHead( num_instance=num_instance, num_shared_fcs=2, reg_class_agnostic=True, num_classes=num_classes) s = 128 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] num_samples = 2 rois = [torch.rand((num_samples, 5))] cls_scores = [] bbox_preds = [] for k in range(num_instance): cls_scores.append(torch.rand((num_samples, num_classes + 1))) bbox_preds.append(torch.rand((num_samples, 4))) cls_scores = [torch.cat(cls_scores, dim=1)] bbox_preds = [torch.cat(bbox_preds, dim=1)] # with nms rcnn_test_cfg = ConfigDict( nms=dict(type='nms', iou_threshold=0.5), score_thr=0.01, max_per_img=500) result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas, rcnn_test_cfg=rcnn_test_cfg) self.assertLessEqual( len(result_list[0]), num_samples * num_instance * num_classes) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(result_list[0].bboxes.shape[1], 4) self.assertEqual(len(result_list[0].scores.shape), 1) self.assertEqual(len(result_list[0].labels.shape), 1) # without nms result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), num_samples * num_instance) self.assertIsNone(result_list[0].get('label', None)) # num_samples is 0 num_samples = 0 rois = [torch.rand((num_samples, 5))] cls_scores = [] bbox_preds = [] for k in range(num_instance): cls_scores.append(torch.rand((num_samples, num_classes + 1))) bbox_preds.append(torch.rand((num_samples, 4))) cls_scores = [torch.cat(cls_scores, dim=1)] bbox_preds = [torch.cat(bbox_preds, dim=1)] # with nms rcnn_test_cfg = ConfigDict( score_thr=0., nms=dict(type='nms', iou_threshold=0.5), max_per_img=100) result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas, rcnn_test_cfg=rcnn_test_cfg) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), 0) self.assertEqual(result_list[0].bboxes.shape[1], 4) # without nms result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), 0 * num_instance) self.assertIsNone(result_list[0].get('label', None)) ================================================ FILE: tests/test_models/test_roi_heads/test_bbox_heads/test_sabl_bbox_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.roi_heads.bbox_heads import SABLHead from mmdet.models.task_modules.samplers import SamplingResult class TestSABLBboxHead(TestCase): def test_init(self): bbox_head = SABLHead( cls_in_channels=1, cls_out_channels=1, reg_in_channels=1, reg_offset_out_channels=1, reg_cls_out_channels=1, num_classes=4) self.assertTrue(bbox_head.fc_cls) self.assertTrue(hasattr(bbox_head, 'reg_cls_fcs')) self.assertTrue(hasattr(bbox_head, 'reg_offset_fcs')) self.assertFalse(hasattr(bbox_head, 'fc_reg')) def test_bbox_head_get_results(self): num_classes = 6 bbox_head = SABLHead(reg_class_agnostic=True, num_classes=num_classes) s = 128 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] num_samples = 2 rois = [torch.rand((num_samples, 5))] cls_scores = [torch.rand((num_samples, num_classes + 1))] bbox_preds = [(torch.rand( (num_samples, 28)), torch.rand((num_samples, 28)))] # with nms rcnn_test_cfg = ConfigDict( score_thr=0., nms=dict(type='nms', iou_threshold=0.5), max_per_img=100) result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas, rcnn_test_cfg=rcnn_test_cfg) self.assertLessEqual(len(result_list[0]), num_samples * num_classes) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(result_list[0].bboxes.shape[1], 4) self.assertEqual(len(result_list[0].scores.shape), 1) self.assertEqual(len(result_list[0].labels.shape), 1) # without nms result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), num_samples) self.assertEqual(result_list[0].scores.shape, cls_scores[0].shape) self.assertIsNone(result_list[0].get('label', None)) # num_samples is 0 num_samples = 0 rois = [torch.rand((num_samples, 5))] cls_scores = [torch.rand((num_samples, num_classes + 1))] bbox_preds = [(torch.rand( (num_samples, 28)), torch.rand((num_samples, 28)))] # with nms rcnn_test_cfg = ConfigDict( score_thr=0., nms=dict(type='nms', iou_threshold=0.5), max_per_img=100) result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas, rcnn_test_cfg=rcnn_test_cfg) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), 0) self.assertEqual(result_list[0].bboxes.shape[1], 4) # without nms result_list = bbox_head.predict_by_feat( rois=tuple(rois), cls_scores=tuple(cls_scores), bbox_preds=tuple(bbox_preds), batch_img_metas=img_metas) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), 0) self.assertIsNone(result_list[0].get('label', None)) def test_bbox_head_refine_bboxes(self): num_classes = 8 bbox_head = SABLHead(reg_class_agnostic=True, num_classes=num_classes) s = 20 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] sampling_results = [SamplingResult.random()] num_samples = 20 rois = torch.rand((num_samples, 4)) roi_img_ids = torch.zeros(num_samples, 1) rois = torch.cat((roi_img_ids, rois), dim=1) cls_scores = torch.rand((num_samples, num_classes + 1)) cls_preds = torch.rand((num_samples, 28)) offset_preds = torch.rand((num_samples, 28)) labels = torch.randint(0, num_classes + 1, (num_samples, )).long() bbox_targets = (labels, None, None, None) bbox_results = dict( rois=rois, bbox_pred=(cls_preds, offset_preds), cls_score=cls_scores, bbox_targets=bbox_targets) bbox_list = bbox_head.refine_bboxes( sampling_results=sampling_results, bbox_results=bbox_results, batch_img_metas=img_metas) self.assertGreaterEqual(num_samples, len(bbox_list[0])) self.assertIsInstance(bbox_list[0], InstanceData) self.assertEqual(bbox_list[0].bboxes.shape[1], 4) ================================================ FILE: tests/test_models/test_roi_heads/test_bbox_heads/test_scnet_bbox_head.py ================================================ import unittest import torch from mmdet.models.roi_heads.bbox_heads import SCNetBBoxHead class TestSCNetBBoxHead(unittest.TestCase): def test_forward(self): x = torch.rand((2, 1, 16, 16)) bbox_head = SCNetBBoxHead( num_shared_fcs=2, in_channels=1, roi_feat_size=16, conv_out_channels=1, fc_out_channels=256, ) results = bbox_head(x, return_shared_feat=False) self.assertEqual(len(results), 2) results = bbox_head(x, return_shared_feat=True) self.assertEqual(len(results), 3) ================================================ FILE: tests/test_models/test_roi_heads/test_cascade_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.models.roi_heads import StandardRoIHead # noqa from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals, get_roi_head_cfg class TestCascadeRoIHead(TestCase): @parameterized.expand( ['cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py']) def test_init(self, cfg_file): """Test init standard RoI head.""" # Normal Cascade Mask R-CNN RoI head roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) assert roi_head.with_bbox assert roi_head.with_mask @parameterized.expand( ['cascade_rcnn/cascade-mask-rcnn_r50_fpn_1x_coco.py']) def test_cascade_roi_head_loss(self, cfg_file): """Tests standard roi head loss when truth is empty and non-empty.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() feats = [] for i in range(len(roi_head_cfg.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) feats = tuple(feats) # When truth is non-empty then both cls, box, and mask loss # should be nonzero for random inputs img_shape_list = [(3, s, s) for _ in img_metas] proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[1], num_classes=4, with_mask=True, device='cuda')['data_samples'] out = roi_head.loss(feats, proposal_list, batch_data_samples) for name, value in out.items(): if 'loss' in name: self.assertGreaterEqual( value.sum(), 0, msg='loss should be non-zero') # When there is no truth, the cls loss should be nonzero but # there should be no box and mask loss. proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[0], num_classes=4, with_mask=True, device='cuda')['data_samples'] out = roi_head.loss(feats, proposal_list, batch_data_samples) for name, value in out.items(): if 'loss_cls' in name: self.assertGreaterEqual( value.sum(), 0, msg='loss should be non-zero') elif 'loss_bbox' in name or 'loss_mask' in name: self.assertEqual(value.sum(), 0) ================================================ FILE: tests/test_models/test_roi_heads/test_dynamic_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals, get_roi_head_cfg from mmdet.utils import register_all_modules class TestDynamicRoIHead(TestCase): def setUp(self): register_all_modules() self.roi_head_cfg = get_roi_head_cfg( 'dynamic_rcnn/dynamic-rcnn_r50_fpn_1x_coco.py') def test_init(self): roi_head = MODELS.build(self.roi_head_cfg) self.assertTrue(roi_head.with_bbox) @parameterized.expand(['cpu', 'cuda']) def test_dynamic_roi_head_loss(self, device): """Tests trident roi head predict.""" if not torch.cuda.is_available() and device == 'cuda': # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') roi_head = MODELS.build(self.roi_head_cfg) roi_head = roi_head.to(device=device) s = 256 feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device=device)) image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[1], num_classes=4, with_mask=True, device=device)['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device=device) out = roi_head.loss(feats, proposals_list, batch_data_samples) loss_cls = out['loss_cls'] loss_bbox = out['loss_bbox'] self.assertGreater(loss_cls.sum(), 0, 'cls loss should be non-zero') self.assertGreater(loss_bbox.sum(), 0, 'box loss should be non-zero') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[0], num_classes=4, with_mask=True, device=device)['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device=device) out = roi_head.loss(feats, proposals_list, batch_data_samples) empty_cls_loss = out['loss_cls'] empty_bbox_loss = out['loss_bbox'] self.assertGreater(empty_cls_loss.sum(), 0, 'cls loss should be non-zero') self.assertEqual( empty_bbox_loss.sum(), 0, 'there should be no box loss when there are no true boxes') ================================================ FILE: tests/test_models/test_roi_heads/test_grid_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals, get_roi_head_cfg from mmdet.utils import register_all_modules class TestGridRoIHead(TestCase): def setUp(self): register_all_modules() self.roi_head_cfg = get_roi_head_cfg( 'grid_rcnn/grid-rcnn_r50_fpn_gn-head_2x_coco.py') def test_init(self): roi_head = MODELS.build(self.roi_head_cfg) self.assertTrue(roi_head.with_bbox) @parameterized.expand(['cpu', 'cuda']) def test_grid_roi_head_loss(self, device): """Tests trident roi head predict.""" if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') roi_head = MODELS.build(self.roi_head_cfg) roi_head = roi_head.to(device=device) s = 256 feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device=device)) image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[1], num_classes=4, with_mask=True, device=device)['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device=device) out = roi_head.loss(feats, proposals_list, batch_data_samples) loss_cls = out['loss_cls'] loss_grid = out['loss_grid'] self.assertGreater(loss_cls.sum(), 0, 'cls loss should be non-zero') self.assertGreater(loss_grid.sum(), 0, 'grid loss should be non-zero') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[0], num_classes=4, with_mask=True, device=device)['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device=device) out = roi_head.loss(feats, proposals_list, batch_data_samples) empty_cls_loss = out['loss_cls'] self.assertGreater(empty_cls_loss.sum(), 0, 'cls loss should be non-zero') self.assertNotIn( 'loss_grid', out, 'grid loss should be passed when there are no true boxes') @parameterized.expand(['cpu', 'cuda']) def test_grid_roi_head_predict(self, device): """Tests trident roi head predict.""" if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') roi_head = MODELS.build(self.roi_head_cfg) roi_head = roi_head.to(device=device) s = 256 feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device=device)) image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[0], num_classes=4, with_mask=True, device=device)['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device=device) roi_head.predict(feats, proposals_list, batch_data_samples) @parameterized.expand(['cpu', 'cuda']) def test_grid_roi_head_forward(self, device): """Tests trident roi head forward.""" if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') roi_head = MODELS.build(self.roi_head_cfg) roi_head = roi_head.to(device=device) s = 256 feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device=device)) image_shapes = [(3, s, s)] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device=device) roi_head.forward(feats, proposals_list) ================================================ FILE: tests/test_models/test_roi_heads/test_htc_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.models.roi_heads import HybridTaskCascadeRoIHead # noqa from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals, get_roi_head_cfg class TestHTCRoIHead(TestCase): @parameterized.expand(['htc/htc_r50_fpn_1x_coco.py']) def test_init(self, cfg_file): """Test init htc RoI head.""" # Normal HTC RoI head roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) assert roi_head.with_bbox assert roi_head.with_mask assert roi_head.with_semantic @parameterized.expand(['htc/htc_r50_fpn_1x_coco.py']) def test_htc_roi_head_loss(self, cfg_file): """Tests htc roi head loss when truth is empty and non-empty.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() feats = [] for i in range(len(roi_head_cfg.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) feats = tuple(feats) # When truth is non-empty then both cls, box, and mask loss # should be nonzero for random inputs img_shape_list = [(3, s, s) for _ in img_metas] proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[1], num_classes=4, with_mask=True, with_semantic=True, device='cuda')['data_samples'] out = roi_head.loss(feats, proposal_list, batch_data_samples) for name, value in out.items(): if 'loss' in name: self.assertGreaterEqual( value.sum(), 0, msg='loss should be non-zero') # When there is no truth, the cls loss should be nonzero but # there should be no box and mask loss. proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[0], num_classes=4, with_mask=True, with_semantic=True, device='cuda')['data_samples'] out = roi_head.loss(feats, proposal_list, batch_data_samples) for name, value in out.items(): if 'loss_cls' in name: self.assertGreaterEqual( value.sum(), 0, msg='loss should be non-zero') elif 'loss_bbox' in name or 'loss_mask' in name: self.assertEqual(value.sum(), 0) @parameterized.expand(['htc/htc_r50_fpn_1x_coco.py']) def test_htc_roi_head_predict(self, cfg_file): if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() feats = [] for i in range(len(roi_head_cfg.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) feats = tuple(feats) img_shape_list = [(3, s, s) for _ in img_metas] proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[1], num_classes=4, with_mask=True, device='cuda')['data_samples'] results = roi_head.predict( feats, proposal_list, batch_data_samples, rescale=True) self.assertEqual(results[0].masks.shape[-2:], (s, s)) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_heads/test_coarse_mask_head.py ================================================ import unittest import torch from parameterized import parameterized from mmdet.models.roi_heads.mask_heads import CoarseMaskHead class TestCoarseMaskHead(unittest.TestCase): def test_init(self): with self.assertRaises(AssertionError): CoarseMaskHead(num_fcs=0) with self.assertRaises(AssertionError): CoarseMaskHead(downsample_factor=0.5) @parameterized.expand(['cpu', 'cuda']) def test_forward(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') x = torch.rand((1, 32, 7, 7)).to(device) mask_head = CoarseMaskHead( downsample_factor=2, in_channels=32, conv_out_channels=32, roi_feat_size=7).to(device) mask_head.init_weights() res = mask_head(x) self.assertEqual(res.shape[-2:], (3, 3)) mask_head = CoarseMaskHead( downsample_factor=1, in_channels=32, conv_out_channels=32, roi_feat_size=7).to(device) mask_head.init_weights() res = mask_head(x) self.assertEqual(res.shape[-2:], (7, 7)) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_heads/test_fcn_mask_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from parameterized import parameterized from mmdet.models.roi_heads.mask_heads import FCNMaskHead class TestFCNMaskHead(TestCase): @parameterized.expand(['cpu', 'cuda']) def test_get_seg_masks(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') num_classes = 6 mask_head = FCNMaskHead( num_convs=1, in_channels=1, conv_out_channels=1, num_classes=num_classes) rcnn_test_cfg = ConfigDict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100, mask_thr_binary=0.5) s = 128 img_metas = { 'img_shape': (s, s, 3), 'scale_factor': (1, 1), 'ori_shape': (s, s, 3) } result = InstanceData(metainfo=img_metas) num_samples = 2 mask_pred = [torch.rand((num_samples, num_classes, 14, 14)).to(device)] result.bboxes = torch.rand((num_samples, 4)).to(device) result.labels = torch.randint( num_classes, (num_samples, ), dtype=torch.long).to(device) mask_head.to(device=device) result_list = mask_head.predict_by_feat( mask_preds=tuple(mask_pred), results_list=[result], batch_img_metas=[img_metas], rcnn_test_cfg=rcnn_test_cfg) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), num_samples) self.assertEqual(result_list[0].masks.shape, (num_samples, s, s)) # test with activate_map, `mask_pred` has been activated before num_samples = 2 mask_pred = [torch.rand((num_samples, num_classes, 14, 14)).to(device)] mask_pred = [m.sigmoid().detach() for m in mask_pred] result.bboxes = torch.rand((num_samples, 4)).to(device) result.labels = torch.randint( num_classes, (num_samples, ), dtype=torch.long).to(device) mask_head.to(device=device) result_list = mask_head.predict_by_feat( mask_preds=tuple(mask_pred), results_list=[result], batch_img_metas=[img_metas], rcnn_test_cfg=rcnn_test_cfg, activate_map=True) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), num_samples) self.assertEqual(result_list[0].masks.shape, (num_samples, s, s)) # num_samples is 0 num_samples = 0 result = InstanceData(metainfo=img_metas) mask_pred = [torch.rand((num_samples, num_classes, 14, 14)).to(device)] result.bboxes = torch.zeros((num_samples, 4)).to(device) result.labels = torch.zeros((num_samples, )).to(device) result_list = mask_head.predict_by_feat( mask_preds=tuple(mask_pred), results_list=[result], batch_img_metas=[img_metas], rcnn_test_cfg=rcnn_test_cfg) self.assertIsInstance(result_list[0], InstanceData) self.assertEqual(len(result_list[0]), num_samples) self.assertEqual(result_list[0].masks.shape, (num_samples, s, s)) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_heads/test_feature_relay_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from torch import Tensor from mmdet.models.roi_heads.mask_heads import FeatureRelayHead class TestFeatureRelayHead(TestCase): @parameterized.expand(['cpu', 'cuda']) def test_forward(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') mask_head = FeatureRelayHead(in_channels=10, out_conv_channels=10) x = torch.rand((1, 10)) results = mask_head(x) self.assertIsInstance(results, Tensor) x = torch.empty((0, 10)) results = mask_head(x) self.assertEqual(results, None) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_heads/test_fused_semantic_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from torch import Tensor from mmdet.models.roi_heads.mask_heads import FusedSemanticHead class TestFusedSemanticHead(TestCase): @parameterized.expand(['cpu', 'cuda']) def test_forward_loss(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') semantic_head = FusedSemanticHead( num_ins=5, fusion_level=1, in_channels=4, conv_out_channels=4, num_classes=6) feats = [ torch.rand((1, 4, 32 // 2**(i + 1), 32 // 2**(i + 1))) for i in range(5) ] mask_pred, x = semantic_head(feats) labels = torch.randint(0, 6, (1, 1, 64, 64)) loss = semantic_head.loss(mask_pred, labels) self.assertIsInstance(loss, Tensor) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_heads/test_global_context_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from torch import Tensor from mmdet.models.roi_heads.mask_heads import GlobalContextHead class TestGlobalContextHead(TestCase): @parameterized.expand(['cpu', 'cuda']) def test_forward_loss(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') head = GlobalContextHead( num_convs=1, in_channels=4, conv_out_channels=4, num_classes=10) feats = [ torch.rand((1, 4, 64 // 2**(i + 1), 64 // 2**(i + 1))) for i in range(5) ] mc_pred, x = head(feats) labels = [torch.randint(0, 10, (10, ))] loss = head.loss(mc_pred, labels) self.assertIsInstance(loss, Tensor) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_heads/test_grid_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from parameterized import parameterized from mmdet.models.roi_heads.mask_heads import GridHead from mmdet.models.utils import unpack_gt_instances from mmdet.testing import (demo_mm_inputs, demo_mm_proposals, demo_mm_sampling_results) class TestGridHead(TestCase): @parameterized.expand(['cpu', 'cuda']) def test_grid_head_loss(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') grid_head = GridHead() grid_head.to(device=device) s = 256 image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[1], num_classes=4, with_mask=True, device=device)['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device=device) train_cfg = ConfigDict(dict(pos_radius=1)) # prepare ground truth (batch_gt_instances, batch_gt_instances_ignore, _) = unpack_gt_instances(batch_data_samples) sampling_results = demo_mm_sampling_results( proposals_list=proposals_list, batch_gt_instances=batch_gt_instances, batch_gt_instances_ignore=batch_gt_instances_ignore) # prepare grid feats pos_bboxes = torch.cat([res.pos_bboxes for res in sampling_results]) grid_feats = torch.rand((pos_bboxes.size(0), 256, 14, 14)).to(device) sample_idx = torch.arange(0, pos_bboxes.size(0)) grid_pred = grid_head(grid_feats) grid_head.loss(grid_pred, sample_idx, sampling_results, train_cfg) @parameterized.expand(['cpu', 'cuda']) def test_mask_iou_head_predict_by_feat(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') grid_head = GridHead() grid_head.to(device=device) s = 128 num_samples = 2 num_classes = 4 img_metas = { 'img_shape': (s, s, 3), 'scale_factor': (1, 1), 'ori_shape': (s, s, 3) } results = InstanceData(metainfo=img_metas) results.bboxes = torch.rand((num_samples, 4)).to(device) results.scores = torch.rand((num_samples, )).to(device) results.labels = torch.randint( num_classes, (num_samples, ), dtype=torch.long).to(device) grid_feats = torch.rand((num_samples, 256, 14, 14)).to(device) grid_preds = grid_head(grid_feats) grid_head.predict_by_feat( grid_preds=grid_preds, results_list=[results], batch_img_metas=[img_metas]) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_heads/test_htc_mask_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from torch import Tensor from mmdet.models.roi_heads.mask_heads import HTCMaskHead class TestHTCMaskHead(TestCase): @parameterized.expand(['cpu', 'cuda']) def test_forward(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') num_classes = 6 mask_head = HTCMaskHead( with_conv_res=True, num_convs=1, in_channels=1, conv_out_channels=1, num_classes=num_classes) x = torch.rand((1, 1, 10, 10)) res_feat = torch.rand((1, 1, 10, 10)) with self.assertRaises(AssertionError): mask_head(x, return_logits=False, return_feat=False) results = mask_head(x) self.assertEqual(len(results), 2) results = mask_head(x, res_feat=res_feat) self.assertEqual(len(results), 2) results = mask_head(x, return_logits=False) self.assertIsInstance(results, Tensor) results = mask_head(x, return_feat=False) self.assertIsInstance(results, Tensor) results = mask_head(x, res_feat=res_feat, return_logits=False) self.assertIsInstance(results, Tensor) results = mask_head(x, res_feat=res_feat, return_feat=False) self.assertIsInstance(results, Tensor) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_heads/test_maskiou_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from parameterized import parameterized from mmdet.models.roi_heads.mask_heads import MaskIoUHead from mmdet.models.utils import unpack_gt_instances from mmdet.structures.mask import mask_target from mmdet.testing import (demo_mm_inputs, demo_mm_proposals, demo_mm_sampling_results) class TestMaskIoUHead(TestCase): @parameterized.expand(['cpu', 'cuda']) def test_mask_iou_head_loss_and_target(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') mask_iou_head = MaskIoUHead(num_classes=4) mask_iou_head.to(device=device) s = 256 image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[1], num_classes=4, with_mask=True, device=device)['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device=device) train_cfg = ConfigDict(dict(mask_size=28, mask_thr_binary=0.5)) # prepare ground truth (batch_gt_instances, batch_gt_instances_ignore, _) = unpack_gt_instances(batch_data_samples) sampling_results = demo_mm_sampling_results( proposals_list=proposals_list, batch_gt_instances=batch_gt_instances, batch_gt_instances_ignore=batch_gt_instances_ignore) # prepare mask feats, pred and target pos_proposals = [res.pos_priors for res in sampling_results] pos_assigned_gt_inds = [ res.pos_assigned_gt_inds for res in sampling_results ] gt_masks = [res.masks for res in batch_gt_instances] mask_targets = mask_target(pos_proposals, pos_assigned_gt_inds, gt_masks, train_cfg) mask_feats = torch.rand((mask_targets.size(0), 256, 14, 14)).to(device) mask_preds = torch.rand((mask_targets.size(0), 4, 28, 28)).to(device) pos_labels = torch.cat([res.pos_gt_labels for res in sampling_results]) pos_mask_pred = mask_preds[range(mask_preds.size(0)), pos_labels] mask_iou_pred = mask_iou_head(mask_feats, pos_mask_pred) pos_mask_iou_pred = mask_iou_pred[range(mask_iou_pred.size(0)), pos_labels] mask_iou_head.loss_and_target(pos_mask_iou_pred, pos_mask_pred, mask_targets, sampling_results, batch_gt_instances, train_cfg) @parameterized.expand(['cpu', 'cuda']) def test_mask_iou_head_predict_by_feat(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') mask_iou_head = MaskIoUHead(num_classes=4) mask_iou_head.to(device=device) s = 128 num_samples = 2 num_classes = 4 img_metas = { 'img_shape': (s, s, 3), 'scale_factor': (1, 1), 'ori_shape': (s, s, 3) } results = InstanceData(metainfo=img_metas) results.bboxes = torch.rand((num_samples, 4)).to(device) results.scores = torch.rand((num_samples, )).to(device) results.labels = torch.randint( num_classes, (num_samples, ), dtype=torch.long).to(device) mask_feats = torch.rand((num_samples, 256, 14, 14)).to(device) mask_preds = torch.rand((num_samples, num_classes, 28, 28)).to(device) mask_iou_preds = mask_iou_head( mask_feats, mask_preds[range(results.labels.size(0)), results.labels]) mask_iou_head.predict_by_feat( mask_iou_preds=[mask_iou_preds], results_list=[results]) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_heads/test_scnet_mask_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from torch import Tensor from mmdet.models.roi_heads.mask_heads import SCNetMaskHead class TestSCNetMaskHead(TestCase): @parameterized.expand(['cpu', 'cuda']) def test_forward(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') num_classes = 6 mask_head = SCNetMaskHead( conv_to_res=True, num_convs=1, in_channels=1, conv_out_channels=1, num_classes=num_classes) x = torch.rand((1, 1, 10, 10)) results = mask_head(x) self.assertIsInstance(results, Tensor) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_heads/test_scnet_semantic_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from torch import Tensor from mmdet.models.roi_heads.mask_heads import SCNetSemanticHead class TestSCNetSemanticHead(TestCase): @parameterized.expand(['cpu', 'cuda']) def test_forward_loss(self, device): if device == 'cuda': if not torch.cuda.is_available(): return unittest.skip('test requires GPU and torch+cuda') semantic_head = SCNetSemanticHead( num_ins=5, fusion_level=1, in_channels=4, conv_out_channels=4, num_classes=6) feats = [ torch.rand((1, 4, 32 // 2**(i + 1), 32 // 2**(i + 1))) for i in range(5) ] mask_pred, x = semantic_head(feats) labels = torch.randint(0, 6, (1, 1, 64, 64)) loss = semantic_head.loss(mask_pred, labels) self.assertIsInstance(loss, Tensor) ================================================ FILE: tests/test_models/test_roi_heads/test_mask_scoring_roI_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals, get_roi_head_cfg from mmdet.utils import register_all_modules class TestMaskScoringRoiHead(TestCase): def setUp(self): register_all_modules() self.roi_head_cfg = get_roi_head_cfg( 'ms_rcnn/ms-rcnn_r50_fpn_1x_coco.py') def test_init(self): roi_head = MODELS.build(self.roi_head_cfg) self.assertTrue(roi_head.with_bbox) self.assertTrue(roi_head.with_mask) self.assertTrue(roi_head.mask_iou_head) def test_mask_scoring_roi_head_loss(self): """Tests trident roi head predict.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') roi_head = MODELS.build(self.roi_head_cfg) roi_head = roi_head.cuda() s = 256 feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[1], num_classes=4, with_mask=True, device='cuda')['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device='cuda') out = roi_head.loss(feats, proposals_list, batch_data_samples) loss_cls = out['loss_cls'] loss_bbox = out['loss_bbox'] loss_mask = out['loss_mask'] self.assertGreater(loss_cls.sum(), 0, 'cls loss should be non-zero') self.assertGreater(loss_bbox.sum(), 0, 'box loss should be non-zero') self.assertGreater(loss_mask.sum(), 0, 'mask loss should be non-zero') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[0], num_classes=4, with_mask=True, device='cuda')['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device='cuda') out = roi_head.loss(feats, proposals_list, batch_data_samples) empty_cls_loss = out['loss_cls'] empty_bbox_loss = out['loss_bbox'] empty_mask_loss = out['loss_mask'] self.assertGreater(empty_cls_loss.sum(), 0, 'cls loss should be non-zero') self.assertEqual( empty_bbox_loss.sum(), 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_mask_loss.sum(), 0, 'there should be no mask loss when there are no true boxes') def test_mask_scoring_roi_head_predict(self): """Tests trident roi head predict.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') roi_head = MODELS.build(self.roi_head_cfg) roi_head = roi_head.cuda() s = 256 feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[0], num_classes=4, with_mask=True, device='cuda')['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device='cuda') roi_head.predict(feats, proposals_list, batch_data_samples) def test_mask_scoring_roi_head_forward(self): """Tests trident roi head forward.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') roi_head = MODELS.build(self.roi_head_cfg) roi_head = roi_head.cuda() s = 256 feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) image_shapes = [(3, s, s)] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device='cuda') roi_head.forward(feats, proposals_list) ================================================ FILE: tests/test_models/test_roi_heads/test_multi_instance_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from mmengine.config import Config from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals from mmdet.utils import register_all_modules register_all_modules() def _fake_roi_head(): """Set a fake roi head config.""" roi_head = Config( dict( type='MultiInstanceRoIHead', bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict( type='RoIAlign', output_size=7, sampling_ratio=-1, aligned=True, use_torchvision=True), out_channels=256, featmap_strides=[4, 8, 16, 32]), bbox_head=dict( type='MultiInstanceBBoxHead', with_refine=False, num_shared_fcs=2, in_channels=256, fc_out_channels=1024, roi_feat_size=7, num_classes=1, bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[0., 0., 0., 0.], target_stds=[0.1, 0.1, 0.2, 0.2]), reg_class_agnostic=False, loss_cls=dict( type='CrossEntropyLoss', loss_weight=1.0, use_sigmoid=False, reduction='none'), loss_bbox=dict( type='SmoothL1Loss', loss_weight=1.0, reduction='none')), train_cfg=dict( assigner=dict( type='MultiInstanceAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.3, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='MultiInsRandomSampler', num=512, pos_fraction=0.5, neg_pos_ub=-1, add_gt_as_proposals=False), pos_weight=-1, debug=False), test_cfg=dict( nms=dict(iou_threshold=0.5), score_thr=0.01, max_per_img=500))) return roi_head class TestMultiInstanceRoIHead(TestCase): def test_init(self): """Test init multi instance RoI head.""" roi_head_cfg = _fake_roi_head() roi_head = MODELS.build(roi_head_cfg) self.assertTrue(roi_head.with_bbox) def test_standard_roi_head_loss(self): """Tests multi instance roi head loss when truth is empty and non- empty.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') s = 256 roi_head_cfg = _fake_roi_head() roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) feats = tuple(feats) # When truth is non-empty then emd loss should be nonzero for # random inputs image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[1], num_classes=4, with_mask=False, device='cuda')['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device='cuda') out = roi_head.loss(feats, proposals_list, batch_data_samples) loss = out['loss_rcnn_emd'] self.assertGreater(loss.sum(), 0, 'loss should be non-zero') # When there is no truth, the emd loss should be zero. batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[0], num_classes=4, with_mask=True, device='cuda')['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device='cuda') out = roi_head.loss(feats, proposals_list, batch_data_samples) empty_loss = out['loss_rcnn_emd'] self.assertEqual( empty_loss.sum(), 0, 'there should be no emd loss when there are no true boxes') ================================================ FILE: tests/test_models/test_roi_heads/test_pisa_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals, get_roi_head_cfg from mmdet.utils import register_all_modules class TestPISARoIHead(TestCase): def setUp(self): register_all_modules() self.roi_head_cfg = get_roi_head_cfg( 'pisa/faster-rcnn_r50_fpn_pisa_1x_coco.py') def test_init(self): roi_head = MODELS.build(self.roi_head_cfg) self.assertTrue(roi_head.with_bbox) @parameterized.expand(['cpu', 'cuda']) def test_pisa_roi_head(self, device): """Tests trident roi head predict.""" if not torch.cuda.is_available() and device == 'cuda': # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') roi_head = MODELS.build(self.roi_head_cfg) roi_head = roi_head.to(device=device) s = 256 feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device=device)) image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[1], num_classes=4, with_mask=True, device=device)['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device=device) out = roi_head.loss(feats, proposals_list, batch_data_samples) loss_cls = out['loss_cls'] loss_bbox = out['loss_bbox'] self.assertGreater(loss_cls.sum(), 0, 'cls loss should be non-zero') self.assertGreater(loss_bbox.sum(), 0, 'box loss should be non-zero') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[0], num_classes=4, with_mask=True, device=device)['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device=device) out = roi_head.loss(feats, proposals_list, batch_data_samples) empty_cls_loss = out['loss_cls'] empty_bbox_loss = out['loss_bbox'] self.assertGreater(empty_cls_loss.sum(), 0, 'cls loss should be non-zero') self.assertEqual( empty_bbox_loss.sum(), 0, 'there should be no box loss when there are no true boxes') ================================================ FILE: tests/test_models/test_roi_heads/test_point_rend_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.models.roi_heads import PointRendRoIHead # noqa from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals, get_roi_head_cfg class TestHTCRoIHead(TestCase): @parameterized.expand( ['point_rend/point-rend_r50-caffe_fpn_ms-1x_coco.py']) def test_init(self, cfg_file): """Test init Point rend RoI head.""" # Normal HTC RoI head roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) assert roi_head.with_bbox assert roi_head.with_mask @parameterized.expand( ['point_rend/point-rend_r50-caffe_fpn_ms-1x_coco.py']) def test_point_rend_roi_head_loss(self, cfg_file): """Tests htc roi head loss when truth is empty and non-empty.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() feats = [] for i in range(len(roi_head_cfg.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) feats = tuple(feats) # When truth is non-empty then both cls, box, and mask loss # should be nonzero for random inputs img_shape_list = [img_meta['img_shape'] for img_meta in img_metas] proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[1], num_classes=4, with_mask=True, device='cuda')['data_samples'] out = roi_head.loss(feats, proposal_list, batch_data_samples) for name, value in out.items(): if 'loss' in name: self.assertGreaterEqual( value.sum(), 0, msg='loss should be non-zero') # Positive rois must not be empty proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[0], num_classes=4, with_mask=True, device='cuda')['data_samples'] with self.assertRaises(AssertionError): out = roi_head.loss(feats, proposal_list, batch_data_samples) @parameterized.expand( ['point_rend/point-rend_r50-caffe_fpn_ms-1x_coco.py']) def test_point_rend_roi_head_predict(self, cfg_file): if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() feats = [] for i in range(len(roi_head_cfg.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) feats = tuple(feats) img_shape_list = [img_meta['img_shape'] for img_meta in img_metas] proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[1], num_classes=4, with_mask=True, device='cuda')['data_samples'] results = roi_head.predict( feats, proposal_list, batch_data_samples, rescale=True) self.assertEqual(results[0].masks.shape[-2:], (s, s)) ================================================ FILE: tests/test_models/test_roi_heads/test_roi_extractors/test_generic_roi_extractor.py ================================================ import unittest import torch from mmdet.models.roi_heads.roi_extractors import GenericRoIExtractor class TestGenericRoIExtractor(unittest.TestCase): def test_init(self): with self.assertRaises(AssertionError): GenericRoIExtractor( aggregation='other', roi_layer=dict( type='RoIAlign', output_size=7, sampling_ratio=2), out_channels=16, featmap_strides=[4, 8, 16, 32]) roi_extractor = GenericRoIExtractor( roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=2), out_channels=16, featmap_strides=[4, 8, 16, 32]) self.assertFalse(roi_extractor.with_pre) self.assertFalse(roi_extractor.with_post) def test_forward(self): # test with pre/post cfg = dict( roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=2), out_channels=16, featmap_strides=[4, 8, 16, 32], pre_cfg=dict( type='ConvModule', in_channels=16, out_channels=16, kernel_size=5, padding=2, inplace=False), post_cfg=dict( type='ConvModule', in_channels=16, out_channels=16, kernel_size=5, padding=2, inplace=False)) roi_extractor = GenericRoIExtractor(**cfg) # empty rois feats = ( torch.rand((1, 16, 200, 336)), torch.rand((1, 16, 100, 168)), ) rois = torch.empty((0, 5), dtype=torch.float32) res = roi_extractor(feats, rois) self.assertEqual(len(res), 0) # single scale feature rois = torch.tensor([[0.0000, 587.8285, 52.1405, 886.2484, 341.5644]]) feats = (torch.rand((1, 16, 200, 336)), ) res = roi_extractor(feats, rois) self.assertEqual(res.shape, (1, 16, 7, 7)) # multi-scale features feats = ( torch.rand((1, 16, 200, 336)), torch.rand((1, 16, 100, 168)), torch.rand((1, 16, 50, 84)), torch.rand((1, 16, 25, 42)), ) res = roi_extractor(feats, rois) self.assertEqual(res.shape, (1, 16, 7, 7)) # test w.o. pre/post concat cfg = dict( aggregation='concat', roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=2), out_channels=16 * 4, featmap_strides=[4, 8, 16, 32]) roi_extractor = GenericRoIExtractor(**cfg) res = roi_extractor(feats, rois) self.assertEqual(res.shape, (1, 64, 7, 7)) # test concat channels number cfg = dict( aggregation='concat', roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=2), out_channels=256 * 5, # 256*5 != 256*4 featmap_strides=[4, 8, 16, 32]) roi_extractor = GenericRoIExtractor(**cfg) feats = ( torch.rand((1, 256, 200, 336)), torch.rand((1, 256, 100, 168)), torch.rand((1, 256, 50, 84)), torch.rand((1, 256, 25, 42)), ) # out_channels does not sum of feat channels with self.assertRaises(AssertionError): roi_extractor(feats, rois) ================================================ FILE: tests/test_models/test_roi_heads/test_roi_extractors/test_single_level_roi_extractor.py ================================================ import unittest import torch from mmdet.models.roi_heads.roi_extractors import SingleRoIExtractor class TestSingleRoIExtractor(unittest.TestCase): def test_forward(self): cfg = dict( roi_layer=dict(type='RoIAlign', output_size=7, sampling_ratio=2), out_channels=16, featmap_strides=[4, 8, 16, 32]) roi_extractor = SingleRoIExtractor(**cfg) # empty rois feats = (torch.rand((1, 16, 200, 336)), ) rois = torch.empty((0, 5), dtype=torch.float32) res = roi_extractor(feats, rois) self.assertEqual(len(res), 0) # single scale feature rois = torch.tensor([[0.0000, 587.8285, 52.1405, 886.2484, 341.5644]]) res = roi_extractor(feats, rois) self.assertEqual(res.shape, (1, 16, 7, 7)) # multi-scale features feats = ( torch.rand((1, 16, 200, 336)), torch.rand((1, 16, 100, 168)), torch.rand((1, 16, 50, 84)), torch.rand((1, 16, 25, 42)), ) res = roi_extractor(feats, rois) self.assertEqual(res.shape, (1, 16, 7, 7)) res = roi_extractor(feats, rois, roi_scale_factor=2.0) self.assertEqual(res.shape, (1, 16, 7, 7)) ================================================ FILE: tests/test_models/test_roi_heads/test_scnet_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from parameterized import parameterized from mmdet.models.roi_heads import SCNetRoIHead # noqa from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals, get_roi_head_cfg class TestSCNetRoIHead(TestCase): @parameterized.expand(['scnet/scnet_r50_fpn_1x_coco.py']) def test_init(self, cfg_file): """Test init scnet RoI head.""" # Normal Cascade Mask R-CNN RoI head roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) assert roi_head.with_bbox assert roi_head.with_mask assert roi_head.with_semantic assert roi_head.with_feat_relay assert roi_head.with_glbctx @parameterized.expand(['scnet/scnet_r50_fpn_1x_coco.py']) def test_scnet_roi_head_loss(self, cfg_file): """Tests htc roi head loss when truth is empty and non-empty.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() feats = [] for i in range(len(roi_head_cfg.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) feats = tuple(feats) # When truth is non-empty then both cls, box, and mask loss # should be nonzero for random inputs img_shape_list = [(3, s, s) for _ in img_metas] proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[1], num_classes=4, with_mask=True, with_semantic=True, device='cuda')['data_samples'] out = roi_head.loss(feats, proposal_list, batch_data_samples) for name, value in out.items(): if 'loss' in name: self.assertGreaterEqual( value.sum(), 0, msg='loss should be non-zero') # When there is no truth, the cls loss should be nonzero but # there should be no box and mask loss. proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[0], num_classes=4, with_mask=True, with_semantic=True, device='cuda')['data_samples'] out = roi_head.loss(feats, proposal_list, batch_data_samples) for name, value in out.items(): if 'loss_cls' in name: self.assertGreaterEqual( value.sum(), 0, msg='loss should be non-zero') elif 'loss_bbox' in name or 'loss_mask' in name: self.assertEqual(value.sum(), 0) @parameterized.expand(['scnet/scnet_r50_fpn_1x_coco.py']) def test_scnet_roi_head_predict(self, cfg_file): if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() feats = [] for i in range(len(roi_head_cfg.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 256, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) feats = tuple(feats) img_shape_list = [(3, s, s) for _ in img_metas] proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[1], num_classes=4, with_mask=True, device='cuda')['data_samples'] results = roi_head.predict( feats, proposal_list, batch_data_samples, rescale=True) self.assertEqual(results[0].masks.shape[-2:], (s, s)) ================================================ FILE: tests/test_models/test_roi_heads/test_sparse_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch import torch.nn as nn from parameterized import parameterized from mmdet.models.roi_heads import StandardRoIHead # noqa from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals, get_roi_head_cfg class TestCascadeRoIHead(TestCase): @parameterized.expand(['queryinst/queryinst_r50_fpn_1x_coco.py']) def test_init(self, cfg_file): """Test init standard RoI head.""" # Normal Cascade Mask R-CNN RoI head roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) roi_head.init_weights() assert roi_head.with_bbox assert roi_head.with_mask @parameterized.expand(['queryinst/queryinst_r50_fpn_1x_coco.py']) def test_cascade_roi_head_loss(self, cfg_file): """Tests standard roi head loss when truth is empty and non-empty.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') s = 256 img_metas = [{ 'img_shape': (s, s, 3), 'scale_factor': 1, }] roi_head_cfg = get_roi_head_cfg(cfg_file) roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() feats = [] for i in range(len(roi_head_cfg.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) feats = tuple(feats) # When truth is non-empty then both cls, box, and mask loss # should be nonzero for random inputs img_shape_list = [(3, s, s) for _ in img_metas] proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') # add import elements into proposal init_proposal_features = nn.Embedding(100, 256).cuda().weight.clone() for proposal in proposal_list: proposal.features = init_proposal_features proposal.imgs_whwh = feats[0].new_tensor([[s, s, s, s]]).repeat(100, 1) batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[1], num_classes=4, with_mask=True, device='cuda')['data_samples'] out = roi_head.loss(feats, proposal_list, batch_data_samples) for name, value in out.items(): if 'loss' in name: self.assertGreaterEqual( value.sum(), 0, msg='loss should be non-zero') # When there is no truth, the cls loss should be nonzero but # there should be no box and mask loss. proposal_list = demo_mm_proposals(img_shape_list, 100, device='cuda') # add import elements into proposal init_proposal_features = nn.Embedding(100, 256).cuda().weight.clone() for proposal in proposal_list: proposal.features = init_proposal_features proposal.imgs_whwh = feats[0].new_tensor([[s, s, s, s]]).repeat(100, 1) batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=[(3, s, s)], num_items=[0], num_classes=4, with_mask=True, device='cuda')['data_samples'] out = roi_head.loss(feats, proposal_list, batch_data_samples) for name, value in out.items(): if 'loss_cls' in name: self.assertGreaterEqual( value.sum(), 0, msg='loss should be non-zero') elif 'loss_bbox' in name or 'loss_mask' in name: self.assertEqual(value.sum(), 0) ================================================ FILE: tests/test_models/test_roi_heads/test_standard_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest from unittest import TestCase import torch from mmengine.config import Config from parameterized import parameterized from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals from mmdet.utils import register_all_modules register_all_modules() def _fake_roi_head(with_shared_head=False): """Set a fake roi head config.""" if not with_shared_head: roi_head = Config( dict( type='StandardRoIHead', bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict( type='RoIAlign', output_size=7, sampling_ratio=0), out_channels=1, featmap_strides=[4, 8, 16, 32]), bbox_head=dict( type='Shared2FCBBoxHead', in_channels=1, fc_out_channels=1, num_classes=4), mask_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict( type='RoIAlign', output_size=14, sampling_ratio=0), out_channels=1, featmap_strides=[4, 8, 16, 32]), mask_head=dict( type='FCNMaskHead', num_convs=1, in_channels=1, conv_out_channels=1, num_classes=4), train_cfg=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=True, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), mask_size=28, pos_weight=-1, debug=False), test_cfg=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100, mask_thr_binary=0.5))) else: roi_head = Config( dict( type='StandardRoIHead', shared_head=dict( type='ResLayer', depth=50, stage=3, stride=2, dilation=1, style='caffe', norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True), bbox_roi_extractor=dict( type='SingleRoIExtractor', roi_layer=dict( type='RoIAlign', output_size=14, sampling_ratio=0), out_channels=1, featmap_strides=[16]), bbox_head=dict( type='BBoxHead', with_avg_pool=True, in_channels=2048, roi_feat_size=7, num_classes=4), mask_roi_extractor=None, mask_head=dict( type='FCNMaskHead', num_convs=0, in_channels=2048, conv_out_channels=1, num_classes=4), train_cfg=dict( assigner=dict( type='MaxIoUAssigner', pos_iou_thr=0.5, neg_iou_thr=0.5, min_pos_iou=0.5, match_low_quality=False, ignore_iof_thr=-1), sampler=dict( type='RandomSampler', num=512, pos_fraction=0.25, neg_pos_ub=-1, add_gt_as_proposals=True), mask_size=14, pos_weight=-1, debug=False), test_cfg=dict( score_thr=0.05, nms=dict(type='nms', iou_threshold=0.5), max_per_img=100, mask_thr_binary=0.5))) return roi_head class TestStandardRoIHead(TestCase): def test_init(self): """Test init standard RoI head.""" # Normal Mask R-CNN RoI head roi_head_cfg = _fake_roi_head() roi_head = MODELS.build(roi_head_cfg) self.assertTrue(roi_head.with_bbox) self.assertTrue(roi_head.with_mask) # Mask R-CNN RoI head with shared_head roi_head_cfg = _fake_roi_head(with_shared_head=True) roi_head = MODELS.build(roi_head_cfg) self.assertTrue(roi_head.with_bbox) self.assertTrue(roi_head.with_mask) self.assertTrue(roi_head.with_shared_head) @parameterized.expand([(False, ), (True, )]) def test_standard_roi_head_loss(self, with_shared_head): """Tests standard roi head loss when truth is empty and non-empty.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') s = 256 roi_head_cfg = _fake_roi_head(with_shared_head=with_shared_head) roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): if not with_shared_head: feats.append( torch.rand(1, 1, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) else: feats.append( torch.rand(1, 1024, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) feats = tuple(feats) # When truth is non-empty then both cls, box, and mask loss # should be nonzero for random inputs image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[1], num_classes=4, with_mask=True, device='cuda')['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device='cuda') out = roi_head.loss(feats, proposals_list, batch_data_samples) loss_cls = out['loss_cls'] loss_bbox = out['loss_bbox'] loss_mask = out['loss_mask'] self.assertGreater(loss_cls.sum(), 0, 'cls loss should be non-zero') self.assertGreater(loss_bbox.sum(), 0, 'box loss should be non-zero') self.assertGreater(loss_mask.sum(), 0, 'mask loss should be non-zero') # When there is no truth, the cls loss should be nonzero but # there should be no box and mask loss. batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[0], num_classes=4, with_mask=True, device='cuda')['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device='cuda') out = roi_head.loss(feats, proposals_list, batch_data_samples) empty_cls_loss = out['loss_cls'] empty_bbox_loss = out['loss_bbox'] empty_mask_loss = out['loss_mask'] self.assertGreater(empty_cls_loss.sum(), 0, 'cls loss should be non-zero') self.assertEqual( empty_bbox_loss.sum(), 0, 'there should be no box loss when there are no true boxes') self.assertEqual( empty_mask_loss.sum(), 0, 'there should be no mask loss when there are no true boxes') ================================================ FILE: tests/test_models/test_roi_heads/test_trident_roi_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import unittest from unittest import TestCase import torch from mmdet.registry import MODELS from mmdet.testing import demo_mm_inputs, demo_mm_proposals, get_roi_head_cfg from mmdet.utils import register_all_modules class TestTridentRoIHead(TestCase): def setUp(self): register_all_modules() self.roi_head_cfg = get_roi_head_cfg( 'tridentnet/tridentnet_r50-caffe_1x_coco.py') def test_init(self): roi_head = MODELS.build(self.roi_head_cfg) self.assertTrue(roi_head.with_bbox) self.assertTrue(roi_head.with_shared_head) def test_trident_roi_head_predict(self): """Tests trident roi head predict.""" if not torch.cuda.is_available(): # RoI pooling only support in GPU return unittest.skip('test requires GPU and torch+cuda') roi_head_cfg = copy.deepcopy(self.roi_head_cfg) roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() s = 256 feats = [] for i in range(len(roi_head.bbox_roi_extractor.featmap_strides)): feats.append( torch.rand(1, 1024, s // (2**(i + 2)), s // (2**(i + 2))).to(device='cuda')) image_shapes = [(3, s, s)] batch_data_samples = demo_mm_inputs( batch_size=1, image_shapes=image_shapes, num_items=[0], num_classes=4, with_mask=True, device='cuda')['data_samples'] proposals_list = demo_mm_proposals( image_shapes=image_shapes, num_proposals=100, device='cuda') # When `test_branch_idx == 1` roi_head.predict(feats, proposals_list, batch_data_samples) # When `test_branch_idx == -1` roi_head_cfg.test_branch_idx = -1 roi_head = MODELS.build(roi_head_cfg) roi_head = roi_head.cuda() roi_head.predict(feats, proposals_list, batch_data_samples) ================================================ FILE: tests/test_models/test_seg_heads/test_heuristic_fusion_head.py ================================================ import unittest import torch from mmengine.config import Config from mmengine.structures import InstanceData from mmengine.testing import assert_allclose from mmdet.evaluation import INSTANCE_OFFSET from mmdet.models.seg_heads.panoptic_fusion_heads import HeuristicFusionHead class TestHeuristicFusionHead(unittest.TestCase): def test_loss(self): head = HeuristicFusionHead(num_things_classes=2, num_stuff_classes=2) result = head.loss() self.assertTrue(not head.with_loss) self.assertDictEqual(result, dict()) def test_predict(self): test_cfg = Config(dict(mask_overlap=0.5, stuff_area_limit=1)) head = HeuristicFusionHead( num_things_classes=2, num_stuff_classes=2, test_cfg=test_cfg) mask_results = InstanceData() mask_results.bboxes = torch.tensor([[0, 0, 1, 1], [1, 1, 2, 2]]) mask_results.labels = torch.tensor([0, 1]) mask_results.scores = torch.tensor([0.8, 0.7]) mask_results.masks = torch.tensor([[[1, 0], [0, 0]], [[0, 0], [0, 1]]]).bool() seg_preds_list = [ torch.tensor([[[0.2, 0.7], [0.3, 0.1]], [[0.2, 0.2], [0.6, 0.1]], [[0.6, 0.1], [0.1, 0.8]]]) ] target_list = [ torch.tensor([[0 + 1 * INSTANCE_OFFSET, 2], [3, 1 + 2 * INSTANCE_OFFSET]]) ] results_list = head.predict([mask_results], seg_preds_list) for target, result in zip(target_list, results_list): assert_allclose(result.sem_seg[0], target) # test with no thing head = HeuristicFusionHead( num_things_classes=2, num_stuff_classes=2, test_cfg=test_cfg) mask_results = InstanceData() mask_results.bboxes = torch.zeros((0, 4)) mask_results.labels = torch.zeros((0, )).long() mask_results.scores = torch.zeros((0, )) mask_results.masks = torch.zeros((0, 2, 2), dtype=torch.bool) seg_preds_list = [ torch.tensor([[[0.2, 0.7], [0.3, 0.1]], [[0.2, 0.2], [0.6, 0.1]], [[0.6, 0.1], [0.1, 0.8]]]) ] target_list = [torch.tensor([[4, 2], [3, 4]])] results_list = head.predict([mask_results], seg_preds_list) for target, result in zip(target_list, results_list): assert_allclose(result.sem_seg[0], target) ================================================ FILE: tests/test_models/test_seg_heads/test_maskformer_fusion_head.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest import torch from mmengine.config import Config from mmdet.models.seg_heads.panoptic_fusion_heads import MaskFormerFusionHead from mmdet.structures import DetDataSample class TestMaskFormerFusionHead(unittest.TestCase): def test_loss(self): head = MaskFormerFusionHead(num_things_classes=2, num_stuff_classes=2) result = head.loss() self.assertTrue(not head.with_loss) self.assertDictEqual(result, dict()) def test_predict(self): mask_cls_results = torch.rand((2, 10, 5)) mask_pred_results = torch.rand((2, 10, 32, 32)) batch_data_samples = [ DetDataSample( metainfo={ 'batch_input_shape': (32, 32), 'img_shape': (32, 30), 'ori_shape': (30, 30) }), DetDataSample( metainfo={ 'batch_input_shape': (32, 32), 'img_shape': (32, 30), 'ori_shape': (29, 30) }) ] # get panoptic and instance segmentation results test_cfg = Config( dict( panoptic_on=True, semantic_on=False, instance_on=True, max_per_image=10, object_mask_thr=0.3, iou_thr=0.3, filter_low_score=False)) head = MaskFormerFusionHead( num_things_classes=2, num_stuff_classes=2, test_cfg=test_cfg) results = head.predict( mask_cls_results, mask_pred_results, batch_data_samples, rescale=False) for i in range(len(results)): self.assertEqual(results[i]['pan_results'].sem_seg.shape[-2:], batch_data_samples[i].img_shape) self.assertEqual(results[i]['ins_results'].masks.shape[-2:], batch_data_samples[i].img_shape) results = head.predict( mask_cls_results, mask_pred_results, batch_data_samples, rescale=True) for i in range(len(results)): self.assertEqual(results[i]['pan_results'].sem_seg.shape[-2:], batch_data_samples[i].ori_shape) self.assertEqual(results[i]['ins_results'].masks.shape[-2:], batch_data_samples[i].ori_shape) # get empty results test_cfg = Config( dict( panoptic_on=False, semantic_on=False, instance_on=False, max_per_image=10, object_mask_thr=0.3, iou_thr=0.3, filter_low_score=False)) head = MaskFormerFusionHead( num_things_classes=2, num_stuff_classes=2, test_cfg=test_cfg) results = head.predict( mask_cls_results, mask_pred_results, batch_data_samples, rescale=True) for i in range(len(results)): self.assertEqual(results[i], dict()) # semantic segmentation is not supported test_cfg = Config( dict( panoptic_on=False, semantic_on=True, instance_on=False, max_per_image=10, object_mask_thr=0.3, iou_thr=0.3, filter_low_score=False)) head = MaskFormerFusionHead( num_things_classes=2, num_stuff_classes=2, test_cfg=test_cfg) with self.assertRaises(AssertionError): results = head.predict( mask_cls_results, mask_pred_results, batch_data_samples, rescale=True) ================================================ FILE: tests/test_models/test_seg_heads/test_panoptic_fpn_head.py ================================================ import unittest import torch from mmengine.structures import PixelData from mmengine.testing import assert_allclose from mmdet.models.seg_heads import PanopticFPNHead from mmdet.structures import DetDataSample class TestPanopticFPNHead(unittest.TestCase): def test_init_weights(self): head = PanopticFPNHead( num_things_classes=2, num_stuff_classes=2, in_channels=32, inner_channels=32) head.init_weights() assert_allclose(head.conv_logits.bias.data, torch.zeros_like(head.conv_logits.bias.data)) def test_loss(self): head = PanopticFPNHead( num_things_classes=2, num_stuff_classes=2, in_channels=32, inner_channels=32, start_level=0, end_level=1) x = [torch.rand((2, 32, 8, 8)), torch.rand((2, 32, 4, 4))] data_sample1 = DetDataSample() data_sample1.gt_sem_seg = PixelData( sem_seg=torch.randint(0, 4, (1, 7, 8))) data_sample2 = DetDataSample() data_sample2.gt_sem_seg = PixelData( sem_seg=torch.randint(0, 4, (1, 7, 8))) batch_data_samples = [data_sample1, data_sample2] results = head.loss(x, batch_data_samples) self.assertIsInstance(results, dict) def test_predict(self): head = PanopticFPNHead( num_things_classes=2, num_stuff_classes=2, in_channels=32, inner_channels=32, start_level=0, end_level=1) x = [torch.rand((2, 32, 8, 8)), torch.rand((2, 32, 4, 4))] img_meta1 = { 'batch_input_shape': (16, 16), 'img_shape': (14, 14), 'ori_shape': (12, 12), } img_meta2 = { 'batch_input_shape': (16, 16), 'img_shape': (16, 16), 'ori_shape': (16, 16), } batch_img_metas = [img_meta1, img_meta2] head.eval() with torch.no_grad(): seg_preds = head.predict(x, batch_img_metas, rescale=False) self.assertTupleEqual(seg_preds[0].shape[-2:], (16, 16)) self.assertTupleEqual(seg_preds[1].shape[-2:], (16, 16)) seg_preds = head.predict(x, batch_img_metas, rescale=True) self.assertTupleEqual(seg_preds[0].shape[-2:], (12, 12)) self.assertTupleEqual(seg_preds[1].shape[-2:], (16, 16)) ================================================ FILE: tests/test_models/test_task_modules/__init__.py ================================================ ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_approx_max_iou_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.models.task_modules.assigners import ApproxMaxIoUAssigner class TestApproxIoUAssigner(TestCase): def test_approx_iou_assigner(self): assigner = ApproxMaxIoUAssigner( pos_iou_thr=0.5, neg_iou_thr=0.5, ) bboxes = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData() pred_instances.priors = bboxes pred_instances.approxs = bboxes[:, None, :] gt_instances = InstanceData() gt_instances.bboxes = gt_bboxes gt_instances.labels = gt_labels assign_result = assigner.assign(pred_instances, gt_instances) expected_gt_inds = torch.LongTensor([1, 0, 2, 0]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) def test_approx_iou_assigner_with_empty_gt(self): """Test corner case where an image might have no true detections.""" assigner = ApproxMaxIoUAssigner( pos_iou_thr=0.5, neg_iou_thr=0.5, ) bboxes = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.FloatTensor([]) gt_labels = torch.LongTensor([]) pred_instances = InstanceData() pred_instances.priors = bboxes pred_instances.approxs = bboxes[:, None, :] gt_instances = InstanceData() gt_instances.bboxes = gt_bboxes gt_instances.labels = gt_labels assign_result = assigner.assign(pred_instances, gt_instances) expected_gt_inds = torch.LongTensor([0, 0, 0, 0]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) def test_approx_iou_assigner_with_empty_boxes(self): """Test corner case where an network might predict no boxes.""" assigner = ApproxMaxIoUAssigner( pos_iou_thr=0.5, neg_iou_thr=0.5, ) bboxes = torch.empty((0, 4)) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData() pred_instances.priors = bboxes pred_instances.approxs = bboxes[:, None, :] gt_instances = InstanceData() gt_instances.bboxes = gt_bboxes gt_instances.labels = gt_labels assign_result = assigner.assign(pred_instances, gt_instances) self.assertEqual(len(assign_result.gt_inds), 0) def test_approx_iou_assigner_with_empty_boxes_and_gt(self): """Test corner case where an network might predict no boxes and no gt.""" assigner = ApproxMaxIoUAssigner( pos_iou_thr=0.5, neg_iou_thr=0.5, ) bboxes = torch.empty((0, 4)) gt_bboxes = torch.empty((0, 4)) gt_labels = torch.LongTensor([]) pred_instances = InstanceData() pred_instances.priors = bboxes pred_instances.approxs = bboxes[:, None, :] gt_instances = InstanceData() gt_instances.bboxes = gt_bboxes gt_instances.labels = gt_labels assign_result = assigner.assign(pred_instances, gt_instances) self.assertEqual(len(assign_result.gt_inds), 0) ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_atss_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.models.task_modules.assigners import ATSSAssigner class TestATSSAssigner(TestCase): def test_atss_assigner(self): atss_assigner = ATSSAssigner(topk=9) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) num_level_bboxes = [4] assign_result = atss_assigner.assign(pred_instances, num_level_bboxes, gt_instances) self.assertEqual(len(assign_result.gt_inds), 4) self.assertEqual(len(assign_result.labels), 4) expected_gt_inds = torch.LongTensor([1, 0, 0, 0]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) def test_atss_assigner_with_ignore(self): atss_assigner = ATSSAssigner(topk=9) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [30, 32, 40, 42], ]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) gt_bboxes_ignore = torch.Tensor([ [30, 30, 40, 40], ]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) gt_instances_ignore = InstanceData(bboxes=gt_bboxes_ignore) num_level_bboxes = [4] assign_result = atss_assigner.assign( pred_instances, num_level_bboxes, gt_instances, gt_instances_ignore=gt_instances_ignore) expected_gt_inds = torch.LongTensor([1, 0, 0, 0]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) def test_atss_assigner_with_empty_gt(self): """Test corner case where an image might have no true detections.""" atss_assigner = ATSSAssigner(topk=9) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.empty(0, 4) gt_labels = torch.empty(0) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) num_level_bboxes = [4] assign_result = atss_assigner.assign(pred_instances, num_level_bboxes, gt_instances) expected_gt_inds = torch.LongTensor([0, 0, 0, 0]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) def test_atss_assigner_with_empty_boxes(self): """Test corner case where a network might predict no boxes.""" atss_assigner = ATSSAssigner(topk=9) priors = torch.empty((0, 4)) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) num_level_bboxes = [0] assign_result = atss_assigner.assign(pred_instances, num_level_bboxes, gt_instances) self.assertEqual(len(assign_result.gt_inds), 0) self.assertTrue(tuple(assign_result.labels.shape) == (0, )) def test_atss_assigner_with_empty_boxes_and_ignore(self): """Test corner case where a network might predict no boxes and ignore_iof_thr is on.""" atss_assigner = ATSSAssigner(topk=9) priors = torch.empty((0, 4)) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_bboxes_ignore = torch.Tensor([ [30, 30, 40, 40], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) gt_instances_ignore = InstanceData(bboxes=gt_bboxes_ignore) num_level_bboxes = [0] assign_result = atss_assigner.assign( pred_instances, num_level_bboxes, gt_instances, gt_instances_ignore=gt_instances_ignore) self.assertEqual(len(assign_result.gt_inds), 0) self.assertTrue(tuple(assign_result.labels.shape) == (0, )) def test_atss_assigner_with_empty_boxes_and_gt(self): """Test corner case where a network might predict no boxes and no gt.""" atss_assigner = ATSSAssigner(topk=9) priors = torch.empty((0, 4)) gt_bboxes = torch.empty((0, 4)) gt_labels = torch.empty(0) num_level_bboxes = [0] pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) assign_result = atss_assigner.assign(pred_instances, num_level_bboxes, gt_instances) self.assertEqual(len(assign_result.gt_inds), 0) ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_center_region_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.models.task_modules.assigners import CenterRegionAssigner class TestCenterRegionAssigner(TestCase): def test_center_region_assigner(self): center_region_assigner = CenterRegionAssigner( pos_scale=0.2, neg_scale=0.2, min_pos_iof=0.01) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) assign_result = center_region_assigner.assign(pred_instances, gt_instances) self.assertEqual(len(assign_result.gt_inds), 4) self.assertEqual(len(assign_result.labels), 4) expected_gt_inds = torch.LongTensor([1, 0, 0, 0]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) expected_shadowed_labels = torch.LongTensor([[2, 3]]) shadowed_labels = assign_result.get_extra_property('shadowed_labels') self.assertTrue(torch.all(shadowed_labels == expected_shadowed_labels)) def test_center_region_assigner_with_ignore(self): center_region_assigner = CenterRegionAssigner( pos_scale=0.2, neg_scale=0.2, min_pos_iof=0.01) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [30, 32, 40, 42], ]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) gt_bboxes_ignore = torch.Tensor([ [30, 30, 40, 40], ]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) gt_instances_ignore = InstanceData(bboxes=gt_bboxes_ignore) assign_result = center_region_assigner.assign( pred_instances, gt_instances, gt_instances_ignore=gt_instances_ignore) expected_gt_inds = torch.LongTensor([1, 0, 0, -1]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) def test_center_region_assigner_with_empty_gt(self): """Test corner case where an image might have no true detections.""" center_region_assigner = CenterRegionAssigner( pos_scale=0.2, neg_scale=0.2, min_pos_iof=0.01) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.empty(0, 4) gt_labels = torch.empty(0) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) assign_result = center_region_assigner.assign(pred_instances, gt_instances) expected_gt_inds = torch.LongTensor([0, 0, 0, 0]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) def test_center_region_assigner_with_empty_boxes(self): """Test corner case where a network might predict no boxes.""" center_region_assigner = CenterRegionAssigner( pos_scale=0.2, neg_scale=0.2, min_pos_iof=0.01) priors = torch.empty((0, 4)) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) assign_result = center_region_assigner.assign(pred_instances, gt_instances) self.assertEqual(len(assign_result.gt_inds), 0) self.assertTrue(tuple(assign_result.labels.shape) == (0, )) def test_center_region_assigner_with_empty_boxes_and_ignore(self): """Test corner case where a network might predict no boxes and ignore_iof_thr is on.""" center_region_assigner = CenterRegionAssigner( pos_scale=0.2, neg_scale=0.2, min_pos_iof=0.01) priors = torch.empty((0, 4)) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_bboxes_ignore = torch.Tensor([ [30, 30, 40, 40], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) gt_instances_ignore = InstanceData(bboxes=gt_bboxes_ignore) assign_result = center_region_assigner.assign( pred_instances, gt_instances, gt_instances_ignore=gt_instances_ignore) self.assertEqual(len(assign_result.gt_inds), 0) self.assertTrue(tuple(assign_result.labels.shape) == (0, )) def test_center_region_assigner_with_empty_boxes_and_gt(self): """Test corner case where a network might predict no boxes and no gt.""" center_region_assigner = CenterRegionAssigner( pos_scale=0.2, neg_scale=0.2, min_pos_iof=0.01) priors = torch.empty((0, 4)) gt_bboxes = torch.empty((0, 4)) gt_labels = torch.empty(0) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) assign_result = center_region_assigner.assign(pred_instances, gt_instances) self.assertEqual(len(assign_result.gt_inds), 0) ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_dynamic_soft_label_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmengine.testing import assert_allclose from mmdet.models.task_modules.assigners import DynamicSoftLabelAssigner from mmdet.structures.bbox import HorizontalBoxes class TestDynamicSoftLabelAssigner(TestCase): def test_assign(self): assigner = DynamicSoftLabelAssigner( soft_center_radius=3.0, topk=1, iou_weight=3.0) pred_instances = InstanceData( bboxes=torch.Tensor([[23, 23, 43, 43], [4, 5, 6, 7]]), scores=torch.FloatTensor([[0.2], [0.8]]), priors=torch.Tensor([[30, 30, 8, 8], [4, 5, 6, 7]])) gt_instances = InstanceData( bboxes=torch.Tensor([[23, 23, 43, 43]]), labels=torch.LongTensor([0])) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([1, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_assign_with_no_valid_bboxes(self): assigner = DynamicSoftLabelAssigner( soft_center_radius=3.0, topk=1, iou_weight=3.0) pred_instances = InstanceData( bboxes=torch.Tensor([[123, 123, 143, 143], [114, 151, 161, 171]]), scores=torch.FloatTensor([[0.2], [0.8]]), priors=torch.Tensor([[30, 30, 8, 8], [55, 55, 8, 8]])) gt_instances = InstanceData( bboxes=torch.Tensor([[0, 0, 1, 1]]), labels=torch.LongTensor([0])) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([0, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_assign_with_empty_gt(self): assigner = DynamicSoftLabelAssigner( soft_center_radius=3.0, topk=1, iou_weight=3.0) pred_instances = InstanceData( bboxes=torch.Tensor([[[30, 40, 50, 60]], [[4, 5, 6, 7]]]), scores=torch.FloatTensor([[0.2], [0.8]]), priors=torch.Tensor([[0, 12, 23, 34], [4, 5, 6, 7]])) gt_instances = InstanceData( bboxes=torch.empty(0, 4), labels=torch.empty(0)) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([0, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_box_type_input(self): assigner = DynamicSoftLabelAssigner( soft_center_radius=3.0, topk=1, iou_weight=3.0) pred_instances = InstanceData( bboxes=torch.Tensor([[23, 23, 43, 43], [4, 5, 6, 7]]), scores=torch.FloatTensor([[0.2], [0.8]]), priors=torch.Tensor([[30, 30, 8, 8], [4, 5, 6, 7]])) gt_instances = InstanceData( bboxes=HorizontalBoxes(torch.Tensor([[23, 23, 43, 43]])), labels=torch.LongTensor([0])) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([1, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_grid_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmengine.testing import assert_allclose from mmdet.models.task_modules.assigners import GridAssigner class TestGridAssigner(TestCase): def test_assign(self): assigner = GridAssigner(pos_iou_thr=0.5, neg_iou_thr=0.3) pred_instances = InstanceData( priors=torch.Tensor([[23, 23, 43, 43], [4, 5, 6, 7]]), responsible_flags=torch.BoolTensor([1, 1])) gt_instances = InstanceData( bboxes=torch.Tensor([[23, 23, 43, 43]]), labels=torch.LongTensor([0])) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([1, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) # invalid neg_iou_thr with self.assertRaises(AssertionError): assigner = GridAssigner( pos_iou_thr=0.5, neg_iou_thr=[0.3, 0.1, 0.4]) assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) # multi-neg_iou_thr assigner = GridAssigner(pos_iou_thr=0.5, neg_iou_thr=(0.1, 0.3)) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([1, -1]) assert_allclose(assign_result.gt_inds, expected_gt_inds) # gt_max_assign_all=False assigner = GridAssigner( pos_iou_thr=0.5, neg_iou_thr=0.3, gt_max_assign_all=False) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([1, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) # large min_pos_iou assigner = GridAssigner( pos_iou_thr=0.5, neg_iou_thr=0.3, min_pos_iou=1) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([1, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_assign_with_empty_gt(self): assigner = GridAssigner(pos_iou_thr=0.5, neg_iou_thr=0.3) pred_instances = InstanceData( priors=torch.Tensor([[0, 12, 23, 34], [4, 5, 6, 7]]), responsible_flags=torch.BoolTensor([1, 1])) gt_instances = InstanceData( bboxes=torch.empty(0, 4), labels=torch.empty(0)) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([0, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_assign_with_empty_priors(self): assigner = GridAssigner(pos_iou_thr=0.5, neg_iou_thr=0.3) pred_instances = InstanceData( priors=torch.Tensor(torch.empty(0, 4)), responsible_flags=torch.empty(0)) gt_instances = InstanceData( bboxes=torch.Tensor([[23, 23, 43, 43]]), labels=torch.LongTensor([0])) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([]) assert_allclose(assign_result.gt_inds, expected_gt_inds) ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_hungarian_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import ConfigDict from mmengine.structures import InstanceData from mmdet.models.task_modules.assigners import HungarianAssigner class TestHungarianAssigner(TestCase): def test_init(self): with self.assertRaises(AssertionError): HungarianAssigner([]) def test_hungarian_match_assigner(self): assigner = HungarianAssigner([ dict(type='ClassificationCost', weight=1.), dict(type='BBoxL1Cost', weight=5.0), dict(type='IoUCost', iou_mode='giou', weight=2.0) ]) # test no gt bboxes gt_instances = InstanceData() gt_instances.bboxes = torch.empty((0, 4)).float() gt_instances.labels = torch.empty((0, )).long() pred_instances = InstanceData() pred_instances.scores = torch.rand((10, 81)) pred_instances.bboxes = torch.rand((10, 4)) img_meta = dict(img_shape=(10, 8)) assign_result = assigner.assign( pred_instances, gt_instances, img_meta=img_meta) self.assertTrue(torch.all(assign_result.gt_inds == 0)) self.assertTrue(torch.all(assign_result.labels == -1)) # test with gt bboxes gt_instances.bboxes = torch.FloatTensor([[0, 0, 5, 7], [3, 5, 7, 8]]) gt_instances.labels = torch.LongTensor([1, 20]) assign_result = assigner.assign( pred_instances, gt_instances, img_meta=img_meta) self.assertTrue(torch.all(assign_result.gt_inds > -1)) self.assertEqual((assign_result.gt_inds > 0).sum(), gt_instances.bboxes.size(0)) self.assertEqual((assign_result.labels > -1).sum(), gt_instances.bboxes.size(0)) def test_bbox_match_cost(self): gt_instances = InstanceData() gt_instances.bboxes = torch.FloatTensor([[0, 0, 5, 7], [3, 5, 7, 8]]) gt_instances.labels = torch.LongTensor([1, 20]) pred_instances = InstanceData() pred_instances.scores = torch.rand((10, 81)) pred_instances.bboxes = torch.rand((10, 4)) img_meta = dict(img_shape=(10, 8)) # test IoUCost assigner = HungarianAssigner( ConfigDict(dict(type='IoUCost', iou_mode='iou'))) assign_result = assigner.assign( pred_instances, gt_instances, img_meta=img_meta) self.assertTrue(torch.all(assign_result.gt_inds > -1)) self.assertEqual((assign_result.gt_inds > 0).sum(), gt_instances.bboxes.size(0)) self.assertEqual((assign_result.labels > -1).sum(), gt_instances.bboxes.size(0)) # test BBoxL1Cost assigner = HungarianAssigner(ConfigDict(dict(type='BBoxL1Cost'))) assign_result = assigner.assign( pred_instances, gt_instances, img_meta=img_meta) self.assertTrue(torch.all(assign_result.gt_inds > -1)) self.assertEqual((assign_result.gt_inds > 0).sum(), gt_instances.bboxes.size(0)) self.assertEqual((assign_result.labels > -1).sum(), gt_instances.bboxes.size(0)) def test_cls_match_cost(self): gt_instances = InstanceData() gt_instances.bboxes = torch.FloatTensor([[0, 0, 5, 7], [3, 5, 7, 8]]) gt_instances.labels = torch.LongTensor([1, 20]) pred_instances = InstanceData() pred_instances.scores = torch.rand((10, 81)) pred_instances.bboxes = torch.rand((10, 4)) img_meta = dict(img_shape=(10, 8)) # test FocalLossCost assigner = HungarianAssigner(dict(type='FocalLossCost')) assign_result = assigner.assign( pred_instances, gt_instances, img_meta=img_meta) self.assertTrue(torch.all(assign_result.gt_inds > -1)) self.assertEqual((assign_result.gt_inds > 0).sum(), gt_instances.bboxes.size(0)) self.assertEqual((assign_result.labels > -1).sum(), gt_instances.bboxes.size(0)) # test ClassificationCost assigner = HungarianAssigner(dict(type='ClassificationCost')) assign_result = assigner.assign( pred_instances, gt_instances, img_meta=img_meta) self.assertTrue(torch.all(assign_result.gt_inds > -1)) self.assertEqual((assign_result.gt_inds > 0).sum(), gt_instances.bboxes.size(0)) self.assertEqual((assign_result.labels > -1).sum(), gt_instances.bboxes.size(0)) def test_mask_match_cost(self): gt_instances = InstanceData() gt_instances.masks = torch.randint(0, 2, (2, 10, 10)).long() gt_instances.labels = torch.LongTensor([1, 20]) pred_instances = InstanceData() pred_instances.masks = torch.rand((4, 10, 10)) pred_instances.scores = torch.rand((4, 25)) img_meta = dict(img_shape=(10, 10)) # test DiceCost assigner = HungarianAssigner(dict(type='DiceCost')) assign_result = assigner.assign( pred_instances, gt_instances, img_meta=img_meta) self.assertTrue(torch.all(assign_result.gt_inds > -1)) self.assertEqual((assign_result.gt_inds > 0).sum(), gt_instances.masks.size(0)) self.assertEqual((assign_result.labels > -1).sum(), gt_instances.masks.size(0)) # test CrossEntropyLossCost assigner = HungarianAssigner(dict(type='CrossEntropyLossCost')) assign_result = assigner.assign( pred_instances, gt_instances, img_meta=img_meta) self.assertTrue(torch.all(assign_result.gt_inds > -1)) self.assertEqual((assign_result.gt_inds > 0).sum(), gt_instances.masks.size(0)) self.assertEqual((assign_result.labels > -1).sum(), gt_instances.masks.size(0)) # test FocalLossCost assigner = HungarianAssigner( dict(type='FocalLossCost', binary_input=True)) assign_result = assigner.assign( pred_instances, gt_instances, img_meta=img_meta) self.assertTrue(torch.all(assign_result.gt_inds > -1)) self.assertEqual((assign_result.gt_inds > 0).sum(), gt_instances.masks.size(0)) self.assertEqual((assign_result.labels > -1).sum(), gt_instances.masks.size(0)) ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_max_iou_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """Tests the Assigner objects. CommandLine: pytest tests/test_core/test_bbox/test_assigners/test_max_iou_assigner.py xdoctest tests/test_core/test_bbox/test_assigners/test_max_iou_assigner.py zero """ # noqa import pytest import torch from mmengine.structures import InstanceData from mmdet.models.task_modules.assigners import MaxIoUAssigner @pytest.mark.parametrize('neg_iou_thr', [0.5, (0, 0.5)]) def test_max_iou_assigner(neg_iou_thr): self = MaxIoUAssigner( pos_iou_thr=0.5, neg_iou_thr=neg_iou_thr, ) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) assign_result = self.assign(pred_instances, gt_instances) assert len(assign_result.gt_inds) == 4 assert len(assign_result.labels) == 4 expected_gt_inds = torch.LongTensor([1, 0, 2, 0]) assert torch.all(assign_result.gt_inds == expected_gt_inds) def test_max_iou_assigner_with_ignore(): self = MaxIoUAssigner( pos_iou_thr=0.5, neg_iou_thr=0.5, ignore_iof_thr=0.5, ignore_wrt_candidates=False, ) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [30, 32, 40, 42], ]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) gt_bboxes_ignore = torch.Tensor([ [30, 30, 40, 40], ]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) gt_instances_ignore = InstanceData(bboxes=gt_bboxes_ignore) assign_result = self.assign( pred_instances, gt_instances, gt_instances_ignore=gt_instances_ignore) expected_gt_inds = torch.LongTensor([1, 0, 2, -1]) assert torch.all(assign_result.gt_inds == expected_gt_inds) def test_max_iou_assigner_with_empty_gt(): """Test corner case where an image might have no true detections.""" self = MaxIoUAssigner( pos_iou_thr=0.5, neg_iou_thr=0.5, ) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.empty(0, 4) gt_labels = torch.empty(0) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) assign_result = self.assign(pred_instances, gt_instances) expected_gt_inds = torch.LongTensor([0, 0, 0, 0]) assert torch.all(assign_result.gt_inds == expected_gt_inds) def test_max_iou_assigner_with_empty_priors(): """Test corner case where a network might predict no boxes.""" self = MaxIoUAssigner( pos_iou_thr=0.5, neg_iou_thr=0.5, ) priors = torch.empty((0, 4)) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) # Test with gt_labels assign_result = self.assign(pred_instances, gt_instances) assert len(assign_result.gt_inds) == 0 assert tuple(assign_result.labels.shape) == (0, ) def test_max_iou_assigner_with_empty_boxes_and_ignore(): """Test corner case where a network might predict no boxes and ignore_iof_thr is on.""" self = MaxIoUAssigner( pos_iou_thr=0.5, neg_iou_thr=0.5, ignore_iof_thr=0.5, ) priors = torch.empty((0, 4)) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_bboxes_ignore = torch.Tensor([ [30, 30, 40, 40], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) gt_instances_ignore = InstanceData(bboxes=gt_bboxes_ignore) # Test with gt_labels assign_result = self.assign( pred_instances, gt_instances, gt_instances_ignore=gt_instances_ignore) assert len(assign_result.gt_inds) == 0 assert tuple(assign_result.labels.shape) == (0, ) def test_max_iou_assigner_with_empty_priors_and_gt(): """Test corner case where a network might predict no boxes and no gt.""" self = MaxIoUAssigner( pos_iou_thr=0.5, neg_iou_thr=0.5, ) priors = torch.empty(0, 4) gt_bboxes = torch.empty(0, 4) gt_labels = torch.empty(0) pred_instances = InstanceData(priors=priors) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) assign_result = self.assign(pred_instances, gt_instances) assert len(assign_result.gt_inds) == 0 ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_point_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import unittest import torch from mmengine.structures import InstanceData from mmengine.testing import assert_allclose from mmdet.models.task_modules.assigners import PointAssigner class TestPointAssigner(unittest.TestCase): def test_point_assigner(self): assigner = PointAssigner() pred_instances = InstanceData() pred_instances.priors = torch.FloatTensor([ # [x, y, stride] [0, 0, 1], [10, 10, 1], [5, 5, 1], [32, 32, 1], ]) gt_instances = InstanceData() gt_instances.bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_instances.labels = torch.LongTensor([0, 1]) assign_result = assigner.assign(pred_instances, gt_instances) expected_gt_inds = torch.LongTensor([1, 2, 1, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_point_assigner_with_empty_gt(self): """Test corner case where an image might have no true detections.""" assigner = PointAssigner() pred_instances = InstanceData() pred_instances.priors = torch.FloatTensor([ # [x, y, stride] [0, 0, 1], [10, 10, 1], [5, 5, 1], [32, 32, 1], ]) gt_instances = InstanceData() gt_instances.bboxes = torch.FloatTensor([]) gt_instances.labels = torch.LongTensor([]) assign_result = assigner.assign(pred_instances, gt_instances) expected_gt_inds = torch.LongTensor([0, 0, 0, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_point_assigner_with_empty_boxes_and_gt(self): """Test corner case where an image might predict no points and no gt.""" assigner = PointAssigner() pred_instances = InstanceData() pred_instances.priors = torch.FloatTensor([]) gt_instances = InstanceData() gt_instances.bboxes = torch.FloatTensor([]) gt_instances.labels = torch.LongTensor([]) assign_result = assigner.assign(pred_instances, gt_instances) self.assertEqual(len(assign_result.gt_inds), 0) ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_region_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.config import ConfigDict from mmengine.structures import InstanceData from mmdet.models.task_modules.assigners import RegionAssigner class TestRegionAssigner(TestCase): def setUp(self): self.img_meta = ConfigDict(dict(img_shape=(256, 256))) self.featmap_sizes = [(64, 64)] self.anchor_scale = 10 self.anchor_strides = [1] def test_region_assigner(self): region_assigner = RegionAssigner(center_ratio=0.5, ignore_ratio=0.8) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) valid_flags = torch.BoolTensor([1, 1, 1, 1]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors, valid_flags=valid_flags) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) num_level_anchors = [4] assign_result = region_assigner.assign( pred_instances, gt_instances, self.img_meta, self.featmap_sizes, num_level_anchors, self.anchor_scale, self.anchor_strides) self.assertEqual(len(assign_result.gt_inds), 4) self.assertEqual(len(assign_result.labels), 4) expected_gt_inds = torch.LongTensor([1, 0, 0, 0]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) def test_region_assigner_with_ignore(self): region_assigner = RegionAssigner(center_ratio=0.5) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [30, 32, 40, 42], ]) valid_flags = torch.BoolTensor([1, 1, 1, 1]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) gt_bboxes_ignore = torch.Tensor([ [30, 30, 40, 40], ]) pred_instances = InstanceData(priors=priors, valid_flags=valid_flags) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) gt_instances_ignore = InstanceData(bboxes=gt_bboxes_ignore) num_level_anchors = [4] with self.assertRaises(NotImplementedError): region_assigner.assign( pred_instances, gt_instances, self.img_meta, self.featmap_sizes, num_level_anchors, self.anchor_scale, self.anchor_strides, gt_instances_ignore=gt_instances_ignore) def test_region_assigner_with_empty_gt(self): """Test corner case where an image might have no true detections.""" region_assigner = RegionAssigner(center_ratio=0.5) priors = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) valid_flags = torch.BoolTensor([1, 1, 1, 1]) gt_bboxes = torch.empty(0, 4) gt_labels = torch.empty(0) pred_instances = InstanceData(priors=priors, valid_flags=valid_flags) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) num_level_anchors = [4] assign_result = region_assigner.assign( pred_instances, gt_instances, self.img_meta, self.featmap_sizes, num_level_anchors, self.anchor_scale, self.anchor_strides) expected_gt_inds = torch.LongTensor([0, 0, 0, 0]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) def test_atss_assigner_with_empty_boxes(self): """Test corner case where a network might predict no boxes.""" region_assigner = RegionAssigner(center_ratio=0.5) priors = torch.empty((0, 4)) valid_flags = torch.BoolTensor([]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData(priors=priors, valid_flags=valid_flags) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) num_level_anchors = [0] assign_result = region_assigner.assign( pred_instances, gt_instances, self.img_meta, self.featmap_sizes, num_level_anchors, self.anchor_scale, self.anchor_strides) self.assertEqual(len(assign_result.gt_inds), 0) self.assertTrue(tuple(assign_result.labels.shape) == (0, )) def test_atss_assigner_with_empty_boxes_and_gt(self): """Test corner case where a network might predict no boxes and no gt.""" region_assigner = RegionAssigner(center_ratio=0.5) priors = torch.empty((0, 4)) valid_flags = torch.BoolTensor([]) gt_bboxes = torch.empty((0, 4)) gt_labels = torch.empty(0) num_level_anchors = [0] pred_instances = InstanceData(priors=priors, valid_flags=valid_flags) gt_instances = InstanceData(bboxes=gt_bboxes, labels=gt_labels) assign_result = region_assigner.assign( pred_instances, gt_instances, self.img_meta, self.featmap_sizes, num_level_anchors, self.anchor_scale, self.anchor_strides) self.assertEqual(len(assign_result.gt_inds), 0) ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_simota_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmengine.testing import assert_allclose from mmdet.models.task_modules.assigners import SimOTAAssigner class TestSimOTAAssigner(TestCase): def test_assign(self): assigner = SimOTAAssigner( center_radius=2.5, candidate_topk=1, iou_weight=3.0, cls_weight=1.0) pred_instances = InstanceData( bboxes=torch.Tensor([[23, 23, 43, 43], [4, 5, 6, 7]]), scores=torch.FloatTensor([[0.2], [0.8]]), priors=torch.Tensor([[30, 30, 8, 8], [4, 5, 6, 7]])) gt_instances = InstanceData( bboxes=torch.Tensor([[23, 23, 43, 43]]), labels=torch.LongTensor([0])) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([1, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_assign_with_no_valid_bboxes(self): assigner = SimOTAAssigner( center_radius=2.5, candidate_topk=1, iou_weight=3.0, cls_weight=1.0) pred_instances = InstanceData( bboxes=torch.Tensor([[123, 123, 143, 143], [114, 151, 161, 171]]), scores=torch.FloatTensor([[0.2], [0.8]]), priors=torch.Tensor([[30, 30, 8, 8], [55, 55, 8, 8]])) gt_instances = InstanceData( bboxes=torch.Tensor([[0, 0, 1, 1]]), labels=torch.LongTensor([0])) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([0, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_assign_with_empty_gt(self): assigner = SimOTAAssigner( center_radius=2.5, candidate_topk=1, iou_weight=3.0, cls_weight=1.0) pred_instances = InstanceData( bboxes=torch.Tensor([[[30, 40, 50, 60]], [[4, 5, 6, 7]]]), scores=torch.FloatTensor([[0.2], [0.8]]), priors=torch.Tensor([[0, 12, 23, 34], [4, 5, 6, 7]])) gt_instances = InstanceData( bboxes=torch.empty(0, 4), labels=torch.empty(0)) assign_result = assigner.assign( pred_instances=pred_instances, gt_instances=gt_instances) expected_gt_inds = torch.LongTensor([0, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_task_aligned_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmdet.models.task_modules.assigners import TaskAlignedAssigner class TestTaskAlignedAssigner(TestCase): def test_task_aligned_assigner(self): with self.assertRaises(AssertionError): TaskAlignedAssigner(topk=0) assigner = TaskAlignedAssigner(topk=13) pred_score = torch.FloatTensor([[0.1, 0.2], [0.2, 0.3], [0.3, 0.4], [0.4, 0.5]]) pred_bbox = torch.FloatTensor([ [1, 1, 12, 8], [4, 4, 20, 20], [1, 5, 15, 15], [30, 5, 32, 42], ]) anchor = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([0, 1]) pred_instances = InstanceData() pred_instances.priors = anchor pred_instances.bboxes = pred_bbox pred_instances.scores = pred_score gt_instances = InstanceData() gt_instances.bboxes = gt_bboxes gt_instances.labels = gt_labels assign_result = assigner.assign(pred_instances, gt_instances) self.assertEqual(len(assign_result.gt_inds), 4) self.assertEqual(len(assign_result.labels), 4) # test empty gt gt_bboxes = torch.empty(0, 4) gt_labels = torch.empty(0, 2).long() gt_instances = InstanceData() gt_instances.bboxes = gt_bboxes gt_instances.labels = gt_labels assign_result = assigner.assign(pred_instances, gt_instances) expected_gt_inds = torch.LongTensor([0, 0, 0, 0]) self.assertTrue(torch.all(assign_result.gt_inds == expected_gt_inds)) ================================================ FILE: tests/test_models/test_task_modules/test_assigners/test_task_uniform_assigner.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine.structures import InstanceData from mmengine.testing import assert_allclose from mmdet.models.task_modules.assigners import UniformAssigner class TestUniformAssigner(TestCase): def test_uniform_assigner(self): assigner = UniformAssigner(0.15, 0.7, 1) pred_bbox = torch.FloatTensor([ [1, 1, 12, 8], [4, 4, 20, 20], [1, 5, 15, 15], [30, 5, 32, 42], ]) anchor = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData() pred_instances.priors = anchor pred_instances.decoder_priors = pred_bbox gt_instances = InstanceData() gt_instances.bboxes = gt_bboxes gt_instances.labels = gt_labels assign_result = assigner.assign(pred_instances, gt_instances) self.assertEqual(len(assign_result.gt_inds), 4) self.assertEqual(len(assign_result.labels), 4) expected_gt_inds = torch.LongTensor([-1, 0, 2, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_uniform_assigner_with_empty_gt(self): """Test corner case where an image might have no true detections.""" assigner = UniformAssigner(0.15, 0.7, 1) pred_bbox = torch.FloatTensor([ [1, 1, 12, 8], [4, 4, 20, 20], [1, 5, 15, 15], [30, 5, 32, 42], ]) anchor = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [5, 5, 15, 15], [32, 32, 38, 42], ]) gt_bboxes = torch.empty(0, 4) gt_labels = torch.empty(0) pred_instances = InstanceData() pred_instances.priors = anchor pred_instances.decoder_priors = pred_bbox gt_instances = InstanceData() gt_instances.bboxes = gt_bboxes gt_instances.labels = gt_labels assign_result = assigner.assign(pred_instances, gt_instances) expected_gt_inds = torch.LongTensor([0, 0, 0, 0]) assert_allclose(assign_result.gt_inds, expected_gt_inds) def test_uniform_assigner_with_empty_boxes(self): """Test corner case where a network might predict no boxes.""" assigner = UniformAssigner(0.15, 0.7, 1) pred_bbox = torch.empty((0, 4)) anchor = torch.empty((0, 4)) gt_bboxes = torch.FloatTensor([ [0, 0, 10, 9], [0, 10, 10, 19], ]) gt_labels = torch.LongTensor([2, 3]) pred_instances = InstanceData() pred_instances.priors = anchor pred_instances.decoder_priors = pred_bbox gt_instances = InstanceData() gt_instances.bboxes = gt_bboxes gt_instances.labels = gt_labels # Test with gt_labels assign_result = assigner.assign(pred_instances, gt_instances) self.assertEqual(len(assign_result.gt_inds), 0) self.assertEqual(tuple(assign_result.labels.shape), (0, )) ================================================ FILE: tests/test_models/test_task_modules/test_coder/test_delta_xywh_bbox_coder.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import pytest import torch from mmdet.models.task_modules.coders import DeltaXYWHBBoxCoder def test_delta_bbox_coder(): coder = DeltaXYWHBBoxCoder() rois = torch.Tensor([[0., 0., 1., 1.], [0., 0., 1., 1.], [0., 0., 1., 1.], [5., 5., 5., 5.]]) deltas = torch.Tensor([[0., 0., 0., 0.], [1., 1., 1., 1.], [0., 0., 2., -1.], [0.7, -1.9, -0.5, 0.3]]) expected_decode_bboxes = torch.Tensor([[0.0000, 0.0000, 1.0000, 1.0000], [0.1409, 0.1409, 2.8591, 2.8591], [0.0000, 0.3161, 4.1945, 0.6839], [5.0000, 5.0000, 5.0000, 5.0000]]) out = coder.decode(rois, deltas, max_shape=(32, 32)) assert expected_decode_bboxes.allclose(out, atol=1e-04) out = coder.decode(rois, deltas, max_shape=torch.Tensor((32, 32))) assert expected_decode_bboxes.allclose(out, atol=1e-04) batch_rois = rois.unsqueeze(0).repeat(2, 1, 1) batch_deltas = deltas.unsqueeze(0).repeat(2, 1, 1) batch_out = coder.decode(batch_rois, batch_deltas, max_shape=(32, 32))[0] assert out.allclose(batch_out) batch_out = coder.decode( batch_rois, batch_deltas, max_shape=[(32, 32), (32, 32)])[0] assert out.allclose(batch_out) # test max_shape is not equal to batch with pytest.raises(AssertionError): coder.decode( batch_rois, batch_deltas, max_shape=[(32, 32), (32, 32), (32, 32)]) rois = torch.zeros((0, 4)) deltas = torch.zeros((0, 4)) out = coder.decode(rois, deltas, max_shape=(32, 32)) assert rois.shape == out.shape # test add_ctr_clamp coder = DeltaXYWHBBoxCoder(add_ctr_clamp=True, ctr_clamp=2) rois = torch.Tensor([[0., 0., 6., 6.], [0., 0., 1., 1.], [0., 0., 1., 1.], [5., 5., 5., 5.]]) deltas = torch.Tensor([[1., 1., 2., 2.], [1., 1., 1., 1.], [0., 0., 2., -1.], [0.7, -1.9, -0.5, 0.3]]) expected_decode_bboxes = torch.Tensor([[0.0000, 0.0000, 27.1672, 27.1672], [0.1409, 0.1409, 2.8591, 2.8591], [0.0000, 0.3161, 4.1945, 0.6839], [5.0000, 5.0000, 5.0000, 5.0000]]) out = coder.decode(rois, deltas, max_shape=(32, 32)) assert expected_decode_bboxes.allclose(out, atol=1e-04) ================================================ FILE: tests/test_models/test_task_modules/test_iou2d_calculator.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import pytest import torch from mmdet.evaluation import bbox_overlaps as recall_overlaps from mmdet.models.task_modules import BboxOverlaps2D from mmdet.structures.bbox import bbox_overlaps def test_bbox_overlaps_2d(eps=1e-7): def _construct_bbox(num_bbox=None): img_h = int(np.random.randint(3, 1000)) img_w = int(np.random.randint(3, 1000)) if num_bbox is None: num_bbox = np.random.randint(1, 10) x1y1 = torch.rand((num_bbox, 2)) x2y2 = torch.max(torch.rand((num_bbox, 2)), x1y1) bboxes = torch.cat((x1y1, x2y2), -1) bboxes[:, 0::2] *= img_w bboxes[:, 1::2] *= img_h return bboxes, num_bbox # is_aligned is True, bboxes.size(-1) == 5 (include score) self = BboxOverlaps2D() bboxes1, num_bbox = _construct_bbox() bboxes2, _ = _construct_bbox(num_bbox) bboxes1 = torch.cat((bboxes1, torch.rand((num_bbox, 1))), 1) bboxes2 = torch.cat((bboxes2, torch.rand((num_bbox, 1))), 1) gious = self(bboxes1, bboxes2, 'giou', True) assert gious.size() == (num_bbox, ), gious.size() assert torch.all(gious >= -1) and torch.all(gious <= 1) # is_aligned is True, bboxes1.size(-2) == 0 bboxes1 = torch.empty((0, 4)) bboxes2 = torch.empty((0, 4)) gious = self(bboxes1, bboxes2, 'giou', True) assert gious.size() == (0, ), gious.size() assert torch.all(gious == torch.empty((0, ))) assert torch.all(gious >= -1) and torch.all(gious <= 1) # is_aligned is True, and bboxes.ndims > 2 bboxes1, num_bbox = _construct_bbox() bboxes2, _ = _construct_bbox(num_bbox) bboxes1 = bboxes1.unsqueeze(0).repeat(2, 1, 1) # test assertion when batch dim is not the same with pytest.raises(AssertionError): self(bboxes1, bboxes2.unsqueeze(0).repeat(3, 1, 1), 'giou', True) bboxes2 = bboxes2.unsqueeze(0).repeat(2, 1, 1) gious = self(bboxes1, bboxes2, 'giou', True) assert torch.all(gious >= -1) and torch.all(gious <= 1) assert gious.size() == (2, num_bbox) bboxes1 = bboxes1.unsqueeze(0).repeat(2, 1, 1, 1) bboxes2 = bboxes2.unsqueeze(0).repeat(2, 1, 1, 1) gious = self(bboxes1, bboxes2, 'giou', True) assert torch.all(gious >= -1) and torch.all(gious <= 1) assert gious.size() == (2, 2, num_bbox) # is_aligned is False bboxes1, num_bbox1 = _construct_bbox() bboxes2, num_bbox2 = _construct_bbox() gious = self(bboxes1, bboxes2, 'giou') assert torch.all(gious >= -1) and torch.all(gious <= 1) assert gious.size() == (num_bbox1, num_bbox2) # is_aligned is False, and bboxes.ndims > 2 bboxes1 = bboxes1.unsqueeze(0).repeat(2, 1, 1) bboxes2 = bboxes2.unsqueeze(0).repeat(2, 1, 1) gious = self(bboxes1, bboxes2, 'giou') assert torch.all(gious >= -1) and torch.all(gious <= 1) assert gious.size() == (2, num_bbox1, num_bbox2) bboxes1 = bboxes1.unsqueeze(0) bboxes2 = bboxes2.unsqueeze(0) gious = self(bboxes1, bboxes2, 'giou') assert torch.all(gious >= -1) and torch.all(gious <= 1) assert gious.size() == (1, 2, num_bbox1, num_bbox2) # is_aligned is False, bboxes1.size(-2) == 0 gious = self(torch.empty(1, 2, 0, 4), bboxes2, 'giou') assert torch.all(gious == torch.empty(1, 2, 0, bboxes2.size(-2))) assert torch.all(gious >= -1) and torch.all(gious <= 1) # test allclose between bbox_overlaps and the original official # implementation. bboxes1 = torch.FloatTensor([ [0, 0, 10, 10], [10, 10, 20, 20], [32, 32, 38, 42], ]) bboxes2 = torch.FloatTensor([ [0, 0, 10, 20], [0, 10, 10, 19], [10, 10, 20, 20], ]) gious = bbox_overlaps(bboxes1, bboxes2, 'giou', is_aligned=True, eps=eps) gious = gious.numpy().round(4) # the gt is got with four decimal precision. expected_gious = np.array([0.5000, -0.0500, -0.8214]) assert np.allclose(gious, expected_gious, rtol=0, atol=eps) # test mode 'iof' ious = bbox_overlaps(bboxes1, bboxes2, 'iof', is_aligned=True, eps=eps) assert torch.all(ious >= -1) and torch.all(ious <= 1) assert ious.size() == (bboxes1.size(0), ) ious = bbox_overlaps(bboxes1, bboxes2, 'iof', eps=eps) assert torch.all(ious >= -1) and torch.all(ious <= 1) assert ious.size() == (bboxes1.size(0), bboxes2.size(0)) def test_voc_recall_overlaps(): def _construct_bbox(num_bbox=None): img_h = int(np.random.randint(3, 1000)) img_w = int(np.random.randint(3, 1000)) if num_bbox is None: num_bbox = np.random.randint(1, 10) x1y1 = torch.rand((num_bbox, 2)) x2y2 = torch.max(torch.rand((num_bbox, 2)), x1y1) bboxes = torch.cat((x1y1, x2y2), -1) bboxes[:, 0::2] *= img_w bboxes[:, 1::2] *= img_h return bboxes.numpy(), num_bbox bboxes1, num_bbox = _construct_bbox() bboxes2, _ = _construct_bbox(num_bbox) ious = recall_overlaps( bboxes1, bboxes2, 'iou', use_legacy_coordinate=False) assert ious.shape == (num_bbox, num_bbox) assert np.all(ious >= -1) and np.all(ious <= 1) ious = recall_overlaps(bboxes1, bboxes2, 'iou', use_legacy_coordinate=True) assert ious.shape == (num_bbox, num_bbox) assert np.all(ious >= -1) and np.all(ious <= 1) ================================================ FILE: tests/test_models/test_task_modules/test_prior_generators/test_anchor_generator.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """ CommandLine: pytest tests/test_utils/test_anchor.py xdoctest tests/test_utils/test_anchor.py zero """ import pytest import torch def test_standard_points_generator(): from mmdet.models.task_modules import build_prior_generator # teat init anchor_generator_cfg = dict( type='MlvlPointGenerator', strides=[4, 8], offset=0) anchor_generator = build_prior_generator(anchor_generator_cfg) assert anchor_generator is not None assert anchor_generator.num_base_priors == [1, 1] # test_stride from mmdet.models.task_modules.prior_generators import MlvlPointGenerator # Square strides mlvl_points = MlvlPointGenerator(strides=[4, 10], offset=0) mlvl_points_half_stride_generator = MlvlPointGenerator( strides=[4, 10], offset=0.5) assert mlvl_points.num_levels == 2 # assert self.num_levels == len(featmap_sizes) with pytest.raises(AssertionError): mlvl_points.grid_priors(featmap_sizes=[(2, 2)], device='cpu') priors = mlvl_points.grid_priors( featmap_sizes=[(2, 2), (4, 8)], device='cpu') priors_with_stride = mlvl_points.grid_priors( featmap_sizes=[(2, 2), (4, 8)], with_stride=True, device='cpu') assert len(priors) == 2 # assert last dimension is (coord_x, coord_y, stride_w, stride_h). assert priors_with_stride[0].size(1) == 4 assert priors_with_stride[0][0][2] == 4 assert priors_with_stride[0][0][3] == 4 assert priors_with_stride[1][0][2] == 10 assert priors_with_stride[1][0][3] == 10 stride_4_feat_2_2 = priors[0] assert (stride_4_feat_2_2[1] - stride_4_feat_2_2[0]).sum() == 4 assert stride_4_feat_2_2.size(0) == 4 assert stride_4_feat_2_2.size(1) == 2 stride_10_feat_4_8 = priors[1] assert (stride_10_feat_4_8[1] - stride_10_feat_4_8[0]).sum() == 10 assert stride_10_feat_4_8.size(0) == 4 * 8 assert stride_10_feat_4_8.size(1) == 2 # assert the offset of 0.5 * stride priors_half_offset = mlvl_points_half_stride_generator.grid_priors( featmap_sizes=[(2, 2), (4, 8)], device='cpu') assert (priors_half_offset[0][0] - priors[0][0]).sum() == 4 * 0.5 * 2 assert (priors_half_offset[1][0] - priors[1][0]).sum() == 10 * 0.5 * 2 if torch.cuda.is_available(): anchor_generator_cfg = dict( type='MlvlPointGenerator', strides=[4, 8], offset=0) anchor_generator = build_prior_generator(anchor_generator_cfg) assert anchor_generator is not None # Square strides mlvl_points = MlvlPointGenerator(strides=[4, 10], offset=0) mlvl_points_half_stride_generator = MlvlPointGenerator( strides=[4, 10], offset=0.5) assert mlvl_points.num_levels == 2 # assert self.num_levels == len(featmap_sizes) with pytest.raises(AssertionError): mlvl_points.grid_priors(featmap_sizes=[(2, 2)], device='cuda') priors = mlvl_points.grid_priors( featmap_sizes=[(2, 2), (4, 8)], device='cuda') priors_with_stride = mlvl_points.grid_priors( featmap_sizes=[(2, 2), (4, 8)], with_stride=True, device='cuda') assert len(priors) == 2 # assert last dimension is (coord_x, coord_y, stride_w, stride_h). assert priors_with_stride[0].size(1) == 4 assert priors_with_stride[0][0][2] == 4 assert priors_with_stride[0][0][3] == 4 assert priors_with_stride[1][0][2] == 10 assert priors_with_stride[1][0][3] == 10 stride_4_feat_2_2 = priors[0] assert (stride_4_feat_2_2[1] - stride_4_feat_2_2[0]).sum() == 4 assert stride_4_feat_2_2.size(0) == 4 assert stride_4_feat_2_2.size(1) == 2 stride_10_feat_4_8 = priors[1] assert (stride_10_feat_4_8[1] - stride_10_feat_4_8[0]).sum() == 10 assert stride_10_feat_4_8.size(0) == 4 * 8 assert stride_10_feat_4_8.size(1) == 2 # assert the offset of 0.5 * stride priors_half_offset = mlvl_points_half_stride_generator.grid_priors( featmap_sizes=[(2, 2), (4, 8)], device='cuda') assert (priors_half_offset[0][0] - priors[0][0]).sum() == 4 * 0.5 * 2 assert (priors_half_offset[1][0] - priors[1][0]).sum() == 10 * 0.5 * 2 def test_sparse_prior(): from mmdet.models.task_modules.prior_generators import MlvlPointGenerator mlvl_points = MlvlPointGenerator(strides=[4, 10], offset=0) prior_indexs = torch.Tensor([0, 2, 4, 5, 6, 9]).long() featmap_sizes = [(3, 5), (6, 4)] grid_anchors = mlvl_points.grid_priors( featmap_sizes=featmap_sizes, with_stride=False, device='cpu') sparse_prior = mlvl_points.sparse_priors( prior_idxs=prior_indexs, featmap_size=featmap_sizes[0], level_idx=0, device='cpu') assert not sparse_prior.is_cuda assert (sparse_prior == grid_anchors[0][prior_indexs]).all() sparse_prior = mlvl_points.sparse_priors( prior_idxs=prior_indexs, featmap_size=featmap_sizes[1], level_idx=1, device='cpu') assert (sparse_prior == grid_anchors[1][prior_indexs]).all() from mmdet.models.task_modules.prior_generators import AnchorGenerator mlvl_anchors = AnchorGenerator( strides=[16, 32], ratios=[1.], scales=[1.], base_sizes=[4, 8]) prior_indexs = torch.Tensor([0, 2, 4, 5, 6, 9]).long() featmap_sizes = [(3, 5), (6, 4)] grid_anchors = mlvl_anchors.grid_priors( featmap_sizes=featmap_sizes, device='cpu') sparse_prior = mlvl_anchors.sparse_priors( prior_idxs=prior_indexs, featmap_size=featmap_sizes[0], level_idx=0, device='cpu') assert (sparse_prior == grid_anchors[0][prior_indexs]).all() sparse_prior = mlvl_anchors.sparse_priors( prior_idxs=prior_indexs, featmap_size=featmap_sizes[1], level_idx=1, device='cpu') assert (sparse_prior == grid_anchors[1][prior_indexs]).all() # for ssd from mmdet.models.task_modules.prior_generators import SSDAnchorGenerator featmap_sizes = [(38, 38), (19, 19), (10, 10)] anchor_generator = SSDAnchorGenerator( scale_major=False, input_size=300, basesize_ratio_range=(0.15, 0.9), strides=[8, 16, 32], ratios=[[2], [2, 3], [2, 3]]) ssd_anchors = anchor_generator.grid_anchors(featmap_sizes, device='cpu') for i in range(len(featmap_sizes)): sparse_ssd_anchors = anchor_generator.sparse_priors( prior_idxs=prior_indexs, level_idx=i, featmap_size=featmap_sizes[i], device='cpu') assert (sparse_ssd_anchors == ssd_anchors[i][prior_indexs]).all() # for yolo from mmdet.models.task_modules.prior_generators import YOLOAnchorGenerator featmap_sizes = [(38, 38), (19, 19), (10, 10)] anchor_generator = YOLOAnchorGenerator( strides=[32, 16, 8], base_sizes=[ [(116, 90), (156, 198), (373, 326)], [(30, 61), (62, 45), (59, 119)], [(10, 13), (16, 30), (33, 23)], ]) yolo_anchors = anchor_generator.grid_anchors(featmap_sizes, device='cpu') for i in range(len(featmap_sizes)): sparse_yolo_anchors = anchor_generator.sparse_priors( prior_idxs=prior_indexs, level_idx=i, featmap_size=featmap_sizes[i], device='cpu') assert (sparse_yolo_anchors == yolo_anchors[i][prior_indexs]).all() if torch.cuda.is_available(): mlvl_points = MlvlPointGenerator(strides=[4, 10], offset=0) prior_indexs = torch.Tensor([0, 3, 4, 5, 6, 7, 1, 2, 4, 5, 6, 9]).long().cuda() featmap_sizes = [(6, 8), (6, 4)] grid_anchors = mlvl_points.grid_priors( featmap_sizes=featmap_sizes, with_stride=False, device='cuda') sparse_prior = mlvl_points.sparse_priors( prior_idxs=prior_indexs, featmap_size=featmap_sizes[0], level_idx=0, device='cuda') assert (sparse_prior == grid_anchors[0][prior_indexs]).all() sparse_prior = mlvl_points.sparse_priors( prior_idxs=prior_indexs, featmap_size=featmap_sizes[1], level_idx=1, device='cuda') assert (sparse_prior == grid_anchors[1][prior_indexs]).all() assert sparse_prior.is_cuda mlvl_anchors = AnchorGenerator( strides=[16, 32], ratios=[1., 2.5], scales=[1., 5.], base_sizes=[4, 8]) prior_indexs = torch.Tensor([4, 5, 6, 7, 0, 2, 50, 4, 5, 6, 9]).long().cuda() featmap_sizes = [(13, 5), (16, 4)] grid_anchors = mlvl_anchors.grid_priors( featmap_sizes=featmap_sizes, device='cuda') sparse_prior = mlvl_anchors.sparse_priors( prior_idxs=prior_indexs, featmap_size=featmap_sizes[0], level_idx=0, device='cuda') assert (sparse_prior == grid_anchors[0][prior_indexs]).all() sparse_prior = mlvl_anchors.sparse_priors( prior_idxs=prior_indexs, featmap_size=featmap_sizes[1], level_idx=1, device='cuda') assert (sparse_prior == grid_anchors[1][prior_indexs]).all() # for ssd from mmdet.models.task_modules.prior_generators import \ SSDAnchorGenerator featmap_sizes = [(38, 38), (19, 19), (10, 10)] anchor_generator = SSDAnchorGenerator( scale_major=False, input_size=300, basesize_ratio_range=(0.15, 0.9), strides=[8, 16, 32], ratios=[[2], [2, 3], [2, 3]]) ssd_anchors = anchor_generator.grid_anchors( featmap_sizes, device='cuda') for i in range(len(featmap_sizes)): sparse_ssd_anchors = anchor_generator.sparse_priors( prior_idxs=prior_indexs, level_idx=i, featmap_size=featmap_sizes[i], device='cuda') assert (sparse_ssd_anchors == ssd_anchors[i][prior_indexs]).all() # for yolo from mmdet.models.task_modules.prior_generators import \ YOLOAnchorGenerator featmap_sizes = [(38, 38), (19, 19), (10, 10)] anchor_generator = YOLOAnchorGenerator( strides=[32, 16, 8], base_sizes=[ [(116, 90), (156, 198), (373, 326)], [(30, 61), (62, 45), (59, 119)], [(10, 13), (16, 30), (33, 23)], ]) yolo_anchors = anchor_generator.grid_anchors( featmap_sizes, device='cuda') for i in range(len(featmap_sizes)): sparse_yolo_anchors = anchor_generator.sparse_priors( prior_idxs=prior_indexs, level_idx=i, featmap_size=featmap_sizes[i], device='cuda') assert (sparse_yolo_anchors == yolo_anchors[i][prior_indexs]).all() def test_standard_anchor_generator(): from mmdet.models.task_modules import build_anchor_generator anchor_generator_cfg = dict( type='AnchorGenerator', scales=[8], ratios=[0.5, 1.0, 2.0], strides=[4, 8]) anchor_generator = build_anchor_generator(anchor_generator_cfg) assert anchor_generator.num_base_priors == \ anchor_generator.num_base_anchors assert anchor_generator.num_base_priors == [3, 3] assert anchor_generator is not None def test_strides(): from mmdet.models.task_modules.prior_generators import AnchorGenerator # Square strides self = AnchorGenerator([10], [1.], [1.], [10]) anchors = self.grid_anchors([(2, 2)], device='cpu') expected_anchors = torch.tensor([[-5., -5., 5., 5.], [5., -5., 15., 5.], [-5., 5., 5., 15.], [5., 5., 15., 15.]]) assert torch.equal(anchors[0], expected_anchors) # Different strides in x and y direction self = AnchorGenerator([(10, 20)], [1.], [1.], [10]) anchors = self.grid_anchors([(2, 2)], device='cpu') expected_anchors = torch.tensor([[-5., -5., 5., 5.], [5., -5., 15., 5.], [-5., 15., 5., 25.], [5., 15., 15., 25.]]) assert torch.equal(anchors[0], expected_anchors) def test_ssd_anchor_generator(): from mmdet.models.task_modules import build_anchor_generator if torch.cuda.is_available(): device = 'cuda' else: device = 'cpu' # min_sizes max_sizes must set at the same time with pytest.raises(AssertionError): anchor_generator_cfg = dict( type='SSDAnchorGenerator', scale_major=False, min_sizes=[48, 100, 150, 202, 253, 300], max_sizes=None, strides=[8, 16, 32, 64, 100, 300], ratios=[[2], [2, 3], [2, 3], [2, 3], [2], [2]]) build_anchor_generator(anchor_generator_cfg) # length of min_sizes max_sizes must be the same with pytest.raises(AssertionError): anchor_generator_cfg = dict( type='SSDAnchorGenerator', scale_major=False, min_sizes=[48, 100, 150, 202, 253, 300], max_sizes=[100, 150, 202, 253], strides=[8, 16, 32, 64, 100, 300], ratios=[[2], [2, 3], [2, 3], [2, 3], [2], [2]]) build_anchor_generator(anchor_generator_cfg) # test setting anchor size manually anchor_generator_cfg = dict( type='SSDAnchorGenerator', scale_major=False, min_sizes=[48, 100, 150, 202, 253, 304], max_sizes=[100, 150, 202, 253, 304, 320], strides=[16, 32, 64, 107, 160, 320], ratios=[[2, 3], [2, 3], [2, 3], [2, 3], [2, 3], [2, 3]]) featmap_sizes = [(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)] anchor_generator = build_anchor_generator(anchor_generator_cfg) expected_base_anchors = [ torch.Tensor([[-16.0000, -16.0000, 32.0000, 32.0000], [-26.6410, -26.6410, 42.6410, 42.6410], [-25.9411, -8.9706, 41.9411, 24.9706], [-8.9706, -25.9411, 24.9706, 41.9411], [-33.5692, -5.8564, 49.5692, 21.8564], [-5.8564, -33.5692, 21.8564, 49.5692]]), torch.Tensor([[-34.0000, -34.0000, 66.0000, 66.0000], [-45.2372, -45.2372, 77.2372, 77.2372], [-54.7107, -19.3553, 86.7107, 51.3553], [-19.3553, -54.7107, 51.3553, 86.7107], [-70.6025, -12.8675, 102.6025, 44.8675], [-12.8675, -70.6025, 44.8675, 102.6025]]), torch.Tensor([[-43.0000, -43.0000, 107.0000, 107.0000], [-55.0345, -55.0345, 119.0345, 119.0345], [-74.0660, -21.0330, 138.0660, 85.0330], [-21.0330, -74.0660, 85.0330, 138.0660], [-97.9038, -11.3013, 161.9038, 75.3013], [-11.3013, -97.9038, 75.3013, 161.9038]]), torch.Tensor([[-47.5000, -47.5000, 154.5000, 154.5000], [-59.5332, -59.5332, 166.5332, 166.5332], [-89.3356, -17.9178, 196.3356, 124.9178], [-17.9178, -89.3356, 124.9178, 196.3356], [-121.4371, -4.8124, 228.4371, 111.8124], [-4.8124, -121.4371, 111.8124, 228.4371]]), torch.Tensor([[-46.5000, -46.5000, 206.5000, 206.5000], [-58.6651, -58.6651, 218.6651, 218.6651], [-98.8980, -9.4490, 258.8980, 169.4490], [-9.4490, -98.8980, 169.4490, 258.8980], [-139.1044, 6.9652, 299.1044, 153.0348], [6.9652, -139.1044, 153.0348, 299.1044]]), torch.Tensor([[8.0000, 8.0000, 312.0000, 312.0000], [4.0513, 4.0513, 315.9487, 315.9487], [-54.9605, 52.5198, 374.9604, 267.4802], [52.5198, -54.9605, 267.4802, 374.9604], [-103.2717, 72.2428, 423.2717, 247.7572], [72.2428, -103.2717, 247.7572, 423.2717]]) ] base_anchors = anchor_generator.base_anchors for i, base_anchor in enumerate(base_anchors): assert base_anchor.allclose(expected_base_anchors[i]) # check valid flags expected_valid_pixels = [2400, 600, 150, 54, 24, 6] multi_level_valid_flags = anchor_generator.valid_flags( featmap_sizes, (320, 320), device) for i, single_level_valid_flag in enumerate(multi_level_valid_flags): assert single_level_valid_flag.sum() == expected_valid_pixels[i] # check number of base anchors for each level assert anchor_generator.num_base_anchors == [6, 6, 6, 6, 6, 6] # check anchor generation anchors = anchor_generator.grid_anchors(featmap_sizes, device) assert len(anchors) == 6 # test vgg ssd anchor setting anchor_generator_cfg = dict( type='SSDAnchorGenerator', scale_major=False, input_size=300, basesize_ratio_range=(0.15, 0.9), strides=[8, 16, 32, 64, 100, 300], ratios=[[2], [2, 3], [2, 3], [2, 3], [2], [2]]) featmap_sizes = [(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)] anchor_generator = build_anchor_generator(anchor_generator_cfg) # check base anchors expected_base_anchors = [ torch.Tensor([[-6.5000, -6.5000, 14.5000, 14.5000], [-11.3704, -11.3704, 19.3704, 19.3704], [-10.8492, -3.4246, 18.8492, 11.4246], [-3.4246, -10.8492, 11.4246, 18.8492]]), torch.Tensor([[-14.5000, -14.5000, 30.5000, 30.5000], [-25.3729, -25.3729, 41.3729, 41.3729], [-23.8198, -7.9099, 39.8198, 23.9099], [-7.9099, -23.8198, 23.9099, 39.8198], [-30.9711, -4.9904, 46.9711, 20.9904], [-4.9904, -30.9711, 20.9904, 46.9711]]), torch.Tensor([[-33.5000, -33.5000, 65.5000, 65.5000], [-45.5366, -45.5366, 77.5366, 77.5366], [-54.0036, -19.0018, 86.0036, 51.0018], [-19.0018, -54.0036, 51.0018, 86.0036], [-69.7365, -12.5788, 101.7365, 44.5788], [-12.5788, -69.7365, 44.5788, 101.7365]]), torch.Tensor([[-44.5000, -44.5000, 108.5000, 108.5000], [-56.9817, -56.9817, 120.9817, 120.9817], [-76.1873, -22.0937, 140.1873, 86.0937], [-22.0937, -76.1873, 86.0937, 140.1873], [-100.5019, -12.1673, 164.5019, 76.1673], [-12.1673, -100.5019, 76.1673, 164.5019]]), torch.Tensor([[-53.5000, -53.5000, 153.5000, 153.5000], [-66.2185, -66.2185, 166.2185, 166.2185], [-96.3711, -23.1855, 196.3711, 123.1855], [-23.1855, -96.3711, 123.1855, 196.3711]]), torch.Tensor([[19.5000, 19.5000, 280.5000, 280.5000], [6.6342, 6.6342, 293.3658, 293.3658], [-34.5549, 57.7226, 334.5549, 242.2774], [57.7226, -34.5549, 242.2774, 334.5549]]), ] base_anchors = anchor_generator.base_anchors for i, base_anchor in enumerate(base_anchors): assert base_anchor.allclose(expected_base_anchors[i]) # check valid flags expected_valid_pixels = [5776, 2166, 600, 150, 36, 4] multi_level_valid_flags = anchor_generator.valid_flags( featmap_sizes, (300, 300), device) for i, single_level_valid_flag in enumerate(multi_level_valid_flags): assert single_level_valid_flag.sum() == expected_valid_pixels[i] # check number of base anchors for each level assert anchor_generator.num_base_anchors == [4, 6, 6, 6, 4, 4] # check anchor generation anchors = anchor_generator.grid_anchors(featmap_sizes, device) assert len(anchors) == 6 def test_anchor_generator_with_tuples(): from mmdet.models.task_modules import build_anchor_generator if torch.cuda.is_available(): device = 'cuda' else: device = 'cpu' anchor_generator_cfg = dict( type='SSDAnchorGenerator', scale_major=False, input_size=300, basesize_ratio_range=(0.15, 0.9), strides=[8, 16, 32, 64, 100, 300], ratios=[[2], [2, 3], [2, 3], [2, 3], [2], [2]]) featmap_sizes = [(38, 38), (19, 19), (10, 10), (5, 5), (3, 3), (1, 1)] anchor_generator = build_anchor_generator(anchor_generator_cfg) anchors = anchor_generator.grid_anchors(featmap_sizes, device) anchor_generator_cfg_tuples = dict( type='SSDAnchorGenerator', scale_major=False, input_size=300, basesize_ratio_range=(0.15, 0.9), strides=[(8, 8), (16, 16), (32, 32), (64, 64), (100, 100), (300, 300)], ratios=[[2], [2, 3], [2, 3], [2, 3], [2], [2]]) anchor_generator_tuples = build_anchor_generator( anchor_generator_cfg_tuples) anchors_tuples = anchor_generator_tuples.grid_anchors( featmap_sizes, device) for anchor, anchor_tuples in zip(anchors, anchors_tuples): assert torch.equal(anchor, anchor_tuples) def test_yolo_anchor_generator(): from mmdet.models.task_modules import build_anchor_generator if torch.cuda.is_available(): device = 'cuda' else: device = 'cpu' anchor_generator_cfg = dict( type='YOLOAnchorGenerator', strides=[32, 16, 8], base_sizes=[ [(116, 90), (156, 198), (373, 326)], [(30, 61), (62, 45), (59, 119)], [(10, 13), (16, 30), (33, 23)], ]) featmap_sizes = [(14, 18), (28, 36), (56, 72)] anchor_generator = build_anchor_generator(anchor_generator_cfg) # check base anchors expected_base_anchors = [ torch.Tensor([[-42.0000, -29.0000, 74.0000, 61.0000], [-62.0000, -83.0000, 94.0000, 115.0000], [-170.5000, -147.0000, 202.5000, 179.0000]]), torch.Tensor([[-7.0000, -22.5000, 23.0000, 38.5000], [-23.0000, -14.5000, 39.0000, 30.5000], [-21.5000, -51.5000, 37.5000, 67.5000]]), torch.Tensor([[-1.0000, -2.5000, 9.0000, 10.5000], [-4.0000, -11.0000, 12.0000, 19.0000], [-12.5000, -7.5000, 20.5000, 15.5000]]) ] base_anchors = anchor_generator.base_anchors for i, base_anchor in enumerate(base_anchors): assert base_anchor.allclose(expected_base_anchors[i]) # check number of base anchors for each level assert anchor_generator.num_base_anchors == [3, 3, 3] # check anchor generation anchors = anchor_generator.grid_anchors(featmap_sizes, device) assert len(anchors) == 3 def test_retina_anchor(): from mmdet.registry import MODELS if torch.cuda.is_available(): device = 'cuda' else: device = 'cpu' # head configs modified from # configs/nas_fpn/retinanet_r50_fpn_crop640_50e.py bbox_head = dict( type='RetinaSepBNHead', num_classes=4, num_ins=5, in_channels=4, stacked_convs=1, feat_channels=4, anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), bbox_coder=dict( type='DeltaXYWHBBoxCoder', target_means=[.0, .0, .0, .0], target_stds=[1.0, 1.0, 1.0, 1.0])) retina_head = MODELS.build(bbox_head) assert retina_head.anchor_generator is not None # use the featmap sizes in NASFPN setting to test retina head featmap_sizes = [(80, 80), (40, 40), (20, 20), (10, 10), (5, 5)] # check base anchors expected_base_anchors = [ torch.Tensor([[-22.6274, -11.3137, 22.6274, 11.3137], [-28.5088, -14.2544, 28.5088, 14.2544], [-35.9188, -17.9594, 35.9188, 17.9594], [-16.0000, -16.0000, 16.0000, 16.0000], [-20.1587, -20.1587, 20.1587, 20.1587], [-25.3984, -25.3984, 25.3984, 25.3984], [-11.3137, -22.6274, 11.3137, 22.6274], [-14.2544, -28.5088, 14.2544, 28.5088], [-17.9594, -35.9188, 17.9594, 35.9188]]), torch.Tensor([[-45.2548, -22.6274, 45.2548, 22.6274], [-57.0175, -28.5088, 57.0175, 28.5088], [-71.8376, -35.9188, 71.8376, 35.9188], [-32.0000, -32.0000, 32.0000, 32.0000], [-40.3175, -40.3175, 40.3175, 40.3175], [-50.7968, -50.7968, 50.7968, 50.7968], [-22.6274, -45.2548, 22.6274, 45.2548], [-28.5088, -57.0175, 28.5088, 57.0175], [-35.9188, -71.8376, 35.9188, 71.8376]]), torch.Tensor([[-90.5097, -45.2548, 90.5097, 45.2548], [-114.0350, -57.0175, 114.0350, 57.0175], [-143.6751, -71.8376, 143.6751, 71.8376], [-64.0000, -64.0000, 64.0000, 64.0000], [-80.6349, -80.6349, 80.6349, 80.6349], [-101.5937, -101.5937, 101.5937, 101.5937], [-45.2548, -90.5097, 45.2548, 90.5097], [-57.0175, -114.0350, 57.0175, 114.0350], [-71.8376, -143.6751, 71.8376, 143.6751]]), torch.Tensor([[-181.0193, -90.5097, 181.0193, 90.5097], [-228.0701, -114.0350, 228.0701, 114.0350], [-287.3503, -143.6751, 287.3503, 143.6751], [-128.0000, -128.0000, 128.0000, 128.0000], [-161.2699, -161.2699, 161.2699, 161.2699], [-203.1873, -203.1873, 203.1873, 203.1873], [-90.5097, -181.0193, 90.5097, 181.0193], [-114.0350, -228.0701, 114.0350, 228.0701], [-143.6751, -287.3503, 143.6751, 287.3503]]), torch.Tensor([[-362.0387, -181.0193, 362.0387, 181.0193], [-456.1401, -228.0701, 456.1401, 228.0701], [-574.7006, -287.3503, 574.7006, 287.3503], [-256.0000, -256.0000, 256.0000, 256.0000], [-322.5398, -322.5398, 322.5398, 322.5398], [-406.3747, -406.3747, 406.3747, 406.3747], [-181.0193, -362.0387, 181.0193, 362.0387], [-228.0701, -456.1401, 228.0701, 456.1401], [-287.3503, -574.7006, 287.3503, 574.7006]]) ] base_anchors = retina_head.anchor_generator.base_anchors for i, base_anchor in enumerate(base_anchors): assert base_anchor.allclose(expected_base_anchors[i]) # check valid flags expected_valid_pixels = [57600, 14400, 3600, 900, 225] multi_level_valid_flags = retina_head.anchor_generator.valid_flags( featmap_sizes, (640, 640), device) for i, single_level_valid_flag in enumerate(multi_level_valid_flags): assert single_level_valid_flag.sum() == expected_valid_pixels[i] # check number of base anchors for each level assert retina_head.anchor_generator.num_base_anchors == [9, 9, 9, 9, 9] # check anchor generation anchors = retina_head.anchor_generator.grid_anchors(featmap_sizes, device) assert len(anchors) == 5 def test_guided_anchor(): from mmdet.registry import MODELS if torch.cuda.is_available(): device = 'cuda' else: device = 'cpu' # head configs modified from # configs/guided_anchoring/ga-retinanet_r50_fpn_1x_coco.py bbox_head = dict( type='GARetinaHead', num_classes=8, in_channels=4, stacked_convs=1, feat_channels=4, approx_anchor_generator=dict( type='AnchorGenerator', octave_base_scale=4, scales_per_octave=3, ratios=[0.5, 1.0, 2.0], strides=[8, 16, 32, 64, 128]), square_anchor_generator=dict( type='AnchorGenerator', ratios=[1.0], scales=[4], strides=[8, 16, 32, 64, 128])) ga_retina_head = MODELS.build(bbox_head) assert ga_retina_head.approx_anchor_generator is not None # use the featmap sizes in NASFPN setting to test ga_retina_head featmap_sizes = [(100, 152), (50, 76), (25, 38), (13, 19), (7, 10)] # check base anchors expected_approxs = [ torch.Tensor([[-22.6274, -11.3137, 22.6274, 11.3137], [-28.5088, -14.2544, 28.5088, 14.2544], [-35.9188, -17.9594, 35.9188, 17.9594], [-16.0000, -16.0000, 16.0000, 16.0000], [-20.1587, -20.1587, 20.1587, 20.1587], [-25.3984, -25.3984, 25.3984, 25.3984], [-11.3137, -22.6274, 11.3137, 22.6274], [-14.2544, -28.5088, 14.2544, 28.5088], [-17.9594, -35.9188, 17.9594, 35.9188]]), torch.Tensor([[-45.2548, -22.6274, 45.2548, 22.6274], [-57.0175, -28.5088, 57.0175, 28.5088], [-71.8376, -35.9188, 71.8376, 35.9188], [-32.0000, -32.0000, 32.0000, 32.0000], [-40.3175, -40.3175, 40.3175, 40.3175], [-50.7968, -50.7968, 50.7968, 50.7968], [-22.6274, -45.2548, 22.6274, 45.2548], [-28.5088, -57.0175, 28.5088, 57.0175], [-35.9188, -71.8376, 35.9188, 71.8376]]), torch.Tensor([[-90.5097, -45.2548, 90.5097, 45.2548], [-114.0350, -57.0175, 114.0350, 57.0175], [-143.6751, -71.8376, 143.6751, 71.8376], [-64.0000, -64.0000, 64.0000, 64.0000], [-80.6349, -80.6349, 80.6349, 80.6349], [-101.5937, -101.5937, 101.5937, 101.5937], [-45.2548, -90.5097, 45.2548, 90.5097], [-57.0175, -114.0350, 57.0175, 114.0350], [-71.8376, -143.6751, 71.8376, 143.6751]]), torch.Tensor([[-181.0193, -90.5097, 181.0193, 90.5097], [-228.0701, -114.0350, 228.0701, 114.0350], [-287.3503, -143.6751, 287.3503, 143.6751], [-128.0000, -128.0000, 128.0000, 128.0000], [-161.2699, -161.2699, 161.2699, 161.2699], [-203.1873, -203.1873, 203.1873, 203.1873], [-90.5097, -181.0193, 90.5097, 181.0193], [-114.0350, -228.0701, 114.0350, 228.0701], [-143.6751, -287.3503, 143.6751, 287.3503]]), torch.Tensor([[-362.0387, -181.0193, 362.0387, 181.0193], [-456.1401, -228.0701, 456.1401, 228.0701], [-574.7006, -287.3503, 574.7006, 287.3503], [-256.0000, -256.0000, 256.0000, 256.0000], [-322.5398, -322.5398, 322.5398, 322.5398], [-406.3747, -406.3747, 406.3747, 406.3747], [-181.0193, -362.0387, 181.0193, 362.0387], [-228.0701, -456.1401, 228.0701, 456.1401], [-287.3503, -574.7006, 287.3503, 574.7006]]) ] approxs = ga_retina_head.approx_anchor_generator.base_anchors for i, base_anchor in enumerate(approxs): assert base_anchor.allclose(expected_approxs[i]) # check valid flags expected_valid_pixels = [136800, 34200, 8550, 2223, 630] multi_level_valid_flags = ga_retina_head.approx_anchor_generator \ .valid_flags(featmap_sizes, (800, 1216), device) for i, single_level_valid_flag in enumerate(multi_level_valid_flags): assert single_level_valid_flag.sum() == expected_valid_pixels[i] # check number of base anchors for each level assert ga_retina_head.approx_anchor_generator.num_base_anchors == [ 9, 9, 9, 9, 9 ] # check approx generation squares = ga_retina_head.square_anchor_generator.grid_anchors( featmap_sizes, device) assert len(squares) == 5 expected_squares = [ torch.Tensor([[-16., -16., 16., 16.]]), torch.Tensor([[-32., -32., 32., 32]]), torch.Tensor([[-64., -64., 64., 64.]]), torch.Tensor([[-128., -128., 128., 128.]]), torch.Tensor([[-256., -256., 256., 256.]]) ] squares = ga_retina_head.square_anchor_generator.base_anchors for i, base_anchor in enumerate(squares): assert base_anchor.allclose(expected_squares[i]) # square_anchor_generator does not check valid flags # check number of base anchors for each level assert (ga_retina_head.square_anchor_generator.num_base_anchors == [ 1, 1, 1, 1, 1 ]) # check square generation anchors = ga_retina_head.square_anchor_generator.grid_anchors( featmap_sizes, device) assert len(anchors) == 5 ================================================ FILE: tests/test_models/test_task_modules/test_samplers/test_pesudo_sampler.py ================================================ # TODO: follow up ================================================ FILE: tests/test_models/test_tta/test_det_tta.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from unittest import TestCase import torch from mmengine import ConfigDict from mmdet.models import DetTTAModel from mmdet.registry import MODELS from mmdet.structures import DetDataSample from mmdet.testing import get_detector_cfg from mmdet.utils import register_all_modules class TestDetTTAModel(TestCase): def setUp(self): register_all_modules() def test_det_tta_model(self): detector_cfg = get_detector_cfg( 'retinanet/retinanet_r18_fpn_1x_coco.py') cfg = ConfigDict( type='DetTTAModel', module=detector_cfg, tta_cfg=dict( nms=dict(type='nms', iou_threshold=0.5), max_per_img=100)) model: DetTTAModel = MODELS.build(cfg) imgs = [] data_samples = [] directions = ['horizontal', 'vertical'] for i in range(12): flip_direction = directions[0] if i % 3 == 0 else directions[1] imgs.append(torch.randn(1, 3, 100 + 10 * i, 100 + 10 * i)) data_samples.append([ DetDataSample( metainfo=dict( ori_shape=(100, 100), img_shape=(100 + 10 * i, 100 + 10 * i), scale_factor=((100 + 10 * i) / 100, (100 + 10 * i) / 100), flip=(i % 2 == 0), flip_direction=flip_direction), ) ]) model.test_step(dict(inputs=imgs, data_samples=data_samples)) ================================================ FILE: tests/test_models/test_utils/test_misc.py ================================================ import copy import pytest import torch from mmengine.structures import InstanceData from mmdet.models.utils import (empty_instances, filter_gt_instances, rename_loss_dict, reweight_loss_dict, unpack_gt_instances) from mmdet.testing import demo_mm_inputs def test_parse_gt_instance_info(): packed_inputs = demo_mm_inputs()['data_samples'] batch_gt_instances, batch_gt_instances_ignore, batch_img_metas \ = unpack_gt_instances(packed_inputs) assert len(batch_gt_instances) == len(packed_inputs) assert len(batch_gt_instances_ignore) == len(packed_inputs) assert len(batch_img_metas) == len(packed_inputs) def test_process_empty_roi(): batch_size = 2 batch_img_metas = [{'ori_shape': (10, 12)}] * batch_size device = torch.device('cpu') results_list = empty_instances(batch_img_metas, device, task_type='bbox') assert len(results_list) == batch_size for results in results_list: assert isinstance(results, InstanceData) assert len(results) == 0 assert torch.allclose(results.bboxes, torch.zeros(0, 4, device=device)) results_list = empty_instances( batch_img_metas, device, task_type='mask', instance_results=results_list, mask_thr_binary=0.5) assert len(results_list) == batch_size for results in results_list: assert isinstance(results, InstanceData) assert len(results) == 0 assert results.masks.shape == (0, 10, 12) # batch_img_metas and instance_results length must be the same with pytest.raises(AssertionError): empty_instances( batch_img_metas, device, task_type='mask', instance_results=[results_list[0]] * 3) def test_filter_gt_instances(): packed_inputs = demo_mm_inputs()['data_samples'] score_thr = 0.7 with pytest.raises(AssertionError): filter_gt_instances(packed_inputs, score_thr=score_thr) # filter no instances by score for inputs in packed_inputs: inputs.gt_instances.scores = torch.ones_like( inputs.gt_instances.labels).float() filtered_packed_inputs = filter_gt_instances( copy.deepcopy(packed_inputs), score_thr=score_thr) for filtered_inputs, inputs in zip(filtered_packed_inputs, packed_inputs): assert len(filtered_inputs.gt_instances) == len(inputs.gt_instances) # filter all instances for inputs in packed_inputs: inputs.gt_instances.scores = torch.zeros_like( inputs.gt_instances.labels).float() filtered_packed_inputs = filter_gt_instances( copy.deepcopy(packed_inputs), score_thr=score_thr) for filtered_inputs in filtered_packed_inputs: assert len(filtered_inputs.gt_instances) == 0 packed_inputs = demo_mm_inputs()['data_samples'] # filter no instances by size wh_thr = (0, 0) filtered_packed_inputs = filter_gt_instances( copy.deepcopy(packed_inputs), wh_thr=wh_thr) for filtered_inputs, inputs in zip(filtered_packed_inputs, packed_inputs): assert len(filtered_inputs.gt_instances) == len(inputs.gt_instances) # filter all instances by size for inputs in packed_inputs: img_shape = inputs.img_shape wh_thr = (max(wh_thr[0], img_shape[0]), max(wh_thr[1], img_shape[1])) filtered_packed_inputs = filter_gt_instances( copy.deepcopy(packed_inputs), wh_thr=wh_thr) for filtered_inputs in filtered_packed_inputs: assert len(filtered_inputs.gt_instances) == 0 def test_rename_loss_dict(): prefix = 'sup_' losses = {'cls_loss': torch.tensor(2.), 'reg_loss': torch.tensor(1.)} sup_losses = rename_loss_dict(prefix, losses) for name in losses.keys(): assert sup_losses[prefix + name] == losses[name] def test_reweight_loss_dict(): weight = 4 losses = {'cls_loss': torch.tensor(2.), 'reg_loss': torch.tensor(1.)} weighted_losses = reweight_loss_dict(copy.deepcopy(losses), weight) for name in losses.keys(): assert weighted_losses[name] == losses[name] * weight ================================================ FILE: tests/test_models/test_utils/test_model_misc.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np import torch from torch.autograd import gradcheck from mmdet.models.utils import interpolate_as, sigmoid_geometric_mean def test_interpolate_as(): source = torch.rand((1, 5, 4, 4)) target = torch.rand((1, 1, 16, 16)) # Test 4D source and target result = interpolate_as(source, target) assert result.shape == torch.Size((1, 5, 16, 16)) # Test 3D target result = interpolate_as(source, target.squeeze(0)) assert result.shape == torch.Size((1, 5, 16, 16)) # Test 3D source result = interpolate_as(source.squeeze(0), target) assert result.shape == torch.Size((5, 16, 16)) # Test type(target) == np.ndarray target = np.random.rand(16, 16) result = interpolate_as(source.squeeze(0), target) assert result.shape == torch.Size((5, 16, 16)) def test_sigmoid_geometric_mean(): x = torch.randn(20, 20, dtype=torch.double, requires_grad=True) y = torch.randn(20, 20, dtype=torch.double, requires_grad=True) inputs = (x, y) test = gradcheck(sigmoid_geometric_mean, inputs, eps=1e-6, atol=1e-4) assert test ================================================ FILE: tests/test_structures/__init__.py ================================================ ================================================ FILE: tests/test_structures/test_bbox/__init__.py ================================================ ================================================ FILE: tests/test_structures/test_bbox/test_base_boxes.py ================================================ from unittest import TestCase import numpy as np import torch from mmengine.testing import assert_allclose from .utils import ToyBaseBoxes class TestBaseBoxes(TestCase): def test_init(self): box_tensor = torch.rand((3, 4, 4)) boxes = ToyBaseBoxes(box_tensor) boxes = ToyBaseBoxes(box_tensor, dtype=torch.float64) self.assertEqual(boxes.tensor.dtype, torch.float64) if torch.cuda.is_available(): boxes = ToyBaseBoxes(box_tensor, device='cuda') self.assertTrue(boxes.tensor.is_cuda) with self.assertRaises(AssertionError): box_tensor = torch.rand((4, )) boxes = ToyBaseBoxes(box_tensor) with self.assertRaises(AssertionError): box_tensor = torch.rand((3, 4, 3)) boxes = ToyBaseBoxes(box_tensor) def test_getitem(self): boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) # test single dimension index # int new_boxes = boxes[0] self.assertIsInstance(new_boxes, ToyBaseBoxes) self.assertEqual(new_boxes.tensor.shape, (4, 4)) # list new_boxes = boxes[[0, 2]] self.assertIsInstance(new_boxes, ToyBaseBoxes) self.assertEqual(new_boxes.tensor.shape, (2, 4, 4)) # slice new_boxes = boxes[0:2] self.assertIsInstance(new_boxes, ToyBaseBoxes) self.assertEqual(new_boxes.tensor.shape, (2, 4, 4)) # torch.LongTensor new_boxes = boxes[torch.LongTensor([0, 1])] self.assertIsInstance(new_boxes, ToyBaseBoxes) self.assertEqual(new_boxes.tensor.shape, (2, 4, 4)) # torch.BoolTensor new_boxes = boxes[torch.BoolTensor([True, False, True])] self.assertIsInstance(new_boxes, ToyBaseBoxes) self.assertEqual(new_boxes.tensor.shape, (2, 4, 4)) with self.assertRaises(AssertionError): index = torch.rand((2, 4, 4)) > 0 new_boxes = boxes[index] # test multiple dimension index # select single box new_boxes = boxes[1, 2] self.assertIsInstance(new_boxes, ToyBaseBoxes) self.assertEqual(new_boxes.tensor.shape, (1, 4)) # select the last dimension with self.assertRaises(AssertionError): new_boxes = boxes[1, 2, 1] # has Ellipsis new_boxes = boxes[None, ...] self.assertIsInstance(new_boxes, ToyBaseBoxes) self.assertEqual(new_boxes.tensor.shape, (1, 3, 4, 4)) with self.assertRaises(AssertionError): new_boxes = boxes[..., None] def test_setitem(self): values = ToyBaseBoxes(torch.rand(3, 4, 4)) tensor = torch.rand(3, 4, 4) # only support BaseBoxes type with self.assertRaises(AssertionError): boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) boxes[0:2] = tensor[0:2] # test single dimension index # int boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) boxes[1] = values[1] assert_allclose(boxes.tensor[1], values.tensor[1]) # list boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) boxes[[1, 2]] = values[[1, 2]] assert_allclose(boxes.tensor[[1, 2]], values.tensor[[1, 2]]) # slice boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) boxes[0:2] = values[0:2] assert_allclose(boxes.tensor[0:2], values.tensor[0:2]) # torch.BoolTensor boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) index = torch.rand(3, 4) > 0.5 boxes[index] = values[index] assert_allclose(boxes.tensor[index], values.tensor[index]) # multiple dimension index boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) boxes[0:2, 0:2] = values[0:2, 0:2] assert_allclose(boxes.tensor[0:2, 0:2], values.tensor[0:2, 0:2]) # select single box boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) boxes[1, 1] = values[1, 1] assert_allclose(boxes.tensor[1, 1], values.tensor[1, 1]) # select the last dimension with self.assertRaises(AssertionError): boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) boxes[1, 1, 1] = values[1, 1, 1] # has Ellipsis boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) boxes[0:2, ...] = values[0:2, ...] assert_allclose(boxes.tensor[0:2, ...], values.tensor[0:2, ...]) def test_tensor_like_functions(self): boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) # new_tensor boxes.new_tensor([1, 2, 3]) # new_full boxes.new_full((3, 4), 0) # new_empty boxes.new_empty((3, 4)) # new_ones boxes.new_ones((3, 4)) # new_zeros boxes.new_zeros((3, 4)) # size self.assertEqual(boxes.size(0), 3) self.assertEqual(tuple(boxes.size()), (3, 4, 4)) # dim self.assertEqual(boxes.dim(), 3) # device self.assertIsInstance(boxes.device, torch.device) # dtype self.assertIsInstance(boxes.dtype, torch.dtype) # numpy np_boxes = boxes.numpy() self.assertIsInstance(np_boxes, np.ndarray) self.assertTrue((np_boxes == np_boxes).all()) # to new_boxes = boxes.to(torch.uint8) self.assertEqual(new_boxes.tensor.dtype, torch.uint8) if torch.cuda.is_available(): new_boxes = boxes.to(device='cuda') self.assertTrue(new_boxes.tensor.is_cuda) # cpu if torch.cuda.is_available(): new_boxes = boxes.to(device='cuda') new_boxes = new_boxes.cpu() self.assertFalse(new_boxes.tensor.is_cuda) # cuda if torch.cuda.is_available(): new_boxes = boxes.cuda() self.assertTrue(new_boxes.tensor.is_cuda) # clone boxes.clone() # detach boxes.detach() # view new_boxes = boxes.view(12, 4) self.assertEqual(tuple(new_boxes.size()), (12, 4)) new_boxes = boxes.view(-1, 4) self.assertEqual(tuple(new_boxes.size()), (12, 4)) with self.assertRaises(AssertionError): new_boxes = boxes.view(-1) # reshape new_boxes = boxes.reshape(12, 4) self.assertEqual(tuple(new_boxes.size()), (12, 4)) new_boxes = boxes.reshape(-1, 4) self.assertEqual(tuple(new_boxes.size()), (12, 4)) with self.assertRaises(AssertionError): new_boxes = boxes.reshape(-1) # expand new_boxes = boxes[None, ...].expand(4, -1, -1, -1) self.assertEqual(tuple(new_boxes.size()), (4, 3, 4, 4)) # repeat new_boxes = boxes.repeat(2, 2, 1) self.assertEqual(tuple(new_boxes.size()), (6, 8, 4)) with self.assertRaises(AssertionError): new_boxes = boxes.repeat(2, 2, 2) # transpose new_boxes = boxes.transpose(0, 1) self.assertEqual(tuple(new_boxes.size()), (4, 3, 4)) with self.assertRaises(AssertionError): new_boxes = boxes.transpose(1, 2) # permute new_boxes = boxes.permute(1, 0, 2) self.assertEqual(tuple(new_boxes.size()), (4, 3, 4)) with self.assertRaises(AssertionError): new_boxes = boxes.permute(2, 1, 0) # split boxes_list = boxes.split(1, dim=0) for box in boxes_list: self.assertIsInstance(box, ToyBaseBoxes) self.assertEqual(tuple(box.size()), (1, 4, 4)) boxes_list = boxes.split([1, 2], dim=0) with self.assertRaises(AssertionError): boxes_list = boxes.split(1, dim=2) # chunk boxes_list = boxes.split(3, dim=1) self.assertEqual(len(boxes_list), 2) for box in boxes_list: self.assertIsInstance(box, ToyBaseBoxes) with self.assertRaises(AssertionError): boxes_list = boxes.split(3, dim=2) # unbind boxes_list = boxes.unbind(dim=1) self.assertEqual(len(boxes_list), 4) for box in boxes_list: self.assertIsInstance(box, ToyBaseBoxes) self.assertEqual(tuple(box.size()), (3, 4)) with self.assertRaises(AssertionError): boxes_list = boxes.unbind(dim=2) # flatten new_boxes = boxes.flatten() self.assertEqual(tuple(new_boxes.size()), (12, 4)) with self.assertRaises(AssertionError): new_boxes = boxes.flatten(end_dim=2) # squeeze boxes = ToyBaseBoxes(torch.rand(1, 3, 1, 4, 4)) new_boxes = boxes.squeeze() self.assertEqual(tuple(new_boxes.size()), (3, 4, 4)) new_boxes = boxes.squeeze(dim=2) self.assertEqual(tuple(new_boxes.size()), (1, 3, 4, 4)) # unsqueeze boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) new_boxes = boxes.unsqueeze(0) self.assertEqual(tuple(new_boxes.size()), (1, 3, 4, 4)) with self.assertRaises(AssertionError): new_boxes = boxes.unsqueeze(3) # cat with self.assertRaises(ValueError): ToyBaseBoxes.cat([]) box_list = [] box_list.append(ToyBaseBoxes(torch.rand(3, 4, 4))) box_list.append(ToyBaseBoxes(torch.rand(1, 4, 4))) with self.assertRaises(AssertionError): ToyBaseBoxes.cat(box_list, dim=2) cat_boxes = ToyBaseBoxes.cat(box_list, dim=0) self.assertIsInstance(cat_boxes, ToyBaseBoxes) self.assertEqual((cat_boxes.size()), (4, 4, 4)) # stack with self.assertRaises(ValueError): ToyBaseBoxes.stack([]) box_list = [] box_list.append(ToyBaseBoxes(torch.rand(3, 4, 4))) box_list.append(ToyBaseBoxes(torch.rand(3, 4, 4))) with self.assertRaises(AssertionError): ToyBaseBoxes.stack(box_list, dim=3) stack_boxes = ToyBaseBoxes.stack(box_list, dim=1) self.assertIsInstance(stack_boxes, ToyBaseBoxes) self.assertEqual((stack_boxes.size()), (3, 2, 4, 4)) def test_misc(self): boxes = ToyBaseBoxes(torch.rand(3, 4, 4)) # __len__ self.assertEqual(len(boxes), 3) # __repr__ repr(boxes) # fake_boxes new_boxes = boxes.fake_boxes((3, 4, 4), 1) self.assertEqual(tuple(new_boxes.size()), (3, 4, 4)) self.assertEqual(boxes.dtype, new_boxes.dtype) self.assertEqual(boxes.device, new_boxes.device) self.assertTrue((new_boxes.tensor == 1).all()) with self.assertRaises(AssertionError): new_boxes = boxes.fake_boxes((3, 4, 1)) new_boxes = boxes.fake_boxes((3, 4, 4), dtype=torch.uint8) self.assertEqual(new_boxes.dtype, torch.uint8) if torch.cuda.is_available(): new_boxes = boxes.fake_boxes((3, 4, 4), device='cuda') self.assertTrue(new_boxes.tensor.is_cuda) ================================================ FILE: tests/test_structures/test_bbox/test_box_type.py ================================================ from unittest import TestCase from unittest.mock import MagicMock import torch from mmdet.structures.bbox.box_type import (_box_type_to_name, box_converters, box_types, convert_box_type, get_box_type, register_box, register_box_converter) from .utils import ToyBaseBoxes class TestBoxType(TestCase): def setUp(self): self.box_types = box_types.copy() self.box_converters = box_converters.copy() self._box_type_to_name = _box_type_to_name.copy() def tearDown(self): # Clear registered items box_types.clear() box_converters.clear() _box_type_to_name.clear() # Restore original items box_types.update(self.box_types) box_converters.update(self.box_converters) _box_type_to_name.update(self._box_type_to_name) def test_register_box(self): # test usage of decorator @register_box('A') class A(ToyBaseBoxes): pass # test usage of normal function class B(ToyBaseBoxes): pass register_box('B', B) # register class without inheriting from BaseBoxes with self.assertRaises(AssertionError): @register_box('C') class C: pass # test register registered class with self.assertRaises(KeyError): @register_box('A') class AA(ToyBaseBoxes): pass with self.assertRaises(KeyError): register_box('BB', B) @register_box('A', force=True) class AAA(ToyBaseBoxes): pass self.assertIs(box_types['a'], AAA) self.assertEqual(_box_type_to_name[AAA], 'a') register_box('BB', B, force=True) self.assertIs(box_types['bb'], B) self.assertEqual(_box_type_to_name[B], 'bb') self.assertEqual(len(box_types), len(_box_type_to_name)) def test_register_box_converter(self): @register_box('A') class A(ToyBaseBoxes): pass @register_box('B') class B(ToyBaseBoxes): pass @register_box('C') class C(ToyBaseBoxes): pass # test usage of decorator @register_box_converter('A', 'B') def converter_A(bboxes): return bboxes # test usage of normal function def converter_B(bboxes): return bboxes register_box_converter('B' 'A', converter_B) # register uncallable object with self.assertRaises(AssertionError): register_box_converter('A', 'C', 'uncallable str') # test register unregistered bbox mode with self.assertRaises(AssertionError): @register_box_converter('A', 'D') def converter_C(bboxes): return bboxes # test register registered converter with self.assertRaises(KeyError): @register_box_converter('A', 'B') def converter_D(bboxes): return bboxes @register_box_converter('A', 'B', force=True) def converter_E(bboxes): return bboxes self.assertIs(box_converters['a2b'], converter_E) def test_get_box_type(self): @register_box('A') class A(ToyBaseBoxes): pass mode_name, mode_cls = get_box_type('A') self.assertEqual(mode_name, 'a') self.assertIs(mode_cls, A) mode_name, mode_cls = get_box_type(A) self.assertEqual(mode_name, 'a') self.assertIs(mode_cls, A) # get unregistered mode class B(ToyBaseBoxes): pass with self.assertRaises(AssertionError): mode_name, mode_cls = get_box_type('B') with self.assertRaises(AssertionError): mode_name, mode_cls = get_box_type(B) def test_convert_box_type(self): @register_box('A') class A(ToyBaseBoxes): pass @register_box('B') class B(ToyBaseBoxes): pass @register_box('C') class C(ToyBaseBoxes): pass converter = MagicMock() converter.return_value = torch.rand(3, 4, 4) register_box_converter('A', 'B', converter) bboxes_a = A(torch.rand(3, 4, 4)) th_bboxes_a = bboxes_a.tensor np_bboxes_a = th_bboxes_a.numpy() # test convert to mode convert_box_type(bboxes_a, dst_type='B') self.assertTrue(converter.called) converted_bboxes = convert_box_type(bboxes_a, dst_type='A') self.assertIs(converted_bboxes, bboxes_a) # test convert to unregistered mode with self.assertRaises(AssertionError): convert_box_type(bboxes_a, dst_type='C') # test convert tensor and ndarray # without specific src_type with self.assertRaises(AssertionError): convert_box_type(th_bboxes_a, dst_type='B') with self.assertRaises(AssertionError): convert_box_type(np_bboxes_a, dst_type='B') # test np.ndarray convert_box_type(np_bboxes_a, src_type='A', dst_type='B') converted_bboxes = convert_box_type( np_bboxes_a, src_type='A', dst_type='A') self.assertIs(converted_bboxes, np_bboxes_a) # test tensor convert_box_type(th_bboxes_a, src_type='A', dst_type='B') converted_bboxes = convert_box_type( th_bboxes_a, src_type='A', dst_type='A') self.assertIs(converted_bboxes, th_bboxes_a) # test other type with self.assertRaises(TypeError): convert_box_type([[1, 2, 3, 4]], src_type='A', dst_type='B') ================================================ FILE: tests/test_structures/test_bbox/test_horizontal_boxes.py ================================================ import random from math import sqrt from unittest import TestCase import cv2 import numpy as np import torch from mmengine.testing import assert_allclose from mmdet.structures.bbox import HorizontalBoxes from mmdet.structures.mask import BitmapMasks, PolygonMasks class TestHorizontalBoxes(TestCase): def test_init(self): th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) th_boxes_cxcywh = torch.Tensor([15, 15, 10, 10]).reshape(1, 1, 4) boxes = HorizontalBoxes(th_boxes) assert_allclose(boxes.tensor, th_boxes) boxes = HorizontalBoxes(th_boxes, in_mode='xyxy') assert_allclose(boxes.tensor, th_boxes) boxes = HorizontalBoxes(th_boxes_cxcywh, in_mode='cxcywh') assert_allclose(boxes.tensor, th_boxes) with self.assertRaises(ValueError): boxes = HorizontalBoxes(th_boxes, in_mode='invalid') def test_cxcywh(self): th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) th_boxes_cxcywh = torch.Tensor([15, 15, 10, 10]).reshape(1, 1, 4) boxes = HorizontalBoxes(th_boxes) assert_allclose( HorizontalBoxes.xyxy_to_cxcywh(th_boxes), th_boxes_cxcywh) assert_allclose(th_boxes, HorizontalBoxes.cxcywh_to_xyxy(th_boxes_cxcywh)) assert_allclose(boxes.cxcywh, th_boxes_cxcywh) def test_propoerty(self): th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) boxes = HorizontalBoxes(th_boxes) # Centers centers = torch.Tensor([15, 15]).reshape(1, 1, 2) assert_allclose(boxes.centers, centers) # Areas areas = torch.Tensor([100]).reshape(1, 1) assert_allclose(boxes.areas, areas) # widths widths = torch.Tensor([10]).reshape(1, 1) assert_allclose(boxes.widths, widths) # heights heights = torch.Tensor([10]).reshape(1, 1) assert_allclose(boxes.heights, heights) def test_flip(self): img_shape = [50, 85] # horizontal flip th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) flipped_boxes_th = torch.Tensor([65, 10, 75, 20]).reshape(1, 1, 4) boxes = HorizontalBoxes(th_boxes) boxes.flip_(img_shape, direction='horizontal') assert_allclose(boxes.tensor, flipped_boxes_th) # vertical flip th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) flipped_boxes_th = torch.Tensor([10, 30, 20, 40]).reshape(1, 1, 4) boxes = HorizontalBoxes(th_boxes) boxes.flip_(img_shape, direction='vertical') assert_allclose(boxes.tensor, flipped_boxes_th) # diagonal flip th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) flipped_boxes_th = torch.Tensor([65, 30, 75, 40]).reshape(1, 1, 4) boxes = HorizontalBoxes(th_boxes) boxes.flip_(img_shape, direction='diagonal') assert_allclose(boxes.tensor, flipped_boxes_th) def test_translate(self): th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) boxes = HorizontalBoxes(th_boxes) boxes.translate_([23, 46]) translated_boxes_th = torch.Tensor([33, 56, 43, 66]).reshape(1, 1, 4) assert_allclose(boxes.tensor, translated_boxes_th) def test_clip(self): th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) img_shape = [13, 14] boxes = HorizontalBoxes(th_boxes) boxes.clip_(img_shape) cliped_boxes_th = torch.Tensor([10, 10, 14, 13]).reshape(1, 1, 4) assert_allclose(boxes.tensor, cliped_boxes_th) def test_rotate(self): th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) center = (15, 15) angle = -45 boxes = HorizontalBoxes(th_boxes) boxes.rotate_(center, angle) rotated_boxes_th = torch.Tensor([ 15 - 5 * sqrt(2), 15 - 5 * sqrt(2), 15 + 5 * sqrt(2), 15 + 5 * sqrt(2) ]).reshape(1, 1, 4) assert_allclose(boxes.tensor, rotated_boxes_th) def test_project(self): th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) boxes1 = HorizontalBoxes(th_boxes) boxes2 = boxes1.clone() matrix = np.zeros((3, 3), dtype=np.float32) center = [random.random() * 80, random.random() * 80] angle = random.random() * 180 matrix[:2, :3] = cv2.getRotationMatrix2D(center, angle, 1) x_translate = random.random() * 40 y_translate = random.random() * 40 matrix[0, 2] = matrix[0, 2] + x_translate matrix[1, 2] = matrix[1, 2] + y_translate scale_factor = random.random() * 2 matrix[2, 2] = 1 / scale_factor boxes1.project_(matrix) boxes2.rotate_(center, -angle) boxes2.translate_([x_translate, y_translate]) boxes2.rescale_([scale_factor, scale_factor]) assert_allclose(boxes1.tensor, boxes2.tensor) # test empty boxes empty_boxes = HorizontalBoxes(torch.zeros((0, 4))) empty_boxes.project_(matrix) def test_rescale(self): scale_factor = [0.4, 0.8] # rescale th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) boxes = HorizontalBoxes(th_boxes) boxes.rescale_(scale_factor) rescaled_boxes_th = torch.Tensor([4, 8, 8, 16]).reshape(1, 1, 4) assert_allclose(boxes.tensor, rescaled_boxes_th) def test_resize(self): scale_factor = [0.4, 0.8] th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 1, 4) boxes = HorizontalBoxes(th_boxes) boxes.resize_(scale_factor) resized_boxes_th = torch.Tensor([13, 11, 17, 19]).reshape(1, 1, 4) assert_allclose(boxes.tensor, resized_boxes_th) def test_is_inside(self): th_boxes = torch.Tensor([[10, 10, 20, 20], [-5, -5, 15, 15], [45, 45, 55, 55]]).reshape(1, 3, 4) img_shape = [30, 30] boxes = HorizontalBoxes(th_boxes) index = boxes.is_inside(img_shape) index_th = torch.BoolTensor([True, True, False]).reshape(1, 3) assert_allclose(index, index_th) def test_find_inside_points(self): th_boxes = torch.Tensor([10, 10, 20, 20]).reshape(1, 4) boxes = HorizontalBoxes(th_boxes) points = torch.Tensor([[0, 0], [0, 15], [15, 0], [15, 15]]) index = boxes.find_inside_points(points) index_th = torch.BoolTensor([False, False, False, True]).reshape(4, 1) assert_allclose(index, index_th) # is_aligned boxes = boxes.expand(4, 4) index = boxes.find_inside_points(points, is_aligned=True) index_th = torch.BoolTensor([False, False, False, True]) assert_allclose(index, index_th) def test_from_instance_masks(self): bitmap_masks = BitmapMasks.random() boxes = HorizontalBoxes.from_instance_masks(bitmap_masks) self.assertIsInstance(boxes, HorizontalBoxes) self.assertEqual(len(boxes), len(bitmap_masks)) polygon_masks = PolygonMasks.random() boxes = HorizontalBoxes.from_instance_masks(polygon_masks) self.assertIsInstance(boxes, HorizontalBoxes) self.assertEqual(len(boxes), len(bitmap_masks)) # zero length masks bitmap_masks = BitmapMasks.random(num_masks=0) boxes = HorizontalBoxes.from_instance_masks(bitmap_masks) self.assertIsInstance(boxes, HorizontalBoxes) self.assertEqual(len(boxes), 0) polygon_masks = PolygonMasks.random(num_masks=0) boxes = HorizontalBoxes.from_instance_masks(polygon_masks) self.assertIsInstance(boxes, HorizontalBoxes) self.assertEqual(len(boxes), 0) ================================================ FILE: tests/test_structures/test_bbox/utils.py ================================================ from mmdet.structures.bbox import BaseBoxes class ToyBaseBoxes(BaseBoxes): box_dim = 4 @property def centers(self): pass @property def areas(self): pass @property def widths(self): pass @property def heights(self): pass def flip_(self, img_shape, direction='horizontal'): pass def translate_(self, distances): pass def clip_(self, img_shape): pass def rotate_(self, center, angle): pass def project_(self, homography_matrix): pass def rescale_(self, scale_factor): pass def resize_(self, scale_factor): pass def is_inside(self, img_shape): pass def find_inside_points(self, points, is_aligned=False): pass def overlaps(bboxes1, bboxes2, mode='iou', is_aligned=False, eps=1e-6): pass def from_instance_masks(masks): pass ================================================ FILE: tests/test_structures/test_det_data_sample.py ================================================ from unittest import TestCase import numpy as np import pytest import torch from mmengine.structures import InstanceData, PixelData from mmdet.structures import DetDataSample def _equal(a, b): if isinstance(a, (torch.Tensor, np.ndarray)): return (a == b).all() else: return a == b class TestDetDataSample(TestCase): def test_init(self): meta_info = dict( img_size=[256, 256], scale_factor=np.array([1.5, 1.5]), img_shape=torch.rand(4)) det_data_sample = DetDataSample(metainfo=meta_info) assert 'img_size' in det_data_sample assert det_data_sample.img_size == [256, 256] assert det_data_sample.get('img_size') == [256, 256] def test_setter(self): det_data_sample = DetDataSample() # test gt_instances gt_instances_data = dict( bboxes=torch.rand(4, 4), labels=torch.rand(4), masks=np.random.rand(4, 2, 2)) gt_instances = InstanceData(**gt_instances_data) det_data_sample.gt_instances = gt_instances assert 'gt_instances' in det_data_sample assert _equal(det_data_sample.gt_instances.bboxes, gt_instances_data['bboxes']) assert _equal(det_data_sample.gt_instances.labels, gt_instances_data['labels']) assert _equal(det_data_sample.gt_instances.masks, gt_instances_data['masks']) # test pred_instances pred_instances_data = dict( bboxes=torch.rand(2, 4), labels=torch.rand(2), masks=np.random.rand(2, 2, 2)) pred_instances = InstanceData(**pred_instances_data) det_data_sample.pred_instances = pred_instances assert 'pred_instances' in det_data_sample assert _equal(det_data_sample.pred_instances.bboxes, pred_instances_data['bboxes']) assert _equal(det_data_sample.pred_instances.labels, pred_instances_data['labels']) assert _equal(det_data_sample.pred_instances.masks, pred_instances_data['masks']) # test proposals proposals_data = dict(bboxes=torch.rand(4, 4), labels=torch.rand(4)) proposals = InstanceData(**proposals_data) det_data_sample.proposals = proposals assert 'proposals' in det_data_sample assert _equal(det_data_sample.proposals.bboxes, proposals_data['bboxes']) assert _equal(det_data_sample.proposals.labels, proposals_data['labels']) # test ignored_instances ignored_instances_data = dict( bboxes=torch.rand(4, 4), labels=torch.rand(4)) ignored_instances = InstanceData(**ignored_instances_data) det_data_sample.ignored_instances = ignored_instances assert 'ignored_instances' in det_data_sample assert _equal(det_data_sample.ignored_instances.bboxes, ignored_instances_data['bboxes']) assert _equal(det_data_sample.ignored_instances.labels, ignored_instances_data['labels']) # test gt_panoptic_seg gt_panoptic_seg_data = dict(panoptic_seg=torch.rand(5, 4)) gt_panoptic_seg = PixelData(**gt_panoptic_seg_data) det_data_sample.gt_panoptic_seg = gt_panoptic_seg assert 'gt_panoptic_seg' in det_data_sample assert _equal(det_data_sample.gt_panoptic_seg.panoptic_seg, gt_panoptic_seg_data['panoptic_seg']) # test pred_panoptic_seg pred_panoptic_seg_data = dict(panoptic_seg=torch.rand(5, 4)) pred_panoptic_seg = PixelData(**pred_panoptic_seg_data) det_data_sample.pred_panoptic_seg = pred_panoptic_seg assert 'pred_panoptic_seg' in det_data_sample assert _equal(det_data_sample.pred_panoptic_seg.panoptic_seg, pred_panoptic_seg_data['panoptic_seg']) # test gt_sem_seg gt_segm_seg_data = dict(segm_seg=torch.rand(5, 4, 2)) gt_segm_seg = PixelData(**gt_segm_seg_data) det_data_sample.gt_segm_seg = gt_segm_seg assert 'gt_segm_seg' in det_data_sample assert _equal(det_data_sample.gt_segm_seg.segm_seg, gt_segm_seg_data['segm_seg']) # test pred_segm_seg pred_segm_seg_data = dict(segm_seg=torch.rand(5, 4, 2)) pred_segm_seg = PixelData(**pred_segm_seg_data) det_data_sample.pred_segm_seg = pred_segm_seg assert 'pred_segm_seg' in det_data_sample assert _equal(det_data_sample.pred_segm_seg.segm_seg, pred_segm_seg_data['segm_seg']) # test type error with pytest.raises(AssertionError): det_data_sample.pred_instances = torch.rand(2, 4) with pytest.raises(AssertionError): det_data_sample.pred_panoptic_seg = torch.rand(2, 4) with pytest.raises(AssertionError): det_data_sample.pred_sem_seg = torch.rand(2, 4) def test_deleter(self): gt_instances_data = dict( bboxes=torch.rand(4, 4), labels=torch.rand(4), masks=np.random.rand(4, 2, 2)) det_data_sample = DetDataSample() gt_instances = InstanceData(data=gt_instances_data) det_data_sample.gt_instances = gt_instances assert 'gt_instances' in det_data_sample del det_data_sample.gt_instances assert 'gt_instances' not in det_data_sample pred_panoptic_seg_data = torch.rand(5, 4) pred_panoptic_seg = PixelData(data=pred_panoptic_seg_data) det_data_sample.pred_panoptic_seg = pred_panoptic_seg assert 'pred_panoptic_seg' in det_data_sample del det_data_sample.pred_panoptic_seg assert 'pred_panoptic_seg' not in det_data_sample pred_segm_seg_data = dict(segm_seg=torch.rand(5, 4, 2)) pred_segm_seg = PixelData(**pred_segm_seg_data) det_data_sample.pred_segm_seg = pred_segm_seg assert 'pred_segm_seg' in det_data_sample del det_data_sample.pred_segm_seg assert 'pred_segm_seg' not in det_data_sample ================================================ FILE: tests/test_structures/test_mask/test_mask_structures.py ================================================ from unittest import TestCase import numpy as np from mmengine.testing import assert_allclose from mmdet.structures.mask import BitmapMasks, PolygonMasks class TestMaskStructures(TestCase): def test_bitmap_translate_same_size(self): mask_array = np.zeros((5, 10, 10), dtype=np.uint8) mask_array[:, 0:5, 0:5] = 1 mask_target = np.zeros((5, 10, 10), dtype=np.uint8) mask_target[:, 0:5, 5:10] = 1 mask = BitmapMasks(mask_array, 10, 10) mask = mask.translate((10, 10), 5) assert mask.masks.shape == (5, 10, 10) assert_allclose(mask_target, mask.masks) def test_bitmap_translate_diff_size(self): # test out shape larger mask_array = np.zeros((5, 10, 10), dtype=np.uint8) mask_array[:, 0:5, 0:5] = 1 mask_target = np.zeros((5, 20, 20), dtype=np.uint8) mask_target[:, 0:5, 5:10] = 1 mask = BitmapMasks(mask_array, 10, 10) mask = mask.translate((20, 20), 5) assert mask.masks.shape == (5, 20, 20) assert_allclose(mask_target, mask.masks) # test out shape smaller mask_array = np.zeros((5, 10, 10), dtype=np.uint8) mask_array[:, 0:5, 0:5] = 1 mask_target = np.zeros((5, 20, 8), dtype=np.uint8) mask_target[:, 0:5, 5:] = 1 mask = BitmapMasks(mask_array, 10, 10) mask = mask.translate((20, 8), 5) assert mask.masks.shape == (5, 20, 8) assert_allclose(mask_target, mask.masks) def test_bitmap_cat(self): # test invalid inputs with self.assertRaises(AssertionError): BitmapMasks.cat(BitmapMasks.random(4)) with self.assertRaises(ValueError): BitmapMasks.cat([]) with self.assertRaises(AssertionError): BitmapMasks.cat([BitmapMasks.random(2), PolygonMasks.random(3)]) masks = [BitmapMasks.random(num_masks=3) for _ in range(5)] cat_mask = BitmapMasks.cat(masks) assert len(cat_mask) == 3 * 5 for i, m in enumerate(masks): assert_allclose(m.masks, cat_mask.masks[i * 3:(i + 1) * 3]) def test_polygon_cat(self): # test invalid inputs with self.assertRaises(AssertionError): PolygonMasks.cat(PolygonMasks.random(4)) with self.assertRaises(ValueError): PolygonMasks.cat([]) with self.assertRaises(AssertionError): PolygonMasks.cat([BitmapMasks.random(2), PolygonMasks.random(3)]) masks = [PolygonMasks.random(num_masks=3) for _ in range(5)] cat_mask = PolygonMasks.cat(masks) assert len(cat_mask) == 3 * 5 for i, m in enumerate(masks): assert_allclose(m.masks, cat_mask.masks[i * 3:(i + 1) * 3]) ================================================ FILE: tests/test_utils/test_benchmark.py ================================================ import copy import os import tempfile import unittest import torch import torch.nn as nn from mmengine import Config, MMLogger from mmengine.dataset import Compose from torch.utils.data import Dataset from mmdet.registry import DATASETS, MODELS from mmdet.utils import register_all_modules from mmdet.utils.benchmark import (DataLoaderBenchmark, DatasetBenchmark, InferenceBenchmark) @MODELS.register_module() class ToyDetector(nn.Module): def __init__(self, *args, **kwargs): super().__init__() def forward(self, data_batch, return_loss=False): pass @DATASETS.register_module() class ToyDataset(Dataset): METAINFO = dict() # type: ignore data = torch.randn(12, 2) label = torch.ones(12) def __init__(self): self.pipeline = Compose([lambda x: x]) def __len__(self): return self.data.size(0) def get_data_info(self, index): return dict(inputs=self.data[index], data_sample=self.label[index]) def __getitem__(self, index): return dict(inputs=self.data[index], data_sample=self.label[index]) @DATASETS.register_module() class ToyFullInitDataset(Dataset): METAINFO = dict() # type: ignore data = torch.randn(12, 2) label = torch.ones(12) def __init__(self): self.pipeline = Compose([lambda x: x]) def __len__(self): return self.data.size(0) def get_data_info(self, index): return dict(inputs=self.data[index], data_sample=self.label[index]) def full_init(self): pass def __getitem__(self, index): return dict(inputs=self.data[index], data_sample=self.label[index]) class TestInferenceBenchmark(unittest.TestCase): def setUp(self) -> None: register_all_modules() self.cfg = Config( dict( model=dict(type='ToyDetector'), test_dataloader=dict( dataset=dict(type='ToyDataset'), sampler=dict(type='DefaultSampler', shuffle=False), batch_size=3, num_workers=1), env_cfg=dict(dist_cfg=dict(backend='nccl')))) self.max_iter = 10 self.log_interval = 5 @unittest.skipIf(not torch.cuda.is_available(), 'test requires GPU and torch+cuda') def test_init_and_run(self): checkpoint_path = os.path.join(tempfile.gettempdir(), 'checkpoint.pth') torch.save(ToyDetector().state_dict(), checkpoint_path) cfg = copy.deepcopy(self.cfg) inference_benchmark = InferenceBenchmark(cfg, checkpoint_path, False, False, self.max_iter, self.log_interval) results = inference_benchmark.run() self.assertTrue(isinstance(results, dict)) self.assertTrue('avg_fps' in results) self.assertTrue('fps_list' in results) self.assertEqual(len(results['fps_list']), 1) self.assertTrue(inference_benchmark.data_loader.num_workers == 0) self.assertTrue(inference_benchmark.data_loader.batch_size == 1) results = inference_benchmark.run(1) self.assertTrue('avg_fps' in results) self.assertTrue('fps_list' in results) self.assertEqual(len(results['fps_list']), 1) self.assertTrue(inference_benchmark.data_loader.num_workers == 0) self.assertTrue(inference_benchmark.data_loader.batch_size == 1) # test repeat results = inference_benchmark.run(3) self.assertTrue('avg_fps' in results) self.assertTrue('fps_list' in results) self.assertEqual(len(results['fps_list']), 3) # test cudnn_benchmark cfg = copy.deepcopy(self.cfg) cfg.env_cfg.cudnn_benchmark = True inference_benchmark = InferenceBenchmark(cfg, checkpoint_path, False, False, self.max_iter, self.log_interval) inference_benchmark.run(1) # test mp_cfg cfg = copy.deepcopy(self.cfg) cfg.env_cfg.cudnn_benchmark = True cfg.env_cfg.mp_cfg = { 'mp_start_method': 'fork', 'opencv_num_threads': 1 } inference_benchmark = InferenceBenchmark(cfg, checkpoint_path, False, False, self.max_iter, self.log_interval) inference_benchmark.run(1) # test fp16 cfg = copy.deepcopy(self.cfg) cfg.fp16 = True inference_benchmark = InferenceBenchmark(cfg, checkpoint_path, False, False, self.max_iter, self.log_interval) inference_benchmark.run(1) # test logger logger = MMLogger.get_instance( 'mmdet', log_file='temp.log', log_level='INFO') inference_benchmark = InferenceBenchmark( cfg, checkpoint_path, False, False, self.max_iter, self.log_interval, logger=logger) inference_benchmark.run(1) self.assertTrue(os.path.exists('temp.log')) os.remove(checkpoint_path) os.remove('temp.log') class TestDataLoaderBenchmark(unittest.TestCase): def setUp(self) -> None: register_all_modules() self.cfg = Config( dict( model=dict(type='ToyDetector'), train_dataloader=dict( dataset=dict(type='ToyDataset'), sampler=dict(type='DefaultSampler', shuffle=True), batch_size=2, num_workers=1), val_dataloader=dict( dataset=dict(type='ToyDataset'), sampler=dict(type='DefaultSampler', shuffle=False), batch_size=1, num_workers=2), test_dataloader=dict( dataset=dict(type='ToyDataset'), sampler=dict(type='DefaultSampler', shuffle=False), batch_size=3, num_workers=1), env_cfg=dict(dist_cfg=dict(backend='nccl')))) self.max_iter = 5 self.log_interval = 1 self.num_warmup = 1 def test_init_and_run(self): cfg = copy.deepcopy(self.cfg) dataloader_benchmark = DataLoaderBenchmark(cfg, False, 'train', self.max_iter, self.log_interval, self.num_warmup) results = dataloader_benchmark.run(1) self.assertTrue('avg_fps' in results) self.assertTrue('fps_list' in results) self.assertEqual(len(results['fps_list']), 1) self.assertTrue(dataloader_benchmark.data_loader.num_workers == 1) self.assertTrue(dataloader_benchmark.data_loader.batch_size == 2) # test repeat results = dataloader_benchmark.run(3) self.assertTrue('avg_fps' in results) self.assertTrue('fps_list' in results) self.assertEqual(len(results['fps_list']), 3) # test dataset_type input parameters error with self.assertRaises(AssertionError): DataLoaderBenchmark(cfg, False, 'training', self.max_iter, self.log_interval, self.num_warmup) dataloader_benchmark = DataLoaderBenchmark(cfg, False, 'val', self.max_iter, self.log_interval, self.num_warmup) self.assertTrue(dataloader_benchmark.data_loader.num_workers == 2) self.assertTrue(dataloader_benchmark.data_loader.batch_size == 1) dataloader_benchmark = DataLoaderBenchmark(cfg, False, 'test', self.max_iter, self.log_interval, self.num_warmup) self.assertTrue(dataloader_benchmark.data_loader.num_workers == 1) self.assertTrue(dataloader_benchmark.data_loader.batch_size == 3) # test mp_cfg cfg = copy.deepcopy(self.cfg) cfg.env_cfg.mp_cfg = { 'mp_start_method': 'fork', 'opencv_num_threads': 1 } dataloader_benchmark = DataLoaderBenchmark(cfg, False, 'train', self.max_iter, self.log_interval, self.num_warmup) dataloader_benchmark.run(1) class TestDatasetBenchmark(unittest.TestCase): def setUp(self) -> None: register_all_modules() self.cfg = Config( dict( model=dict(type='ToyDetector'), train_dataloader=dict( dataset=dict(type='ToyDataset'), sampler=dict(type='DefaultSampler', shuffle=True), batch_size=2, num_workers=1), val_dataloader=dict( dataset=dict(type='ToyDataset'), sampler=dict(type='DefaultSampler', shuffle=False), batch_size=1, num_workers=2), test_dataloader=dict( dataset=dict(type='ToyDataset'), sampler=dict(type='DefaultSampler', shuffle=False), batch_size=3, num_workers=1))) self.max_iter = 5 self.log_interval = 1 self.num_warmup = 1 def test_init_and_run(self): cfg = copy.deepcopy(self.cfg) dataset_benchmark = DatasetBenchmark(cfg, 'train', self.max_iter, self.log_interval, self.num_warmup) results = dataset_benchmark.run(1) self.assertTrue('avg_fps' in results) self.assertTrue('fps_list' in results) self.assertEqual(len(results['fps_list']), 1) # test repeat results = dataset_benchmark.run(3) self.assertTrue('avg_fps' in results) self.assertTrue('fps_list' in results) self.assertEqual(len(results['fps_list']), 3) # test test dataset dataset_benchmark = DatasetBenchmark(cfg, 'test', self.max_iter, self.log_interval, self.num_warmup) dataset_benchmark.run(1) # test val dataset dataset_benchmark = DatasetBenchmark(cfg, 'val', self.max_iter, self.log_interval, self.num_warmup) dataset_benchmark.run(1) # test dataset_type input parameters error with self.assertRaises(AssertionError): DatasetBenchmark(cfg, 'training', self.max_iter, self.log_interval, self.num_warmup) # test full_init cfg = copy.deepcopy(self.cfg) cfg.test_dataloader.dataset = dict(type='ToyFullInitDataset') dataset_benchmark = DatasetBenchmark(cfg, 'train', self.max_iter, self.log_interval, self.num_warmup) dataset_benchmark.run(1) ================================================ FILE: tests/test_utils/test_memory.py ================================================ import numpy as np import pytest import torch from mmdet.utils import AvoidOOM from mmdet.utils.memory import cast_tensor_type def test_avoidoom(): tensor = torch.from_numpy(np.random.random((20, 20))) if torch.cuda.is_available(): tensor = tensor.cuda() # get default result default_result = torch.mm(tensor, tensor.transpose(1, 0)) # when not occurred OOM error AvoidCudaOOM = AvoidOOM() result = AvoidCudaOOM.retry_if_cuda_oom(torch.mm)(tensor, tensor.transpose( 1, 0)) assert default_result.device == result.device and \ default_result.dtype == result.dtype and \ torch.equal(default_result, result) # calculate with fp16 and convert back to source type AvoidCudaOOM = AvoidOOM(test=True) result = AvoidCudaOOM.retry_if_cuda_oom(torch.mm)(tensor, tensor.transpose( 1, 0)) assert default_result.device == result.device and \ default_result.dtype == result.dtype and \ torch.allclose(default_result, result, 1e-3) # calculate on cpu and convert back to source device AvoidCudaOOM = AvoidOOM(test=True) result = AvoidCudaOOM.retry_if_cuda_oom(torch.mm)(tensor, tensor.transpose( 1, 0)) assert result.dtype == default_result.dtype and \ result.device == default_result.device and \ torch.allclose(default_result, result) # do not calculate on cpu and the outputs will be same as input AvoidCudaOOM = AvoidOOM(test=True, to_cpu=False) result = AvoidCudaOOM.retry_if_cuda_oom(torch.mm)(tensor, tensor.transpose( 1, 0)) assert result.dtype == default_result.dtype and \ result.device == default_result.device else: default_result = torch.mm(tensor, tensor.transpose(1, 0)) AvoidCudaOOM = AvoidOOM() result = AvoidCudaOOM.retry_if_cuda_oom(torch.mm)(tensor, tensor.transpose( 1, 0)) assert default_result.device == result.device and \ default_result.dtype == result.dtype and \ torch.equal(default_result, result) def test_cast_tensor_type(): inputs = torch.rand(10) if torch.cuda.is_available(): inputs = inputs.cuda() with pytest.raises(AssertionError): cast_tensor_type(inputs, src_type=None, dst_type=None) # input is a float out = cast_tensor_type(10., dst_type=torch.half) assert out == 10. and isinstance(out, float) # convert Tensor to fp16 and re-convert to fp32 fp16_out = cast_tensor_type(inputs, dst_type=torch.half) assert fp16_out.dtype == torch.half fp32_out = cast_tensor_type(fp16_out, dst_type=torch.float32) assert fp32_out.dtype == torch.float32 # input is a list list_input = [inputs, inputs] list_outs = cast_tensor_type(list_input, dst_type=torch.half) assert len(list_outs) == len(list_input) and \ isinstance(list_outs, list) for out in list_outs: assert out.dtype == torch.half # input is a dict dict_input = {'test1': inputs, 'test2': inputs} dict_outs = cast_tensor_type(dict_input, dst_type=torch.half) assert len(dict_outs) == len(dict_input) and \ isinstance(dict_outs, dict) # convert the input tensor to CPU and re-convert to GPU if torch.cuda.is_available(): cpu_device = torch.empty(0).device gpu_device = inputs.device cpu_out = cast_tensor_type(inputs, dst_type=cpu_device) assert cpu_out.device == cpu_device gpu_out = cast_tensor_type(inputs, dst_type=gpu_device) assert gpu_out.device == gpu_device ================================================ FILE: tests/test_utils/test_replace_cfg_vals.py ================================================ import os.path as osp import tempfile from copy import deepcopy import pytest from mmengine.config import Config from mmdet.utils import replace_cfg_vals def test_replace_cfg_vals(): temp_file = tempfile.NamedTemporaryFile() cfg_path = f'{temp_file.name}.py' with open(cfg_path, 'w') as f: f.write('configs') ori_cfg_dict = dict() ori_cfg_dict['cfg_name'] = osp.basename(temp_file.name) ori_cfg_dict['work_dir'] = 'work_dirs/${cfg_name}/${percent}/${fold}' ori_cfg_dict['percent'] = 5 ori_cfg_dict['fold'] = 1 ori_cfg_dict['model_wrapper'] = dict( type='SoftTeacher', detector='${model}') ori_cfg_dict['model'] = dict( type='FasterRCNN', backbone=dict(type='ResNet'), neck=dict(type='FPN'), rpn_head=dict(type='RPNHead'), roi_head=dict(type='StandardRoIHead'), train_cfg=dict( rpn=dict( assigner=dict(type='MaxIoUAssigner'), sampler=dict(type='RandomSampler'), ), rpn_proposal=dict(nms=dict(type='nms', iou_threshold=0.7)), rcnn=dict( assigner=dict(type='MaxIoUAssigner'), sampler=dict(type='RandomSampler'), ), ), test_cfg=dict( rpn=dict(nms=dict(type='nms', iou_threshold=0.7)), rcnn=dict(nms=dict(type='nms', iou_threshold=0.5)), ), ) ori_cfg_dict['iou_threshold'] = dict( rpn_proposal_nms='${model.train_cfg.rpn_proposal.nms.iou_threshold}', test_rpn_nms='${model.test_cfg.rpn.nms.iou_threshold}', test_rcnn_nms='${model.test_cfg.rcnn.nms.iou_threshold}', ) ori_cfg_dict['str'] = 'Hello, world!' ori_cfg_dict['dict'] = {'Hello': 'world!'} ori_cfg_dict['list'] = [ 'Hello, world!', ] ori_cfg_dict['tuple'] = ('Hello, world!', ) ori_cfg_dict['test_str'] = 'xxx${str}xxx' ori_cfg = Config(ori_cfg_dict, filename=cfg_path) updated_cfg = replace_cfg_vals(deepcopy(ori_cfg)) assert updated_cfg.work_dir \ == f'work_dirs/{osp.basename(temp_file.name)}/5/1' assert updated_cfg.model.detector == ori_cfg.model assert updated_cfg.iou_threshold.rpn_proposal_nms \ == ori_cfg.model.train_cfg.rpn_proposal.nms.iou_threshold assert updated_cfg.test_str == 'xxxHello, world!xxx' ori_cfg_dict['test_dict'] = 'xxx${dict}xxx' ori_cfg_dict['test_list'] = 'xxx${list}xxx' ori_cfg_dict['test_tuple'] = 'xxx${tuple}xxx' with pytest.raises(AssertionError): cfg = deepcopy(ori_cfg) cfg['test_dict'] = 'xxx${dict}xxx' updated_cfg = replace_cfg_vals(cfg) with pytest.raises(AssertionError): cfg = deepcopy(ori_cfg) cfg['test_list'] = 'xxx${list}xxx' updated_cfg = replace_cfg_vals(cfg) with pytest.raises(AssertionError): cfg = deepcopy(ori_cfg) cfg['test_tuple'] = 'xxx${tuple}xxx' updated_cfg = replace_cfg_vals(cfg) ================================================ FILE: tests/test_utils/test_setup_env.py ================================================ import datetime import sys from unittest import TestCase from mmengine import DefaultScope from mmdet.utils import register_all_modules class TestSetupEnv(TestCase): def test_register_all_modules(self): from mmdet.registry import DATASETS # not init default scope sys.modules.pop('mmdet.datasets', None) sys.modules.pop('mmdet.datasets.coco', None) DATASETS._module_dict.pop('CocoDataset', None) self.assertFalse('CocoDataset' in DATASETS.module_dict) register_all_modules(init_default_scope=False) self.assertTrue('CocoDataset' in DATASETS.module_dict) # init default scope sys.modules.pop('mmdet.datasets') sys.modules.pop('mmdet.datasets.coco') DATASETS._module_dict.pop('CocoDataset', None) self.assertFalse('CocoDataset' in DATASETS.module_dict) register_all_modules(init_default_scope=True) self.assertTrue('CocoDataset' in DATASETS.module_dict) self.assertEqual(DefaultScope.get_current_instance().scope_name, 'mmdet') # init default scope when another scope is init name = f'test-{datetime.datetime.now()}' DefaultScope.get_instance(name, scope_name='test') with self.assertWarnsRegex( Warning, 'The current default scope "test" is not "mmdet"'): register_all_modules(init_default_scope=True) ================================================ FILE: tests/test_visualization/test_local_visualizer.py ================================================ import os from unittest import TestCase import cv2 import numpy as np import torch from mmengine.structures import InstanceData, PixelData from mmdet.evaluation import INSTANCE_OFFSET from mmdet.structures import DetDataSample from mmdet.visualization import DetLocalVisualizer def _rand_bboxes(num_boxes, h, w): cx, cy, bw, bh = torch.rand(num_boxes, 4).T tl_x = ((cx * w) - (w * bw / 2)).clamp(0, w) tl_y = ((cy * h) - (h * bh / 2)).clamp(0, h) br_x = ((cx * w) + (w * bw / 2)).clamp(0, w) br_y = ((cy * h) + (h * bh / 2)).clamp(0, h) bboxes = torch.stack([tl_x, tl_y, br_x, br_y], dim=0).T return bboxes def _create_panoptic_data(num_boxes, h, w): sem_seg = np.zeros((h, w), dtype=np.int64) + 2 bboxes = _rand_bboxes(num_boxes, h, w).int() labels = torch.randint(2, (num_boxes, )) for i in range(num_boxes): x, y, w, h = bboxes[i] sem_seg[y:y + h, x:x + w] = (i + 1) * INSTANCE_OFFSET + labels[i] return sem_seg[None] class TestDetLocalVisualizer(TestCase): def test_add_datasample(self): h = 12 w = 10 num_class = 3 num_bboxes = 5 out_file = 'out_file.jpg' image = np.random.randint(0, 256, size=(h, w, 3)).astype('uint8') # test gt_instances gt_instances = InstanceData() gt_instances.bboxes = _rand_bboxes(num_bboxes, h, w) gt_instances.labels = torch.randint(0, num_class, (num_bboxes, )) det_data_sample = DetDataSample() det_data_sample.gt_instances = gt_instances det_local_visualizer = DetLocalVisualizer() det_local_visualizer.add_datasample( 'image', image, det_data_sample, draw_pred=False) # test out_file det_local_visualizer.add_datasample( 'image', image, det_data_sample, draw_pred=False, out_file=out_file) assert os.path.exists(out_file) drawn_img = cv2.imread(out_file) assert drawn_img.shape == (h, w, 3) os.remove(out_file) # test gt_instances and pred_instances pred_instances = InstanceData() pred_instances.bboxes = _rand_bboxes(num_bboxes, h, w) pred_instances.labels = torch.randint(0, num_class, (num_bboxes, )) pred_instances.scores = torch.rand((num_bboxes, )) det_data_sample.pred_instances = pred_instances det_local_visualizer.add_datasample( 'image', image, det_data_sample, out_file=out_file) self._assert_image_and_shape(out_file, (h, w * 2, 3)) det_local_visualizer.add_datasample( 'image', image, det_data_sample, draw_gt=False, out_file=out_file) self._assert_image_and_shape(out_file, (h, w, 3)) det_local_visualizer.add_datasample( 'image', image, det_data_sample, draw_pred=False, out_file=out_file) self._assert_image_and_shape(out_file, (h, w, 3)) # test gt_panoptic_seg and pred_panoptic_seg det_local_visualizer.dataset_meta = dict(classes=('1', '2')) gt_sem_seg = _create_panoptic_data(num_bboxes, h, w) panoptic_seg = PixelData(sem_seg=gt_sem_seg) det_data_sample = DetDataSample() det_data_sample.gt_panoptic_seg = panoptic_seg pred_sem_seg = _create_panoptic_data(num_bboxes, h, w) panoptic_seg = PixelData(sem_seg=pred_sem_seg) det_data_sample.pred_panoptic_seg = panoptic_seg det_local_visualizer.add_datasample( 'image', image, det_data_sample, out_file=out_file) self._assert_image_and_shape(out_file, (h, w * 2, 3)) # class information must be provided det_local_visualizer.dataset_meta = {} with self.assertRaises(AssertionError): det_local_visualizer.add_datasample( 'image', image, det_data_sample, out_file=out_file) def _assert_image_and_shape(self, out_file, out_shape): assert os.path.exists(out_file) drawn_img = cv2.imread(out_file) assert drawn_img.shape == out_shape os.remove(out_file) ================================================ FILE: tests/test_visualization/test_palette.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import numpy as np from mmdet.datasets import CocoDataset from mmdet.visualization import get_palette, jitter_color, palette_val def test_palette(): assert palette_val([(1, 2, 3)])[0] == (1 / 255, 2 / 255, 3 / 255) # test list palette = [(1, 0, 0), (0, 1, 0), (0, 0, 1)] palette_ = get_palette(palette, 3) for color, color_ in zip(palette, palette_): assert color == color_ # test tuple palette = get_palette((1, 2, 3), 3) assert len(palette) == 3 for color in palette: assert color == (1, 2, 3) # test color str palette = get_palette('red', 3) assert len(palette) == 3 for color in palette: assert color == (255, 0, 0) # test dataset str palette = get_palette('coco', len(CocoDataset.METAINFO['classes'])) assert len(palette) == len(CocoDataset.METAINFO['classes']) assert palette[0] == (220, 20, 60) # TODO: Awaiting refactoring # palette = get_palette('coco', len(CocoPanopticDataset.METAINFO['CLASSES'])) # noqa # assert len(palette) == len(CocoPanopticDataset.METAINFO['CLASSES']) # assert palette[-1] == (250, 141, 255) # palette = get_palette('voc', len(VOCDataset.METAINFO['CLASSES'])) # assert len(palette) == len(VOCDataset.METAINFO['CLASSES']) # assert palette[0] == (106, 0, 228) # palette = get_palette('citys', len(CityscapesDataset.METAINFO['CLASSES'])) # noqa # assert len(palette) == len(CityscapesDataset.METAINFO['CLASSES']) # assert palette[0] == (220, 20, 60) # test random palette1 = get_palette('random', 3) palette2 = get_palette(None, 3) for color1, color2 in zip(palette1, palette2): assert isinstance(color1, tuple) assert isinstance(color2, tuple) assert color1 == color2 def test_jitter_color(): color = tuple(np.random.randint(0, 255, 3, np.uint8)) jittered_color = jitter_color(color) for c in jittered_color: assert 0 <= c <= 255 ================================================ FILE: tools/analysis_tools/analyze_logs.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import json from collections import defaultdict import matplotlib.pyplot as plt import numpy as np import seaborn as sns def cal_train_time(log_dicts, args): for i, log_dict in enumerate(log_dicts): print(f'{"-" * 5}Analyze train time of {args.json_logs[i]}{"-" * 5}') all_times = [] for epoch in log_dict.keys(): if args.include_outliers: all_times.append(log_dict[epoch]['time']) else: all_times.append(log_dict[epoch]['time'][1:]) if not all_times: raise KeyError( 'Please reduce the log interval in the config so that' 'interval is less than iterations of one epoch.') epoch_ave_time = np.array(list(map(lambda x: np.mean(x), all_times))) slowest_epoch = epoch_ave_time.argmax() fastest_epoch = epoch_ave_time.argmin() std_over_epoch = epoch_ave_time.std() print(f'slowest epoch {slowest_epoch + 1}, ' f'average time is {epoch_ave_time[slowest_epoch]:.4f} s/iter') print(f'fastest epoch {fastest_epoch + 1}, ' f'average time is {epoch_ave_time[fastest_epoch]:.4f} s/iter') print(f'time std over epochs is {std_over_epoch:.4f}') print(f'average iter time: {np.mean(epoch_ave_time):.4f} s/iter\n') def plot_curve(log_dicts, args): if args.backend is not None: plt.switch_backend(args.backend) sns.set_style(args.style) # if legend is None, use {filename}_{key} as legend legend = args.legend if legend is None: legend = [] for json_log in args.json_logs: for metric in args.keys: legend.append(f'{json_log}_{metric}') assert len(legend) == (len(args.json_logs) * len(args.keys)) metrics = args.keys # TODO: support dynamic eval interval(e.g. RTMDet) when plotting mAP. num_metrics = len(metrics) for i, log_dict in enumerate(log_dicts): epochs = list(log_dict.keys()) for j, metric in enumerate(metrics): print(f'plot curve of {args.json_logs[i]}, metric is {metric}') if metric not in log_dict[epochs[int(args.eval_interval) - 1]]: if 'mAP' in metric: raise KeyError( f'{args.json_logs[i]} does not contain metric ' f'{metric}. Please check if "--no-validate" is ' 'specified when you trained the model. Or check ' f'if the eval_interval {args.eval_interval} in args ' 'is equal to the eval_interval during training.') raise KeyError( f'{args.json_logs[i]} does not contain metric {metric}. ' 'Please reduce the log interval in the config so that ' 'interval is less than iterations of one epoch.') if 'mAP' in metric: xs = [] ys = [] for epoch in epochs: ys += log_dict[epoch][metric] if log_dict[epoch][metric]: xs += [epoch] plt.xlabel('epoch') plt.plot(xs, ys, label=legend[i * num_metrics + j], marker='o') else: xs = [] ys = [] for epoch in epochs: iters = log_dict[epoch]['step'] xs.append(np.array(iters)) ys.append(np.array(log_dict[epoch][metric][:len(iters)])) xs = np.concatenate(xs) ys = np.concatenate(ys) plt.xlabel('iter') plt.plot( xs, ys, label=legend[i * num_metrics + j], linewidth=0.5) plt.legend() if args.title is not None: plt.title(args.title) if args.out is None: plt.show() else: print(f'save curve to: {args.out}') plt.savefig(args.out) plt.cla() def add_plot_parser(subparsers): parser_plt = subparsers.add_parser( 'plot_curve', help='parser for plotting curves') parser_plt.add_argument( 'json_logs', type=str, nargs='+', help='path of train log in json format') parser_plt.add_argument( '--keys', type=str, nargs='+', default=['bbox_mAP'], help='the metric that you want to plot') parser_plt.add_argument( '--start-epoch', type=str, default='1', help='the epoch that you want to start') parser_plt.add_argument( '--eval-interval', type=str, default='1', help='the eval interval when training') parser_plt.add_argument('--title', type=str, help='title of figure') parser_plt.add_argument( '--legend', type=str, nargs='+', default=None, help='legend of each plot') parser_plt.add_argument( '--backend', type=str, default=None, help='backend of plt') parser_plt.add_argument( '--style', type=str, default='dark', help='style of plt') parser_plt.add_argument('--out', type=str, default=None) def add_time_parser(subparsers): parser_time = subparsers.add_parser( 'cal_train_time', help='parser for computing the average time per training iteration') parser_time.add_argument( 'json_logs', type=str, nargs='+', help='path of train log in json format') parser_time.add_argument( '--include-outliers', action='store_true', help='include the first value of every epoch when computing ' 'the average time') def parse_args(): parser = argparse.ArgumentParser(description='Analyze Json Log') # currently only support plot curve and calculate average train time subparsers = parser.add_subparsers(dest='task', help='task parser') add_plot_parser(subparsers) add_time_parser(subparsers) args = parser.parse_args() return args def load_json_logs(json_logs): # load and convert json_logs to log_dict, key is epoch, value is a sub dict # keys of sub dict is different metrics, e.g. memory, bbox_mAP # value of sub dict is a list of corresponding values of all iterations log_dicts = [dict() for _ in json_logs] for json_log, log_dict in zip(json_logs, log_dicts): with open(json_log, 'r') as log_file: epoch = 1 for i, line in enumerate(log_file): log = json.loads(line.strip()) val_flag = False # skip lines only contains one key if not len(log) > 1: continue if epoch not in log_dict: log_dict[epoch] = defaultdict(list) for k, v in log.items(): if '/' in k: log_dict[epoch][k.split('/')[-1]].append(v) val_flag = True elif val_flag: continue else: log_dict[epoch][k].append(v) if 'epoch' in log.keys(): epoch = log['epoch'] return log_dicts def main(): args = parse_args() json_logs = args.json_logs for json_log in json_logs: assert json_log.endswith('.json') log_dicts = load_json_logs(json_logs) eval(args.task)(log_dicts, args) if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/analyze_results.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os.path as osp from multiprocessing import Pool import mmcv import numpy as np from mmengine.config import Config, DictAction from mmengine.fileio import load from mmengine.registry import init_default_scope from mmengine.runner import Runner from mmengine.structures import InstanceData, PixelData from mmengine.utils import ProgressBar, check_file_exist, mkdir_or_exist from mmdet.datasets import get_loading_pipeline from mmdet.evaluation import eval_map from mmdet.registry import DATASETS, RUNNERS from mmdet.structures import DetDataSample from mmdet.utils import replace_cfg_vals, update_data_root from mmdet.visualization import DetLocalVisualizer def bbox_map_eval(det_result, annotation, nproc=4): """Evaluate mAP of single image det result. Args: det_result (list[list]): [[cls1_det, cls2_det, ...], ...]. The outer list indicates images, and the inner list indicates per-class detected bboxes. annotation (dict): Ground truth annotations where keys of annotations are: - bboxes: numpy array of shape (n, 4) - labels: numpy array of shape (n, ) - bboxes_ignore (optional): numpy array of shape (k, 4) - labels_ignore (optional): numpy array of shape (k, ) nproc (int): Processes used for computing mAP. Default: 4. Returns: float: mAP """ # use only bbox det result if isinstance(det_result, tuple): bbox_det_result = [det_result[0]] else: bbox_det_result = [det_result] # mAP iou_thrs = np.linspace( .5, 0.95, int(np.round((0.95 - .5) / .05)) + 1, endpoint=True) processes = [] workers = Pool(processes=nproc) for thr in iou_thrs: p = workers.apply_async(eval_map, (bbox_det_result, [annotation]), { 'iou_thr': thr, 'logger': 'silent', 'nproc': 1 }) processes.append(p) workers.close() workers.join() mean_aps = [] for p in processes: mean_aps.append(p.get()[0]) return sum(mean_aps) / len(mean_aps) class ResultVisualizer: """Display and save evaluation results. Args: show (bool): Whether to show the image. Default: True. wait_time (float): Value of waitKey param. Default: 0. score_thr (float): Minimum score of bboxes to be shown. Default: 0. runner (:obj:`Runner`): The runner of the visualization process. """ def __init__(self, show=False, wait_time=0, score_thr=0, runner=None): self.show = show self.wait_time = wait_time self.score_thr = score_thr self.visualizer = DetLocalVisualizer() self.runner = runner self.evaluator = runner.test_evaluator def _save_image_gts_results(self, dataset, results, performances, out_dir=None, task='det'): """Display or save image with groung truths and predictions from a model. Args: dataset (Dataset): A PyTorch dataset. results (list): Object detection or panoptic segmentation results from test results pkl file. performances (dict): A dict contains samples's indices in dataset and model's performance on them. out_dir (str, optional): The filename to write the image. Defaults: None. task (str): The task to be performed. Defaults: 'det' """ mkdir_or_exist(out_dir) for performance_info in performances: index, performance = performance_info data_info = dataset[index] data_info['gt_instances'] = data_info['instances'] # calc save file path filename = data_info['img_path'] fname, name = osp.splitext(osp.basename(filename)) save_filename = fname + '_' + str(round(performance, 3)) + name out_file = osp.join(out_dir, save_filename) if task == 'det': gt_instances = InstanceData() gt_instances.bboxes = results[index]['gt_instances']['bboxes'] gt_instances.labels = results[index]['gt_instances']['labels'] pred_instances = InstanceData() pred_instances.bboxes = results[index]['pred_instances'][ 'bboxes'] pred_instances.labels = results[index]['pred_instances'][ 'labels'] pred_instances.scores = results[index]['pred_instances'][ 'scores'] data_samples = DetDataSample() data_samples.pred_instances = pred_instances data_samples.gt_instances = gt_instances elif task == 'seg': gt_panoptic_seg = PixelData() gt_panoptic_seg.sem_seg = results[index]['gt_seg_map'] pred_panoptic_seg = PixelData() pred_panoptic_seg.sem_seg = results[index][ 'pred_panoptic_seg']['sem_seg'] data_samples = DetDataSample() data_samples.pred_panoptic_seg = pred_panoptic_seg data_samples.gt_panoptic_seg = gt_panoptic_seg img = mmcv.imread(filename, channel_order='rgb') self.visualizer.add_datasample( 'image', img, data_samples, show=self.show, draw_gt=False, pred_score_thr=self.score_thr, out_file=out_file) def evaluate_and_show(self, dataset, results, topk=20, show_dir='work_dir'): """Evaluate and show results. Args: dataset (Dataset): A PyTorch dataset. results (list): Object detection or panoptic segmentation results from test results pkl file. topk (int): Number of the highest topk and lowest topk after evaluation index sorting. Default: 20. show_dir (str, optional): The filename to write the image. Default: 'work_dir' """ self.visualizer.dataset_meta = dataset.metainfo assert topk > 0 if (topk * 2) > len(dataset): topk = len(dataset) // 2 good_dir = osp.abspath(osp.join(show_dir, 'good')) bad_dir = osp.abspath(osp.join(show_dir, 'bad')) if 'pred_panoptic_seg' in results[0].keys(): good_samples, bad_samples = self.panoptic_evaluate( dataset, results, topk=topk) self._save_image_gts_results( dataset, results, good_samples, good_dir, task='seg') self._save_image_gts_results( dataset, results, bad_samples, bad_dir, task='seg') elif 'pred_instances' in results[0].keys(): good_samples, bad_samples = self.detection_evaluate( dataset, results, topk=topk) self._save_image_gts_results( dataset, results, good_samples, good_dir, task='det') self._save_image_gts_results( dataset, results, bad_samples, bad_dir, task='det') else: raise 'expect \'pred_panoptic_seg\' or \'pred_instances\' \ in dict result' def detection_evaluate(self, dataset, results, topk=20, eval_fn=None): """Evaluation for object detection. Args: dataset (Dataset): A PyTorch dataset. results (list): Object detection results from test results pkl file. topk (int): Number of the highest topk and lowest topk after evaluation index sorting. Default: 20. eval_fn (callable, optional): Eval function, Default: None. Returns: tuple: A tuple contains good samples and bad samples. good_mAPs (dict[int, float]): A dict contains good samples's indices in dataset and model's performance on them. bad_mAPs (dict[int, float]): A dict contains bad samples's indices in dataset and model's performance on them. """ if eval_fn is None: eval_fn = bbox_map_eval else: assert callable(eval_fn) prog_bar = ProgressBar(len(results)) _mAPs = {} data_info = {} for i, (result, ) in enumerate(zip(results)): # self.dataset[i] should not call directly # because there is a risk of mismatch data_info = dataset.prepare_data(i) data_info['bboxes'] = data_info['gt_bboxes'].tensor data_info['labels'] = data_info['gt_bboxes_labels'] pred = result['pred_instances'] pred_bboxes = pred['bboxes'].cpu().numpy() pred_scores = pred['scores'].cpu().numpy() pred_labels = pred['labels'].cpu().numpy() dets = [] for label in range(len(dataset.metainfo['classes'])): index = np.where(pred_labels == label)[0] pred_bbox_scores = np.hstack( [pred_bboxes[index], pred_scores[index].reshape((-1, 1))]) dets.append(pred_bbox_scores) mAP = eval_fn(dets, data_info) _mAPs[i] = mAP prog_bar.update() # descending select topk image _mAPs = list(sorted(_mAPs.items(), key=lambda kv: kv[1])) good_mAPs = _mAPs[-topk:] bad_mAPs = _mAPs[:topk] return good_mAPs, bad_mAPs def panoptic_evaluate(self, dataset, results, topk=20): """Evaluation for panoptic segmentation. Args: dataset (Dataset): A PyTorch dataset. results (list): Panoptic segmentation results from test results pkl file. topk (int): Number of the highest topk and lowest topk after evaluation index sorting. Default: 20. Returns: tuple: A tuple contains good samples and bad samples. good_pqs (dict[int, float]): A dict contains good samples's indices in dataset and model's performance on them. bad_pqs (dict[int, float]): A dict contains bad samples's indices in dataset and model's performance on them. """ pqs = {} prog_bar = ProgressBar(len(results)) for i in range(len(results)): data_sample = {} for k in dataset[i].keys(): data_sample[k] = dataset[i][k] for k in results[i].keys(): data_sample[k] = results[i][k] self.evaluator.process([data_sample]) metrics = self.evaluator.evaluate(1) pqs[i] = metrics['coco_panoptic/PQ'] prog_bar.update() # descending select topk image pqs = list(sorted(pqs.items(), key=lambda kv: kv[1])) good_pqs = pqs[-topk:] bad_pqs = pqs[:topk] return good_pqs, bad_pqs def parse_args(): parser = argparse.ArgumentParser( description='MMDet eval image prediction result for each') parser.add_argument('config', help='test config file path') parser.add_argument( 'prediction_path', help='prediction path where test pkl result') parser.add_argument( 'show_dir', help='directory where painted images will be saved') parser.add_argument('--show', action='store_true', help='show results') parser.add_argument( '--wait-time', type=float, default=0, help='the interval of show (s), 0 is block') parser.add_argument( '--topk', default=20, type=int, help='saved Number of the highest topk ' 'and lowest topk after index sorting') parser.add_argument( '--show-score-thr', type=float, default=0, help='score threshold (default: 0.)') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') args = parser.parse_args() return args def main(): args = parse_args() check_file_exist(args.prediction_path) cfg = Config.fromfile(args.config) # replace the ${key} with the value of cfg.key cfg = replace_cfg_vals(cfg) # update data root according to MMDET_DATASETS update_data_root(cfg) if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) init_default_scope(cfg.get('default_scope', 'mmdet')) cfg.test_dataloader.dataset.test_mode = True cfg.test_dataloader.pop('batch_size', 0) if cfg.train_dataloader.dataset.type in ('MultiImageMixDataset', 'ClassBalancedDataset', 'RepeatDataset', 'ConcatDataset'): cfg.test_dataloader.dataset.pipeline = get_loading_pipeline( cfg.train_dataloader.dataset.dataset.pipeline) else: cfg.test_dataloader.dataset.pipeline = get_loading_pipeline( cfg.train_dataloader.dataset.pipeline) dataset = DATASETS.build(cfg.test_dataloader.dataset) outputs = load(args.prediction_path) cfg.work_dir = args.show_dir # build the runner from config if 'runner_type' not in cfg: # build the default runner runner = Runner.from_cfg(cfg) else: # build customized runner from the registry # if 'runner_type' is set in the cfg runner = RUNNERS.build(cfg) result_visualizer = ResultVisualizer(args.show, args.wait_time, args.show_score_thr, runner) result_visualizer.evaluate_and_show( dataset, outputs, topk=args.topk, show_dir=args.show_dir) if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/benchmark.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os from mmengine import MMLogger from mmengine.config import Config, DictAction from mmengine.dist import init_dist from mmengine.registry import init_default_scope from mmengine.utils import mkdir_or_exist from mmdet.utils.benchmark import (DataLoaderBenchmark, DatasetBenchmark, InferenceBenchmark) def parse_args(): parser = argparse.ArgumentParser(description='MMDet benchmark') parser.add_argument('config', help='test config file path') parser.add_argument('--checkpoint', help='checkpoint file') parser.add_argument( '--task', choices=['inference', 'dataloader', 'dataset'], default='dataloader', help='Which task do you want to go to benchmark') parser.add_argument( '--repeat-num', type=int, default=1, help='number of repeat times of measurement for averaging the results') parser.add_argument( '--max-iter', type=int, default=2000, help='num of max iter') parser.add_argument( '--log-interval', type=int, default=50, help='interval of logging') parser.add_argument( '--num-warmup', type=int, default=5, help='Number of warmup') parser.add_argument( '--fuse-conv-bn', action='store_true', help='Whether to fuse conv and bn, this will slightly increase' 'the inference speed') parser.add_argument( '--dataset-type', choices=['train', 'val', 'test'], default='test', help='Benchmark dataset type. only supports train, val and test') parser.add_argument( '--work-dir', help='the directory to save the file containing ' 'benchmark metrics') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') parser.add_argument( '--launcher', choices=['none', 'pytorch', 'slurm', 'mpi'], default='none', help='job launcher') parser.add_argument('--local_rank', type=int, default=0) args = parser.parse_args() if 'LOCAL_RANK' not in os.environ: os.environ['LOCAL_RANK'] = str(args.local_rank) return args def inference_benchmark(args, cfg, distributed, logger): benchmark = InferenceBenchmark( cfg, args.checkpoint, distributed, args.fuse_conv_bn, args.max_iter, args.log_interval, args.num_warmup, logger=logger) return benchmark def dataloader_benchmark(args, cfg, distributed, logger): benchmark = DataLoaderBenchmark( cfg, distributed, args.dataset_type, args.max_iter, args.log_interval, args.num_warmup, logger=logger) return benchmark def dataset_benchmark(args, cfg, distributed, logger): benchmark = DatasetBenchmark( cfg, args.dataset_type, args.max_iter, args.log_interval, args.num_warmup, logger=logger) return benchmark def main(): args = parse_args() cfg = Config.fromfile(args.config) if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) init_default_scope(cfg.get('default_scope', 'mmdet')) distributed = False if args.launcher != 'none': init_dist(args.launcher, **cfg.get('env_cfg', {}).get('dist_cfg', {})) distributed = True log_file = None if args.work_dir: log_file = os.path.join(args.work_dir, 'benchmark.log') mkdir_or_exist(args.work_dir) logger = MMLogger.get_instance( 'mmdet', log_file=log_file, log_level='INFO') benchmark = eval(f'{args.task}_benchmark')(args, cfg, distributed, logger) benchmark.run(args.repeat_num) if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/browse_dataset.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os.path as osp from mmengine.config import Config, DictAction from mmengine.registry import init_default_scope from mmengine.utils import ProgressBar from mmdet.models.utils import mask2ndarray from mmdet.registry import DATASETS, VISUALIZERS from mmdet.structures.bbox import BaseBoxes def parse_args(): parser = argparse.ArgumentParser(description='Browse a dataset') parser.add_argument('config', help='train config file path') parser.add_argument( '--output-dir', default=None, type=str, help='If there is no display interface, you can save it') parser.add_argument('--not-show', default=False, action='store_true') parser.add_argument( '--show-interval', type=float, default=2, help='the interval of show (s)') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') args = parser.parse_args() return args def main(): args = parse_args() cfg = Config.fromfile(args.config) if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) # register all modules in mmdet into the registries init_default_scope(cfg.get('default_scope', 'mmdet')) dataset = DATASETS.build(cfg.train_dataloader.dataset) visualizer = VISUALIZERS.build(cfg.visualizer) visualizer.dataset_meta = dataset.metainfo progress_bar = ProgressBar(len(dataset)) for item in dataset: img = item['inputs'].permute(1, 2, 0).numpy() data_sample = item['data_samples'].numpy() gt_instances = data_sample.gt_instances img_path = osp.basename(item['data_samples'].img_path) out_file = osp.join( args.output_dir, osp.basename(img_path)) if args.output_dir is not None else None img = img[..., [2, 1, 0]] # bgr to rgb gt_bboxes = gt_instances.get('bboxes', None) if gt_bboxes is not None and isinstance(gt_bboxes, BaseBoxes): gt_instances.bboxes = gt_bboxes.tensor gt_masks = gt_instances.get('masks', None) if gt_masks is not None: masks = mask2ndarray(gt_masks) gt_instances.masks = masks.astype(bool) data_sample.gt_instances = gt_instances visualizer.add_datasample( osp.basename(img_path), img, data_sample, draw_pred=False, show=not args.not_show, wait_time=args.show_interval, out_file=out_file) progress_bar.update() if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/coco_error_analysis.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import copy import os from argparse import ArgumentParser from multiprocessing import Pool import matplotlib.pyplot as plt import numpy as np from pycocotools.coco import COCO from pycocotools.cocoeval import COCOeval def makeplot(rs, ps, outDir, class_name, iou_type): cs = np.vstack([ np.ones((2, 3)), np.array([0.31, 0.51, 0.74]), np.array([0.75, 0.31, 0.30]), np.array([0.36, 0.90, 0.38]), np.array([0.50, 0.39, 0.64]), np.array([1, 0.6, 0]), ]) areaNames = ['allarea', 'small', 'medium', 'large'] types = ['C75', 'C50', 'Loc', 'Sim', 'Oth', 'BG', 'FN'] for i in range(len(areaNames)): area_ps = ps[..., i, 0] figure_title = iou_type + '-' + class_name + '-' + areaNames[i] aps = [ps_.mean() for ps_ in area_ps] ps_curve = [ ps_.mean(axis=1) if ps_.ndim > 1 else ps_ for ps_ in area_ps ] ps_curve.insert(0, np.zeros(ps_curve[0].shape)) fig = plt.figure() ax = plt.subplot(111) for k in range(len(types)): ax.plot(rs, ps_curve[k + 1], color=[0, 0, 0], linewidth=0.5) ax.fill_between( rs, ps_curve[k], ps_curve[k + 1], color=cs[k], label=str(f'[{aps[k]:.3f}]' + types[k]), ) plt.xlabel('recall') plt.ylabel('precision') plt.xlim(0, 1.0) plt.ylim(0, 1.0) plt.title(figure_title) plt.legend() # plt.show() fig.savefig(outDir + f'/{figure_title}.png') plt.close(fig) def autolabel(ax, rects): """Attach a text label above each bar in *rects*, displaying its height.""" for rect in rects: height = rect.get_height() if height > 0 and height <= 1: # for percent values text_label = '{:2.0f}'.format(height * 100) else: text_label = '{:2.0f}'.format(height) ax.annotate( text_label, xy=(rect.get_x() + rect.get_width() / 2, height), xytext=(0, 3), # 3 points vertical offset textcoords='offset points', ha='center', va='bottom', fontsize='x-small', ) def makebarplot(rs, ps, outDir, class_name, iou_type): areaNames = ['allarea', 'small', 'medium', 'large'] types = ['C75', 'C50', 'Loc', 'Sim', 'Oth', 'BG', 'FN'] fig, ax = plt.subplots() x = np.arange(len(areaNames)) # the areaNames locations width = 0.60 # the width of the bars rects_list = [] figure_title = iou_type + '-' + class_name + '-' + 'ap bar plot' for i in range(len(types) - 1): type_ps = ps[i, ..., 0] aps = [ps_.mean() for ps_ in type_ps.T] rects_list.append( ax.bar( x - width / 2 + (i + 1) * width / len(types), aps, width / len(types), label=types[i], )) # Add some text for labels, title and custom x-axis tick labels, etc. ax.set_ylabel('Mean Average Precision (mAP)') ax.set_title(figure_title) ax.set_xticks(x) ax.set_xticklabels(areaNames) ax.legend() # Add score texts over bars for rects in rects_list: autolabel(ax, rects) # Save plot fig.savefig(outDir + f'/{figure_title}.png') plt.close(fig) def get_gt_area_group_numbers(cocoEval): areaRng = cocoEval.params.areaRng areaRngStr = [str(aRng) for aRng in areaRng] areaRngLbl = cocoEval.params.areaRngLbl areaRngStr2areaRngLbl = dict(zip(areaRngStr, areaRngLbl)) areaRngLbl2Number = dict.fromkeys(areaRngLbl, 0) for evalImg in cocoEval.evalImgs: if evalImg: for gtIgnore in evalImg['gtIgnore']: if not gtIgnore: aRngLbl = areaRngStr2areaRngLbl[str(evalImg['aRng'])] areaRngLbl2Number[aRngLbl] += 1 return areaRngLbl2Number def make_gt_area_group_numbers_plot(cocoEval, outDir, verbose=True): areaRngLbl2Number = get_gt_area_group_numbers(cocoEval) areaRngLbl = areaRngLbl2Number.keys() if verbose: print('number of annotations per area group:', areaRngLbl2Number) # Init figure fig, ax = plt.subplots() x = np.arange(len(areaRngLbl)) # the areaNames locations width = 0.60 # the width of the bars figure_title = 'number of annotations per area group' rects = ax.bar(x, areaRngLbl2Number.values(), width) # Add some text for labels, title and custom x-axis tick labels, etc. ax.set_ylabel('Number of annotations') ax.set_title(figure_title) ax.set_xticks(x) ax.set_xticklabels(areaRngLbl) # Add score texts over bars autolabel(ax, rects) # Save plot fig.tight_layout() fig.savefig(outDir + f'/{figure_title}.png') plt.close(fig) def make_gt_area_histogram_plot(cocoEval, outDir): n_bins = 100 areas = [ann['area'] for ann in cocoEval.cocoGt.anns.values()] # init figure figure_title = 'gt annotation areas histogram plot' fig, ax = plt.subplots() # Set the number of bins ax.hist(np.sqrt(areas), bins=n_bins) # Add some text for labels, title and custom x-axis tick labels, etc. ax.set_xlabel('Squareroot Area') ax.set_ylabel('Number of annotations') ax.set_title(figure_title) # Save plot fig.tight_layout() fig.savefig(outDir + f'/{figure_title}.png') plt.close(fig) def analyze_individual_category(k, cocoDt, cocoGt, catId, iou_type, areas=None): nm = cocoGt.loadCats(catId)[0] print(f'--------------analyzing {k + 1}-{nm["name"]}---------------') ps_ = {} dt = copy.deepcopy(cocoDt) nm = cocoGt.loadCats(catId)[0] imgIds = cocoGt.getImgIds() dt_anns = dt.dataset['annotations'] select_dt_anns = [] for ann in dt_anns: if ann['category_id'] == catId: select_dt_anns.append(ann) dt.dataset['annotations'] = select_dt_anns dt.createIndex() # compute precision but ignore superclass confusion gt = copy.deepcopy(cocoGt) child_catIds = gt.getCatIds(supNms=[nm['supercategory']]) for idx, ann in enumerate(gt.dataset['annotations']): if ann['category_id'] in child_catIds and ann['category_id'] != catId: gt.dataset['annotations'][idx]['ignore'] = 1 gt.dataset['annotations'][idx]['iscrowd'] = 1 gt.dataset['annotations'][idx]['category_id'] = catId cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type) cocoEval.params.imgIds = imgIds cocoEval.params.maxDets = [100] cocoEval.params.iouThrs = [0.1] cocoEval.params.useCats = 1 if areas: cocoEval.params.areaRng = [[0**2, areas[2]], [0**2, areas[0]], [areas[0], areas[1]], [areas[1], areas[2]]] cocoEval.evaluate() cocoEval.accumulate() ps_supercategory = cocoEval.eval['precision'][0, :, k, :, :] ps_['ps_supercategory'] = ps_supercategory # compute precision but ignore any class confusion gt = copy.deepcopy(cocoGt) for idx, ann in enumerate(gt.dataset['annotations']): if ann['category_id'] != catId: gt.dataset['annotations'][idx]['ignore'] = 1 gt.dataset['annotations'][idx]['iscrowd'] = 1 gt.dataset['annotations'][idx]['category_id'] = catId cocoEval = COCOeval(gt, copy.deepcopy(dt), iou_type) cocoEval.params.imgIds = imgIds cocoEval.params.maxDets = [100] cocoEval.params.iouThrs = [0.1] cocoEval.params.useCats = 1 if areas: cocoEval.params.areaRng = [[0**2, areas[2]], [0**2, areas[0]], [areas[0], areas[1]], [areas[1], areas[2]]] cocoEval.evaluate() cocoEval.accumulate() ps_allcategory = cocoEval.eval['precision'][0, :, k, :, :] ps_['ps_allcategory'] = ps_allcategory return k, ps_ def analyze_results(res_file, ann_file, res_types, out_dir, extraplots=None, areas=None): for res_type in res_types: assert res_type in ['bbox', 'segm'] if areas: assert len(areas) == 3, '3 integers should be specified as areas, \ representing 3 area regions' directory = os.path.dirname(out_dir + '/') if not os.path.exists(directory): print(f'-------------create {out_dir}-----------------') os.makedirs(directory) cocoGt = COCO(ann_file) cocoDt = cocoGt.loadRes(res_file) imgIds = cocoGt.getImgIds() for res_type in res_types: res_out_dir = out_dir + '/' + res_type + '/' res_directory = os.path.dirname(res_out_dir) if not os.path.exists(res_directory): print(f'-------------create {res_out_dir}-----------------') os.makedirs(res_directory) iou_type = res_type cocoEval = COCOeval( copy.deepcopy(cocoGt), copy.deepcopy(cocoDt), iou_type) cocoEval.params.imgIds = imgIds cocoEval.params.iouThrs = [0.75, 0.5, 0.1] cocoEval.params.maxDets = [100] if areas: cocoEval.params.areaRng = [[0**2, areas[2]], [0**2, areas[0]], [areas[0], areas[1]], [areas[1], areas[2]]] cocoEval.evaluate() cocoEval.accumulate() ps = cocoEval.eval['precision'] ps = np.vstack([ps, np.zeros((4, *ps.shape[1:]))]) catIds = cocoGt.getCatIds() recThrs = cocoEval.params.recThrs with Pool(processes=48) as pool: args = [(k, cocoDt, cocoGt, catId, iou_type, areas) for k, catId in enumerate(catIds)] analyze_results = pool.starmap(analyze_individual_category, args) for k, catId in enumerate(catIds): nm = cocoGt.loadCats(catId)[0] print(f'--------------saving {k + 1}-{nm["name"]}---------------') analyze_result = analyze_results[k] assert k == analyze_result[0] ps_supercategory = analyze_result[1]['ps_supercategory'] ps_allcategory = analyze_result[1]['ps_allcategory'] # compute precision but ignore superclass confusion ps[3, :, k, :, :] = ps_supercategory # compute precision but ignore any class confusion ps[4, :, k, :, :] = ps_allcategory # fill in background and false negative errors and plot ps[ps == -1] = 0 ps[5, :, k, :, :] = ps[4, :, k, :, :] > 0 ps[6, :, k, :, :] = 1.0 makeplot(recThrs, ps[:, :, k], res_out_dir, nm['name'], iou_type) if extraplots: makebarplot(recThrs, ps[:, :, k], res_out_dir, nm['name'], iou_type) makeplot(recThrs, ps, res_out_dir, 'allclass', iou_type) if extraplots: makebarplot(recThrs, ps, res_out_dir, 'allclass', iou_type) make_gt_area_group_numbers_plot( cocoEval=cocoEval, outDir=res_out_dir, verbose=True) make_gt_area_histogram_plot(cocoEval=cocoEval, outDir=res_out_dir) def main(): parser = ArgumentParser(description='COCO Error Analysis Tool') parser.add_argument('result', help='result file (json format) path') parser.add_argument('out_dir', help='dir to save analyze result images') parser.add_argument( '--ann', default='data/coco/annotations/instances_val2017.json', help='annotation file path') parser.add_argument( '--types', type=str, nargs='+', default=['bbox'], help='result types') parser.add_argument( '--extraplots', action='store_true', help='export extra bar/stat plots') parser.add_argument( '--areas', type=int, nargs='+', default=[1024, 9216, 10000000000], help='area regions') args = parser.parse_args() analyze_results( args.result, args.ann, args.types, out_dir=args.out_dir, extraplots=args.extraplots, areas=args.areas) if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/coco_occluded_separated_recall.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from argparse import ArgumentParser import mmengine from mmengine.logging import print_log from mmdet.datasets import CocoDataset from mmdet.evaluation import CocoOccludedSeparatedMetric def main(): parser = ArgumentParser( description='Compute recall of COCO occluded and separated masks ' 'presented in paper https://arxiv.org/abs/2210.10046.') parser.add_argument('result', help='result file (pkl format) path') parser.add_argument('--out', help='file path to save evaluation results') parser.add_argument( '--score-thr', type=float, default=0.3, help='Score threshold for the recall calculation. Defaults to 0.3') parser.add_argument( '--iou-thr', type=float, default=0.75, help='IoU threshold for the recall calculation. Defaults to 0.75.') parser.add_argument( '--ann', default='data/coco/annotations/instances_val2017.json', help='coco annotation file path') args = parser.parse_args() results = mmengine.load(args.result) assert 'masks' in results[0]['pred_instances'], \ 'The results must be predicted by instance segmentation model.' metric = CocoOccludedSeparatedMetric( ann_file=args.ann, iou_thr=args.iou_thr, score_thr=args.score_thr) metric.dataset_meta = CocoDataset.METAINFO for datasample in results: metric.process(data_batch=None, data_samples=[datasample]) metric_res = metric.compute_metrics(metric.results) if args.out is not None: mmengine.dump(metric_res, args.out) print_log(f'Evaluation results have been saved to {args.out}.') if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/confusion_matrix.py ================================================ import argparse import os import matplotlib.pyplot as plt import numpy as np from matplotlib.ticker import MultipleLocator from mmcv.ops import nms from mmengine import Config, DictAction from mmengine.fileio import load from mmengine.registry import init_default_scope from mmengine.utils import ProgressBar from mmdet.evaluation import bbox_overlaps from mmdet.registry import DATASETS from mmdet.utils import replace_cfg_vals, update_data_root def parse_args(): parser = argparse.ArgumentParser( description='Generate confusion matrix from detection results') parser.add_argument('config', help='test config file path') parser.add_argument( 'prediction_path', help='prediction path where test .pkl result') parser.add_argument( 'save_dir', help='directory where confusion matrix will be saved') parser.add_argument( '--show', action='store_true', help='show confusion matrix') parser.add_argument( '--color-theme', default='plasma', help='theme of the matrix color map') parser.add_argument( '--score-thr', type=float, default=0.3, help='score threshold to filter detection bboxes') parser.add_argument( '--tp-iou-thr', type=float, default=0.5, help='IoU threshold to be considered as matched') parser.add_argument( '--nms-iou-thr', type=float, default=None, help='nms IoU threshold, only applied when users want to change the' 'nms IoU threshold.') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') args = parser.parse_args() return args def calculate_confusion_matrix(dataset, results, score_thr=0, nms_iou_thr=None, tp_iou_thr=0.5): """Calculate the confusion matrix. Args: dataset (Dataset): Test or val dataset. results (list[ndarray]): A list of detection results in each image. score_thr (float|optional): Score threshold to filter bboxes. Default: 0. nms_iou_thr (float|optional): nms IoU threshold, the detection results have done nms in the detector, only applied when users want to change the nms IoU threshold. Default: None. tp_iou_thr (float|optional): IoU threshold to be considered as matched. Default: 0.5. """ num_classes = len(dataset.metainfo['classes']) confusion_matrix = np.zeros(shape=[num_classes + 1, num_classes + 1]) assert len(dataset) == len(results) prog_bar = ProgressBar(len(results)) for idx, per_img_res in enumerate(results): res_bboxes = per_img_res['pred_instances'] gts = dataset.get_data_info(idx)['instances'] analyze_per_img_dets(confusion_matrix, gts, res_bboxes, score_thr, tp_iou_thr, nms_iou_thr) prog_bar.update() return confusion_matrix def analyze_per_img_dets(confusion_matrix, gts, result, score_thr=0, tp_iou_thr=0.5, nms_iou_thr=None): """Analyze detection results on each image. Args: confusion_matrix (ndarray): The confusion matrix, has shape (num_classes + 1, num_classes + 1). gt_bboxes (ndarray): Ground truth bboxes, has shape (num_gt, 4). gt_labels (ndarray): Ground truth labels, has shape (num_gt). result (ndarray): Detection results, has shape (num_classes, num_bboxes, 5). score_thr (float): Score threshold to filter bboxes. Default: 0. tp_iou_thr (float): IoU threshold to be considered as matched. Default: 0.5. nms_iou_thr (float|optional): nms IoU threshold, the detection results have done nms in the detector, only applied when users want to change the nms IoU threshold. Default: None. """ true_positives = np.zeros(len(gts)) gt_bboxes = [] gt_labels = [] for gt in gts: gt_bboxes.append(gt['bbox']) gt_labels.append(gt['bbox_label']) gt_bboxes = np.array(gt_bboxes) gt_labels = np.array(gt_labels) unique_label = np.unique(result['labels'].numpy()) for det_label in unique_label: mask = (result['labels'] == det_label) det_bboxes = result['bboxes'][mask].numpy() det_scores = result['scores'][mask].numpy() if nms_iou_thr: det_bboxes, _ = nms( det_bboxes, det_scores, nms_iou_thr, score_threshold=score_thr) ious = bbox_overlaps(det_bboxes[:, :4], gt_bboxes) for i, score in enumerate(det_scores): det_match = 0 if score >= score_thr: for j, gt_label in enumerate(gt_labels): if ious[i, j] >= tp_iou_thr: det_match += 1 if gt_label == det_label: true_positives[j] += 1 # TP confusion_matrix[gt_label, det_label] += 1 if det_match == 0: # BG FP confusion_matrix[-1, det_label] += 1 for num_tp, gt_label in zip(true_positives, gt_labels): if num_tp == 0: # FN confusion_matrix[gt_label, -1] += 1 def plot_confusion_matrix(confusion_matrix, labels, save_dir=None, show=True, title='Normalized Confusion Matrix', color_theme='plasma'): """Draw confusion matrix with matplotlib. Args: confusion_matrix (ndarray): The confusion matrix. labels (list[str]): List of class names. save_dir (str|optional): If set, save the confusion matrix plot to the given path. Default: None. show (bool): Whether to show the plot. Default: True. title (str): Title of the plot. Default: `Normalized Confusion Matrix`. color_theme (str): Theme of the matrix color map. Default: `plasma`. """ # normalize the confusion matrix per_label_sums = confusion_matrix.sum(axis=1)[:, np.newaxis] confusion_matrix = \ confusion_matrix.astype(np.float32) / per_label_sums * 100 num_classes = len(labels) fig, ax = plt.subplots( figsize=(0.5 * num_classes, 0.5 * num_classes * 0.8), dpi=180) cmap = plt.get_cmap(color_theme) im = ax.imshow(confusion_matrix, cmap=cmap) plt.colorbar(mappable=im, ax=ax) title_font = {'weight': 'bold', 'size': 12} ax.set_title(title, fontdict=title_font) label_font = {'size': 10} plt.ylabel('Ground Truth Label', fontdict=label_font) plt.xlabel('Prediction Label', fontdict=label_font) # draw locator xmajor_locator = MultipleLocator(1) xminor_locator = MultipleLocator(0.5) ax.xaxis.set_major_locator(xmajor_locator) ax.xaxis.set_minor_locator(xminor_locator) ymajor_locator = MultipleLocator(1) yminor_locator = MultipleLocator(0.5) ax.yaxis.set_major_locator(ymajor_locator) ax.yaxis.set_minor_locator(yminor_locator) # draw grid ax.grid(True, which='minor', linestyle='-') # draw label ax.set_xticks(np.arange(num_classes)) ax.set_yticks(np.arange(num_classes)) ax.set_xticklabels(labels) ax.set_yticklabels(labels) ax.tick_params( axis='x', bottom=False, top=True, labelbottom=False, labeltop=True) plt.setp( ax.get_xticklabels(), rotation=45, ha='left', rotation_mode='anchor') # draw confution matrix value for i in range(num_classes): for j in range(num_classes): ax.text( j, i, '{}%'.format( int(confusion_matrix[ i, j]) if not np.isnan(confusion_matrix[i, j]) else -1), ha='center', va='center', color='w', size=7) ax.set_ylim(len(confusion_matrix) - 0.5, -0.5) # matplotlib>3.1.1 fig.tight_layout() if save_dir is not None: plt.savefig( os.path.join(save_dir, 'confusion_matrix.png'), format='png') if show: plt.show() def main(): args = parse_args() cfg = Config.fromfile(args.config) # replace the ${key} with the value of cfg.key cfg = replace_cfg_vals(cfg) # update data root according to MMDET_DATASETS update_data_root(cfg) if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) init_default_scope(cfg.get('default_scope', 'mmdet')) results = load(args.prediction_path) if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) dataset = DATASETS.build(cfg.test_dataloader.dataset) confusion_matrix = calculate_confusion_matrix(dataset, results, args.score_thr, args.nms_iou_thr, args.tp_iou_thr) plot_confusion_matrix( confusion_matrix, dataset.metainfo['classes'] + ('background', ), save_dir=args.save_dir, show=args.show, color_theme=args.color_theme) if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/eval_metric.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import mmengine from mmengine import Config, DictAction from mmengine.evaluator import Evaluator from mmengine.registry import init_default_scope from mmdet.registry import DATASETS def parse_args(): parser = argparse.ArgumentParser(description='Evaluate metric of the ' 'results saved in pkl format') parser.add_argument('config', help='Config of the model') parser.add_argument('pkl_results', help='Results in pickle format') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') args = parser.parse_args() return args def main(): args = parse_args() cfg = Config.fromfile(args.config) init_default_scope(cfg.get('default_scope', 'mmdet')) if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) dataset = DATASETS.build(cfg.test_dataloader.dataset) predictions = mmengine.load(args.pkl_results) evaluator = Evaluator(cfg.val_evaluator) evaluator.dataset_meta = dataset.metainfo eval_results = evaluator.offline_evaluate(predictions) print(eval_results) if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/get_flops.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import tempfile from functools import partial from pathlib import Path import torch from mmengine.config import Config, DictAction from mmengine.logging import MMLogger from mmengine.model import revert_sync_batchnorm from mmengine.registry import init_default_scope from mmengine.runner import Runner from mmdet.registry import MODELS try: from mmengine.analysis import get_model_complexity_info from mmengine.analysis.print_helper import _format_size except ImportError: raise ImportError('Please upgrade mmengine >= 0.6.0') def parse_args(): parser = argparse.ArgumentParser(description='Get a detector flops') parser.add_argument('config', help='train config file path') parser.add_argument( '--shape', type=int, nargs='+', default=[1280, 800], help='input image size') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') args = parser.parse_args() return args def inference(args, logger): if str(torch.__version__) < '1.12': logger.warning( 'Some config files, such as configs/yolact and configs/detectors,' 'may have compatibility issues with torch.jit when torch<1.12. ' 'If you want to calculate flops for these models, ' 'please make sure your pytorch version is >=1.12.') config_name = Path(args.config) if not config_name.exists(): logger.error(f'{config_name} not found.') cfg = Config.fromfile(args.config) cfg.work_dir = tempfile.TemporaryDirectory().name cfg.log_level = 'WARN' if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) init_default_scope(cfg.get('default_scope', 'mmdet')) # TODO: The following usage is temporary and not safe # use hard code to convert mmSyncBN to SyncBN. This is a known # bug in mmengine, mmSyncBN requires a distributed environment， # this question involves models like configs/strong_baselines if hasattr(cfg, 'head_norm_cfg'): cfg['head_norm_cfg'] = dict(type='SyncBN', requires_grad=True) cfg['model']['roi_head']['bbox_head']['norm_cfg'] = dict( type='SyncBN', requires_grad=True) cfg['model']['roi_head']['mask_head']['norm_cfg'] = dict( type='SyncBN', requires_grad=True) if len(args.shape) == 1: h = w = args.shape[0] elif len(args.shape) == 2: h, w = args.shape else: raise ValueError('invalid input shape') result = {} # Supports two ways to calculate flops, # 1. randomly generate a picture # 2. load a picture from the dataset # In two stage detectors, _forward need batch_samples to get # rpn_results_list, then use rpn_results_list to compute flops, # so only the second way is supported try: model = MODELS.build(cfg.model) if torch.cuda.is_available(): model.cuda() model = revert_sync_batchnorm(model) data_batch = {'inputs': [torch.rand(3, h, w)], 'batch_samples': [None]} data = model.data_preprocessor(data_batch) result['ori_shape'] = (h, w) result['pad_shape'] = data['inputs'].shape[-2:] model.eval() outputs = get_model_complexity_info( model, None, inputs=data['inputs'], show_table=False, show_arch=False) flops = outputs['flops'] params = outputs['params'] result['compute_type'] = 'direct: randomly generate a picture' except TypeError: logger.warning( 'Failed to directly get FLOPs, try to get flops with real data') data_loader = Runner.build_dataloader(cfg.val_dataloader) data_batch = next(iter(data_loader)) model = MODELS.build(cfg.model) if torch.cuda.is_available(): model = model.cuda() model = revert_sync_batchnorm(model) model.eval() _forward = model.forward data = model.data_preprocessor(data_batch) result['ori_shape'] = data['data_samples'][0].ori_shape result['pad_shape'] = data['data_samples'][0].pad_shape del data_loader model.forward = partial(_forward, data_samples=data['data_samples']) outputs = get_model_complexity_info( model, None, inputs=data['inputs'], show_table=False, show_arch=False) flops = outputs['flops'] params = outputs['params'] result['compute_type'] = 'dataloader: load a picture from the dataset' flops = _format_size(flops) params = _format_size(params) result['flops'] = flops result['params'] = params return result def main(): args = parse_args() logger = MMLogger.get_instance(name='MMLogger') result = inference(args, logger) split_line = '=' * 30 ori_shape = result['ori_shape'] pad_shape = result['pad_shape'] flops = result['flops'] params = result['params'] compute_type = result['compute_type'] if pad_shape != ori_shape: print(f'{split_line}\nUse size divisor set input shape ' f'from {ori_shape} to {pad_shape}') print(f'{split_line}\nCompute type: {compute_type}\n' f'Input shape: {pad_shape}\nFlops: {flops}\n' f'Params: {params}\n{split_line}') print('!!!Please be cautious if you use the results in papers. ' 'You may need to check if all ops are supported and verify ' 'that the flops computation is correct.') if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/optimize_anchors.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """Optimize anchor settings on a specific dataset. This script provides two method to optimize YOLO anchors including k-means anchor cluster and differential evolution. You can use ``--algorithm k-means`` and ``--algorithm differential_evolution`` to switch two method. Example: Use k-means anchor cluster:: python tools/analysis_tools/optimize_anchors.py ${CONFIG} \ --algorithm k-means --input-shape ${INPUT_SHAPE [WIDTH HEIGHT]} \ --output-dir ${OUTPUT_DIR} Use differential evolution to optimize anchors:: python tools/analysis_tools/optimize_anchors.py ${CONFIG} \ --algorithm differential_evolution \ --input-shape ${INPUT_SHAPE [WIDTH HEIGHT]} \ --output-dir ${OUTPUT_DIR} """ import argparse import os.path as osp import numpy as np import torch from mmengine.config import Config from mmengine.fileio import dump from mmengine.logging import MMLogger from mmengine.registry import init_default_scope from mmengine.utils import ProgressBar from scipy.optimize import differential_evolution from mmdet.registry import DATASETS from mmdet.structures.bbox import (bbox_cxcywh_to_xyxy, bbox_overlaps, bbox_xyxy_to_cxcywh) from mmdet.utils import replace_cfg_vals, update_data_root def parse_args(): parser = argparse.ArgumentParser(description='Optimize anchor parameters.') parser.add_argument('config', help='Train config file path.') parser.add_argument( '--device', default='cuda:0', help='Device used for calculating.') parser.add_argument( '--input-shape', type=int, nargs='+', default=[608, 608], help='input image size') parser.add_argument( '--algorithm', default='differential_evolution', help='Algorithm used for anchor optimizing.' 'Support k-means and differential_evolution for YOLO.') parser.add_argument( '--iters', default=1000, type=int, help='Maximum iterations for optimizer.') parser.add_argument( '--output-dir', default=None, type=str, help='Path to save anchor optimize result.') args = parser.parse_args() return args class BaseAnchorOptimizer: """Base class for anchor optimizer. Args: dataset (obj:`Dataset`): Dataset object. input_shape (list[int]): Input image shape of the model. Format in [width, height]. logger (obj:`logging.Logger`): The logger for logging. device (str, optional): Device used for calculating. Default: 'cuda:0' out_dir (str, optional): Path to save anchor optimize result. Default: None """ def __init__(self, dataset, input_shape, logger, device='cuda:0', out_dir=None): self.dataset = dataset self.input_shape = input_shape self.logger = logger self.device = device self.out_dir = out_dir bbox_whs, img_shapes = self.get_whs_and_shapes() ratios = img_shapes.max(1, keepdims=True) / np.array([input_shape]) # resize to input shape self.bbox_whs = bbox_whs / ratios def get_whs_and_shapes(self): """Get widths and heights of bboxes and shapes of images. Returns: tuple[np.ndarray]: Array of bbox shapes and array of image shapes with shape (num_bboxes, 2) in [width, height] format. """ self.logger.info('Collecting bboxes from annotation...') bbox_whs = [] img_shapes = [] prog_bar = ProgressBar(len(self.dataset)) for idx in range(len(self.dataset)): data_info = self.dataset.get_data_info(idx) img_shape = np.array([data_info['width'], data_info['height']]) gt_instances = data_info['instances'] for instance in gt_instances: bbox = np.array(instance['bbox']) wh = bbox[2:4] - bbox[0:2] img_shapes.append(img_shape) bbox_whs.append(wh) prog_bar.update() print('\n') bbox_whs = np.array(bbox_whs) img_shapes = np.array(img_shapes) self.logger.info(f'Collected {bbox_whs.shape[0]} bboxes.') return bbox_whs, img_shapes def get_zero_center_bbox_tensor(self): """Get a tensor of bboxes centered at (0, 0). Returns: Tensor: Tensor of bboxes with shape (num_bboxes, 4) in [xmin, ymin, xmax, ymax] format. """ whs = torch.from_numpy(self.bbox_whs).to( self.device, dtype=torch.float32) bboxes = bbox_cxcywh_to_xyxy( torch.cat([torch.zeros_like(whs), whs], dim=1)) return bboxes def optimize(self): raise NotImplementedError def save_result(self, anchors, path=None): anchor_results = [] for w, h in anchors: anchor_results.append([round(w), round(h)]) self.logger.info(f'Anchor optimize result:{anchor_results}') if path: json_path = osp.join(path, 'anchor_optimize_result.json') dump(anchor_results, json_path) self.logger.info(f'Result saved in {json_path}') class YOLOKMeansAnchorOptimizer(BaseAnchorOptimizer): r"""YOLO anchor optimizer using k-means. Code refer to `AlexeyAB/darknet. `_. Args: num_anchors (int) : Number of anchors. iters (int): Maximum iterations for k-means. """ def __init__(self, num_anchors, iters, **kwargs): super(YOLOKMeansAnchorOptimizer, self).__init__(**kwargs) self.num_anchors = num_anchors self.iters = iters def optimize(self): anchors = self.kmeans_anchors() self.save_result(anchors, self.out_dir) def kmeans_anchors(self): self.logger.info( f'Start cluster {self.num_anchors} YOLO anchors with K-means...') bboxes = self.get_zero_center_bbox_tensor() cluster_center_idx = torch.randint( 0, bboxes.shape[0], (self.num_anchors, )).to(self.device) assignments = torch.zeros((bboxes.shape[0], )).to(self.device) cluster_centers = bboxes[cluster_center_idx] if self.num_anchors == 1: cluster_centers = self.kmeans_maximization(bboxes, assignments, cluster_centers) anchors = bbox_xyxy_to_cxcywh(cluster_centers)[:, 2:].cpu().numpy() anchors = sorted(anchors, key=lambda x: x[0] * x[1]) return anchors prog_bar = ProgressBar(self.iters) for i in range(self.iters): converged, assignments = self.kmeans_expectation( bboxes, assignments, cluster_centers) if converged: self.logger.info(f'K-means process has converged at iter {i}.') break cluster_centers = self.kmeans_maximization(bboxes, assignments, cluster_centers) prog_bar.update() print('\n') avg_iou = bbox_overlaps(bboxes, cluster_centers).max(1)[0].mean().item() anchors = bbox_xyxy_to_cxcywh(cluster_centers)[:, 2:].cpu().numpy() anchors = sorted(anchors, key=lambda x: x[0] * x[1]) self.logger.info(f'Anchor cluster finish. Average IOU: {avg_iou}') return anchors def kmeans_maximization(self, bboxes, assignments, centers): """Maximization part of EM algorithm(Expectation-Maximization)""" new_centers = torch.zeros_like(centers) for i in range(centers.shape[0]): mask = (assignments == i) if mask.sum(): new_centers[i, :] = bboxes[mask].mean(0) return new_centers def kmeans_expectation(self, bboxes, assignments, centers): """Expectation part of EM algorithm(Expectation-Maximization)""" ious = bbox_overlaps(bboxes, centers) closest = ious.argmax(1) converged = (closest == assignments).all() return converged, closest class YOLODEAnchorOptimizer(BaseAnchorOptimizer): """YOLO anchor optimizer using differential evolution algorithm. Args: num_anchors (int) : Number of anchors. iters (int): Maximum iterations for k-means. strategy (str): The differential evolution strategy to use. Should be one of: - 'best1bin' - 'best1exp' - 'rand1exp' - 'randtobest1exp' - 'currenttobest1exp' - 'best2exp' - 'rand2exp' - 'randtobest1bin' - 'currenttobest1bin' - 'best2bin' - 'rand2bin' - 'rand1bin' Default: 'best1bin'. population_size (int): Total population size of evolution algorithm. Default: 15. convergence_thr (float): Tolerance for convergence, the optimizing stops when ``np.std(pop) <= abs(convergence_thr) + convergence_thr * np.abs(np.mean(population_energies))``, respectively. Default: 0.0001. mutation (tuple[float]): Range of dithering randomly changes the mutation constant. Default: (0.5, 1). recombination (float): Recombination constant of crossover probability. Default: 0.7. """ def __init__(self, num_anchors, iters, strategy='best1bin', population_size=15, convergence_thr=0.0001, mutation=(0.5, 1), recombination=0.7, **kwargs): super(YOLODEAnchorOptimizer, self).__init__(**kwargs) self.num_anchors = num_anchors self.iters = iters self.strategy = strategy self.population_size = population_size self.convergence_thr = convergence_thr self.mutation = mutation self.recombination = recombination def optimize(self): anchors = self.differential_evolution() self.save_result(anchors, self.out_dir) def differential_evolution(self): bboxes = self.get_zero_center_bbox_tensor() bounds = [] for i in range(self.num_anchors): bounds.extend([(0, self.input_shape[0]), (0, self.input_shape[1])]) result = differential_evolution( func=self.avg_iou_cost, bounds=bounds, args=(bboxes, ), strategy=self.strategy, maxiter=self.iters, popsize=self.population_size, tol=self.convergence_thr, mutation=self.mutation, recombination=self.recombination, updating='immediate', disp=True) self.logger.info( f'Anchor evolution finish. Average IOU: {1 - result.fun}') anchors = [(w, h) for w, h in zip(result.x[::2], result.x[1::2])] anchors = sorted(anchors, key=lambda x: x[0] * x[1]) return anchors @staticmethod def avg_iou_cost(anchor_params, bboxes): assert len(anchor_params) % 2 == 0 anchor_whs = torch.tensor( [[w, h] for w, h in zip(anchor_params[::2], anchor_params[1::2])]).to( bboxes.device, dtype=bboxes.dtype) anchor_boxes = bbox_cxcywh_to_xyxy( torch.cat([torch.zeros_like(anchor_whs), anchor_whs], dim=1)) ious = bbox_overlaps(bboxes, anchor_boxes) max_ious, _ = ious.max(1) cost = 1 - max_ious.mean().item() return cost def main(): logger = MMLogger.get_current_instance() args = parse_args() cfg = args.config cfg = Config.fromfile(cfg) init_default_scope(cfg.get('default_scope', 'mmdet')) # replace the ${key} with the value of cfg.key cfg = replace_cfg_vals(cfg) # update data root according to MMDET_DATASETS update_data_root(cfg) input_shape = args.input_shape assert len(input_shape) == 2 anchor_type = cfg.model.bbox_head.anchor_generator.type assert anchor_type == 'YOLOAnchorGenerator', \ f'Only support optimize YOLOAnchor, but get {anchor_type}.' base_sizes = cfg.model.bbox_head.anchor_generator.base_sizes num_anchors = sum([len(sizes) for sizes in base_sizes]) train_data_cfg = cfg.train_dataloader while 'dataset' in train_data_cfg: train_data_cfg = train_data_cfg['dataset'] dataset = DATASETS.build(train_data_cfg) if args.algorithm == 'k-means': optimizer = YOLOKMeansAnchorOptimizer( dataset=dataset, input_shape=input_shape, device=args.device, num_anchors=num_anchors, iters=args.iters, logger=logger, out_dir=args.output_dir) elif args.algorithm == 'differential_evolution': optimizer = YOLODEAnchorOptimizer( dataset=dataset, input_shape=input_shape, device=args.device, num_anchors=num_anchors, iters=args.iters, logger=logger, out_dir=args.output_dir) else: raise NotImplementedError( f'Only support k-means and differential_evolution, ' f'but get {args.algorithm}') optimizer.optimize() if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/robustness_eval.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import os.path as osp from argparse import ArgumentParser import numpy as np from mmengine.fileio import load def print_coco_results(results): def _print(result, ap=1, iouThr=None, areaRng='all', maxDets=100): titleStr = 'Average Precision' if ap == 1 else 'Average Recall' typeStr = '(AP)' if ap == 1 else '(AR)' iouStr = '0.50:0.95' \ if iouThr is None else f'{iouThr:0.2f}' iStr = f' {titleStr:<18} {typeStr} @[ IoU={iouStr:<9} | ' iStr += f'area={areaRng:>6s} | maxDets={maxDets:>3d} ] = {result:0.3f}' print(iStr) stats = np.zeros((12, )) stats[0] = _print(results[0], 1) stats[1] = _print(results[1], 1, iouThr=.5) stats[2] = _print(results[2], 1, iouThr=.75) stats[3] = _print(results[3], 1, areaRng='small') stats[4] = _print(results[4], 1, areaRng='medium') stats[5] = _print(results[5], 1, areaRng='large') # TODO support recall metric ''' stats[6] = _print(results[6], 0, maxDets=1) stats[7] = _print(results[7], 0, maxDets=10) stats[8] = _print(results[8], 0) stats[9] = _print(results[9], 0, areaRng='small') stats[10] = _print(results[10], 0, areaRng='medium') stats[11] = _print(results[11], 0, areaRng='large') ''' def get_coco_style_results(filename, task='bbox', metric=None, prints='mPC', aggregate='benchmark'): assert aggregate in ['benchmark', 'all'] if prints == 'all': prints = ['P', 'mPC', 'rPC'] elif isinstance(prints, str): prints = [prints] for p in prints: assert p in ['P', 'mPC', 'rPC'] if metric is None: metrics = [ 'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l', ] elif isinstance(metric, list): metrics = metric else: metrics = [metric] for metric_name in metrics: assert metric_name in [ 'mAP', 'mAP_50', 'mAP_75', 'mAP_s', 'mAP_m', 'mAP_l' ] eval_output = load(filename) num_distortions = len(list(eval_output.keys())) results = np.zeros((num_distortions, 6, len(metrics)), dtype='float32') for corr_i, distortion in enumerate(eval_output): for severity in eval_output[distortion]: for metric_j, metric_name in enumerate(metrics): metric_dict = eval_output[distortion][severity] new_metric_dict = {} for k, v in metric_dict.items(): if '/' in k: new_metric_dict[k.split('/')[-1]] = v mAP = new_metric_dict['_'.join((task, metric_name))] results[corr_i, severity, metric_j] = mAP P = results[0, 0, :] if aggregate == 'benchmark': mPC = np.mean(results[:15, 1:, :], axis=(0, 1)) else: mPC = np.mean(results[:, 1:, :], axis=(0, 1)) rPC = mPC / P print(f'\nmodel: {osp.basename(filename)}') if metric is None: if 'P' in prints: print(f'Performance on Clean Data [P] ({task})') print_coco_results(P) if 'mPC' in prints: print(f'Mean Performance under Corruption [mPC] ({task})') print_coco_results(mPC) if 'rPC' in prints: print(f'Relative Performance under Corruption [rPC] ({task})') print_coco_results(rPC) else: if 'P' in prints: print(f'Performance on Clean Data [P] ({task})') for metric_i, metric_name in enumerate(metrics): print(f'{metric_name:5} = {P[metric_i]:0.3f}') if 'mPC' in prints: print(f'Mean Performance under Corruption [mPC] ({task})') for metric_i, metric_name in enumerate(metrics): print(f'{metric_name:5} = {mPC[metric_i]:0.3f}') if 'rPC' in prints: print(f'Relative Performance under Corruption [rPC] ({task})') for metric_i, metric_name in enumerate(metrics): print(f'{metric_name:5} => {rPC[metric_i] * 100:0.1f} %') return results def get_voc_style_results(filename, prints='mPC', aggregate='benchmark'): assert aggregate in ['benchmark', 'all'] if prints == 'all': prints = ['P', 'mPC', 'rPC'] elif isinstance(prints, str): prints = [prints] for p in prints: assert p in ['P', 'mPC', 'rPC'] eval_output = load(filename) num_distortions = len(list(eval_output.keys())) results = np.zeros((num_distortions, 6, 20), dtype='float32') for i, distortion in enumerate(eval_output): for severity in eval_output[distortion]: mAP = [ eval_output[distortion][severity][j]['ap'] for j in range(len(eval_output[distortion][severity])) ] results[i, severity, :] = mAP P = results[0, 0, :] if aggregate == 'benchmark': mPC = np.mean(results[:15, 1:, :], axis=(0, 1)) else: mPC = np.mean(results[:, 1:, :], axis=(0, 1)) rPC = mPC / P print(f'\nmodel: {osp.basename(filename)}') if 'P' in prints: print(f'Performance on Clean Data [P] in AP50 = {np.mean(P):0.3f}') if 'mPC' in prints: print('Mean Performance under Corruption [mPC] in AP50 = ' f'{np.mean(mPC):0.3f}') if 'rPC' in prints: print('Relative Performance under Corruption [rPC] in % = ' f'{np.mean(rPC) * 100:0.1f}') return np.mean(results, axis=2, keepdims=True) def get_results(filename, dataset='coco', task='bbox', metric=None, prints='mPC', aggregate='benchmark'): assert dataset in ['coco', 'voc', 'cityscapes'] if dataset in ['coco', 'cityscapes']: results = get_coco_style_results( filename, task=task, metric=metric, prints=prints, aggregate=aggregate) elif dataset == 'voc': if task != 'bbox': print('Only bbox analysis is supported for Pascal VOC') print('Will report bbox results\n') if metric not in [None, ['AP'], ['AP50']]: print('Only the AP50 metric is supported for Pascal VOC') print('Will report AP50 metric\n') results = get_voc_style_results( filename, prints=prints, aggregate=aggregate) return results def get_distortions_from_file(filename): eval_output = load(filename) return get_distortions_from_results(eval_output) def get_distortions_from_results(eval_output): distortions = [] for i, distortion in enumerate(eval_output): distortions.append(distortion.replace('_', ' ')) return distortions def main(): parser = ArgumentParser(description='Corruption Result Analysis') parser.add_argument('filename', help='result file path') parser.add_argument( '--dataset', type=str, choices=['coco', 'voc', 'cityscapes'], default='coco', help='dataset type') parser.add_argument( '--task', type=str, nargs='+', choices=['bbox', 'segm'], default=['bbox'], help='task to report') parser.add_argument( '--metric', nargs='+', choices=[ None, 'AP', 'AP50', 'AP75', 'APs', 'APm', 'APl', 'AR1', 'AR10', 'AR100', 'ARs', 'ARm', 'ARl' ], default=None, help='metric to report') parser.add_argument( '--prints', type=str, nargs='+', choices=['P', 'mPC', 'rPC'], default='mPC', help='corruption benchmark metric to print') parser.add_argument( '--aggregate', type=str, choices=['all', 'benchmark'], default='benchmark', help='aggregate all results or only those \ for benchmark corruptions') args = parser.parse_args() for task in args.task: get_results( args.filename, dataset=args.dataset, task=task, metric=args.metric, prints=args.prints, aggregate=args.aggregate) if __name__ == '__main__': main() ================================================ FILE: tools/analysis_tools/test_robustness.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import copy import os import os.path as osp from mmengine.config import Config, DictAction from mmengine.dist import get_dist_info from mmengine.evaluator import DumpResults from mmengine.fileio import dump from mmengine.runner import Runner from mmdet.engine.hooks.utils import trigger_visualization_hook from mmdet.registry import RUNNERS from tools.analysis_tools.robustness_eval import get_results def parse_args(): parser = argparse.ArgumentParser(description='MMDet test detector') parser.add_argument('config', help='test config file path') parser.add_argument('checkpoint', help='checkpoint file') parser.add_argument( '--out', type=str, help='dump predictions to a pickle file for offline evaluation') parser.add_argument( '--corruptions', type=str, nargs='+', default='benchmark', choices=[ 'all', 'benchmark', 'noise', 'blur', 'weather', 'digital', 'holdout', 'None', 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', 'brightness', 'contrast', 'elastic_transform', 'pixelate', 'jpeg_compression', 'speckle_noise', 'gaussian_blur', 'spatter', 'saturate' ], help='corruptions') parser.add_argument( '--work-dir', help='the directory to save the file containing evaluation metrics') parser.add_argument( '--severities', type=int, nargs='+', default=[0, 1, 2, 3, 4, 5], help='corruption severity levels') parser.add_argument( '--summaries', type=bool, default=False, help='Print summaries for every corruption and severity') parser.add_argument('--show', action='store_true', help='show results') parser.add_argument( '--show-dir', help='directory where painted images will be saved') parser.add_argument( '--wait-time', type=float, default=2, help='the interval of show (s)') parser.add_argument('--seed', type=int, default=None, help='random seed') parser.add_argument( '--launcher', choices=['none', 'pytorch', 'slurm', 'mpi'], default='none', help='job launcher') parser.add_argument('--local_rank', type=int, default=0) parser.add_argument( '--final-prints', type=str, nargs='+', choices=['P', 'mPC', 'rPC'], default='mPC', help='corruption benchmark metric to print at the end') parser.add_argument( '--final-prints-aggregate', type=str, choices=['all', 'benchmark'], default='benchmark', help='aggregate all results or only those for benchmark corruptions') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') args = parser.parse_args() if 'LOCAL_RANK' not in os.environ: os.environ['LOCAL_RANK'] = str(args.local_rank) return args def main(): args = parse_args() assert args.out or args.show or args.show_dir, \ ('Please specify at least one operation (save or show the results) ' 'with the argument "--out", "--show" or "show-dir"') # load config cfg = Config.fromfile(args.config) cfg.launcher = args.launcher if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) # work_dir is determined in this priority: CLI > segment in file > filename if args.work_dir is not None: # update configs according to CLI args if args.work_dir is not None cfg.work_dir = args.work_dir elif cfg.get('work_dir', None) is None: # use config filename as default work_dir if cfg.work_dir is None cfg.work_dir = osp.join('./work_dirs', osp.splitext(osp.basename(args.config))[0]) cfg.model.backbone.init_cfg.type = None cfg.test_dataloader.dataset.test_mode = True cfg.load_from = args.checkpoint if args.show or args.show_dir: cfg = trigger_visualization_hook(cfg, args) # build the runner from config if 'runner_type' not in cfg: # build the default runner runner = Runner.from_cfg(cfg) else: # build customized runner from the registry # if 'runner_type' is set in the cfg runner = RUNNERS.build(cfg) # add `DumpResults` dummy metric if args.out is not None: assert args.out.endswith(('.pkl', '.pickle')), \ 'The dump file must be a pkl file.' runner.test_evaluator.metrics.append( DumpResults(out_file_path=args.out)) if 'all' in args.corruptions: corruptions = [ 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', 'brightness', 'contrast', 'elastic_transform', 'pixelate', 'jpeg_compression', 'speckle_noise', 'gaussian_blur', 'spatter', 'saturate' ] elif 'benchmark' in args.corruptions: corruptions = [ 'gaussian_noise', 'shot_noise', 'impulse_noise', 'defocus_blur', 'glass_blur', 'motion_blur', 'zoom_blur', 'snow', 'frost', 'fog', 'brightness', 'contrast', 'elastic_transform', 'pixelate', 'jpeg_compression' ] elif 'noise' in args.corruptions: corruptions = ['gaussian_noise', 'shot_noise', 'impulse_noise'] elif 'blur' in args.corruptions: corruptions = [ 'defocus_blur', 'glass_blur', 'motion_blur', 'zoom_blur' ] elif 'weather' in args.corruptions: corruptions = ['snow', 'frost', 'fog', 'brightness'] elif 'digital' in args.corruptions: corruptions = [ 'contrast', 'elastic_transform', 'pixelate', 'jpeg_compression' ] elif 'holdout' in args.corruptions: corruptions = ['speckle_noise', 'gaussian_blur', 'spatter', 'saturate'] elif 'None' in args.corruptions: corruptions = ['None'] args.severities = [0] else: corruptions = args.corruptions aggregated_results = {} for corr_i, corruption in enumerate(corruptions): aggregated_results[corruption] = {} for sev_i, corruption_severity in enumerate(args.severities): # evaluate severity 0 (= no corruption) only once if corr_i > 0 and corruption_severity == 0: aggregated_results[corruption][0] = \ aggregated_results[corruptions[0]][0] continue test_loader_cfg = copy.deepcopy(cfg.test_dataloader) # assign corruption and severity if corruption_severity > 0: corruption_trans = dict( type='Corrupt', corruption=corruption, severity=corruption_severity) # TODO: hard coded "1", we assume that the first step is # loading images, which needs to be fixed in the future test_loader_cfg.dataset.pipeline.insert(1, corruption_trans) test_loader = runner.build_dataloader(test_loader_cfg) runner.test_loop.dataloader = test_loader # set random seeds if args.seed is not None: runner.set_randomness(args.seed) # print info print(f'\nTesting {corruption} at severity {corruption_severity}') eval_results = runner.test() if args.out: eval_results_filename = ( osp.splitext(args.out)[0] + '_results' + osp.splitext(args.out)[1]) aggregated_results[corruption][ corruption_severity] = eval_results dump(aggregated_results, eval_results_filename) rank, _ = get_dist_info() if rank == 0: eval_results_filename = ( osp.splitext(args.out)[0] + '_results' + osp.splitext(args.out)[1]) # print final results print('\nAggregated results:') prints = args.final_prints aggregate = args.final_prints_aggregate if cfg.dataset_type == 'VOCDataset': get_results( eval_results_filename, dataset='voc', prints=prints, aggregate=aggregate) else: get_results( eval_results_filename, dataset='coco', prints=prints, aggregate=aggregate) if __name__ == '__main__': main() ================================================ FILE: tools/dataset_converters/cityscapes.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import glob import os.path as osp import cityscapesscripts.helpers.labels as CSLabels import mmcv import numpy as np import pycocotools.mask as maskUtils from mmengine.fileio import dump from mmengine.utils import (Timer, mkdir_or_exist, track_parallel_progress, track_progress) def collect_files(img_dir, gt_dir): suffix = 'leftImg8bit.png' files = [] for img_file in glob.glob(osp.join(img_dir, '**/*.png')): assert img_file.endswith(suffix), img_file inst_file = gt_dir + img_file[ len(img_dir):-len(suffix)] + 'gtFine_instanceIds.png' # Note that labelIds are not converted to trainId for seg map segm_file = gt_dir + img_file[ len(img_dir):-len(suffix)] + 'gtFine_labelIds.png' files.append((img_file, inst_file, segm_file)) assert len(files), f'No images found in {img_dir}' print(f'Loaded {len(files)} images from {img_dir}') return files def collect_annotations(files, nproc=1): print('Loading annotation images') if nproc > 1: images = track_parallel_progress(load_img_info, files, nproc=nproc) else: images = track_progress(load_img_info, files) return images def load_img_info(files): img_file, inst_file, segm_file = files inst_img = mmcv.imread(inst_file, 'unchanged') # ids < 24 are stuff labels (filtering them first is about 5% faster) unique_inst_ids = np.unique(inst_img[inst_img >= 24]) anno_info = [] for inst_id in unique_inst_ids: # For non-crowd annotations, inst_id // 1000 is the label_id # Crowd annotations have <1000 instance ids label_id = inst_id // 1000 if inst_id >= 1000 else inst_id label = CSLabels.id2label[label_id] if not label.hasInstances or label.ignoreInEval: continue category_id = label.id iscrowd = int(inst_id < 1000) mask = np.asarray(inst_img == inst_id, dtype=np.uint8, order='F') mask_rle = maskUtils.encode(mask[:, :, None])[0] area = maskUtils.area(mask_rle) # convert to COCO style XYWH format bbox = maskUtils.toBbox(mask_rle) # for json encoding mask_rle['counts'] = mask_rle['counts'].decode() anno = dict( iscrowd=iscrowd, category_id=category_id, bbox=bbox.tolist(), area=area.tolist(), segmentation=mask_rle) anno_info.append(anno) video_name = osp.basename(osp.dirname(img_file)) img_info = dict( # remove img_prefix for filename file_name=osp.join(video_name, osp.basename(img_file)), height=inst_img.shape[0], width=inst_img.shape[1], anno_info=anno_info, segm_file=osp.join(video_name, osp.basename(segm_file))) return img_info def cvt_annotations(image_infos, out_json_name): out_json = dict() img_id = 0 ann_id = 0 out_json['images'] = [] out_json['categories'] = [] out_json['annotations'] = [] for image_info in image_infos: image_info['id'] = img_id anno_infos = image_info.pop('anno_info') out_json['images'].append(image_info) for anno_info in anno_infos: anno_info['image_id'] = img_id anno_info['id'] = ann_id out_json['annotations'].append(anno_info) ann_id += 1 img_id += 1 for label in CSLabels.labels: if label.hasInstances and not label.ignoreInEval: cat = dict(id=label.id, name=label.name) out_json['categories'].append(cat) if len(out_json['annotations']) == 0: out_json.pop('annotations') dump(out_json, out_json_name) return out_json def parse_args(): parser = argparse.ArgumentParser( description='Convert Cityscapes annotations to COCO format') parser.add_argument('cityscapes_path', help='cityscapes data path') parser.add_argument('--img-dir', default='leftImg8bit', type=str) parser.add_argument('--gt-dir', default='gtFine', type=str) parser.add_argument('-o', '--out-dir', help='output path') parser.add_argument( '--nproc', default=1, type=int, help='number of process') args = parser.parse_args() return args def main(): args = parse_args() cityscapes_path = args.cityscapes_path out_dir = args.out_dir if args.out_dir else cityscapes_path mkdir_or_exist(out_dir) img_dir = osp.join(cityscapes_path, args.img_dir) gt_dir = osp.join(cityscapes_path, args.gt_dir) set_name = dict( train='instancesonly_filtered_gtFine_train.json', val='instancesonly_filtered_gtFine_val.json', test='instancesonly_filtered_gtFine_test.json') for split, json_name in set_name.items(): print(f'Converting {split} into {json_name}') with Timer(print_tmpl='It took {}s to convert Cityscapes annotation'): files = collect_files( osp.join(img_dir, split), osp.join(gt_dir, split)) image_infos = collect_annotations(files, nproc=args.nproc) cvt_annotations(image_infos, osp.join(out_dir, json_name)) if __name__ == '__main__': main() ================================================ FILE: tools/dataset_converters/images2coco.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os from mmengine.fileio import dump, list_from_file from mmengine.utils import mkdir_or_exist, scandir, track_iter_progress from PIL import Image def parse_args(): parser = argparse.ArgumentParser( description='Convert images to coco format without annotations') parser.add_argument('img_path', help='The root path of images') parser.add_argument( 'classes', type=str, help='The text file name of storage class list') parser.add_argument( 'out', type=str, help='The output annotation json file name, The save dir is in the ' 'same directory as img_path') parser.add_argument( '-e', '--exclude-extensions', type=str, nargs='+', help='The suffix of images to be excluded, such as "png" and "bmp"') args = parser.parse_args() return args def collect_image_infos(path, exclude_extensions=None): img_infos = [] images_generator = scandir(path, recursive=True) for image_path in track_iter_progress(list(images_generator)): if exclude_extensions is None or ( exclude_extensions is not None and not image_path.lower().endswith(exclude_extensions)): image_path = os.path.join(path, image_path) img_pillow = Image.open(image_path) img_info = { 'filename': image_path, 'width': img_pillow.width, 'height': img_pillow.height, } img_infos.append(img_info) return img_infos def cvt_to_coco_json(img_infos, classes): image_id = 0 coco = dict() coco['images'] = [] coco['type'] = 'instance' coco['categories'] = [] coco['annotations'] = [] image_set = set() for category_id, name in enumerate(classes): category_item = dict() category_item['supercategory'] = str('none') category_item['id'] = int(category_id) category_item['name'] = str(name) coco['categories'].append(category_item) for img_dict in img_infos: file_name = img_dict['filename'] assert file_name not in image_set image_item = dict() image_item['id'] = int(image_id) image_item['file_name'] = str(file_name) image_item['height'] = int(img_dict['height']) image_item['width'] = int(img_dict['width']) coco['images'].append(image_item) image_set.add(file_name) image_id += 1 return coco def main(): args = parse_args() assert args.out.endswith( 'json'), 'The output file name must be json suffix' # 1 load image list info img_infos = collect_image_infos(args.img_path, args.exclude_extensions) # 2 convert to coco format data classes = list_from_file(args.classes) coco_info = cvt_to_coco_json(img_infos, classes) # 3 dump save_dir = os.path.join(args.img_path, '..', 'annotations') mkdir_or_exist(save_dir) save_path = os.path.join(save_dir, args.out) dump(coco_info, save_path) print(f'save json file: {save_path}') if __name__ == '__main__': main() ================================================ FILE: tools/dataset_converters/pascal_voc.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os.path as osp import xml.etree.ElementTree as ET import numpy as np from mmengine.fileio import dump, list_from_file from mmengine.utils import mkdir_or_exist, track_progress from mmdet.evaluation import voc_classes label_ids = {name: i for i, name in enumerate(voc_classes())} def parse_xml(args): xml_path, img_path = args tree = ET.parse(xml_path) root = tree.getroot() size = root.find('size') w = int(size.find('width').text) h = int(size.find('height').text) bboxes = [] labels = [] bboxes_ignore = [] labels_ignore = [] for obj in root.findall('object'): name = obj.find('name').text label = label_ids[name] difficult = int(obj.find('difficult').text) bnd_box = obj.find('bndbox') bbox = [ int(bnd_box.find('xmin').text), int(bnd_box.find('ymin').text), int(bnd_box.find('xmax').text), int(bnd_box.find('ymax').text) ] if difficult: bboxes_ignore.append(bbox) labels_ignore.append(label) else: bboxes.append(bbox) labels.append(label) if not bboxes: bboxes = np.zeros((0, 4)) labels = np.zeros((0, )) else: bboxes = np.array(bboxes, ndmin=2) - 1 labels = np.array(labels) if not bboxes_ignore: bboxes_ignore = np.zeros((0, 4)) labels_ignore = np.zeros((0, )) else: bboxes_ignore = np.array(bboxes_ignore, ndmin=2) - 1 labels_ignore = np.array(labels_ignore) annotation = { 'filename': img_path, 'width': w, 'height': h, 'ann': { 'bboxes': bboxes.astype(np.float32), 'labels': labels.astype(np.int64), 'bboxes_ignore': bboxes_ignore.astype(np.float32), 'labels_ignore': labels_ignore.astype(np.int64) } } return annotation def cvt_annotations(devkit_path, years, split, out_file): if not isinstance(years, list): years = [years] annotations = [] for year in years: filelist = osp.join(devkit_path, f'VOC{year}/ImageSets/Main/{split}.txt') if not osp.isfile(filelist): print(f'filelist does not exist: {filelist}, ' f'skip voc{year} {split}') return img_names = list_from_file(filelist) xml_paths = [ osp.join(devkit_path, f'VOC{year}/Annotations/{img_name}.xml') for img_name in img_names ] img_paths = [ f'VOC{year}/JPEGImages/{img_name}.jpg' for img_name in img_names ] part_annotations = track_progress(parse_xml, list(zip(xml_paths, img_paths))) annotations.extend(part_annotations) if out_file.endswith('json'): annotations = cvt_to_coco_json(annotations) dump(annotations, out_file) return annotations def cvt_to_coco_json(annotations): image_id = 0 annotation_id = 0 coco = dict() coco['images'] = [] coco['type'] = 'instance' coco['categories'] = [] coco['annotations'] = [] image_set = set() def addAnnItem(annotation_id, image_id, category_id, bbox, difficult_flag): annotation_item = dict() annotation_item['segmentation'] = [] seg = [] # bbox[] is x1,y1,x2,y2 # left_top seg.append(int(bbox[0])) seg.append(int(bbox[1])) # left_bottom seg.append(int(bbox[0])) seg.append(int(bbox[3])) # right_bottom seg.append(int(bbox[2])) seg.append(int(bbox[3])) # right_top seg.append(int(bbox[2])) seg.append(int(bbox[1])) annotation_item['segmentation'].append(seg) xywh = np.array( [bbox[0], bbox[1], bbox[2] - bbox[0], bbox[3] - bbox[1]]) annotation_item['area'] = int(xywh[2] * xywh[3]) if difficult_flag == 1: annotation_item['ignore'] = 0 annotation_item['iscrowd'] = 1 else: annotation_item['ignore'] = 0 annotation_item['iscrowd'] = 0 annotation_item['image_id'] = int(image_id) annotation_item['bbox'] = xywh.astype(int).tolist() annotation_item['category_id'] = int(category_id) annotation_item['id'] = int(annotation_id) coco['annotations'].append(annotation_item) return annotation_id + 1 for category_id, name in enumerate(voc_classes()): category_item = dict() category_item['supercategory'] = str('none') category_item['id'] = int(category_id) category_item['name'] = str(name) coco['categories'].append(category_item) for ann_dict in annotations: file_name = ann_dict['filename'] ann = ann_dict['ann'] assert file_name not in image_set image_item = dict() image_item['id'] = int(image_id) image_item['file_name'] = str(file_name) image_item['height'] = int(ann_dict['height']) image_item['width'] = int(ann_dict['width']) coco['images'].append(image_item) image_set.add(file_name) bboxes = ann['bboxes'][:, :4] labels = ann['labels'] for bbox_id in range(len(bboxes)): bbox = bboxes[bbox_id] label = labels[bbox_id] annotation_id = addAnnItem( annotation_id, image_id, label, bbox, difficult_flag=0) bboxes_ignore = ann['bboxes_ignore'][:, :4] labels_ignore = ann['labels_ignore'] for bbox_id in range(len(bboxes_ignore)): bbox = bboxes_ignore[bbox_id] label = labels_ignore[bbox_id] annotation_id = addAnnItem( annotation_id, image_id, label, bbox, difficult_flag=1) image_id += 1 return coco def parse_args(): parser = argparse.ArgumentParser( description='Convert PASCAL VOC annotations to mmdetection format') parser.add_argument('devkit_path', help='pascal voc devkit path') parser.add_argument('-o', '--out-dir', help='output path') parser.add_argument( '--out-format', default='pkl', choices=('pkl', 'coco'), help='output format, "coco" indicates coco annotation format') args = parser.parse_args() return args def main(): args = parse_args() devkit_path = args.devkit_path out_dir = args.out_dir if args.out_dir else devkit_path mkdir_or_exist(out_dir) years = [] if osp.isdir(osp.join(devkit_path, 'VOC2007')): years.append('2007') if osp.isdir(osp.join(devkit_path, 'VOC2012')): years.append('2012') if '2007' in years and '2012' in years: years.append(['2007', '2012']) if not years: raise IOError(f'The devkit path {devkit_path} contains neither ' '"VOC2007" nor "VOC2012" subfolder') out_fmt = f'.{args.out_format}' if args.out_format == 'coco': out_fmt = '.json' for year in years: if year == '2007': prefix = 'voc07' elif year == '2012': prefix = 'voc12' elif year == ['2007', '2012']: prefix = 'voc0712' for split in ['train', 'val', 'trainval']: dataset_name = prefix + '_' + split print(f'processing {dataset_name} ...') cvt_annotations(devkit_path, year, split, osp.join(out_dir, dataset_name + out_fmt)) if not isinstance(year, list): dataset_name = prefix + '_test' print(f'processing {dataset_name} ...') cvt_annotations(devkit_path, year, 'test', osp.join(out_dir, dataset_name + out_fmt)) print('Done!') if __name__ == '__main__': main() ================================================ FILE: tools/deployment/mmdet2torchserve.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. from argparse import ArgumentParser, Namespace from pathlib import Path from tempfile import TemporaryDirectory from mmengine.config import Config from mmengine.utils import mkdir_or_exist try: from model_archiver.model_packaging import package_model from model_archiver.model_packaging_utils import ModelExportUtils except ImportError: package_model = None def mmdet2torchserve( config_file: str, checkpoint_file: str, output_folder: str, model_name: str, model_version: str = '1.0', force: bool = False, ): """Converts MMDetection model (config + checkpoint) to TorchServe `.mar`. Args: config_file: In MMDetection config format. The contents vary for each task repository. checkpoint_file: In MMDetection checkpoint format. The contents vary for each task repository. output_folder: Folder where `{model_name}.mar` will be created. The file created will be in TorchServe archive format. model_name: If not None, used for naming the `{model_name}.mar` file that will be created under `output_folder`. If None, `{Path(checkpoint_file).stem}` will be used. model_version: Model's version. force: If True, if there is an existing `{model_name}.mar` file under `output_folder` it will be overwritten. """ mkdir_or_exist(output_folder) config = Config.fromfile(config_file) with TemporaryDirectory() as tmpdir: config.dump(f'{tmpdir}/config.py') args = Namespace( **{ 'model_file': f'{tmpdir}/config.py', 'serialized_file': checkpoint_file, 'handler': f'{Path(__file__).parent}/mmdet_handler.py', 'model_name': model_name or Path(checkpoint_file).stem, 'version': model_version, 'export_path': output_folder, 'force': force, 'requirements_file': None, 'extra_files': None, 'runtime': 'python', 'archive_format': 'default' }) manifest = ModelExportUtils.generate_manifest_json(args) package_model(args, manifest) def parse_args(): parser = ArgumentParser( description='Convert MMDetection models to TorchServe `.mar` format.') parser.add_argument('config', type=str, help='config file path') parser.add_argument('checkpoint', type=str, help='checkpoint file path') parser.add_argument( '--output-folder', type=str, required=True, help='Folder where `{model_name}.mar` will be created.') parser.add_argument( '--model-name', type=str, default=None, help='If not None, used for naming the `{model_name}.mar`' 'file that will be created under `output_folder`.' 'If None, `{Path(checkpoint_file).stem}` will be used.') parser.add_argument( '--model-version', type=str, default='1.0', help='Number used for versioning.') parser.add_argument( '-f', '--force', action='store_true', help='overwrite the existing `{model_name}.mar`') args = parser.parse_args() return args if __name__ == '__main__': args = parse_args() if package_model is None: raise ImportError('`torch-model-archiver` is required.' 'Try: pip install torch-model-archiver') mmdet2torchserve(args.config, args.checkpoint, args.output_folder, args.model_name, args.model_version, args.force) ================================================ FILE: tools/deployment/mmdet_handler.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import base64 import os import mmcv import numpy as np import torch from ts.torch_handler.base_handler import BaseHandler from mmdet.apis import inference_detector, init_detector class MMdetHandler(BaseHandler): threshold = 0.5 def initialize(self, context): properties = context.system_properties self.map_location = 'cuda' if torch.cuda.is_available() else 'cpu' self.device = torch.device(self.map_location + ':' + str(properties.get('gpu_id')) if torch.cuda. is_available() else self.map_location) self.manifest = context.manifest model_dir = properties.get('model_dir') serialized_file = self.manifest['model']['serializedFile'] checkpoint = os.path.join(model_dir, serialized_file) self.config_file = os.path.join(model_dir, 'config.py') self.model = init_detector(self.config_file, checkpoint, self.device) self.initialized = True def preprocess(self, data): images = [] for row in data: image = row.get('data') or row.get('body') if isinstance(image, str): image = base64.b64decode(image) image = mmcv.imfrombytes(image) images.append(image) return images def inference(self, data, *args, **kwargs): results = inference_detector(self.model, data) return results def postprocess(self, data): # Format output following the example ObjectDetectionHandler format output = [] for data_sample in data: pred_instances = data_sample.pred_instances bboxes = pred_instances.bboxes.cpu().numpy().astype( np.float32).tolist() labels = pred_instances.labels.cpu().numpy().astype( np.int32).tolist() scores = pred_instances.scores.cpu().numpy().astype( np.float32).tolist() preds = [] for idx in range(len(labels)): cls_score, bbox, cls_label = scores[idx], bboxes[idx], labels[ idx] if cls_score >= self.threshold: class_name = self.model.dataset_meta['classes'][cls_label] result = dict( class_label=cls_label, class_name=class_name, bbox=bbox, score=cls_score) preds.append(result) output.append(preds) return output ================================================ FILE: tools/deployment/test_torchserver.py ================================================ import os from argparse import ArgumentParser import mmcv import requests import torch from mmengine.structures import InstanceData from mmdet.apis import inference_detector, init_detector from mmdet.registry import VISUALIZERS from mmdet.structures import DetDataSample def parse_args(): parser = ArgumentParser() parser.add_argument('img', help='Image file') parser.add_argument('config', help='Config file') parser.add_argument('checkpoint', help='Checkpoint file') parser.add_argument('model_name', help='The model name in the server') parser.add_argument( '--inference-addr', default='127.0.0.1:8080', help='Address and port of the inference server') parser.add_argument( '--device', default='cuda:0', help='Device used for inference') parser.add_argument( '--score-thr', type=float, default=0.5, help='bbox score threshold') parser.add_argument( '--work-dir', type=str, default=None, help='output directory to save drawn results.') args = parser.parse_args() return args def align_ts_output(inputs, metainfo, device): bboxes = [] labels = [] scores = [] for i, pred in enumerate(inputs): bboxes.append(pred['bbox']) labels.append(pred['class_label']) scores.append(pred['score']) pred_instances = InstanceData(metainfo=metainfo) pred_instances.bboxes = torch.tensor( bboxes, dtype=torch.float32, device=device) pred_instances.labels = torch.tensor( labels, dtype=torch.int64, device=device) pred_instances.scores = torch.tensor( scores, dtype=torch.float32, device=device) ts_data_sample = DetDataSample(pred_instances=pred_instances) return ts_data_sample def main(args): # build the model from a config file and a checkpoint file model = init_detector(args.config, args.checkpoint, device=args.device) # test a single image pytorch_results = inference_detector(model, args.img) keep = pytorch_results.pred_instances.scores >= args.score_thr pytorch_results.pred_instances = pytorch_results.pred_instances[keep] # init visualizer visualizer = VISUALIZERS.build(model.cfg.visualizer) # the dataset_meta is loaded from the checkpoint and # then pass to the model in init_detector visualizer.dataset_meta = model.dataset_meta # show the results img = mmcv.imread(args.img) img = mmcv.imconvert(img, 'bgr', 'rgb') pt_out_file = None ts_out_file = None if args.work_dir is not None: os.makedirs(args.work_dir, exist_ok=True) pt_out_file = os.path.join(args.work_dir, 'pytorch_result.png') ts_out_file = os.path.join(args.work_dir, 'torchserve_result.png') visualizer.add_datasample( 'pytorch_result', img.copy(), data_sample=pytorch_results, draw_gt=False, out_file=pt_out_file, show=True, wait_time=0) url = 'http://' + args.inference_addr + '/predictions/' + args.model_name with open(args.img, 'rb') as image: response = requests.post(url, image) metainfo = pytorch_results.pred_instances.metainfo ts_results = align_ts_output(response.json(), metainfo, args.device) visualizer.add_datasample( 'torchserve_result', img, data_sample=ts_results, draw_gt=False, out_file=ts_out_file, show=True, wait_time=0) assert torch.allclose(pytorch_results.pred_instances.bboxes, ts_results.pred_instances.bboxes) assert torch.allclose(pytorch_results.pred_instances.labels, ts_results.pred_instances.labels) assert torch.allclose(pytorch_results.pred_instances.scores, ts_results.pred_instances.scores) if __name__ == '__main__': args = parse_args() main(args) ================================================ FILE: tools/dist_test.sh ================================================ #!/usr/bin/env bash CONFIG=$1 CHECKPOINT=$2 GPUS=$3 NNODES=${NNODES:-1} NODE_RANK=${NODE_RANK:-0} PORT=${PORT:-29500} MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ python -m torch.distributed.launch \ --nnodes=$NNODES \ --node_rank=$NODE_RANK \ --master_addr=$MASTER_ADDR \ --nproc_per_node=$GPUS \ --master_port=$PORT \ $(dirname "$0")/test.py \ $CONFIG \ $CHECKPOINT \ --launcher pytorch \ ${@:4} ================================================ FILE: tools/dist_train.sh ================================================ #!/usr/bin/env bash CONFIG=$1 GPUS=$2 NNODES=${NNODES:-1} NODE_RANK=${NODE_RANK:-0} PORT=${PORT:-29500} MASTER_ADDR=${MASTER_ADDR:-"127.0.0.1"} PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ python -m torch.distributed.launch \ --nnodes=$NNODES \ --node_rank=$NODE_RANK \ --master_addr=$MASTER_ADDR \ --nproc_per_node=$GPUS \ --master_port=$PORT \ $(dirname "$0")/train.py \ $CONFIG \ --launcher pytorch ${@:3} ================================================ FILE: tools/misc/download_dataset.py ================================================ import argparse import tarfile from itertools import repeat from multiprocessing.pool import ThreadPool from pathlib import Path from tarfile import TarFile from zipfile import ZipFile import torch from mmengine.utils.path import mkdir_or_exist def parse_args(): parser = argparse.ArgumentParser( description='Download datasets for training') parser.add_argument( '--dataset-name', type=str, help='dataset name', default='coco2017') parser.add_argument( '--save-dir', type=str, help='the dir to save dataset', default='data/coco') parser.add_argument( '--unzip', action='store_true', help='whether unzip dataset or not, zipped files will be saved') parser.add_argument( '--delete', action='store_true', help='delete the download zipped files') parser.add_argument( '--threads', type=int, help='number of threading', default=4) args = parser.parse_args() return args def download(url, dir, unzip=True, delete=False, threads=1): def download_one(url, dir): f = dir / Path(url).name if Path(url).is_file(): Path(url).rename(f) elif not f.exists(): print(f'Downloading {url} to {f}') torch.hub.download_url_to_file(url, f, progress=True) if unzip and f.suffix in ('.zip', '.tar'): print(f'Unzipping {f.name}') if f.suffix == '.zip': ZipFile(f).extractall(path=dir) elif f.suffix == '.tar': TarFile(f).extractall(path=dir) if delete: f.unlink() print(f'Delete {f}') dir = Path(dir) if threads > 1: pool = ThreadPool(threads) pool.imap(lambda x: download_one(*x), zip(url, repeat(dir))) pool.close() pool.join() else: for u in [url] if isinstance(url, (str, Path)) else url: download_one(u, dir) def download_objects365v2(url, dir, unzip=True, delete=False, threads=1): def download_single(url, dir): if 'train' in url: saving_dir = dir / Path('train_zip') mkdir_or_exist(saving_dir) f = saving_dir / Path(url).name unzip_dir = dir / Path('train') mkdir_or_exist(unzip_dir) elif 'val' in url: saving_dir = dir / Path('val') mkdir_or_exist(saving_dir) f = saving_dir / Path(url).name unzip_dir = dir / Path('val') mkdir_or_exist(unzip_dir) else: raise NotImplementedError if Path(url).is_file(): Path(url).rename(f) elif not f.exists(): print(f'Downloading {url} to {f}') torch.hub.download_url_to_file(url, f, progress=True) if unzip and str(f).endswith('.tar.gz'): print(f'Unzipping {f.name}') tar = tarfile.open(f) tar.extractall(path=unzip_dir) if delete: f.unlink() print(f'Delete {f}') # process annotations full_url = [] for _url in url: if 'zhiyuan_objv2_train.tar.gz' in _url or \ 'zhiyuan_objv2_val.json' in _url: full_url.append(_url) elif 'train' in _url: for i in range(51): full_url.append(f'{_url}patch{i}.tar.gz') elif 'val/images/v1' in _url: for i in range(16): full_url.append(f'{_url}patch{i}.tar.gz') elif 'val/images/v2' in _url: for i in range(16, 44): full_url.append(f'{_url}patch{i}.tar.gz') else: raise NotImplementedError dir = Path(dir) if threads > 1: pool = ThreadPool(threads) pool.imap(lambda x: download_single(*x), zip(full_url, repeat(dir))) pool.close() pool.join() else: for u in full_url: download_single(u, dir) def main(): args = parse_args() path = Path(args.save_dir) if not path.exists(): path.mkdir(parents=True, exist_ok=True) data2url = dict( # TODO: Support for downloading Panoptic Segmentation of COCO coco2017=[ 'http://images.cocodataset.org/zips/train2017.zip', 'http://images.cocodataset.org/zips/val2017.zip', 'http://images.cocodataset.org/zips/test2017.zip', 'http://images.cocodataset.org/zips/unlabeled2017.zip', 'http://images.cocodataset.org/annotations/annotations_trainval2017.zip', # noqa 'http://images.cocodataset.org/annotations/stuff_annotations_trainval2017.zip', # noqa 'http://images.cocodataset.org/annotations/panoptic_annotations_trainval2017.zip', # noqa 'http://images.cocodataset.org/annotations/image_info_test2017.zip', # noqa 'http://images.cocodataset.org/annotations/image_info_unlabeled2017.zip', # noqa ], lvis=[ 'https://s3-us-west-2.amazonaws.com/dl.fbaipublicfiles.com/LVIS/lvis_v1_train.json.zip', # noqa 'https://s3-us-west-2.amazonaws.com/dl.fbaipublicfiles.com/LVIS/lvis_v1_train.json.zip', # noqa ], voc2007=[ 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtrainval_06-Nov-2007.tar', # noqa 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCtest_06-Nov-2007.tar', # noqa 'http://host.robots.ox.ac.uk/pascal/VOC/voc2007/VOCdevkit_08-Jun-2007.tar', # noqa ], # Note: There is no download link for Objects365-V1 right now. If you # would like to download Objects365-V1, please visit # http://www.objects365.org/ to concat the author. objects365v2=[ # training annotations 'https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/train/zhiyuan_objv2_train.tar.gz', # noqa # validation annotations 'https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/val/zhiyuan_objv2_val.json', # noqa # training url root 'https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/train/', # noqa # validation url root_1 'https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/val/images/v1/', # noqa # validation url root_2 'https://dorc.ks3-cn-beijing.ksyun.com/data-set/2020Objects365%E6%95%B0%E6%8D%AE%E9%9B%86/val/images/v2/' # noqa ]) url = data2url.get(args.dataset_name, None) if url is None: print('Only support COCO, VOC, LVIS, and Objects365v2 now!') return if args.dataset_name == 'objects365v2': download_objects365v2( url, dir=path, unzip=args.unzip, delete=args.delete, threads=args.threads) else: download( url, dir=path, unzip=args.unzip, delete=args.delete, threads=args.threads) if __name__ == '__main__': main() ================================================ FILE: tools/misc/gen_coco_panoptic_test_info.py ================================================ import argparse import os.path as osp from mmengine.fileio import dump, load def parse_args(): parser = argparse.ArgumentParser( description='Generate COCO test image information ' 'for COCO panoptic segmentation.') parser.add_argument('data_root', help='Path to COCO annotation directory.') args = parser.parse_args() return args def main(): args = parse_args() data_root = args.data_root val_info = load(osp.join(data_root, 'panoptic_val2017.json')) test_old_info = load(osp.join(data_root, 'image_info_test-dev2017.json')) # replace categories from image_info_test-dev2017.json # with categories from panoptic_val2017.json which # has attribute `isthing`. test_info = test_old_info test_info.update({'categories': val_info['categories']}) dump(test_info, osp.join(data_root, 'panoptic_image_info_test-dev2017.json')) if __name__ == '__main__': main() ================================================ FILE: tools/misc/get_crowdhuman_id_hw.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """Get image shape on CrowdHuman dataset. Here is an example to run this script. Example: python tools/misc/get_crowdhuman_id_hw.py ${CONFIG} \ --dataset ${DATASET_TYPE} """ import argparse import json import logging import os.path as osp from multiprocessing import Pool import mmcv from mmengine.config import Config from mmengine.fileio import FileClient, dump from mmengine.logging import print_log def parse_args(): parser = argparse.ArgumentParser(description='Collect image metas') parser.add_argument('config', help='Config file path') parser.add_argument( '--dataset', choices=['train', 'val'], help='Collect image metas from which dataset') parser.add_argument( '--nproc', default=10, type=int, help='Processes used for get image metas') args = parser.parse_args() return args def get_image_metas(anno_str, img_prefix): id_hw = {} file_client = FileClient(backend='disk') anno_dict = json.loads(anno_str) img_path = osp.join(img_prefix, f"{anno_dict['ID']}.jpg") img_id = anno_dict['ID'] img_bytes = file_client.get(img_path) img = mmcv.imfrombytes(img_bytes, backend='cv2') id_hw[img_id] = img.shape[:2] return id_hw def main(): args = parse_args() # get ann_file and img_prefix from config files cfg = Config.fromfile(args.config) file_client_args = cfg.get('file_client_args', dict(backend='disk')) file_client = FileClient(**file_client_args) dataset = args.dataset dataloader_cfg = cfg.get(f'{dataset}_dataloader') ann_file = osp.join(dataloader_cfg.dataset.data_root, dataloader_cfg.dataset.ann_file) img_prefix = osp.join(dataloader_cfg.dataset.data_root, dataloader_cfg.dataset.data_prefix['img']) # load image metas print_log( f'loading CrowdHuman {dataset} annotation...', level=logging.INFO) anno_strs = file_client.get_text(ann_file).strip().split('\n') pool = Pool(args.nproc) # get image metas with multiple processes id_hw_temp = pool.starmap( get_image_metas, zip(anno_strs, [img_prefix for _ in range(len(anno_strs))]), ) pool.close() # save image metas id_hw = {} for sub_dict in id_hw_temp: id_hw.update(sub_dict) data_root = osp.dirname(ann_file) save_path = osp.join(data_root, f'id_hw_{dataset}.json') print_log( f'\nsaving "id_hw_{dataset}.json" in "{data_root}"', level=logging.INFO) dump(id_hw, save_path, file_format='json') if __name__ == '__main__': main() ================================================ FILE: tools/misc/get_image_metas.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. """Get image metas on a specific dataset. Here is an example to run this script. Example: python tools/misc/get_image_metas.py ${CONFIG} \ --out ${OUTPUT FILE NAME} """ import argparse import csv import os.path as osp from multiprocessing import Pool import mmcv from mmengine.config import Config from mmengine.fileio import FileClient, dump def parse_args(): parser = argparse.ArgumentParser(description='Collect image metas') parser.add_argument('config', help='Config file path') parser.add_argument( '--dataset', default='val', choices=['train', 'val', 'test'], help='Collect image metas from which dataset') parser.add_argument( '--out', default='validation-image-metas.pkl', help='The output image metas file name. The save dir is in the ' 'same directory as `dataset.ann_file` path') parser.add_argument( '--nproc', default=4, type=int, help='Processes used for get image metas') args = parser.parse_args() return args def get_metas_from_csv_style_ann_file(ann_file): data_infos = [] cp_filename = None with open(ann_file, 'r') as f: reader = csv.reader(f) for i, line in enumerate(reader): if i == 0: continue img_id = line[0] filename = f'{img_id}.jpg' if filename != cp_filename: data_infos.append(dict(filename=filename)) cp_filename = filename return data_infos def get_metas_from_txt_style_ann_file(ann_file): with open(ann_file) as f: lines = f.readlines() i = 0 data_infos = [] while i < len(lines): filename = lines[i].rstrip() data_infos.append(dict(filename=filename)) skip_lines = int(lines[i + 2]) + 3 i += skip_lines return data_infos def get_image_metas(data_info, img_prefix): file_client = FileClient(backend='disk') filename = data_info.get('filename', None) if filename is not None: if img_prefix is not None: filename = osp.join(img_prefix, filename) img_bytes = file_client.get(filename) img = mmcv.imfrombytes(img_bytes, flag='color') shape = img.shape meta = dict(filename=filename, ori_shape=shape) else: raise NotImplementedError('Missing `filename` in data_info') return meta def main(): args = parse_args() assert args.out.endswith('pkl'), 'The output file name must be pkl suffix' # load config files cfg = Config.fromfile(args.config) dataloader_cfg = cfg.get(f'{args.dataset}_dataloader') ann_file = osp.join(dataloader_cfg.dataset.data_root, dataloader_cfg.dataset.ann_file) img_prefix = osp.join(dataloader_cfg.dataset.data_root, dataloader_cfg.dataset.data_prefix['img']) print(f'{"-" * 5} Start Processing {"-" * 5}') if ann_file.endswith('csv'): data_infos = get_metas_from_csv_style_ann_file(ann_file) elif ann_file.endswith('txt'): data_infos = get_metas_from_txt_style_ann_file(ann_file) else: shuffix = ann_file.split('.')[-1] raise NotImplementedError('File name must be csv or txt suffix but ' f'get {shuffix}') print(f'Successfully load annotation file from {ann_file}') print(f'Processing {len(data_infos)} images...') pool = Pool(args.nproc) # get image metas with multiple processes image_metas = pool.starmap( get_image_metas, zip(data_infos, [img_prefix for _ in range(len(data_infos))]), ) pool.close() # save image metas root_path = dataloader_cfg.dataset.ann_file.rsplit('/', 1)[0] save_path = osp.join(root_path, args.out) dump(image_metas, save_path, protocol=4) print(f'Image meta file save to: {save_path}') if __name__ == '__main__': main() ================================================ FILE: tools/misc/print_config.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os from mmengine import Config, DictAction from mmdet.utils import replace_cfg_vals, update_data_root def parse_args(): parser = argparse.ArgumentParser(description='Print the whole config') parser.add_argument('config', help='config file path') parser.add_argument( '--save-path', default=None, help='save path of whole config, suffixed with .py, .json or .yml') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') args = parser.parse_args() return args def main(): args = parse_args() cfg = Config.fromfile(args.config) # replace the ${key} with the value of cfg.key cfg = replace_cfg_vals(cfg) # update data root according to MMDET_DATASETS update_data_root(cfg) if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) print(f'Config:\n{cfg.pretty_text}') if args.save_path is not None: save_path = args.save_path suffix = os.path.splitext(save_path)[-1] assert suffix in ['.py', '.json', '.yml'] if not os.path.exists(os.path.split(save_path)[0]): os.makedirs(os.path.split(save_path)[0]) cfg.dump(save_path) print(f'Config saving at {save_path}') if __name__ == '__main__': main() ================================================ FILE: tools/misc/split_coco.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os.path as osp import numpy as np from mmengine.fileio import dump, load from mmengine.utils import mkdir_or_exist, track_parallel_progress prog_description = '''K-Fold coco split. To split coco data for semi-supervised object detection: python tools/misc/split_coco.py ''' def parse_args(): parser = argparse.ArgumentParser() parser.add_argument( '--data-root', type=str, help='The data root of coco dataset.', default='./data/coco/') parser.add_argument( '--out-dir', type=str, help='The output directory of coco semi-supervised annotations.', default='./data/coco/semi_anns/') parser.add_argument( '--labeled-percent', type=float, nargs='+', help='The percentage of labeled data in the training set.', default=[1, 2, 5, 10]) parser.add_argument( '--fold', type=int, help='K-fold cross validation for semi-supervised object detection.', default=5) args = parser.parse_args() return args def split_coco(data_root, out_dir, percent, fold): """Split COCO data for Semi-supervised object detection. Args: data_root (str): The data root of coco dataset. out_dir (str): The output directory of coco semi-supervised annotations. percent (float): The percentage of labeled data in the training set. fold (int): The fold of dataset and set as random seed for data split. """ def save_anns(name, images, annotations): sub_anns = dict() sub_anns['images'] = images sub_anns['annotations'] = annotations sub_anns['licenses'] = anns['licenses'] sub_anns['categories'] = anns['categories'] sub_anns['info'] = anns['info'] mkdir_or_exist(out_dir) dump(sub_anns, f'{out_dir}/{name}.json') # set random seed with the fold np.random.seed(fold) ann_file = osp.join(data_root, 'annotations/instances_train2017.json') anns = load(ann_file) image_list = anns['images'] labeled_total = int(percent / 100. * len(image_list)) labeled_inds = set( np.random.choice(range(len(image_list)), size=labeled_total)) labeled_ids, labeled_images, unlabeled_images = [], [], [] for i in range(len(image_list)): if i in labeled_inds: labeled_images.append(image_list[i]) labeled_ids.append(image_list[i]['id']) else: unlabeled_images.append(image_list[i]) # get all annotations of labeled images labeled_ids = set(labeled_ids) labeled_annotations, unlabeled_annotations = [], [] for ann in anns['annotations']: if ann['image_id'] in labeled_ids: labeled_annotations.append(ann) else: unlabeled_annotations.append(ann) # save labeled and unlabeled labeled_name = f'instances_train2017.{fold}@{percent}' unlabeled_name = f'instances_train2017.{fold}@{percent}-unlabeled' save_anns(labeled_name, labeled_images, labeled_annotations) save_anns(unlabeled_name, unlabeled_images, unlabeled_annotations) def multi_wrapper(args): return split_coco(*args) if __name__ == '__main__': args = parse_args() arguments_list = [(args.data_root, args.out_dir, p, f) for f in range(1, args.fold + 1) for p in args.labeled_percent] track_parallel_progress(multi_wrapper, arguments_list, args.fold) ================================================ FILE: tools/model_converters/detectron2_to_mmdet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse from collections import OrderedDict import torch from mmengine.fileio import load from mmengine.runner import save_checkpoint def convert(src: str, dst: str, prefix: str = 'd2_model') -> None: """Convert Detectron2 checkpoint to MMDetection style. Args: src (str): The Detectron2 checkpoint path, should endswith `pkl`. dst (str): The MMDetection checkpoint path. prefix (str): The prefix of MMDetection model, defaults to 'd2_model'. """ # load arch_settings assert src.endswith('pkl'), \ 'the source Detectron2 checkpoint should endswith `pkl`.' d2_model = load(src, encoding='latin1').get('model') assert d2_model is not None # convert to mmdet style dst_state_dict = OrderedDict() for name, value in d2_model.items(): if not isinstance(value, torch.Tensor): value = torch.from_numpy(value) dst_state_dict[f'{prefix}.{name}'] = value mmdet_model = dict(state_dict=dst_state_dict, meta=dict()) save_checkpoint(mmdet_model, dst) print(f'Convert Detectron2 model {src} to MMDetection model {dst}') def main(): parser = argparse.ArgumentParser( description='Convert Detectron2 checkpoint to MMDetection style') parser.add_argument('src', help='Detectron2 model path') parser.add_argument('dst', help='MMDetectron model save path') parser.add_argument( '--prefix', default='d2_model', type=str, help='prefix of the model') args = parser.parse_args() convert(args.src, args.dst, args.prefix) if __name__ == '__main__': main() ================================================ FILE: tools/model_converters/detectron2pytorch.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse from collections import OrderedDict import torch from mmengine.fileio import load arch_settings = {50: (3, 4, 6, 3), 101: (3, 4, 23, 3)} def convert_bn(blobs, state_dict, caffe_name, torch_name, converted_names): # detectron replace bn with affine channel layer state_dict[torch_name + '.bias'] = torch.from_numpy(blobs[caffe_name + '_b']) state_dict[torch_name + '.weight'] = torch.from_numpy(blobs[caffe_name + '_s']) bn_size = state_dict[torch_name + '.weight'].size() state_dict[torch_name + '.running_mean'] = torch.zeros(bn_size) state_dict[torch_name + '.running_var'] = torch.ones(bn_size) converted_names.add(caffe_name + '_b') converted_names.add(caffe_name + '_s') def convert_conv_fc(blobs, state_dict, caffe_name, torch_name, converted_names): state_dict[torch_name + '.weight'] = torch.from_numpy(blobs[caffe_name + '_w']) converted_names.add(caffe_name + '_w') if caffe_name + '_b' in blobs: state_dict[torch_name + '.bias'] = torch.from_numpy(blobs[caffe_name + '_b']) converted_names.add(caffe_name + '_b') def convert(src, dst, depth): """Convert keys in detectron pretrained ResNet models to pytorch style.""" # load arch_settings if depth not in arch_settings: raise ValueError('Only support ResNet-50 and ResNet-101 currently') block_nums = arch_settings[depth] # load caffe model caffe_model = load(src, encoding='latin1') blobs = caffe_model['blobs'] if 'blobs' in caffe_model else caffe_model # convert to pytorch style state_dict = OrderedDict() converted_names = set() convert_conv_fc(blobs, state_dict, 'conv1', 'conv1', converted_names) convert_bn(blobs, state_dict, 'res_conv1_bn', 'bn1', converted_names) for i in range(1, len(block_nums) + 1): for j in range(block_nums[i - 1]): if j == 0: convert_conv_fc(blobs, state_dict, f'res{i + 1}_{j}_branch1', f'layer{i}.{j}.downsample.0', converted_names) convert_bn(blobs, state_dict, f'res{i + 1}_{j}_branch1_bn', f'layer{i}.{j}.downsample.1', converted_names) for k, letter in enumerate(['a', 'b', 'c']): convert_conv_fc(blobs, state_dict, f'res{i + 1}_{j}_branch2{letter}', f'layer{i}.{j}.conv{k+1}', converted_names) convert_bn(blobs, state_dict, f'res{i + 1}_{j}_branch2{letter}_bn', f'layer{i}.{j}.bn{k + 1}', converted_names) # check if all layers are converted for key in blobs: if key not in converted_names: print(f'Not Convert: {key}') # save checkpoint checkpoint = dict() checkpoint['state_dict'] = state_dict torch.save(checkpoint, dst) def main(): parser = argparse.ArgumentParser(description='Convert model keys') parser.add_argument('src', help='src detectron model path') parser.add_argument('dst', help='save path') parser.add_argument('depth', type=int, help='ResNet model depth') args = parser.parse_args() convert(args.src, args.dst, args.depth) if __name__ == '__main__': main() ================================================ FILE: tools/model_converters/publish_model.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import subprocess import torch from mmengine.logging import print_log def parse_args(): parser = argparse.ArgumentParser( description='Process a checkpoint to be published') parser.add_argument('in_file', help='input checkpoint filename') parser.add_argument('out_file', help='output checkpoint filename') parser.add_argument( '--save-keys', nargs='+', type=str, default=['meta', 'state_dict'], help='keys to save in the published checkpoint') args = parser.parse_args() return args def process_checkpoint(in_file, out_file, save_keys=['meta', 'state_dict']): checkpoint = torch.load(in_file, map_location='cpu') # only keep `meta` and `state_dict` for smaller file size ckpt_keys = list(checkpoint.keys()) for k in ckpt_keys: if k not in save_keys: print_log( f'Key `{k}` will be removed because it is not in ' f'save_keys. If you want to keep it, ' f'please set --save-keys.', logger='current') checkpoint.pop(k, None) # if it is necessary to remove some sensitive data in checkpoint['meta'], # add the code here. if torch.__version__ >= '1.6': torch.save(checkpoint, out_file, _use_new_zipfile_serialization=False) else: torch.save(checkpoint, out_file) sha = subprocess.check_output(['sha256sum', out_file]).decode() if out_file.endswith('.pth'): out_file_name = out_file[:-4] else: out_file_name = out_file final_file = out_file_name + f'-{sha[:8]}.pth' subprocess.Popen(['mv', out_file, final_file]) print_log( f'The published model is saved at {final_file}.', logger='current') def main(): args = parse_args() process_checkpoint(args.in_file, args.out_file, args.save_keys) if __name__ == '__main__': main() ================================================ FILE: tools/model_converters/regnet2mmdet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse from collections import OrderedDict import torch def convert_stem(model_key, model_weight, state_dict, converted_names): new_key = model_key.replace('stem.conv', 'conv1') new_key = new_key.replace('stem.bn', 'bn1') state_dict[new_key] = model_weight converted_names.add(model_key) print(f'Convert {model_key} to {new_key}') def convert_head(model_key, model_weight, state_dict, converted_names): new_key = model_key.replace('head.fc', 'fc') state_dict[new_key] = model_weight converted_names.add(model_key) print(f'Convert {model_key} to {new_key}') def convert_reslayer(model_key, model_weight, state_dict, converted_names): split_keys = model_key.split('.') layer, block, module = split_keys[:3] block_id = int(block[1:]) layer_name = f'layer{int(layer[1:])}' block_name = f'{block_id - 1}' if block_id == 1 and module == 'bn': new_key = f'{layer_name}.{block_name}.downsample.1.{split_keys[-1]}' elif block_id == 1 and module == 'proj': new_key = f'{layer_name}.{block_name}.downsample.0.{split_keys[-1]}' elif module == 'f': if split_keys[3] == 'a_bn': module_name = 'bn1' elif split_keys[3] == 'b_bn': module_name = 'bn2' elif split_keys[3] == 'c_bn': module_name = 'bn3' elif split_keys[3] == 'a': module_name = 'conv1' elif split_keys[3] == 'b': module_name = 'conv2' elif split_keys[3] == 'c': module_name = 'conv3' new_key = f'{layer_name}.{block_name}.{module_name}.{split_keys[-1]}' else: raise ValueError(f'Unsupported conversion of key {model_key}') print(f'Convert {model_key} to {new_key}') state_dict[new_key] = model_weight converted_names.add(model_key) def convert(src, dst): """Convert keys in pycls pretrained RegNet models to mmdet style.""" # load caffe model regnet_model = torch.load(src) blobs = regnet_model['model_state'] # convert to pytorch style state_dict = OrderedDict() converted_names = set() for key, weight in blobs.items(): if 'stem' in key: convert_stem(key, weight, state_dict, converted_names) elif 'head' in key: convert_head(key, weight, state_dict, converted_names) elif key.startswith('s'): convert_reslayer(key, weight, state_dict, converted_names) # check if all layers are converted for key in blobs: if key not in converted_names: print(f'not converted: {key}') # save checkpoint checkpoint = dict() checkpoint['state_dict'] = state_dict torch.save(checkpoint, dst) def main(): parser = argparse.ArgumentParser(description='Convert model keys') parser.add_argument('src', help='src detectron model path') parser.add_argument('dst', help='save path') args = parser.parse_args() convert(args.src, args.dst) if __name__ == '__main__': main() ================================================ FILE: tools/model_converters/selfsup2mmdet.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse from collections import OrderedDict import torch def moco_convert(src, dst): """Convert keys in pycls pretrained moco models to mmdet style.""" # load caffe model moco_model = torch.load(src) blobs = moco_model['state_dict'] # convert to pytorch style state_dict = OrderedDict() for k, v in blobs.items(): if not k.startswith('module.encoder_q.'): continue old_k = k k = k.replace('module.encoder_q.', '') state_dict[k] = v print(old_k, '->', k) # save checkpoint checkpoint = dict() checkpoint['state_dict'] = state_dict torch.save(checkpoint, dst) def main(): parser = argparse.ArgumentParser(description='Convert model keys') parser.add_argument('src', help='src detectron model path') parser.add_argument('dst', help='save path') parser.add_argument( '--selfsup', type=str, choices=['moco', 'swav'], help='save path') args = parser.parse_args() if args.selfsup == 'moco': moco_convert(args.src, args.dst) elif args.selfsup == 'swav': print('SWAV does not need to convert the keys') if __name__ == '__main__': main() ================================================ FILE: tools/model_converters/upgrade_model_version.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import re import tempfile from collections import OrderedDict import torch from mmengine import Config def is_head(key): valid_head_list = [ 'bbox_head', 'mask_head', 'semantic_head', 'grid_head', 'mask_iou_head' ] return any(key.startswith(h) for h in valid_head_list) def parse_config(config_strings): temp_file = tempfile.NamedTemporaryFile() config_path = f'{temp_file.name}.py' with open(config_path, 'w') as f: f.write(config_strings) config = Config.fromfile(config_path) is_two_stage = True is_ssd = False is_retina = False reg_cls_agnostic = False if 'rpn_head' not in config.model: is_two_stage = False # check whether it is SSD if config.model.bbox_head.type == 'SSDHead': is_ssd = True elif config.model.bbox_head.type == 'RetinaHead': is_retina = True elif isinstance(config.model['bbox_head'], list): reg_cls_agnostic = True elif 'reg_class_agnostic' in config.model.bbox_head: reg_cls_agnostic = config.model.bbox_head \ .reg_class_agnostic temp_file.close() return is_two_stage, is_ssd, is_retina, reg_cls_agnostic def reorder_cls_channel(val, num_classes=81): # bias if val.dim() == 1: new_val = torch.cat((val[1:], val[:1]), dim=0) # weight else: out_channels, in_channels = val.shape[:2] # conv_cls for softmax output if out_channels != num_classes and out_channels % num_classes == 0: new_val = val.reshape(-1, num_classes, in_channels, *val.shape[2:]) new_val = torch.cat((new_val[:, 1:], new_val[:, :1]), dim=1) new_val = new_val.reshape(val.size()) # fc_cls elif out_channels == num_classes: new_val = torch.cat((val[1:], val[:1]), dim=0) # agnostic | retina_cls | rpn_cls else: new_val = val return new_val def truncate_cls_channel(val, num_classes=81): # bias if val.dim() == 1: if val.size(0) % num_classes == 0: new_val = val[:num_classes - 1] else: new_val = val # weight else: out_channels, in_channels = val.shape[:2] # conv_logits if out_channels % num_classes == 0: new_val = val.reshape(num_classes, in_channels, *val.shape[2:])[1:] new_val = new_val.reshape(-1, *val.shape[1:]) # agnostic else: new_val = val return new_val def truncate_reg_channel(val, num_classes=81): # bias if val.dim() == 1: # fc_reg | rpn_reg if val.size(0) % num_classes == 0: new_val = val.reshape(num_classes, -1)[:num_classes - 1] new_val = new_val.reshape(-1) # agnostic else: new_val = val # weight else: out_channels, in_channels = val.shape[:2] # fc_reg | rpn_reg if out_channels % num_classes == 0: new_val = val.reshape(num_classes, -1, in_channels, *val.shape[2:])[1:] new_val = new_val.reshape(-1, *val.shape[1:]) # agnostic else: new_val = val return new_val def convert(in_file, out_file, num_classes): """Convert keys in checkpoints. There can be some breaking changes during the development of mmdetection, and this tool is used for upgrading checkpoints trained with old versions to the latest one. """ checkpoint = torch.load(in_file) in_state_dict = checkpoint.pop('state_dict') out_state_dict = OrderedDict() meta_info = checkpoint['meta'] is_two_stage, is_ssd, is_retina, reg_cls_agnostic = parse_config( '#' + meta_info['config']) if meta_info['mmdet_version'] <= '0.5.3' and is_retina: upgrade_retina = True else: upgrade_retina = False # MMDetection v2.5.0 unifies the class order in RPN # if the model is trained in version=2.5.0 if meta_info['mmdet_version'] < '2.5.0': upgrade_rpn = True else: upgrade_rpn = False for key, val in in_state_dict.items(): new_key = key new_val = val if is_two_stage and is_head(key): new_key = 'roi_head.{}'.format(key) # classification if upgrade_rpn: m = re.search( r'(conv_cls|retina_cls|rpn_cls|fc_cls|fcos_cls|' r'fovea_cls).(weight|bias)', new_key) else: m = re.search( r'(conv_cls|retina_cls|fc_cls|fcos_cls|' r'fovea_cls).(weight|bias)', new_key) if m is not None: print(f'reorder cls channels of {new_key}') new_val = reorder_cls_channel(val, num_classes) # regression if upgrade_rpn: m = re.search(r'(fc_reg).(weight|bias)', new_key) else: m = re.search(r'(fc_reg|rpn_reg).(weight|bias)', new_key) if m is not None and not reg_cls_agnostic: print(f'truncate regression channels of {new_key}') new_val = truncate_reg_channel(val, num_classes) # mask head m = re.search(r'(conv_logits).(weight|bias)', new_key) if m is not None: print(f'truncate mask prediction channels of {new_key}') new_val = truncate_cls_channel(val, num_classes) m = re.search(r'(cls_convs|reg_convs).\d.(weight|bias)', key) # Legacy issues in RetinaNet since V1.x # Use ConvModule instead of nn.Conv2d in RetinaNet # cls_convs.0.weight -> cls_convs.0.conv.weight if m is not None and upgrade_retina: param = m.groups()[1] new_key = key.replace(param, f'conv.{param}') out_state_dict[new_key] = val print(f'rename the name of {key} to {new_key}') continue m = re.search(r'(cls_convs).\d.(weight|bias)', key) if m is not None and is_ssd: print(f'reorder cls channels of {new_key}') new_val = reorder_cls_channel(val, num_classes) out_state_dict[new_key] = new_val checkpoint['state_dict'] = out_state_dict torch.save(checkpoint, out_file) def main(): parser = argparse.ArgumentParser(description='Upgrade model version') parser.add_argument('in_file', help='input checkpoint file') parser.add_argument('out_file', help='output checkpoint file') parser.add_argument( '--num-classes', type=int, default=81, help='number of classes of the original model') args = parser.parse_args() convert(args.in_file, args.out_file, args.num_classes) if __name__ == '__main__': main() ================================================ FILE: tools/model_converters/upgrade_ssd_version.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import tempfile from collections import OrderedDict import torch from mmengine import Config def parse_config(config_strings): temp_file = tempfile.NamedTemporaryFile() config_path = f'{temp_file.name}.py' with open(config_path, 'w') as f: f.write(config_strings) config = Config.fromfile(config_path) # check whether it is SSD if config.model.bbox_head.type != 'SSDHead': raise AssertionError('This is not a SSD model.') def convert(in_file, out_file): checkpoint = torch.load(in_file) in_state_dict = checkpoint.pop('state_dict') out_state_dict = OrderedDict() meta_info = checkpoint['meta'] parse_config('#' + meta_info['config']) for key, value in in_state_dict.items(): if 'extra' in key: layer_idx = int(key.split('.')[2]) new_key = 'neck.extra_layers.{}.{}.conv.'.format( layer_idx // 2, layer_idx % 2) + key.split('.')[-1] elif 'l2_norm' in key: new_key = 'neck.l2_norm.weight' elif 'bbox_head' in key: new_key = key[:21] + '.0' + key[21:] else: new_key = key out_state_dict[new_key] = value checkpoint['state_dict'] = out_state_dict if torch.__version__ >= '1.6': torch.save(checkpoint, out_file, _use_new_zipfile_serialization=False) else: torch.save(checkpoint, out_file) def main(): parser = argparse.ArgumentParser(description='Upgrade SSD version') parser.add_argument('in_file', help='input checkpoint file') parser.add_argument('out_file', help='output checkpoint file') args = parser.parse_args() convert(args.in_file, args.out_file) if __name__ == '__main__': main() ================================================ FILE: tools/slurm_test.sh ================================================ #!/usr/bin/env bash set -x PARTITION=$1 JOB_NAME=$2 CONFIG=$3 CHECKPOINT=$4 GPUS=${GPUS:-8} GPUS_PER_NODE=${GPUS_PER_NODE:-8} CPUS_PER_TASK=${CPUS_PER_TASK:-5} PY_ARGS=${@:5} SRUN_ARGS=${SRUN_ARGS:-""} PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ srun -p ${PARTITION} \ --job-name=${JOB_NAME} \ --gres=gpu:${GPUS_PER_NODE} \ --ntasks=${GPUS} \ --ntasks-per-node=${GPUS_PER_NODE} \ --cpus-per-task=${CPUS_PER_TASK} \ --kill-on-bad-exit=1 \ ${SRUN_ARGS} \ python -u tools/test.py ${CONFIG} ${CHECKPOINT} --launcher="slurm" ${PY_ARGS} ================================================ FILE: tools/slurm_train.sh ================================================ #!/usr/bin/env bash set -x PARTITION=$1 JOB_NAME=$2 CONFIG=$3 WORK_DIR=$4 GPUS=${GPUS:-8} GPUS_PER_NODE=${GPUS_PER_NODE:-8} CPUS_PER_TASK=${CPUS_PER_TASK:-5} SRUN_ARGS=${SRUN_ARGS:-""} PY_ARGS=${@:5} PYTHONPATH="$(dirname $0)/..":$PYTHONPATH \ srun -p ${PARTITION} \ --job-name=${JOB_NAME} \ --gres=gpu:${GPUS_PER_NODE} \ --ntasks=${GPUS} \ --ntasks-per-node=${GPUS_PER_NODE} \ --cpus-per-task=${CPUS_PER_TASK} \ --kill-on-bad-exit=1 \ ${SRUN_ARGS} \ python -u tools/train.py ${CONFIG} --work-dir=${WORK_DIR} --launcher="slurm" ${PY_ARGS} ================================================ FILE: tools/test.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import os import os.path as osp import warnings from copy import deepcopy from mmengine import ConfigDict from mmengine.config import Config, DictAction from mmengine.runner import Runner from mmdet.engine.hooks.utils import trigger_visualization_hook from mmdet.evaluation import DumpDetResults from mmdet.registry import RUNNERS # TODO: support fuse_conv_bn and format_only def parse_args(): parser = argparse.ArgumentParser( description='MMDet test (and eval) a model') parser.add_argument('config', help='test config file path') parser.add_argument('checkpoint', help='checkpoint file') parser.add_argument( '--work-dir', help='the directory to save the file containing evaluation metrics') parser.add_argument( '--out', type=str, help='dump predictions to a pickle file for offline evaluation') parser.add_argument( '--show', action='store_true', help='show prediction results') parser.add_argument( '--show-dir', help='directory where painted images will be saved. ' 'If specified, it will be automatically saved ' 'to the work_dir/timestamp/show_dir') parser.add_argument( '--wait-time', type=float, default=2, help='the interval of show (s)') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') parser.add_argument( '--launcher', choices=['none', 'pytorch', 'slurm', 'mpi'], default='none', help='job launcher') parser.add_argument('--tta', action='store_true') parser.add_argument('--local_rank', type=int, default=0) args = parser.parse_args() if 'LOCAL_RANK' not in os.environ: os.environ['LOCAL_RANK'] = str(args.local_rank) return args def main(): args = parse_args() # load config cfg = Config.fromfile(args.config) cfg.launcher = args.launcher if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) # work_dir is determined in this priority: CLI > segment in file > filename if args.work_dir is not None: # update configs according to CLI args if args.work_dir is not None cfg.work_dir = args.work_dir elif cfg.get('work_dir', None) is None: # use config filename as default work_dir if cfg.work_dir is None cfg.work_dir = osp.join('./work_dirs', osp.splitext(osp.basename(args.config))[0]) cfg.load_from = args.checkpoint if args.show or args.show_dir: cfg = trigger_visualization_hook(cfg, args) if args.tta: if 'tta_model' not in cfg: warnings.warn('Cannot find ``tta_model`` in config, ' 'we will set it as default.') cfg.tta_model = dict( type='DetTTAModel', tta_cfg=dict( nms=dict(type='nms', iou_threshold=0.5), max_per_img=100)) if 'tta_pipeline' not in cfg: warnings.warn('Cannot find ``tta_pipeline`` in config, ' 'we will set it as default.') test_data_cfg = cfg.test_dataloader.dataset while 'dataset' in test_data_cfg: test_data_cfg = test_data_cfg['dataset'] cfg.tta_pipeline = deepcopy(test_data_cfg.pipeline) flip_tta = dict( type='TestTimeAug', transforms=[ [ dict(type='RandomFlip', prob=1.), dict(type='RandomFlip', prob=0.) ], [ dict( type='PackDetInputs', meta_keys=('img_id', 'img_path', 'ori_shape', 'img_shape', 'scale_factor', 'flip', 'flip_direction')) ], ]) cfg.tta_pipeline[-1] = flip_tta cfg.model = ConfigDict(**cfg.tta_model, module=cfg.model) cfg.test_dataloader.dataset.pipeline = cfg.tta_pipeline # build the runner from config if 'runner_type' not in cfg: # build the default runner runner = Runner.from_cfg(cfg) else: # build customized runner from the registry # if 'runner_type' is set in the cfg runner = RUNNERS.build(cfg) # add `DumpResults` dummy metric if args.out is not None: assert args.out.endswith(('.pkl', '.pickle')), \ 'The dump file must be a pkl file.' runner.test_evaluator.metrics.append( DumpDetResults(out_file_path=args.out)) # start testing runner.test() if __name__ == '__main__': main() ================================================ FILE: tools/train.py ================================================ # Copyright (c) OpenMMLab. All rights reserved. import argparse import logging import os import os.path as osp from mmengine.config import Config, DictAction from mmengine.logging import print_log from mmengine.registry import RUNNERS from mmengine.runner import Runner def parse_args(): parser = argparse.ArgumentParser(description='Train a detector') parser.add_argument('config', help='train config file path') parser.add_argument('--work-dir', help='the dir to save logs and models') parser.add_argument( '--amp', action='store_true', default=False, help='enable automatic-mixed-precision training') parser.add_argument( '--auto-scale-lr', action='store_true', help='enable automatically scaling LR.') parser.add_argument( '--resume', nargs='?', type=str, const='auto', help='If specify checkpoint path, resume from it, while if not ' 'specify, try to auto resume from the latest checkpoint ' 'in the work directory.') parser.add_argument( '--cfg-options', nargs='+', action=DictAction, help='override some settings in the used config, the key-value pair ' 'in xxx=yyy format will be merged into config file. If the value to ' 'be overwritten is a list, it should be like key="[a,b]" or key=a,b ' 'It also allows nested list/tuple values, e.g. key="[(a,b),(c,d)]" ' 'Note that the quotation marks are necessary and that no white space ' 'is allowed.') parser.add_argument( '--launcher', choices=['none', 'pytorch', 'slurm', 'mpi'], default='none', help='job launcher') parser.add_argument('--local_rank', type=int, default=0) args = parser.parse_args() if 'LOCAL_RANK' not in os.environ: os.environ['LOCAL_RANK'] = str(args.local_rank) return args def main(): args = parse_args() # load config cfg = Config.fromfile(args.config) cfg.launcher = args.launcher if args.cfg_options is not None: cfg.merge_from_dict(args.cfg_options) # work_dir is determined in this priority: CLI > segment in file > filename if args.work_dir is not None: # update configs according to CLI args if args.work_dir is not None cfg.work_dir = args.work_dir elif cfg.get('work_dir', None) is None: # use config filename as default work_dir if cfg.work_dir is None cfg.work_dir = osp.join('./work_dirs', osp.splitext(osp.basename(args.config))[0]) # enable automatic-mixed-precision training if args.amp is True: optim_wrapper = cfg.optim_wrapper.type if optim_wrapper == 'AmpOptimWrapper': print_log( 'AMP training is already enabled in your config.', logger='current', level=logging.WARNING) else: assert optim_wrapper == 'OptimWrapper', ( '`--amp` is only supported when the optimizer wrapper type is ' f'`OptimWrapper` but got {optim_wrapper}.') cfg.optim_wrapper.type = 'AmpOptimWrapper' cfg.optim_wrapper.loss_scale = 'dynamic' # enable automatically scaling LR if args.auto_scale_lr: if 'auto_scale_lr' in cfg and \ 'enable' in cfg.auto_scale_lr and \ 'base_batch_size' in cfg.auto_scale_lr: cfg.auto_scale_lr.enable = True else: raise RuntimeError('Can not find "auto_scale_lr" or ' '"auto_scale_lr.enable" or ' '"auto_scale_lr.base_batch_size" in your' ' configuration file.') # resume is determined in this priority: resume from > auto_resume if args.resume == 'auto': cfg.resume = True cfg.load_from = None elif args.resume is not None: cfg.resume = True cfg.load_from = args.resume # build the runner from config if 'runner_type' not in cfg: # build the default runner runner = Runner.from_cfg(cfg) else: # build customized runner from the registry # if 'runner_type' is set in the cfg runner = RUNNERS.build(cfg) # start training runner.train() if __name__ == '__main__': main()